Upgrade to Python 3.8 + duplicate check when adding in sdp

sdp/sheerkaDataProvider.py

@@ -52,16 +52,61 @@ class Event(object):
         self.message = as_dict["message"]
 
 
-class ObjWithKey:
+class ObjToUpdate:
     """
     Internal key value class to hold the key (and the value)
     when it is detected
     It's created to distinguish from {key, value}
     """
 
-    def __init__(self, key, obj):
-        self.key = key
+    def __init__(self, obj, key=None, digest=None):
         self.obj = obj
+        self.has_key = None
+        self.has_digest = None
+        self._key = None
+        self._digest = None
+        if key is not None:
+            self.set_key(key)
+        if digest is not None:
+            self.set_digest(digest)
+
+    def get_key(self):
+        if self.has_key is None:
+            key = SheerkaDataProvider.get_obj_key(self.obj)
+            if key is None:
+                self.has_key = False
+                return None
+            else:
+                self.has_key = True
+                self._key = key
+                return key
+        elif not self.has_key:
+            return None
+        else:
+            return self._key
+
+    def get_digest(self):
+        if self.has_digest is None:
+            digest = SheerkaDataProvider.get_obj_digest(self.obj)
+            if digest is None:
+                self.has_digest = False
+                return None
+            else:
+                self.has_digest = True
+                self._digest = digest
+                return digest
+        elif not self.has_digest:
+            return None
+        else:
+            return self._digest
+
+    def set_digest(self, digest):
+        self.has_digest = True
+        self._digest = digest
+
+    def set_key(self, key):
+        self.has_key = True
+        self._key = key
 
 
 class State:
@@ -76,15 +121,29 @@ class State:
         self.events = []
         self.data = {}
 
-    def update(self, entry, obj, append=True):
+    @staticmethod
+    def check_duplicate(items, obj: ObjToUpdate, key):
+        digest = obj.get_digest()
+        if digest is None:
+            return
+
+        if not isinstance(items, list):
+            items = [items]
+
+        for item in items:
+            item_digest = SheerkaDataProvider.get_obj_digest(item)
+            if item_digest == digest:
+                raise SheerkaDataProviderDuplicateKeyError("duplicate key", key, obj.obj)
+
+    def update(self, entry, obj: ObjToUpdate, append=True):
         """
         adds obj to entry
         :param entry:
         :param obj:
-        :param append: if True, ducplicate keys will create lists
+        :param append: if True, duplicate keys will create lists
         :return:
         """
-        obj_to_use = {obj.key: obj.obj} if isinstance(obj, ObjWithKey) else obj
+        obj_to_use = {obj.get_key(): obj.obj} if obj.has_key else obj.obj
 
         if entry not in self.data:
             self.data[entry] = obj_to_use
@@ -96,21 +155,25 @@ class State:
                 self.data[entry] = obj_to_use
 
         elif isinstance(self.data[entry], list):
-            self.data[entry].append(obj.obj if isinstance(obj, ObjWithKey) else obj)  # do not use obj_to_use !
+            self.check_duplicate(self.data[entry], obj, entry)
+            self.data[entry].append(obj.obj)
 
         elif isinstance(obj_to_use, dict):
             for k in obj_to_use:
                 if k not in self.data[entry]:
                     self.data[entry][k] = obj_to_use[k]
                 elif isinstance(self.data[entry][k], list):
+                    self.check_duplicate(self.data[entry][k], obj, entry + "." + k)
                     self.data[entry][k].append(obj_to_use[k])
                 else:
+                    self.check_duplicate(self.data[entry][k], obj, entry + "." + k)
                     self.data[entry][k] = [self.data[entry][k], obj_to_use[k]]
 
         elif isinstance(self.data[entry], dict):
-            raise SheerkaDataProviderError(f"Cannot found key on '{obj}' while all other elements have.", obj)
+            raise SheerkaDataProviderError(f"Cannot found key on '{obj.obj}' while all other elements have.", obj.obj)
 
         else:
+            self.check_duplicate(self.data[entry], obj, entry)
             self.data[entry] = [self.data[entry], obj_to_use]
 
     def modify(self, entry, key, obj, obj_key):
@@ -120,7 +183,7 @@ class State:
             self.remove(entry, lambda k, o: k == key)  # modify from on object to another
             append = True
 
-        self.update(entry, ObjWithKey(obj_key, obj), append=append)
+        self.update(entry, ObjToUpdate(obj, obj_key), append=append)
 
     def modify_in_list(self, entry, key, obj, obj_key, obj_origin, load_ref_if_needed, save_ref_if_needed):
         found = False
@@ -135,7 +198,7 @@ class State:
                     self.data[entry][key][i] = obj
                 else:
                     to_remove = i
-                    self.update(entry, ObjWithKey(obj_key, obj), append=True)
+                    self.update(entry, ObjToUpdate(obj, obj_key), append=True)
                 found = True
                 break
 
@@ -194,6 +257,13 @@ class SheerkaDataProviderError(Exception):
         self.obj = obj
 
 
+class SheerkaDataProviderDuplicateKeyError(Exception):
+    def __init__(self, message, key, obj):
+        Exception.__init__(self, message)
+        self.key = key
+        self.obj = obj
+
+
 class SheerkaDataProvider:
     """Manages the state of the system"""
 
@@ -233,7 +303,24 @@ class SheerkaDataProvider:
         :param obj:
         :return: String version of that is found, None otherwise
         """
-        return str(obj.key) if hasattr(obj, "key") else str(obj.get_key()) if hasattr(obj, "get_key") else None
+        return str(obj.key) if hasattr(obj, "key") \
+            else str(obj.get_key()) if hasattr(obj, "get_key") \
+            else None
+
+    @staticmethod
+    def get_obj_digest(obj):
+        """
+        Tries to find the key of an object
+        Look for .digest, .get_digest()
+        :param obj:
+        :return: digest, None otherwise
+        """
+        if isinstance(obj, str) and obj.startswith(SheerkaDataProvider.REF_PREFIX):
+            return obj[len(SheerkaDataProvider.REF_PREFIX):]
+
+        return obj.digest if hasattr(obj, "digest") \
+            else obj.get_digest() if hasattr(obj, "get_digest") \
+            else None
 
     @staticmethod
     def get_stream_digest(stream):
@@ -266,12 +353,15 @@ class SheerkaDataProvider:
 
         log.debug(f"Adding obj '{obj}' in entry '{entry}' (allow_multiple={allow_multiple}, use_ref={use_ref})")
 
+        if not isinstance(obj, ObjToUpdate):
+            obj = ObjToUpdate(obj)
+
         # check uniqueness, cannot add the same key twice if allow_multiple == False
-        key = self.get_obj_key(obj)
+        key = obj.get_key()
         log.debug(f"key found : '{key}'") if key else log.debug("No key found")
         if not allow_multiple:
-            if isinstance(obj, dict):
-                for k in obj:
+            if isinstance(obj.obj, dict):
+                for k in obj.obj:
                     if state.contains(entry, k):
                         raise IndexError(f"{entry}.{k}")
             else:
@@ -284,10 +374,10 @@ class SheerkaDataProvider:
         log.debug(state.data)
 
         if use_ref:
-            digest = self.save_obj(obj)
-            obj = ObjWithKey(key, self.REF_PREFIX + digest) if key else self.REF_PREFIX + digest
+            obj.set_digest(self.save_obj(obj.obj))
+            obj.obj = self.REF_PREFIX + obj.get_digest()
 
-        state.update(entry, obj if (isinstance(obj, ObjWithKey) or key is None) else ObjWithKey(key, obj))
+        state.update(entry, obj)
 
         new_snapshot = self.save_state(state)
         self.set_snapshot(new_snapshot)
@@ -304,7 +394,7 @@ class SheerkaDataProvider:
         next_key = self.get_next_key(entry)
         if hasattr(obj, "set_key"):
             obj.set_key(next_key)
-        self.add(event_digest, entry, ObjWithKey(next_key, obj))
+        self.add(event_digest, entry, ObjToUpdate(obj, next_key))
         return entry, next_key
 
     def add_unique(self, event_digest: str, entry, obj):