Implemented SheerkaOntology

src/cache/BaseCache.py

@@ -1,5 +1,8 @@
 from threading import RLock
 
+from core.global_symbols import NotFound, Removed
+from core.utils import sheerka_deepcopy
+
 MAX_INITIALIZED_KEY = 100
 
 
@@ -10,14 +13,17 @@ class BaseCache:
     When you put the same key twice, the previous element is overridden
     """
 
-    def __init__(self, max_size=None, default=None, extend_exists=None):
+    def __init__(self, max_size=None, default=NotFound, extend_exists=None, alt_sdp_get=None, sdp=None):
         self._cache = {}
         self._max_size = max_size
         self._default = default  # default value to return when key is not found. It can be a callable of key
         self._extend_exists = extend_exists  # search in remote
+        self._alt_sdp_get = alt_sdp_get  # How to get the value when called by alt_sdp
+        self._sdp = sdp  # current instance of SheerkaDataProvider
         self._lock = RLock()
         self._current_size = 0
         self._initialized_keys = set()  # to keep the list of the keys already requested (using get())
+        self._is_cleared = False  # indicate that clear() was called
 
         self.to_add = set()
         self.to_remove = set()
@@ -49,42 +55,75 @@ class BaseCache:
     def __repr__(self):
         return f"{self.__class__.__name__}(size={self._current_size}, #keys={len(self._cache)})"
 
-    def configure(self, max_size=None, default=None, extend_exists=None):
+    def configure(self, max_size=None, default=NotFound, extend_exists=None, alt_sdp_get=None, sdp=None):
         if max_size is not None:
             self._max_size = max_size
 
-        if default is not None:
+        if default is not NotFound:
             self._default = default
 
         if extend_exists is not None:
             self._extend_exists = extend_exists
 
-    def disable_default(self):
-        self._default = None
+        if alt_sdp_get is not None:
+            self._alt_sdp_get = alt_sdp_get
 
-    def put(self, key, value):
+        if sdp is not None:
+            self._sdp = sdp
+
+        return self
+
+    def auto_configure(self, cache_name):
+        """
+        Convenient way to configure the cache
+        :param cache_name:
+        :return:
+        """
+        self._default = lambda sdp, key: sdp.get(cache_name, key)
+        self._extend_exists = lambda sdp, key: sdp.exists(cache_name, key)
+        self._alt_sdp_get = lambda sdp, key: sdp.alt_get(cache_name, key)  # by default, same than get
+
+        return self
+
+    def disable_default(self):
+        self._default = (lambda sdp, key: NotFound) if self._sdp else (lambda key: NotFound)
+
+    def put(self, key, value, alt_sdp=None):
         """
         Add a new entry in cache
         :param key:
         :param value:
+        :param alt_sdp:
         :return:
         """
         with self._lock:
             if self._max_size and self._current_size >= self._max_size:
                 self.evict(self._max_size - self._current_size + 1)
 
-            if self._put(key, value):
+            if self._put(key, value, alt_sdp):
                 self._current_size += 1
 
-    def get(self, key):
+    def get(self, key, alt_sdp=None):
         """
         Retrieve an entry from the cache
         If the entry does not exist, will use the 'default' value or delegate
         :param key:
+        :param alt_sdp: if not found in cache._sdp, look in other repositories
         :return:
         """
         with self._lock:
-            return self._get(key)
+            return self._get(key, alt_sdp)
+
+    def alt_get(self, key):
+        """
+        Alternate way to get an entry, from concept cache
+        This is mainly used for IncCache, in order to get the value without increasing it
+        It used for another cache, it must return the value from key WITHOUT modifying the state of the cache
+        :param key:
+        :return:
+        """
+        with self._lock:
+            return self._alt_get(key)
 
     def get_all(self):
         """
@@ -98,36 +137,63 @@ class BaseCache:
     def inner_get(self, key):
         return self._cache[key]
 
-    def update(self, old_key, old_value, new_key, new_value):
+    def update(self, old_key, old_value, new_key, new_value, alt_sdp=None):
         """
         Update an entry in the cache
         :param old_key: key of the previous version of the entry
         :param old_value: previous version of the entry
         :param new_key: key of the entry
         :param new_value: new value
+        :param alt_sdp: new value
         :return:
         """
         with self._lock:
-            self._update(old_key, old_value, new_key, new_value)
+            self._update(old_key, old_value, new_key, new_value, alt_sdp)
 
-    def delete(self, key, value=None):
+    def delete(self, key, value=None, alt_sdp=None):
         with self._lock:
             try:
-                self._delete(key, value)
+                self._sync(key)
+                self._delete(key, value, alt_sdp)
+                return True
             except KeyError:
-                pass
+                return False
 
-    def populate(self, populate_function, get_key_function):
+    def populate(self, populate_function, get_key_function, reset_events=False):
         """
         Initialise the cache with a bunch of data
         :param populate_function:
         :param get_key_function:
+        :param reset_events:
         :return:
         """
         with self._lock:
-            for item in populate_function():
+            if reset_events:
+                to_add_copy = self.to_add.copy()
+                to_remove_copy = self.to_remove.copy()
+
+            for item in (populate_function(self._sdp) if self._sdp else populate_function()):
                 self.put(get_key_function(item), item)
 
+            if reset_events:
+                self.to_add = to_add_copy
+                self.to_remove = to_remove_copy
+
+    def force_value(self, key, value):
+        """
+        Force a value into a key without raising any event
+        """
+        with self._lock:
+            self._cache[key] = value
+
+    def remove_initialized_key(self, key):
+        """
+        When a value is requested by alt_sdp, we should not keep track of the request
+        As the outcome is not known
+        """
+        with self._lock:
+            self._initialized_keys.remove(key)
+
     def has(self, key):
         """
         Return True if the key is in the cache
@@ -149,7 +215,10 @@ class BaseCache:
             if key in self._cache:
                 return True
 
-            return self._extend_exists(key) if self._extend_exists else False
+            if self._extend_exists:
+                return self._extend_exists(self._sdp, key) if self._sdp else self._extend_exists(key)
+            else:
+                return False
 
     def evict(self, nb_items):
         """
@@ -195,13 +264,16 @@ class BaseCache:
 
         return len(to_delete)
 
-    def clear(self):
+    def clear(self, set_is_cleared=True):
         with self._lock:
+            # Seems that remote sdp is not correctly updated
             self._cache.clear()
             self._current_size = 0
             self._initialized_keys.clear()
             self.to_add.clear()
             self.to_remove.clear()
+            if set_is_cleared:
+                self._is_cleared = True
 
     def dump(self):
         with self._lock:
@@ -225,9 +297,32 @@ class BaseCache:
             self.to_add.clear()
             self.to_remove.clear()
 
+    def reset_initialized_keys(self):
+        """
+        Use when an ontology is put back. Reset all the previous requests as alt_sdp is a new one
+        """
+        with self._lock:
+            self._initialized_keys.clear()
+
+    def is_cleared(self):
+        with self._lock:
+            return self._is_cleared
+
+    def clone(self):
+        return type(self)(self._max_size, self._default, self._extend_exists, self._alt_sdp_get, self._sdp)
+
+    def test_only_reset(self):
+        """
+        Clears the cache, but does not set is_cleared to True
+        It's a convenient way to clear the cache without altering alt_sdp behaviour
+        """
+        self.clear(set_is_cleared=False)
+
     def _sync(self, *keys):
+        # KSI 2020-12-29. DO not try to use alt_sdp here
+        # Sync must only sync with the current sdp
         for key in keys:
-            if key not in self._initialized_keys and self._default:
+            if key not in self._initialized_keys and callable(self._default):
                 # to keep sync with the remote repo is needed
                 # first check self._initialized_keys to prevent infinite loop
                 self.get(key)
@@ -246,7 +341,7 @@ class BaseCache:
         except KeyError:
             pass
 
-    def _get(self, key):
+    def _get(self, key, alt_sdp=None):
         try:
             value = self._cache[key]
         except KeyError:
@@ -254,11 +349,18 @@ class BaseCache:
                 self._initialized_keys.clear()
             if callable(self._default):
                 if key in self._initialized_keys:
-                    return None
+                    # it means that we have already asked the repository
+                    return NotFound
 
-                value = self._default(key)
-                if value is not None:
-                    self._cache[key] = value
+                simple_copy = True
+                value = self._default(self._sdp, key) if self._sdp else self._default(key)
+                if value is NotFound and alt_sdp and not self._is_cleared:
+                    value = self._alt_sdp_get(alt_sdp, key)
+                    simple_copy = False
+
+                if value is not NotFound:
+                    self._cache[key] = value if simple_copy else sheerka_deepcopy(value)
+                    value = self._cache[key]
 
                     # update _current_size
                     if isinstance(value, (list, set)):
@@ -271,11 +373,14 @@ class BaseCache:
 
         return value
 
-    def _put(self, key, value):
+    def _alt_get(self, key):
+        return self._get(key)  # by default, point to _get
+
+    def _put(self, key, value, alt_sdp):
         pass
 
-    def _update(self, old_key, old_value, new_key, new_value):
+    def _update(self, old_key, old_value, new_key, new_value, alt_sdp):
         pass
 
-    def _delete(self, key, value):
+    def _delete(self, key, value, alt_sdp):
         raise NotImplementedError()

src/cache/Cache.py

@@ -1,6 +1,7 @@
 from threading import RLock
 
 from cache.BaseCache import BaseCache
+from core.global_symbols import Removed
 
 
 class Cache(BaseCache):
@@ -10,23 +11,32 @@ class Cache(BaseCache):
     When you put the same key twice, the previous element is overridden
     """
 
-    def _put(self, key, value):
+    def _put(self, key, value, alt_sdp):
         res = key not in self._cache
         self._cache[key] = value
         self._add_to_add(key)
         return res
 
-    def _update(self, old_key, old_value, new_key, new_value):
+    def _update(self, old_key, old_value, new_key, new_value, alt_sdp):
         self._cache[new_key] = new_value
         self._add_to_add(new_key)
 
         if new_key != old_key:
             self._sync(old_key)
-            del (self._cache[old_key])
-            self._add_to_remove(old_key)
-
-    def _delete(self, key, value):
-        del(self._cache[key])
-        self._current_size -= 1
-        self._add_to_remove(key)
+            if not self._is_cleared and alt_sdp and self._extend_exists and self._extend_exists(alt_sdp, old_key):
+                self._cache[old_key] = Removed
+                self._add_to_add(old_key)
+                self._current_size += 1
+            else:
+                del (self._cache[old_key])
+                self._add_to_remove(old_key)
 
+    def _delete(self, key, value, alt_sdp):
+        if not self._is_cleared and alt_sdp and self._extend_exists and self._extend_exists(alt_sdp, key):
+            self._cache[key] = Removed
+            self._add_to_add(key)
+            # do not decrease self._current_size as 'Removed' takes on slot
+        else:
+            del (self._cache[key])
+            self._add_to_remove(key)
+            self._current_size -= 1

src/cache/CacheManager.py

@@ -4,6 +4,7 @@ from typing import Callable
 
 from cache.BaseCache import BaseCache
 from core.concept import Concept
+from core.global_symbols import NotFound
 
 
 @dataclass
@@ -37,8 +38,9 @@ class CacheManager:
     Single class to manage all the caches
     """
 
-    def __init__(self, cache_only):
+    def __init__(self, cache_only, sdp=None):
         self.cache_only = cache_only  # if true disable all remote access when key not found
+        self.sdp = sdp
         self.caches = {}
         self.concept_caches = []
         self.is_dirty = False  # to indicate that the value of a cache has changed
@@ -57,6 +59,8 @@ class CacheManager:
         with self._lock:
             if self.cache_only:
                 cache.disable_default()
+            if self.sdp:
+                cache.configure(sdp=self.sdp)
             self.caches[name] = CacheDefinition(cache, use_ref, get_key)
             self.concept_caches.append(name)
 
@@ -70,8 +74,13 @@ class CacheManager:
         :return:
         """
         with self._lock:
+            if self.sdp:
+                cache.configure(sdp=self.sdp)
+
             if self.cache_only:
                 cache.disable_default()
+                persist = False
+
             self.caches[name] = CacheDefinition(cache, use_ref, None, persist)
 
     def add_concept(self, concept):
@@ -89,11 +98,12 @@ class CacheManager:
 
             self.is_dirty = True
 
-    def update_concept(self, old, new):
+    def update_concept(self, old, new, alt_sdp=None):
         """
         Update a concept.
         :param old:  old version of the concept
         :param new: new version of the concept
+        :param alt_sdp: if not found in self.sdp, look in other repositories
         :return:
         """
         with self._lock:
@@ -103,42 +113,15 @@ class CacheManager:
                 old_key = cache_def.get_key(old)
                 new_key = cache_def.get_key(new)
 
-                cache_def.cache.update(old_key, old, new_key, new)
+                cache_def.cache.update(old_key, old, new_key, new, alt_sdp=alt_sdp)
 
             self.is_dirty = True
 
-        # how can you update an entry it the key may have changed ?
-        # You need to have an invariant. By convention the keys in the first cache cannot change
-        # with self._lock:
-        #     iter_cache_def = iter(self.caches)
-        #
-        #     cache_def = next(iter_cache_def)
-        #     old_key = cache_def.get_key(concept)
-        #
-        #     try:
-        #         while True:
-        #             items = cache_def.cache[old_key]
-        #             if isinstance(items, (list, set)):
-        #                 for item in items:
-        #                     if item.id == concept.id:
-        #                         break
-        #                 else:
-        #                     raise IndexError(f"{old_key=}, id={concept.id}")
-        #
-        #                 cache_def.cache.update(old_key, item, cache_def.get_key(concept), concept)
-        #
-        #             else:
-        #                 cache_def.cache.update(old_key, items, cache_def.get_key(concept), concept)
-        #
-        #             cache_def = next(iter_cache_def)
-        #     except StopIteration:
-        #         pass
-        #     self.is_dirty = True
-
-    def remove_concept(self, concept):
+    def remove_concept(self, concept, alt_sdp=None):
         """
         Remove a concept from all caches
         :param concept:
+        :param alt_sdp: if not found in self.sdp, look in other repositories
         :return:
         """
         with self._lock:
@@ -148,25 +131,66 @@ class CacheManager:
             concept_id = ref_cache_def.get_key(concept)
             ref_concept = ref_cache_def.cache.get(concept_id)
 
-            if ref_concept is None:
+            if ref_concept is NotFound and alt_sdp:
+                ref_concept = alt_sdp.get(self.concept_caches[0], concept_id)
+
+            if ref_concept is NotFound:
                 raise ConceptNotFound(concept)
 
             for cache_name in self.concept_caches:
                 cache_def = self.caches[cache_name]
                 key = cache_def.get_key(ref_concept)
-                cache_def.cache.delete(key, ref_concept)
+                cache_def.cache.delete(key, ref_concept, alt_sdp=alt_sdp)
 
             self.is_dirty = True
 
-    def get(self, cache_name, key):
+    def get(self, cache_name, key, alt_sdp=None):
         """
         From concept cache, get an entry
         :param cache_name:
         :param key:
+        :param alt_sdp: if not found in self.sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            return self.caches[cache_name].cache.get(key, alt_sdp)
+
+    def alt_get(self, cache_name, key):
+        """
+        Alternate way to get an entry, from concept cache
+        This is mainly used for IncCache, in order to get the value without increasing it
+        :param cache_name:
+        :param key:
         :return:
         """
         with self._lock:
-            return self.caches[cache_name].cache.get(key)
+            return self.caches[cache_name].cache.alt_get(key)
+
+    def put(self, cache_name, key, value, alt_sdp=None):
+        """
+        Add to a cache
+        :param cache_name:
+        :param key:
+        :param value:
+        :param alt_sdp: if not found in self.sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            self.caches[cache_name].cache.put(key, value, alt_sdp)
+            self.is_dirty = True
+
+    def delete(self, cache_name, key, value=None, alt_sdp=None):
+        """
+        Delete an entry from the cache
+        :param cache_name:
+        :param key:
+        :param value:
+        :param alt_sdp: if not found in self.sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            if self.caches[cache_name].cache.delete(key, value, alt_sdp):
+                self.is_dirty = True
 
     def get_cache(self, cache_name):
         """
@@ -186,40 +210,31 @@ class CacheManager:
         """
         return self.caches[cache_name].cache.copy()
 
-    def put(self, cache_name, key, value):
-        """
-        Add to a cache
-        :param cache_name:
-        :param key:
-        :param value:
-        :return:
-        """
-        with self._lock:
-            self.caches[cache_name].cache.put(key, value)
-            self.is_dirty = True
-
-    def delete(self, cache_name, key, value=None):
-        """
-        Delete an entry from the cache
-        :param cache_name:
-        :param key:
-        :param value:
-        :return:
-        """
-        with self._lock:
-            self.caches[cache_name].cache.delete(key, value)
-            self.is_dirty = True
-
-    def populate(self, cache_name, populate_function, get_key_function):
+    def populate(self, cache_name, populate_function, get_key_function, reset_events=False):
         """
         Populate a specific cache with a bunch of items
         :param cache_name:
         :param populate_function: how to get the items
         :param get_key_function: how to get the key, out of an item
+        :param reset_events: reset to_add and to_remove events after populate
         :return:
         """
         with self._lock:
-            self.caches[cache_name].cache.init(populate_function, get_key_function)
+            self.caches[cache_name].cache.populate(populate_function, get_key_function, reset_events)
+
+    def force_value(self, cache_name, key, value):
+        """
+        Update the content of the cache, but does not raise any event
+        """
+        with self._lock:
+            self.caches[cache_name].cache.force_value(key, value)
+
+    def remove_initialized_key(self, cache_name, key):
+        """
+
+        """
+        with self._lock:
+            self.caches[cache_name].cache.remove_initialized_key(key)
 
     def has(self, cache_name, key):
         """
@@ -267,7 +282,7 @@ class CacheManager:
             return
 
         with self._lock:
-            with context.sheerka.sdp.get_transaction(context.event.get_digest()) as transaction:
+            with self.sdp.get_transaction(context.event.get_digest()) as transaction:
                 for cache_name, cache_def in self.caches.items():
                     if not cache_def.persist:
                         continue
@@ -287,13 +302,13 @@ class CacheManager:
                     cache_def.cache.reset_events()
             self.is_dirty = False
 
-    def clear(self, cache_name=None):
+    def clear(self, cache_name=None, set_is_cleared=True):
         with self._lock:
             if cache_name:
-                self.caches[cache_name].cache.clear()
+                self.caches[cache_name].cache.clear(set_is_cleared)
             else:
                 for cache_def in self.caches.values():
-                    cache_def.cache.clear()
+                    cache_def.cache.clear(set_is_cleared)
 
     def dump(self):
         """

src/cache/DictionaryCache.py

@@ -1,30 +1,62 @@
-from cache.BaseCache import BaseCache
+from cache.BaseCache import BaseCache, MAX_INITIALIZED_KEY
+from core.global_symbols import NotFound, Removed
+from core.utils import sheerka_deepcopy
 
 
 class DictionaryCache(BaseCache):
-    def _get(self, key):
+    """
+    Kind of all or nothing dictionary database
+    You can get the values key by by
+    But when you want to put, you must put the whole database
+    For this reason, alt_sdp is not supported. The top ontology layer contains the whole database
+    """
+
+    def auto_configure(self, cache_name):
+        """
+        Convenient way to configure the cache
+        :param cache_name:
+        :return:
+        """
+        self._default = lambda sdp, key: sdp.get(cache_name)  # retrieve the whole entry
+        self._extend_exists = None  # not used
+        self._alt_sdp_get = None  # not used
+
+        return self
+
+    def _get(self, key, alt_sdp=None):
         """
         Management of the default is different
         :param key:
         :return:
         """
         try:
-            value = self._cache[key]
-            return value
+            return self._cache[key]
         except KeyError:
+            if key in self._initialized_keys:
+                return NotFound
+
+            if len(self._initialized_keys) == MAX_INITIALIZED_KEY:
+                self._initialized_keys.clear()
+
+            self._initialized_keys.add(key)
+
             if callable(self._default):
-                self._cache = self._default(key) or {}
+                default_values = self._default(self._sdp, key) if self._sdp else self._default(key)
             else:
-                self._cache = self._default.copy() if self._default else {}
+                default_values = self._default
+
+            if isinstance(default_values, dict):
+                self._cache.update(default_values)  # update the whole cache dictionary to resync with remote sdp
 
             self._count_items()
-            return self._cache[key] if key in self._cache else None
+            return self._cache[key] if key in self._cache else NotFound
 
-    def _put(self, key, value):
+    def _put(self, key, value, alt_sdp):
         """
         Adds a whole dictionary
         :param key: True to append, false to reset
         :param value: dictionary
+        :param alt_sdp: NOT SUPPORTED as the values from alt_sdp must be retrieved and computed BEFORE the put
         :return:
         """
         if not isinstance(key, bool):
@@ -33,12 +65,12 @@ class DictionaryCache(BaseCache):
         if not isinstance(value, dict):
             raise ValueError
 
-        if key:
+        if key:  # update the current cache
             if self._cache is None:
                 self._cache = value.copy()
             else:
                 self._cache.update(value)
-        else:
+        else:  # reset the current cache
             self._cache = value
 
         self._count_items()
@@ -47,6 +79,9 @@ class DictionaryCache(BaseCache):
         self._add_to_add("*self*")
         return False
 
+    def _delete(self, key, value, alt_sdp):
+        raise NotImplementedError()
+
     def _count_items(self):
         self._current_size = 0
         for v in self._cache.values():

src/cache/FastCache.py

@@ -1,12 +1,16 @@
+from core.global_symbols import NotFound
+
+
 class FastCache:
     """
     Simplest LRU cache
     """
 
-    def __init__(self, max_size=256):
+    def __init__(self, max_size=256, default=None):
         self.max_size = max_size
         self.cache = {}
         self.lru = []
+        self.default = default
 
     def put(self, key, value):
         if len(self.cache) == self.max_size:
@@ -18,11 +22,19 @@ class FastCache:
         self.cache[key] = value
         self.lru.append(key)
 
+    def has(self, key):
+        return key in self.cache
+
     def get(self, key):
         try:
             return self.cache[key]
         except KeyError:
-            return None
+            if self.default:
+                value = self.default(key)
+                self.put(key, value)
+                return value
+
+            return NotFound
 
     def evict_by_key(self, predicate):
         to_remove = []

src/cache/IncCache.py

@@ -1,4 +1,5 @@
 from cache.Cache import Cache
+from core.global_symbols import NotFound, Removed
 
 
 class IncCache(Cache):
@@ -6,13 +7,18 @@ class IncCache(Cache):
     Increment the value of the key every time it's accessed
     """
 
-    def _get(self, key):
-        value = super()._get(key) or 0
+    def _get(self, key, alt_sdp=None):
+        value = super()._get(key, alt_sdp=alt_sdp)
+        if value in (NotFound, Removed):
+            value = 0
         value += 1
-        self._put(key, value)
+        self._put(key, value, alt_sdp)
         return value
 
-    def _put(self, key, value):
+    def _put(self, key, value, alt_sdp):
         self._cache[key] = value
         self._add_to_add(key)
         return True
+
+    def _alt_get(self, key):
+        return super()._get(key)  # point to parent, not to self

src/cache/ListCache.py

@@ -1,4 +1,6 @@
 from cache.Cache import BaseCache
+from core.global_symbols import Removed, NotFound
+from core.utils import sheerka_deepcopy
 
 
 class ListCache(BaseCache):
@@ -8,12 +10,17 @@ class ListCache(BaseCache):
     Items of this cache are list
     """
 
-    def _put(self, key, value):
+    def _put(self, key, value, alt_sdp):
         if key in self._cache:
             self._cache[key].append(value)
         else:
             self._sync(key)
 
+            if key not in self._cache and alt_sdp and not self._is_cleared:
+                previous = self._alt_sdp_get(alt_sdp, key)
+                if previous not in (NotFound, Removed):
+                    self._cache[key] = sheerka_deepcopy(previous)
+
             if key in self._cache:
                 self._cache[key].append(value)
             else:
@@ -22,18 +29,33 @@ class ListCache(BaseCache):
         self._add_to_add(key)
         return True
 
-    def _update(self, old_key, old_value, new_key, new_value):
+    def _update(self, old_key, old_value, new_key, new_value, alt_sdp):
         self._sync(old_key, new_key)
 
+        if old_key not in self._cache and alt_sdp and not self._is_cleared:
+            # no value found in local cache or remote repository
+            # Use the values from alt_sdp
+            previous = self._alt_sdp_get(alt_sdp, old_key)
+            if previous in (NotFound, Removed):
+                raise KeyError(old_key)
+
+            self._cache[old_key] = sheerka_deepcopy(previous)
+            self._current_size += len(previous)
+
         if old_key != new_key:
             self._cache[old_key].remove(old_value)
             if len(self._cache[old_key]) == 0:
-                del (self._cache[old_key])
-                self._add_to_remove(old_key)
+                if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, old_key):
+                    self._cache[old_key] = Removed
+                    self._add_to_add(old_key)
+                    self._current_size += 1
+                else:
+                    del (self._cache[old_key])
+                    self._add_to_remove(old_key)
             else:
                 self._add_to_add(old_key)
 
-            self._put(new_key, new_value)
+            self._put(new_key, new_value, alt_sdp)
             self._add_to_add(new_key)
         else:
             for i in range(len(self._cache[new_key])):

src/cache/ListIfNeededCache.py

@@ -1,4 +1,6 @@
 from cache.Cache import BaseCache
+from core.global_symbols import Removed, NotFound
+from core.utils import sheerka_deepcopy
 
 
 class ListIfNeededCache(BaseCache):
@@ -8,7 +10,7 @@ class ListIfNeededCache(BaseCache):
     When you put the same key twice, you now have a list of two elements
     """
 
-    def _put(self, key, value):
+    def _put(self, key, value, alt_sdp):
         if key in self._cache:
             if isinstance(self._cache[key], list):
                 self._cache[key].append(value)
@@ -17,6 +19,11 @@ class ListIfNeededCache(BaseCache):
         else:
             self._sync(key)
 
+            if key not in self._cache and alt_sdp and not self._is_cleared:
+                previous = self._alt_sdp_get(alt_sdp, key)
+                if previous not in (NotFound, Removed):
+                    self._cache[key] = sheerka_deepcopy(previous)
+
             if key in self._cache:
                 if isinstance(self._cache[key], list):
                     self._cache[key].append(value)
@@ -27,23 +34,36 @@ class ListIfNeededCache(BaseCache):
         self._add_to_add(key)
         return True
 
-    def _update(self, old_key, old_value, new_key, new_value):
+    def _update(self, old_key, old_value, new_key, new_value, alt_sdp):
 
         self._sync(old_key, new_key)
 
+        if old_key not in self._cache and alt_sdp and not self._is_cleared:
+            # no value found in local cache or remote repository
+            # Use the values from alt_sdp
+            previous = self._alt_sdp_get(alt_sdp, old_key)
+            if previous in (NotFound, Removed):
+                raise KeyError(old_key)
+
+            self._cache[old_key] = sheerka_deepcopy(previous)
+            self._current_size += len(previous) if isinstance(previous, list) else 1
+
         if old_key != new_key:
             if isinstance(self._cache[old_key], list):
                 self._cache[old_key].remove(old_value)
-                if len(self._cache[old_key]) == 0:
+                if len(self._cache[old_key]) == 1:
+                    self._cache[old_key] = self._cache[old_key][0]
+                self._add_to_add(old_key)
+            else:
+                if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, old_key):
+                    self._cache[old_key] = Removed
+                    self._add_to_add(old_key)
+                    self._current_size += 1
+                else:
                     del (self._cache[old_key])
                     self._add_to_remove(old_key)
-                else:
-                    self._add_to_add(old_key)
-            else:
-                del (self._cache[old_key])
-                self._add_to_remove(old_key)
 
-            self._put(new_key, new_value)
+            self._put(new_key, new_value, alt_sdp)
             self._add_to_add(new_key)
         else:
             if isinstance(self._cache[new_key], list):
@@ -55,22 +75,60 @@ class ListIfNeededCache(BaseCache):
                 self._cache[new_key] = new_value
             self._add_to_add(new_key)
 
-    def _delete(self, key, value):
+    def _delete(self, key, value, alt_sdp):
         if value is None:
-            self._current_size -= len(self._cache[key])
-            del self._cache[key]
-            self._add_to_remove(key)
-        else:
-            previous = self._cache[key]
-            if isinstance(previous, list):
-                previous.remove(value)
-                if len(previous) == 1:
-                    self._cache[key] = previous[0]
-                self._current_size -= 1
-                self.to_add.add(key)
-            else:
-                if previous == value:
-                    del self._cache[key]
-                    self._current_size -= 1
-                    self.to_remove.add(key)
+            # Remove the whole key
+            if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key):
+                if key in self._cache:
+                    previous = self._cache[key]
+                    if isinstance(previous, list):
+                        self._current_size -= len(previous) + 1
+                else:
+                    self._current_size += 1
 
+                self._cache[key] = Removed
+                self._add_to_add(key)
+            else:
+                previous = self._cache[key]
+                self._current_size -= len(previous) if isinstance(previous, list) else 1
+                del self._cache[key]
+                self._add_to_remove(key)
+
+        else:
+            # Remove a single value
+            try:
+                previous = self._cache[key]
+                if isinstance(previous, list):
+                    previous.remove(value)
+                    self._cache[key] = previous[0] if len(previous) == 1 else previous
+                    self._current_size -= 1
+                    self.to_add.add(key)
+                else:
+                    if previous == value:
+                        # I am about to delete the entry
+                        if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key):
+                            self._cache[key] = Removed
+                            self.to_add.add(key)
+                            # self._current_size -= 1 # Do not decrease size, as it's replaced by 'Removed'
+                        else:
+                            del self._cache[key]
+                            self._current_size -= 1
+                            self.to_remove.add(key)
+            except KeyError as ex:
+                previous = self._alt_sdp_get(alt_sdp, key) if not self._is_cleared and alt_sdp else NotFound
+                if previous in (NotFound, Removed):
+                    raise ex
+
+                if isinstance(previous, list):
+                    previous = sheerka_deepcopy(previous)
+                    previous.remove(value)  # raise an exception if value in not in the list
+                    self._cache[key] = previous[0] if len(previous) == 1 else previous
+                    self._current_size -= 1
+                    self.to_add.add(key)
+                else:
+                    if previous == value:
+                        self._cache[key] = Removed
+                        self.to_add.add(key)
+                        self._current_size -= 1
+
+        return True

src/cache/SetCache.py

@@ -1,4 +1,6 @@
 from cache.Cache import BaseCache
+from core.global_symbols import NotFound, Removed
+from core.utils import sheerka_deepcopy
 
 
 class SetCache(BaseCache):
@@ -15,7 +17,7 @@ class SetCache(BaseCache):
     >> assert {'value1', 'value2'} == self.get('key')
     """
 
-    def _put(self, key, value):
+    def _put(self, key, value, alt_sdp):
         if key in self._cache:
             if value in self._cache[key]:
                 return False
@@ -23,6 +25,11 @@ class SetCache(BaseCache):
         else:
             self._sync(key)
 
+            if key not in self._cache and alt_sdp and not self._is_cleared:
+                previous = self._alt_sdp_get(alt_sdp, key)
+                if previous not in (NotFound, Removed):
+                    self._cache[key] = sheerka_deepcopy(previous)
+
             if key in self._cache:
                 self._cache[key].add(value)
             else:
@@ -31,35 +38,79 @@ class SetCache(BaseCache):
         self._add_to_add(key)
         return True
 
-    def _update(self, old_key, old_value, new_key, new_value):
+    def _update(self, old_key, old_value, new_key, new_value, alt_sdp):
         self._sync(old_key, new_key)
 
+        if old_key not in self._cache and alt_sdp and not self._is_cleared:
+            # no value found in local cache or remote repository
+            # Use the values from alt_sdp
+            previous = self._alt_sdp_get(alt_sdp, old_key)
+            if previous in (NotFound, Removed):
+                raise KeyError(old_key)
+
+            self._cache[old_key] = sheerka_deepcopy(previous)
+            self._current_size += len(previous)
+
         if old_key != new_key:
             if isinstance(self._cache[old_key], set):
                 self._cache[old_key].remove(old_value)
                 if len(self._cache[old_key]) == 0:
-                    del (self._cache[old_key])
-                    self._add_to_remove(old_key)
+                    if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, old_key):
+                        self._cache[old_key] = Removed
+                        self._add_to_add(old_key)
+                        self._current_size += 1
+                    else:
+                        del (self._cache[old_key])
+                        self._add_to_remove(old_key)
                 else:
                     self._add_to_add(old_key)
 
-            self._put(new_key, new_value)
+            self._put(new_key, new_value, alt_sdp)
             self._add_to_add(new_key)
         else:
             self._cache[new_key].remove(old_value)
-            self._put(new_key, new_value)
+            self._put(new_key, new_value, alt_sdp)
             self._add_to_add(new_key)
 
-    def _delete(self, key, value):
+    def _delete(self, key, value, alt_sdp):
         if value is None:
-            self._current_size -= len(self._cache[key])
-            del self._cache[key]
-            self._add_to_remove(key)
-        else:
-            self._cache[key].remove(value)
-            self._current_size -= 1
-            if len(self._cache[key]) == 0:
+            if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key):
+                self._current_size += 1 - len(self._cache[key]) if key in self._cache else 1
+                self._cache[key] = Removed
+                self._add_to_add(key)
+            else:
+                self._current_size -= len(self._cache[key])
                 del self._cache[key]
                 self._add_to_remove(key)
-            else:
+
+        else:
+            try:
+                self._cache[key].remove(value)
+                if len(self._cache[key]) == 0:
+                    if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key):
+                        self._cache[key] = Removed
+                        self._add_to_add(key)
+                        # self._current_size -= 1 # Do not decrease size, as it's replaced by 'Removed'
+                    else:
+                        del self._cache[key]
+                        self._add_to_remove(key)
+                        self._current_size -= 1
+                else:
+                    self._add_to_add(key)
+                    self._current_size -= 1
+            except KeyError as ex:
+                previous = self._alt_sdp_get(alt_sdp, key) if not self._is_cleared and alt_sdp else NotFound
+                if previous in (NotFound, Removed):
+                    raise ex
+
+                previous = sheerka_deepcopy(previous)
+                previous.remove(value)  # will raise a KeyError if value is not in the set
+                if len(previous) == 0:
+                    self._cache[key] = Removed
+                    self._current_size += 1
+                else:
+                    self._cache[key] = previous
+                    self._current_size += len(previous)
                 self._add_to_add(key)
+
+        return True