Fixed #8

Fixed #12
Fixed #13
Fixed #14

tests/caching/__init__.py

@@ -0,0 +1,18 @@
+class FakeSdp:
+    def __init__(self, /, get_value=None, extend_exists=None, get_alt_value=None, populate=None):
+        self.get_value = get_value
+        self.extend_exists = extend_exists
+        self.populate_function = populate
+        self.get_alt_value = get_alt_value
+
+    def get(self, cache_name, key):
+        return self.get_value(cache_name, key)
+
+    def exists(self, cache_name, key):
+        return self.extend_exists(cache_name, key)
+
+    def alt_get(self, cache_name, key):
+        return self.get_alt_value(cache_name, key)
+
+    def populate(self):
+        return self.populate_function() if callable(self.populate_function) else self.populate_function

tests/caching/test_CacheManager.py

@@ -0,0 +1,288 @@
+from dataclasses import dataclass
+
+import pytest
+
+from base import BaseTest
+from caching.Cache import Cache
+from caching.CacheManager import CacheManager, ConceptNotFound
+from caching.ListIfNeededCache import ListIfNeededCache
+from common.global_symbols import NotFound
+from helpers import get_metadata
+from tests.caching import FakeSdp
+
+
+@dataclass
+class Obj:
+    key: str
+    value: str
+
+
+class TestCacheManager(BaseTest):
+
+    def test_i_can_push_into_sdp(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache(), persist=True)
+        cache_manager.put("test", "key", "value")
+
+        cache_manager.commit(context)
+        assert sdp.exists("test", "key")
+
+    def test_sdp_given_to_the_cache_manager_can_be_used_by_the_cache(self, context, sdp):
+        """
+        When the sdp is given to the cache_manager, the 'Cache' object can use the lambda with two parameters
+        :param context:
+        :type context:
+        :return:
+        :rtype:
+        """
+        cache_manager = CacheManager(sdp)
+        cache = Cache(default=lambda _sdp, k: _sdp.get("test", k))
+        cache_manager.register_cache("test", cache)
+
+        with cache_manager.sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        assert cache_manager.get("test", "key") == "value"
+
+    def test_cache_can_use_auto_configure_when_sdp_is_given_to_the_cache_manager(self, context, sdp):
+        """
+        When sdp is given to the cache manager,
+        Cache can be configured using auto_configure as the sdp will be provided by the cache_manager
+        :param context:
+        :type context:
+        :return:
+        :rtype:
+        """
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        cache_manager = CacheManager(sdp)
+        cache = Cache().auto_configure("test")
+        cache_manager.register_cache("test", cache)
+
+        assert cache_manager.get("test", "key") == "value"
+
+    def test_i_can_get_value_when_the_sdp_is_given_to_the_cache(self, context, sdp):
+        """
+        If the sdp is not given to the cache manager, the cache will need to explicitly provide the sdp
+        during its configuration
+        :param context:
+        :type context:
+        :return:
+        :rtype:
+        """
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        cache_manager = CacheManager()
+        cache = Cache(default=lambda k: sdp.get("test", k))
+        cache_manager.register_cache("test", cache)
+
+        assert cache_manager.get("test", "key") == "value"
+
+    def test_i_can_get_value_from_alt_sdp(self, sdp):
+        """
+        When nothing is found in cache and in sdp, we use alternate sdp
+        :param sdp:
+        :type sdp:
+        :return:
+        :rtype:
+        """
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !")
+        assert cache_manager.get("test", "key", alt_sdp=alt_sdp) == "value found !"
+
+    def test_i_can_commit_simple_cache(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+        cache = cache_manager.caches["test"].cache
+
+        cache_manager.put("test", "key", "value")
+
+        cache_manager.commit(context)
+
+        cache.clear()
+        assert cache_manager.sdp.get("test", "key") == "value"
+        assert cache.get("key") == "value"
+
+        cache.update("key", "value", "key", "another_value")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") == "another_value"
+
+        cache.update("key", "another_value", "key2", "another_value")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") is NotFound
+        assert cache_manager.sdp.get("test", "key2") == "another_value"
+
+        # sanity check
+        # sdp 'test' has value, but sdp '__default__' does not
+        assert cache_manager.sdp.name == "test"
+        assert cache_manager.sdp.state.data == {'test': {'key2': 'another_value'}}
+
+    def test_i_can_use_concept_cache(self):
+        cache_manager = CacheManager()
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("name", ListIfNeededCache(), lambda c: c.name, True)
+
+        # caches are correctly created
+        assert cache_manager.concept_caches == ["id", "name"]
+        assert "id" in cache_manager.caches
+        assert "name" in cache_manager.caches
+
+        # caches are correctly updated on insertion
+        meta1 = get_metadata(id="1", name="foo")
+        meta2 = get_metadata(id="2", name="bar")
+        meta3 = get_metadata(id="3", name="foo")
+
+        for metadata in meta1, meta2, meta3:
+            cache_manager.add_concept(metadata)
+
+        assert cache_manager.get_inner_cache("id").copy() == {"1": meta1, "2": meta2, "3": meta3}
+        assert cache_manager.get_inner_cache("name").copy() == {"foo": [meta1, meta3], "bar": meta2}
+
+        # caches are correctly updated on modification
+        meta3prime = get_metadata(id="3", name="bar")
+        cache_manager.update_concept(meta3, meta3prime)
+
+        assert cache_manager.get_inner_cache("id").copy() == {"1": meta1, "2": meta2, "3": meta3prime}
+        assert cache_manager.get_inner_cache("name").copy() == {"foo": meta1, "bar": [meta2, meta3prime]}
+
+        # caches are correctly updated on removal
+        cache_manager.remove_concept(meta3prime)
+        assert cache_manager.get_inner_cache("id").copy() == {"1": meta1, "2": meta2}
+        assert cache_manager.get_inner_cache("name").copy() == {"foo": meta1, "bar": meta2}
+
+    def test_i_cannot_remove_a_concept_that_does_not_exists(self):
+        cache_manager = CacheManager()
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
+
+        meta1 = get_metadata(id="1", name="foo")
+
+        with pytest.raises(ConceptNotFound) as ex:
+            cache_manager.remove_concept(meta1)
+
+        assert ex.value.concept == meta1
+
+    def test_nothing_is_sent_to_sdp_if_persist_is_false(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache(), persist=False)
+
+        cache_manager.put("test", "key", "value")
+        cache_manager.commit(context)
+
+        value = sdp.get("test", "key")
+        assert value is NotFound
+
+    def test_i_can_delete_from_cache_manager(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache(), persist=True)
+        cache_manager.put("test", "key", "value")
+        cache_manager.commit(context)
+
+        # sanity check
+        assert cache_manager.get("test", "key") == "value"
+        assert sdp.get("test", "key") == "value"
+
+        # I remove but I don't commit
+        cache_manager.delete("test", "key")
+        assert cache_manager.get("test", "key") is NotFound
+        assert sdp.get("test", "key") == "value"
+
+        # commit
+        cache_manager.commit(context)
+        assert cache_manager.get("test", "key") is NotFound
+        assert sdp.get("test", "key") is NotFound
+
+    def test_i_can_get_the_inner_cache(self):
+        cache_manager = CacheManager()
+        cache = Cache()
+        cache_manager.register_cache("test", cache)
+
+        inner_cache = cache_manager.get_inner_cache("test")
+        assert id(inner_cache) == id(cache)
+
+    def test_i_can_get_a_copy_of_a_cache(self):
+        cache_manager = CacheManager()
+        cache = Cache()
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+        cache_manager.register_cache("test", cache)
+
+        copy = cache_manager.copy("test")
+
+        assert isinstance(copy, dict)
+        assert copy == {"key1": "value1", "key2": "value2"}
+
+    def test_i_can_populate(self):
+        cache_manager = CacheManager()
+        cache = Cache()
+        cache_manager.register_cache("test", cache)
+
+        obj1, obj2 = Obj("key1", "value1"), Obj("key2", "value2")
+        cache_manager.populate("test", lambda: [obj1, obj2], lambda o: o.key)
+
+        assert cache_manager.copy("test") == {"key1": obj1, "key2": obj2}
+
+    def test_has(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+
+        with cache_manager.sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key_from_sdp", "value")
+
+        cache_manager.put("test", "key_in_cache", "value")
+
+        assert cache_manager.has("test", "key_in_cache")
+        assert not cache_manager.has("test", "key_from_sdp")
+
+    def test_exist(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+
+        with cache_manager.sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key_from_sdp", "value")
+
+        cache_manager.put("test", "key_in_cache", "value")
+
+        assert cache_manager.exists("test", "key_in_cache")
+        assert cache_manager.exists("test", "key_from_sdp")
+
+    def test_i_can_clear_a_single_cache(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+        cache_manager.put("test", "key1", "value1")
+        cache_manager.put("test", "key2", "value2")
+
+        cache_manager.commit(context)
+
+        cache_manager.clear("test")
+        assert cache_manager.copy("test") == {}
+        assert sdp.exists("test", "key1")
+        assert sdp.exists("test", "key2")
+
+        cache_manager.commit(context)
+        assert cache_manager.copy("test") == {}
+        assert not sdp.exists("test", "key1")
+        assert not sdp.exists("test", "key2")
+
+    def test_i_can_clear_all_caches(self, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test1", Cache())
+        cache_manager.register_cache("test2", Cache())
+        cache_manager.put("test1", "key", "value")
+        cache_manager.put("test2", "key", "value")
+
+        cache_manager.clear()
+        assert cache_manager.copy("test1") == {}
+        assert cache_manager.copy("test2") == {}
+
+    def test_i_cannot_add_null_keys_into_concept_cache(self):
+        cache_manager = CacheManager()
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+
+        with pytest.raises(KeyError):
+            meta1 = get_metadata(name="foo")
+            cache_manager.add_concept(meta1)

tests/caching/test_DictionaryCache.py

@@ -0,0 +1,165 @@
+import pytest
+
+from base import BaseTest
+from caching.DictionaryCache import DictionaryCache
+from common.global_symbols import NotFound
+from tests.caching import FakeSdp
+
+
+class TestDictionaryCache(BaseTest):
+    @pytest.mark.parametrize('key', [None, "str_value", 0, 1.0])
+    def test_key_must_be_true_or_false(self, key):
+        cache = DictionaryCache()
+
+        # key must be true False
+        with pytest.raises(KeyError):
+            cache.put("key", key)
+
+    def test_value_must_be_a_dictionary(self):
+        cache = DictionaryCache()
+
+        with pytest.raises(ValueError):
+            cache.put(True, "value")
+
+        with pytest.raises(ValueError):
+            cache.put(False, "value")
+
+    def test_i_can_put_and_retrieve_value_from_dictionary_cache(self):
+        cache = DictionaryCache()
+
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(False, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) == id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+        # I can append values
+        cache.put(True, {"key": "another_value", "key3": "value3"})
+        assert len(cache) == 4
+        assert cache.get("key") == "another_value"
+        assert cache.get("key2") == ["value21", "value22"]
+        assert cache.get("key3") == "value3"
+
+        # I can reset
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(False, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) == id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+        assert cache.get("key3") is NotFound
+
+        assert cache.copy() == {'key': 'value', 'key2': ['value21', 'value22']}
+
+    def test_i_can_get_a_value_that_does_not_exist_without_compromising_the_cache(self):
+        cache = DictionaryCache()
+        cache.put(False, {"key": "value"})
+
+        assert cache.get("key2") is NotFound
+        assert cache.copy() == {"key": "value"}
+
+    def test_i_can_append_to_a_dictionary_cache_even_if_it_is_new(self):
+        cache = DictionaryCache()
+
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(True, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) != id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+    def test_exists_in_dictionary_cache(self):
+        cache = DictionaryCache()
+        assert not cache.exists("key")
+
+        cache.put(True, {"key": "value"})
+        assert cache.exists("key")
+
+    def test_default_for_dictionary_cache(self):
+        cache = DictionaryCache(default={"key": "value", "key2": "value2"})
+
+        # cache is fully set when the value is found
+        assert cache.get("key") == "value"
+        assert cache.copy() == {"key": "value", "key2": "value2"}
+
+        # cache is fully set when the value is not found
+        cache.test_only_reset()
+        assert cache.get("key3") is NotFound
+        assert cache.copy() == {"key": "value", "key2": "value2"}
+
+        # cache is not corrupted when value is found
+        cache.put(True, {"key3": "value3", "key4": "value4"})
+        assert cache.get("key3") == "value3"
+        assert cache.copy() == {"key": "value", "key2": "value2", "key3": "value3", "key4": "value4"}
+
+        # cache is not corrupted when value is not found
+        cache._cache["key"] = "another value"  # operation that is normally not possible
+        assert cache.get("key5") is NotFound
+        assert cache.copy() == {"key": "value", "key2": "value2", "key3": "value3", "key4": "value4"}
+
+    def test_default_callable_for_dictionary_cache(self):
+        cache = DictionaryCache(default=lambda k: {"key": "value", "key2": "value2"})
+
+        assert cache.get("key") == "value"
+        assert "key2" in cache
+        assert len(cache) == 2
+
+        cache.clear()
+        assert cache.get("key3") is NotFound
+        assert len(cache) == 2
+        assert "key" in cache
+        assert "key2" in cache
+
+    def test_default_callable_with_internal_sdp_for_dictionary_cache(self):
+        cache = DictionaryCache(default=lambda sdp, key: sdp.get("cache_name", key),
+                                sdp=FakeSdp(lambda entry, k: {"key": "value", "key2": "value2"}))
+
+        assert cache.get("key") == "value"
+        assert "key2" in cache
+        assert len(cache) == 2
+
+        cache.clear()
+        assert cache.get("key3") is NotFound
+        assert len(cache) == 2
+        assert "key" in cache
+        assert "key2" in cache
+
+    def test_dictionary_cache_cannot_be_null(self):
+        cache = DictionaryCache(default=lambda k: NotFound)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=NotFound)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=lambda k: None)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=None)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+    def test_auto_configure_retrieves_the_whole_remote_repository(self, sdp, context):
+        cache = DictionaryCache(sdp=sdp).auto_configure("test")
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key1", "value1")
+            transaction.add("test", "key2", "value2")
+
+        # when call for a value that is not in the cache, Dictionary cache is configured to retrieve the repo
+        cache.get("value")
+
+        assert cache.copy() == {'key1': 'value1', 'key2': 'value2'}
+
+    def test_we_do_no_go_twice_in_repo_when_not_found(self, sdp, context):
+        cache = DictionaryCache(sdp=sdp).auto_configure("test")
+
+        assert cache.get("key") is NotFound
+
+        # now add a value in remote repo
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        assert cache.get("key") is NotFound  # the key was previously requested

tests/caching/test_FastCache.py

@@ -0,0 +1,111 @@
+from caching.FastCache import FastCache
+from common.global_symbols import NotFound
+
+
+def test_i_can_put_an_retrieve_values():
+    cache = FastCache()
+    cache.put("key", "value")
+
+    assert cache.get("key") == "value"
+    assert cache.cache == {"key": "value"}
+    assert cache.lru == ["key"]
+
+
+def test_i_can_put_and_retrieve_multiple_items():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    assert cache.cache == {"key1": "value1", "key2": "value2", "key3": "value3"}
+    assert cache.lru == ["key1", "key2", "key3"]
+
+
+def test_i_the_least_used_is_remove_first():
+    cache = FastCache(3)
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    cache.put("key4", "value4")
+    assert cache.cache == {"key4": "value4", "key2": "value2", "key3": "value3"}
+    assert cache.lru == ["key2", "key3", "key4"]
+
+    cache.put("key5", "value5")
+    assert cache.cache == {"key4": "value4", "key5": "value5", "key3": "value3"}
+    assert cache.lru == ["key3", "key4", "key5"]
+
+
+def test_i_can_put_the_same_key_several_times():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key1", "value3")
+
+    assert cache.cache == {"key1": "value3", "key2": "value2"}
+    assert cache.lru == ["key2", "key1"]
+
+
+def test_none_is_returned_when_not_found():
+    cache = FastCache()
+    assert cache.get("foo") is NotFound
+
+
+def test_i_can_evict_by_key():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("to_keep1", "to_keep_value1")
+    cache.put("key2", "value2")
+    cache.put("to_keep2", "to_keep_value2")
+    cache.put("key3", "value3")
+    cache.put("to_keep3", "to_keep_value3")
+
+    cache.evict_by_key(lambda k: k.startswith("key"))
+    assert cache.cache == {"to_keep1": "to_keep_value1",
+                           "to_keep2": "to_keep_value2",
+                           "to_keep3": "to_keep_value3"}
+
+    assert cache.lru == ["to_keep1", "to_keep2", "to_keep3"]
+
+
+def test_i_can_get_default_value():
+    cache = FastCache(max_size=3, default=lambda key: key + 1)
+
+    assert cache.get(1) == 2
+    assert cache.get(2) == 3
+    assert cache.get(3) == 4
+    assert cache.get(4) == 5
+
+    assert cache.cache == {2: 3, 3: 4, 4: 5}  # only 3 values
+
+
+def test_i_can_iter_on_entries():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    res = []
+    for k in cache:
+        assert k in cache
+        res.append(k)
+
+    assert res == ["key1", "key2", "key3"]
+
+
+def test_i_can_count():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    assert len(cache) == 3
+
+
+def test_i_can_copy():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    assert cache.copy() == {"key1": "value1", "key2": "value2", "key3": "value3"}

tests/caching/test_IncCache.py

@@ -0,0 +1,87 @@
+from base import BaseTest
+from caching.IncCache import IncCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class FakeIncSdp:
+    """
+    FakeIncSdp has two levels of "ontology"
+    if a value of
+    """
+    def __init__(self, init_value1, init_value2):
+        self.level1 = IncCache()
+        self.level1.put("key", init_value1)
+
+        self.level2 = IncCache()
+        self.level2.put("key", init_value2)
+
+    def alt_get(self, cache_name, key):
+        for cache in [self.level1, self.level2]:
+            value = cache.alt_get(key)
+            if value is not NotFound:
+                return value
+
+        return NotFound
+
+
+class TestIncCache(BaseTest):
+    def test_i_can_put_and_retrieve_values_from_inc_cache(self):
+        cache = IncCache()
+
+        assert cache.get("key") == 1
+        assert cache.get("key") == 2
+        assert cache.get("key") == 3
+        assert cache.get("key2") == 1
+        assert cache.get("key2") == 2
+
+        cache.put("key", 100)
+        assert cache.get("key") == 101
+
+        assert cache.copy() == {'key': 101, 'key2': 2}
+
+    def test_i_can_alt_get(self):
+        cache = IncCache()
+
+        assert cache.get("key") == 1
+        assert cache.get("key") == 2
+        assert cache.alt_get("key") == 2
+        assert cache.alt_get("key") == 2
+        assert cache.get("key") == 3
+
+    def test_current_cache_takes_precedence_over_alt_sdp(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        assert cache.get("key") == 1
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: 10)
+        assert cache.get("key", alt_sdp=alt_sdp) == 2
+
+    def test_remote_repository_takes_precedence_over_alt_sdp(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: 5)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: 10)
+        assert cache.get("key", alt_sdp=alt_sdp) == 6
+        assert cache.get("key", alt_sdp=alt_sdp) == 7  # then we use the value from the cache
+
+    def test_i_can_take_value_from_alt_sdp(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeIncSdp(10, NotFound)
+        assert cache.get("key", alt_sdp=alt_sdp) == 11
+        assert cache.get("key", alt_sdp=alt_sdp) == 12  # then we use the value from the cache
+
+    def test_i_can_get_when_alt_sdp_and_cache_is_cleared(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        alt_sdp = FakeIncSdp(10, NotFound)
+        assert cache.get("key", alt_sdp=alt_sdp) == 1
+        assert cache.get("key", alt_sdp=alt_sdp) == 2  # then we use the value from the cache
+
+    def test_i_can_manage_when_the_value_from_alt_sdp_is_removed(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeIncSdp(Removed, 10)
+        assert cache.get("key", alt_sdp=alt_sdp) == 1
+        assert cache.get("key", alt_sdp=alt_sdp) == 2  # then we use the value from the cache

tests/caching/test_ListCache.py

@@ -0,0 +1,281 @@
+import pytest
+
+from base import BaseTest
+from caching.ListCache import ListCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class TestListCache(BaseTest):
+
+    def test_i_can_put_and_retrieve_value_from_list_cache(self):
+        cache = ListCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == ["value"]
+        assert len(cache) == 1
+
+        cache.put("key", "value2")  # we can append to this list
+        assert cache.get("key") == ["value", "value2"]
+        assert len(cache) == 2
+
+        cache.put("key2", "value")
+        assert cache.get("key2") == ["value"]
+        assert len(cache) == 3
+
+        # duplicates are allowed
+        cache.put("key", "value")
+        assert cache.get("key") == ["value", "value2", "value"]
+        assert len(cache) == 4
+
+        assert cache.copy() == {'key': ['value', 'value2', 'value'], 'key2': ['value']}
+
+    def test_i_can_put_in_list_cache_when_alt_sdp_returns_values(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]))
+        assert cache.get("key") == ["value1", "value2"]
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == ["value1"]
+
+    def test_i_can_put_in_list_cache_when_alt_sdp_returns_values_and_cache_is_cleared(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]))
+        assert cache.get("key") == ["value2"]
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == ["value1"]
+
+    def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_list_cache(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value1")
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_list_cache(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: ["value1"])).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_i_can_get_when_alt_sdp(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.get("key", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]))
+        assert cache.get("key") == ["value1"]
+
+    def test_i_can_update_from_list_cache(self):
+        cache = ListCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "another value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value2", "value"]  # only the first one is affected
+
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key2") == ["value2"]
+
+        cache.update("key2", "value2", "key3", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key3") == ["value2"]
+        assert cache.get("key2") is NotFound
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self):
+        cache = ListCache(default=lambda sdp, key: sdp.get("cache_name", key),
+                          extend_exists=lambda sdp, key: sdp.exists("cache_name", key),
+                          sdp=FakeSdp(get_value=lambda cache_name, key: NotFound))
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == ["new_value"]
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self):
+        # There is nothing in cache or remote repository.
+        # We must ust the value from alt_sdp
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        previous_value = ["old_1", "old_2", "value"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: previous_value)
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == ["old_1", "old_2", "new_value"]
+        assert previous_value == ["old_1", "old_2", "value"]
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self):
+        # keys are different
+        # make sure that current cache take precedence over alt_sdp
+        # In this test, the values from alt_sdp are never used
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: ["xxx1"] if key == "key1" else NotFound)
+
+        # one values in 'key1'
+        cache.put("key1", "old_1")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2"]
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self):
+        # keys are different
+        # make sure that current repo take precedence over alt_sdp
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1"] if key == "key1" else NotFound)
+        cache = ListCache(sdp=remote).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: ["xxx1"] if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2"] if key == "key1" else NotFound)
+        cache = ListCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2"]
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self):
+        # keys are different
+        # No value found in cache or remote repository,
+        # Will use values from alt_sdp
+        # The old key is the same, so it has to be marked as Removed
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one values in 'key1'
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: ["old_1"] if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.test_only_reset()
+        old_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2"]
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == ["old_1", "old_2"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It takes precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: ["xxx2"] if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.put("key2", "old_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It takes precedence
+        # If no value is found, we must use the value from alt_sdp
+        remote_repo = FakeSdp(get_value=lambda cache_name, key: ["old_value"] if key == "key2" else NotFound)
+        cache = ListCache(sdp=remote_repo).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: ["xxx2"] if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It takes precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key1", "source_value")
+        previous_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ["old_1", "old_2", 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+        assert previous_values == ["old_1", "old_2"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])
+        cache.clear()
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp)
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp)
+
+    def test_default_is_called_before_updating_list_cache(self):
+        cache = ListCache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"])
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == ["new_value", "other old value"]
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else NotFound)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == ["other old value"]
+        assert cache.get("new_key") == ["new_value"]
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else ["other new"])
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == ["other old value"]
+        assert cache.get("new_key") == ["other new", "new_value"]

tests/caching/test_ListIfNeededCache.py

@@ -0,0 +1,648 @@
+import pytest
+
+from base import BaseTest
+from caching.ListIfNeededCache import ListIfNeededCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class TestListIfNeededCache(BaseTest):
+    def test_i_can_put_and_retrieve_value_from_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == "value"
+
+        # second time with the same key creates a list
+        cache.put("key", "value2")
+        assert cache.get("key") == ["value", "value2"]
+        assert len(cache) == 2
+
+        # third time, we now have a list
+        cache.put("key", "value3")
+        assert cache.get("key") == ["value", "value2", "value3"]
+        assert len(cache) == 3
+
+        # other keys are not affected
+        cache.put("key2", "value")
+        assert cache.get("key2") == "value"
+        assert len(cache) == 4
+
+        # duplicates are allowed
+        cache.put("key", "value")
+        assert cache.get("key") == ["value", "value2", "value3", "value"]
+        assert len(cache) == 5
+
+    def test_i_can_put_in_list_if_need_cache_when_alt_sdp_returns_values(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "value1"))
+        assert cache.get("key") == ["value1", "value2"]
+
+        cache.put("key2", "value3", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2"]))
+        assert cache.get("key2") == ["value1", "value2", "value3"]
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == "value1"
+
+    def test_i_can_put_in_list_if_need__cache_when_alt_sdp_returns_values_and_cache_is_cleared(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "value1"))
+        assert cache.get("key") == "value2"
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == "value1"
+
+    def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_list_if_needed_cache(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value1")
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_list_if_needed_cache(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: "value1")).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_i_can_update_from_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value")
+
+        # only the first 'value' is affected
+        cache.update("key", "value", "key", "another value")
+        assert len(cache._cache) == 1
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value2", "value"]
+
+        # change the key
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key2") == "value2"
+
+        # rename the newly created key
+        cache.update("key2", "value2", "key3", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key3") == "value2"
+        assert cache.get("key2") is NotFound
+
+        # from list to single item and vice versa
+        cache.update("key", "value", "key3", "value")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == "another value"  # 'key' is no longer a list
+        assert cache.get("key3") == ["value2", "value"]  # 'key3' is now a list
+        assert cache.get("key2") is NotFound
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == "new_value"
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one value in alt_sdp
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: "old_value")
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == "new_value"
+
+        # multiple values in alt_sdp
+        cache.test_only_reset()
+        previous_value = ["old_1", "old_2", "value"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: previous_value)
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == ["old_1", "old_2", "new_value"]
+        assert previous_value == ["old_1", "old_2", "value"]
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self):
+        # keys are different
+        # make sure that current cache take precedence over alt_sdp
+        # In this test, the values from alt_sdp are never used
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: "xxx1" if key == "key1" else NotFound)
+
+        # one values in 'key1'
+        cache.put("key1", "old_1")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == "old_2"
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # List of values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+        cache.put("key1", "old_3")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2", "old_3"]
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self):
+        # keys are different
+        # make sure that current repo take precedence over alt_sdp
+        remote = FakeSdp(get_value=lambda cache_name, key: "old_1" if key == "key1" else NotFound)
+        cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: "xxx1" if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2"] if key == "key1" else NotFound)
+        cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == "old_2"
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # List of values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2", "old_3"] if key == "key1" else NotFound)
+        cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2", "old_3"]
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self):
+        # keys are different
+        # No value found in cache or remote repository,
+        # Will use values from alt_sdp
+        # The old key is the same, so it has to be marked as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one values in 'key1'
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: "old_1" if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.test_only_reset()
+        old_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == "old_2"
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == ["old_1", "old_2"]  # not modified
+
+        # List of values in 'key1'
+        cache.test_only_reset()
+        old_values = ["old_1", "old_2", "old_3"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2", "old_3"]
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == ["old_1", "old_2", "old_3"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.put("key2", "old_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        remote_repo = FakeSdp(get_value=lambda cache_name, key: "old_value" if key == "key2" else NotFound)
+        cache = ListIfNeededCache(sdp=remote_repo).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        # one value in 'key2'
+        cache.put("key1", "source_value")
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "old_value" if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+        # Multiple values in 'key2'
+        cache.test_only_reset()
+        cache.put("key1", "source_value")
+        previous_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ["old_1", "old_2", 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+        assert previous_values == ["old_1", "old_2"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1")
+        cache.clear()
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp)
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp)
+
+    def test_default_is_called_before_updating_list_if_needed_cache(self):
+        cache = ListIfNeededCache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = ListIfNeededCache(default=lambda k: "old_value")
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == "new_value"
+
+        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"])
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == ["new_value", "other old value"]
+
+        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else NotFound)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == "other old value"
+        assert cache.get("new_key") == "new_value"
+
+    def test_i_can_delete_key_and_values(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+        cache.put("key", "value11")
+        cache.put("key2", "value2")
+        cache.put("key2", "value22")
+        cache.put("key2", "value222")
+        cache.put("key3", "value3")
+        cache.put("key3", "value33")
+        cache.put("key4", "value4")
+        cache.reset_events()
+
+        assert len(cache) == 8
+
+        # I can remove a whole key
+        cache.delete("key")
+        assert cache.get("key") is NotFound
+        assert len(cache) == 6
+        assert cache.to_remove == {"key"}
+        assert cache.to_add == set()
+
+        # I can remove an element while a list is remaining
+        cache.reset_events()
+        cache.delete("key2", "value22")
+        assert cache.get("key2") == ["value2", "value222"]
+        assert len(cache) == 5
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while a single element is remaining
+        cache.reset_events()
+        cache.delete("key3", "value33")
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 4
+        assert cache.to_add == {"key3"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while nothing remains
+        cache.reset_events()
+        cache.delete("key4", "value4")
+        assert cache.get("key4") is NotFound
+        assert len(cache) == 3
+        assert cache.to_remove == {"key4"}
+        assert cache.to_add == set()
+
+        # I do not remove when the value is not the same
+        cache.reset_events()
+        cache.delete("key3", "value33")  # value33 was already remove
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 3
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_and_then_put_back(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)  # remove all values
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_one_value(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+        cache.put("key", "value2")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": "value2"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value3")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ['value2', 'value3']}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure(
+            "cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: "value")).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository_and_then_put_back(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure(
+            "cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value=None, alt_sdp=alt_sdp)  # remove all values
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_remaining_one_value(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure(
+            "cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": "value2"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1, value2"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1",
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_and_then_put_back(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1",
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_one_value_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, one value remains in the cache
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": "value2"}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_multiple_values_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, one value remains in the cache
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2", "value3"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ["value2", "value3"]}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_an_already_removed_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # But the alternate sdp returns Removed, which means that previous value was deleted
+        # It's like there is nothing to delete
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_deleting_an_entry_that_does_not_exist_is_not_an_error(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+
+        cache.reset_events()
+        cache.delete("key3")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key3", "value")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value2")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value",
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.clear()
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: Removed)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository_from_alt_sdp(self):
+        # The key is removed in the sub layers
+        # We can put it back
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.put("key", "value", alt_sdp=alt_sdp)
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}

tests/caching/test_SetCache.py

@@ -0,0 +1,540 @@
+import pytest
+
+from base import BaseTest
+from caching.SetCache import SetCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class TestSetCache(BaseTest):
+
+    def test_i_can_put_and_retrieve_values_from_set_cache(self):
+        cache = SetCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == {"value"}
+        assert len(cache) == 1
+
+        # we can add to this set
+        cache.put("key", "value2")
+        assert cache.get("key") == {"value", "value2"}
+        assert len(cache) == 2
+
+        # other keys are not affected
+        cache.put("key2", "value")
+        assert cache.get("key2") == {"value"}
+        assert len(cache) == 3
+
+        # duplicates are removed
+        cache.put("key", "value")
+        assert cache.get("key") == {"value", "value2"}
+        assert len(cache) == 3
+
+        assert cache.copy() == {'key': {'value', 'value2'}, 'key2': {'value'}}
+
+    def test_i_can_put_in_set_cache_when_alt_sdp_returns_values(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: {"value1"}))
+        assert cache.get("key") == {"value1", "value2"}
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == {"value1"}
+
+    def test_i_can_put_in_set_cache_when_alt_sdp_returns_values_and_cache_is_cleared(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: {"value1"}))
+        assert cache.get("key") == {"value2"}
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == {"value1"}
+
+    def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_set_cache(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value1")
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == {"value1", "value2"}
+
+    def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_set_cache(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: {"value1"})).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == {"value1", "value2"}
+
+    def test_i_can_update_from_set_cache(self):
+        cache = SetCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.update("key", "value", "key", "another value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 2
+        assert cache.get("key") == {"another value", "value2"}
+
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 2
+        assert cache.get("key") == {"another value"}
+        assert cache.get("key2") == {"value2"}
+
+        cache.update("key", "another value", "key3", "another value")
+        assert len(cache._cache) == 2
+        assert len(cache) == 2
+        assert cache.get("key") is NotFound
+        assert cache.get("key2") == {"value2"}
+        assert cache.get("key3") == {"another value"}
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == {"new_value"}
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self):
+        # There is nothing in cache or remote repository.
+        # We must ust the value from alt_sdp
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        previous_value = {"old_1", "old_2", "value"}
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: previous_value)
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == {"old_1", "old_2", "new_value"}
+        assert previous_value == {"old_1", "old_2", "value"}
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self):
+        # keys are different
+        # make sure that current cache take precedence over alt_sdp
+        # In this test, the values from alt_sdp are never used
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: {"xxx1"} if key == "key1" else NotFound)
+
+        # one values in 'key1'
+        cache.put("key1", "old_1")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == {"old_2"}
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self):
+        # keys are different
+        # make sure that current repo take precedence over alt_sdp
+        remote = FakeSdp(get_value=lambda cache_name, key: {"old_1"} if key == "key1" else NotFound)
+        cache = SetCache(sdp=remote).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: {"xxx1"} if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: {"old_1", "old_2"} if key == "key1" else NotFound)
+        cache = SetCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == {"old_2"}
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self):
+        # keys are different
+        # No value found in cache or remote repository,
+        # Will use values from alt_sdp
+        # The old key is the same, so it has to be marked as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one values in 'key1'
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: {"old_1"} if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.test_only_reset()
+        old_values = {"old_1", "old_2"}
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == {"old_2"}
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == {"old_1", "old_2"}  # not modified
+
+    def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: {"xxx2"} if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.put("key2", "old_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == {'old_value', 'new_value'}
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        remote_repo = FakeSdp(get_value=lambda cache_name, key: {"old_value"} if key == "key2" else NotFound)
+        cache = SetCache(sdp=remote_repo).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == {'old_value', 'new_value'}
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key1", "source_value")
+        previous_values = {"old_1", "old_2"}
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == {"old_1", "old_2", 'new_value'}
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+        assert previous_values == {"old_1", "old_2"}  # not modified
+
+    def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])
+        cache.clear()
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp)
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp)
+
+    def test_default_is_called_before_updating_set_cache(self):
+        cache = SetCache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"})
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == {"new_value", "other old value"}
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else NotFound)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == {"other old value"}
+        assert cache.get("new_key") == {"new_value"}
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else {"other new"})
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == {"other old value"}
+        assert cache.get("new_key") == {"other new", "new_value"}
+
+    def test_i_can_delete_values_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+        cache.reset_events()
+
+        cache.delete("key", "fake_value")
+        assert cache.get("key") == {"value1", "value2"}
+        assert len(cache) == 2
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value1")
+        assert cache.get("key") == {"value2"}
+        assert cache.to_add == {"key"}
+        assert len(cache) == 1
+
+        cache.delete("key", "value2")
+        assert cache.get("key") is NotFound
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_key_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+
+        cache.delete("key")
+        assert cache.get("key") is NotFound
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_a_key_that_does_not_exists(self):
+        cache = SetCache()
+        cache.delete("key")
+
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_and_then_put_back(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But there is a value in the current cache after deletion
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+
+        cache.delete("key", value="value1", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": {"value2"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value"})).auto_configure("cache_name")
+
+        cache.delete("key", value="value", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_and_then_put_back(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But there is a value in the current cache after deletion
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name")
+
+        cache.delete("key", value="value1", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": {"value2"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1, value2"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_and_then_put_back(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_one_value_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1", "value2"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": {"value2"}}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_key_that_does_not_exist_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=(lambda cache_name, key: {"value1", "value2"} if key == "key" else NotFound),
+                          extend_exists=lambda cache_name, key: key == "key")
+
+        cache.delete("key2", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_that_does_not_exist_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1", "value2"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value4", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_an_already_removed_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # But the alternate sdp returns Removed, which means that previous value was deleted
+        # It's like there is nothing to delete
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.clear()
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: Removed)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository_from_alt_sdp(self):
+        # The key is removed in the sub layers
+        # We can put it back
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.put("key", "value", alt_sdp=alt_sdp)
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}

tests/caching/test_cache.py

@@ -0,0 +1,512 @@
+import pytest
+
+from caching.BaseCache import BaseCache, MAX_INITIALIZED_KEY
+from caching.Cache import Cache
+from caching.DictionaryCache import DictionaryCache
+from caching.IncCache import IncCache
+from caching.ListCache import ListCache
+from caching.ListIfNeededCache import ListIfNeededCache
+from caching.SetCache import SetCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class TestCache:
+    def test_i_can_configure(self):
+        cache = Cache()
+        cache.configure(max_size=256,
+                        default="default_delegate",
+                        extend_exists="extend_exists_delegate",
+                        alt_sdp_get="alt_sdp_delegate",
+                        sdp=FakeSdp())
+
+        # Caution, in this test, I initialize default, extend_exists and alt_get_delegate with string
+        # to simplify the test, but in real usage, they are lambda
+        # default = lambda sdp, key: sdp.get(cache_name, key) or lambda key: func(key)
+        # extend_exists = lambda sdp, key: sdp.exists(cache_name, key) or lambda key: func(key)
+        # alt_sdp_get = lambda sdp, key: sdp.alt_get(cache_name, key)
+
+        assert cache._max_size == 256
+        assert cache._default == "default_delegate"
+        assert cache._extend_exists == "extend_exists_delegate"
+        assert cache._alt_sdp_get == "alt_sdp_delegate"
+        assert cache._sdp is not None
+
+    def test_i_can_auto_configure(self):
+        sdp = FakeSdp(get_value=lambda cache_name, key: key + 1 if cache_name == "cache_name" else NotFound,
+                      extend_exists=lambda cache_name, key: True if cache_name == "cache_name" else False,
+                      get_alt_value=lambda cache_name, key: key + 2 if cache_name == "cache_name" else NotFound)
+
+        cache = Cache(sdp=sdp).auto_configure("cache_name")
+        assert cache._default(cache._sdp, 10) == 11
+        assert cache._extend_exists(cache._sdp, 10) is True
+        assert cache._alt_sdp_get(cache._sdp, 10) == 12
+
+        cache = Cache(sdp=sdp).auto_configure("another_cache")
+        assert cache._default(cache._sdp, 10) == NotFound
+        assert cache._extend_exists(cache._sdp, 10) is False
+        assert cache._alt_sdp_get(cache._sdp, 10) == NotFound
+
+    def test_i_can_get_an_retrieve_value_from_cache(self):
+        cache = Cache()
+        cache.put("key", "value")
+        assert cache.get("key") == "value"
+        assert len(cache) == 1
+
+        cache.put("key", "another value")  # another value in the cache replace the previous one
+        assert cache.get("key") == "another value"
+        assert len(cache) == 1
+
+        cache.put("key2", "value2")  # another key
+        assert cache.get("key2") == "value2"
+        assert len(cache) == 2
+        assert cache.copy() == {"key": "another value", "key2": "value2"}
+
+    def test_i_do_not_evict_when_put(self):
+        """
+        It's because we evict on get()
+        :return:
+        :rtype:
+        """
+        maxsize = 5
+        cache = Cache(max_size=5)
+
+        for key in range(maxsize + 2):
+            cache.put(str(key), key)
+
+        assert len(cache) == maxsize + 2
+        assert cache.copy() == {
+            "0": 0,
+            "1": 1,
+            "2": 2,
+            "3": 3,
+            "4": 4,
+            "5": 5,
+            "6": 6,
+        }
+
+    def test_i_can_evict_when_get(self):
+        maxsize = 5
+        cache = Cache(max_size=5, default=lambda k: int(k))
+
+        for key in range(maxsize + 2):
+            cache.get(str(key))
+
+        assert len(cache) == maxsize
+        assert cache.copy() == {
+            "2": 2,
+            "3": 3,
+            "4": 4,
+            "5": 5,
+            "6": 6,
+        }
+
+    def test_i_do_not_evict_when_items_are_not_committed(self):
+        maxsize = 5
+        cache = Cache(max_size=5, default=lambda k: k)
+
+        for key in range(maxsize + 2):
+            cache.put(str(key), key)
+
+        assert len(cache) == maxsize + 2
+
+        cache.get("-1")
+        assert len(cache) == maxsize + 2
+        assert cache.copy() == {
+            "0": 0,
+            "1": 1,
+            "2": 2,
+            "3": 3,
+            "4": 4,
+            "5": 5,
+            "6": 6,
+        }
+
+    def test_i_can_get_a_value_from_alt_sdp(self):
+        cache = Cache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !")
+        assert cache.get("key", alt_sdp=alt_sdp) == "value found !"
+
+        # The value is now in cache
+        assert cache.copy() == {'key': 'value found !'}
+
+    def test_i_cannot_get_a_value_from_alt_sdp_when_cache_is_cleared(self):
+        cache = Cache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !")
+        assert cache.get("key", alt_sdp=alt_sdp) is NotFound
+        assert cache.copy() == {}
+
+    def test_i_can_get_default_value_from_simple_cache(self):
+        cache = Cache()
+        assert cache.get("key") is NotFound
+
+        cache = Cache(default=10)
+        assert cache.get("key") == 10
+        assert "key" not in cache  # default value are not put in cache
+
+        cache = Cache(default=lambda key: key + "_not_found")
+        assert cache.get("key") == "key_not_found"
+        assert "key" in cache  # default callable are put in cache
+
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda entry, key: key + "_not_found"))
+        assert cache.get("key") == "key_not_found"
+        assert "key" in cache  # default callable are put in cache
+
+    def test_i_do_not_ask_the_remote_repository_twice(self):
+        nb_request = []
+
+        cache = Cache(default=lambda key: nb_request.append("requested"))
+        assert cache.get("key") is None
+        assert cache.get("key") is None
+        assert len(nb_request) == 1
+
+    def test_i_can_update_from_simple_cache(self):
+        cache = Cache()
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 1
+        assert cache.get("key") == "new_value"
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.reset_events()
+        cache.update("key", "new_value", "another_key", "another_value")
+        assert len(cache._cache) == 1
+        assert len(cache) == 1
+        assert cache.get("key") is NotFound
+        assert cache.get("another_key") == "another_value"
+        assert cache.to_add == {"another_key"}
+        assert cache.to_remove == {"key"}
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_same_keys(self):
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key),
+                      extend_exists=lambda sdp, key: sdp.exists("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda cache_name, key: NotFound))
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == "new_value"
+
+    def test_i_can_update_when_alt_sdp_different_keys(self):
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key),
+                      extend_exists=lambda sdp, key: sdp.exists("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda cache_name, key: NotFound))
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key2", "value2", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key2") == "value2"
+        assert cache.get("key") == Removed
+        assert cache.to_add == {"key", "key2"}
+        assert cache.to_remove == set()
+
+    @pytest.mark.parametrize("cache", [
+        Cache(), ListCache(), ListIfNeededCache(), SetCache(), IncCache()
+    ])
+    def test_i_can_manage_cache_events(self, cache: BaseCache):
+        cache.put("key", "value")
+
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "value", "key", "another value")
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "another value", "key2", "value2")
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key"}
+
+        cache.update("key2", "value2", "key", "value")
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == {"key2"}
+
+    @pytest.mark.parametrize("cache", [
+        ListCache(), SetCache(), ListIfNeededCache()
+    ])
+    def test_i_can_manage_list_and_set_cache_events(self, cache):
+        cache.put("key", "value")
+        cache.put("key", "value2")
+
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "value", "key", "another value")
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "value2", "key2", "value2")
+        assert cache.to_add == {"key", "key2"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "another value", "key3", "another value")
+        assert cache.to_add == {"key2", "key3"}
+        assert cache.to_remove == {"key"}
+
+    @pytest.mark.parametrize("cache", [
+        Cache(), ListCache(), SetCache(), ListIfNeededCache(), IncCache()
+    ])
+    def test_exists(self, cache):
+        assert not cache.exists("key")
+        cache.put("key", "value")
+        assert cache.exists("key")
+
+    def test_exists_extend(self):
+        cache = Cache(extend_exists=lambda k: True if k == "special_key" else False)
+        assert not cache.exists("key")
+        assert cache.exists("special_key")
+
+    def test_i_can_extend_exists_when_internal_sdp(self):
+        cache = Cache(extend_exists=lambda sdp, k: True if k == "special_key" else False, sdp=FakeSdp)
+        assert not cache.exists("key")
+        assert cache.exists("special_key")
+
+    @pytest.mark.parametrize("cache, default, new_value, expected", [
+        (ListCache(), lambda k: NotFound, "value", ["value"]),
+        (ListCache(), lambda k: ["value"], "value", ["value", "value"]),
+        (ListIfNeededCache(), lambda k: NotFound, "value", "value"),
+        (ListIfNeededCache(), lambda k: "value", "value1", ["value", "value1"]),
+        (ListIfNeededCache(), lambda k: ["value1", "value2"], "value1", ["value1", "value2", "value1"]),
+        (SetCache(), lambda k: NotFound, "value", {"value"}),
+        (SetCache(), lambda k: {"value"}, "value", {"value"}),
+        (SetCache(), lambda k: {"value1"}, "value2", {"value1", "value2"}),
+    ])
+    def test_default_is_called_before_put_to_keep_in_sync(self, cache, default, new_value, expected):
+        cache.configure(default=default)
+        cache.put("key", new_value)
+
+        assert cache.get("key") == expected
+
+    def test_default_is_called_before_updating_simple_cache(self):
+        cache = Cache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = Cache(default=lambda k: "old_value")
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("new_key") == "new_value"
+
+    def test_i_can_delete_an_entry_from_cache(self):
+        cache = Cache()
+        cache.put("key", "value")
+
+        assert cache.get("key") == "value"
+        cache.delete("key")
+        assert cache.get("key") is NotFound
+        assert cache.to_remove == {"key"}
+
+    def test_i_can_delete_when_entry_is_only_in_db(self):
+        cache = Cache(default=lambda k: "value" if k == 'key' else NotFound)
+
+        cache.delete("another_key")
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key")
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == {"key"}
+
+    def test_i_can_delete_an_entry_from_cache_when_alt_sdp_and_value_in_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = Cache(extend_exists=lambda sdp, k: sdp.exists("cache_name", k))
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_an_entry_from_cache_when_alt_sdp_when_in_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = Cache(default=lambda k: "value", extend_exists=lambda sdp, k: sdp.exists("cache_name", k))
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_an_entry_from_cache_when_alt_sdp_and_no_value_in_cache_or_remote_repository(self):
+        # alt_cache_manager is used when no value found
+        cache = Cache(default=lambda sdp, k: sdp.get("cache_name", k),
+                      extend_exists=lambda sdp, k: sdp.exists("cache_name", k),
+                      sdp=FakeSdp(get_value=lambda entry, k: NotFound))
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_no_error_when_deleting_a_key_that_does_not_exists_when_alt_sdp(self):
+        # alt_cache_manager is used when no value found
+        cache = Cache(default=lambda sdp, k: sdp.get("cache_name", k),
+                      extend_exists=lambda sdp, k: sdp.exists("cache_name", k),
+                      sdp=FakeSdp(get_value=lambda entry, k: NotFound))
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: False))
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_initialized_key_is_removed_when_the_entry_is_found(self):
+        caches = [Cache(), ListCache(), ListIfNeededCache(), SetCache()]
+
+        for cache in caches:
+            cache.put("key", "value")
+            cache.get("key")
+
+            assert len(cache._initialized_keys) == 0
+
+        cache = IncCache()
+        cache.put("key", 10)
+        cache.get("key")
+        assert len(cache._initialized_keys) == 0
+
+    def test_initialized_keys_are_reset_when_max_length_is_reached(self):
+        cache = Cache()
+        for i in range(MAX_INITIALIZED_KEY):
+            cache.get(str(i))
+
+        assert len(cache._initialized_keys) == MAX_INITIALIZED_KEY
+
+        cache.get(str(MAX_INITIALIZED_KEY + 1))
+        assert len(cache._initialized_keys) == 1
+
+    def test_i_can_populate(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache()
+
+        cache.populate(lambda: items, lambda item: item[0])
+
+        assert len(cache) == 3
+        assert cache.get("1") == ("1", "1")
+        assert cache.get("2") == ("2", "2")
+        assert cache.get("3") == ("3", "3")
+
+        assert cache.to_add == {"1", "2", "3"}
+        assert cache.to_remove == set()
+
+    def test_i_can_populate_using_internal_sdp(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache(sdp=FakeSdp(populate=items))
+
+        cache.populate(lambda sdp: sdp.populate(), lambda item: item[0])
+
+        assert len(cache) == 3
+        assert cache.get("1") == ("1", "1")
+        assert cache.get("2") == ("2", "2")
+        assert cache.get("3") == ("3", "3")
+
+        assert cache.to_add == {"1", "2", "3"}
+        assert cache.to_remove == set()
+
+    def test_i_can_reset_the_event_after_populate(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache()
+        cache.to_add = {"some_value"}
+        cache.to_remove = {"some_other_value"}
+
+        cache.populate(lambda: items, lambda item: item[0], reset_events=True)
+
+        assert len(cache) == 3
+        assert cache.copy() == {"1": ("1", "1"),
+                                "2": ("2", "2"),
+                                "3": ("3", "3")}
+        assert cache.to_add == {"some_value"}
+        assert cache.to_remove == {"some_other_value"}
+
+    def test_i_can_get_all(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache()
+
+        cache.populate(lambda: items, lambda item: item[0])
+
+        res = cache.get_all()
+        assert len(res) == 3
+        assert list(res) == [('1', '1'), ('2', '2'), ('3', '3')]
+
+    def test_i_can_clone_cache(self):
+        cache = Cache(max_size=256,
+                      default=lambda sdp, key: sdp.get("cache_name", key),
+                      extend_exists=False,
+                      alt_sdp_get=lambda sdp, key: sdp.alt_get("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda entry, key: key + "_not_found"))
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+
+        clone = cache.clone()
+        assert type(cache) == type(clone)
+        assert clone._max_size == cache._max_size
+        assert clone._default == cache._default
+        assert clone._extend_exists == cache._extend_exists
+        assert clone._alt_sdp_get == cache._alt_sdp_get
+        assert clone._sdp == cache._sdp
+        assert clone._cache == {}  # value are not copied
+        assert clone._initialized_keys == set()
+        assert clone._current_size == 0
+        assert clone.to_add == set()
+        assert clone.to_remove == set()
+
+        clone.configure(sdp=FakeSdp(lambda entry, key: key + " found !"))
+
+        assert cache.get("key3") == "key3_not_found"
+        assert clone.get("key3") == "key3 found !"
+
+    @pytest.mark.parametrize("cache", [
+        Cache(),
+        DictionaryCache(),
+        IncCache(),
+        ListCache(),
+        ListIfNeededCache()
+    ])
+    def test_i_can_clone_all_caches(self, cache):
+        clone = cache.clone()
+        assert type(clone) == type(cache)
+
+    def test_sanity_check_on_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+        clone = cache.clone()
+
+        clone.put("key", "value1")
+        clone.put("key", "value2")
+
+        assert clone.get("key") == ["value1", "value2"]
+
+    def test_i_can_clear_when_alt_sdp(self):
+        cache = Cache().auto_configure("cache_name")
+
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+
+        cache.clear()
+
+        assert cache.copy() == {}
+        assert cache._is_cleared
+
+    def test_i_can_iter_on_the_content(self):
+        cache = Cache()
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+        cache.put("key3", "value3")
+
+        res = []
+        for k in cache:
+            assert k in cache
+            res.append(k)
+
+        assert res == ["key1", "key2", "key3"]