Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

tests/cache/test_cache.py

@@ -0,0 +1,534 @@
+import pytest
+from cache.Cache import Cache
+from cache.CacheManager import CacheManager
+from cache.DictionaryCache import DictionaryCache
+from cache.IncCache import IncCache
+from cache.ListCache import ListCache
+from cache.ListIfNeededCache import ListIfNeededCache
+from cache.SetCache import SetCache
+from core.concept import Concept
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestCache(TestUsingMemoryBasedSheerka):
+    def test_i_can_get_an_retrieve_value_from_cache(self):
+        cache = Cache()
+        cache.put("key", "value")
+        assert cache.get("key") == "value"
+        assert len(cache) == 1
+
+        cache.put("key", "another value")  # another value in the cache replace the previous one
+        assert cache.get("key") == "another value"
+        assert len(cache) == 1
+
+        cache.put("key2", "value2")  # another key
+        assert cache.get("key2") == "value2"
+        assert len(cache) == 2
+        assert cache.copy() == {"key": "another value", "key2": "value2"}
+
+    def test_i_can_evict(self):
+        maxsize = 5
+        cache = Cache(max_size=5)
+
+        for key in range(maxsize):
+            cache.put(key, key)
+
+        assert len(cache) == maxsize
+        assert cache.has(0)
+
+        for key in range(maxsize, maxsize * 2):
+            cache.put(key, key)
+
+            assert len(cache) == maxsize
+            assert not cache.has(key - maxsize)
+
+    def test_i_can_get_default_value_from_simple_cache(self):
+        cache = Cache()
+        assert cache.get("key") is None
+
+        cache = Cache(default=10)
+        assert cache.get("key") == 10
+        assert "key" not in cache  # default value are not put in cache
+
+        cache = Cache(default=lambda key: key + "_not_found")
+        assert cache.get("key") == "key_not_found"
+        assert "key" in cache  # default callable are put in cache
+
+    def test_i_can_put_and_retrieve_value_from_list_cache(self):
+        cache = ListCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == ["value"]
+        assert len(cache) == 1
+
+        cache.put("key", "value2")  # we can append to this list
+        assert cache.get("key") == ["value", "value2"]
+        assert len(cache) == 2
+
+        cache.put("key2", "value")
+        assert cache.get("key2") == ["value"]
+        assert len(cache) == 3
+
+        # duplicates are allowed
+        cache.put("key", "value")
+        assert cache.get("key") == ["value", "value2", "value"]
+        assert len(cache) == 4
+
+        assert cache.copy() == {'key': ['value', 'value2', 'value'], 'key2': ['value']}
+
+    def test_i_can_put_and_retrieve_value_from_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == "value"
+
+        # second time with the same key creates a list
+        cache.put("key", "value2")
+        assert cache.get("key") == ["value", "value2"]
+        assert len(cache) == 2
+
+        # third time, we now have a list
+        cache.put("key", "value3")
+        assert cache.get("key") == ["value", "value2", "value3"]
+        assert len(cache) == 3
+
+        # other keys are not affected
+        cache.put("key2", "value")
+        assert cache.get("key2") == "value"
+        assert len(cache) == 4
+
+        # duplicates are allowed
+        cache.put("key", "value")
+        assert cache.get("key") == ["value", "value2", "value3", "value"]
+        assert len(cache) == 5
+
+    def test_i_can_put_and_retrieve_values_from_set_cache(self):
+        cache = SetCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == {"value"}
+        assert len(cache) == 1
+
+        # we can add to this set
+        cache.put("key", "value2")
+        assert cache.get("key") == {"value", "value2"}
+        assert len(cache) == 2
+
+        # other keys are not affected
+        cache.put("key2", "value")
+        assert cache.get("key2") == {"value"}
+        assert len(cache) == 3
+
+        # duplicates are removed
+        cache.put("key", "value")
+        assert cache.get("key") == {"value", "value2"}
+        assert len(cache) == 3
+
+        assert cache.copy() == {'key': {'value', 'value2'}, 'key2': {'value'}}
+
+    def test_i_can_put_and_retrieve_value_from_dictionary_cache(self):
+        cache = DictionaryCache()
+
+        # # key must be None
+        # with pytest.raises(KeyError):
+        #     cache.put("key", None)
+        #
+        # # value must be a dictionary
+        # with pytest.raises(ValueError):
+        #     cache.put(True, "value")
+
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(False, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) == id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+        # I can append values
+        cache.put(True, {"key": "another_value", "key3": "value3"})
+        assert len(cache) == 4
+        assert cache.get("key") == "another_value"
+        assert cache.get("key2") == ["value21", "value22"]
+        assert cache.get("key3") == "value3"
+
+        # I can reset
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(False, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) == id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+        assert cache.copy() == {'key': 'value', 'key2': ['value21', 'value22']}
+
+    def test_i_can_put_and_retrieve_values_from_inc_cache(self):
+        cache = IncCache()
+
+        assert cache.get("key") == 1
+        assert cache.get("key") == 2
+        assert cache.get("key") == 3
+        assert cache.get("key2") == 1
+        assert cache.get("key2") == 2
+
+        cache.put("key", 100)
+        assert cache.get("key") == 101
+
+        assert cache.copy() == {'key': 101, 'key2': 2}
+
+    @pytest.mark.parametrize("key", [
+        None,
+        "something"
+    ])
+    def test_keys_have_constraints_when_dictionary_cache(self, key):
+        cache = DictionaryCache()
+        with pytest.raises(KeyError):
+            cache.put(key, None)
+
+    @pytest.mark.parametrize("value", [
+        None,
+        "something"
+    ])
+    def test_values_have_constraints_when_dictionary_cache(self, value):
+        cache = DictionaryCache()
+        with pytest.raises(ValueError):
+            cache.put(True, value)
+
+    def test_i_can_append_to_a_dictionary_cache_even_if_it_s_new(self):
+        cache = DictionaryCache()
+
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(True, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) != id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+    def test_i_can_update_from_simple_cache(self):
+        cache = Cache()
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 1
+        assert cache.get("key") == "new_value"
+
+        cache.update("key", "new_value", "another_key", "another_value")
+        assert len(cache._cache) == 1
+        assert len(cache) == 1
+        assert cache.get("key") is None
+        assert cache.get("another_key") == "another_value"
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_from_list_cache(self):
+        cache = ListCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "another value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value2", "value"]  # only the first one is affected
+
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key2") == ["value2"]
+
+        cache.update("key2", "value2", "key3", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key3") == ["value2"]
+        assert cache.get("key2") is None
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_from_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "another value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value2", "value"]  # only the first one is affected
+
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key2") == "value2"
+
+        cache.update("key2", "value2", "key3", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key3") == "value2"
+        assert cache.get("key2") is None
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_from_set_cache(self):
+        cache = SetCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.update("key", "value", "key", "another value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 2
+        assert cache.get("key") == {"another value", "value2"}
+
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 2
+        assert cache.get("key") == {"another value"}
+        assert cache.get("key2") == {"value2"}
+
+        cache.update("key", "another value", "key3", "another value")
+        assert len(cache._cache) == 2
+        assert len(cache) == 2
+        assert cache.get("key") is None
+        assert cache.get("key2") == {"value2"}
+        assert cache.get("key3") == {"another value"}
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    @pytest.mark.parametrize("cache", [
+        Cache(), ListCache(), ListIfNeededCache(), SetCache(), IncCache()
+    ])
+    def test_i_can_manage_cache_events(self, cache):
+        cache.put("key", "value")
+
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "value", "key", "another value")
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "another value", "key2", "value2")
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key"}
+
+        cache.update("key2", "value2", "key", "value")
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == {"key2"}
+
+    @pytest.mark.parametrize("cache", [
+        ListCache(), SetCache(), ListIfNeededCache()
+    ])
+    def test_i_can_manage_list_and_set_cache_events(self, cache):
+        cache.put("key", "value")
+        cache.put("key", "value2")
+
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "value", "key", "another value")
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "value2", "key2", "value2")
+        assert cache.to_add == {"key", "key2"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "another value", "key3", "another value")
+        assert cache.to_add == {"key2", "key3"}
+        assert cache.to_remove == {"key"}
+
+    @pytest.mark.parametrize("cache", [
+        Cache(), ListCache(), SetCache(), ListIfNeededCache(), IncCache()
+    ])
+    def test_exists(self, cache):
+        assert not cache.exists("key")
+        cache.put("key", "value")
+        assert cache.exists("key")
+
+    def test_exists_in_dictionary_cache(self):
+        cache = DictionaryCache()
+        assert not cache.exists("key")
+
+        cache.put(True, {"key": "value"})
+        assert cache.exists("key")
+
+    def test_exists_extend(self):
+        cache = Cache(extend_exists=lambda k: True if k == "special_key" else False)
+        assert not cache.exists("key")
+        assert cache.exists("special_key")
+
+    def test_add_concept_fills_all_dependent_caches(self):
+        sheerka, context, one, two, two_2, three = self.init_concepts("one", "two", Concept("two"), "three")
+        cache_manager = CacheManager(None)
+
+        cache_manager.register_concept_cache("by_id", Cache(), lambda obj: obj.id, True)
+        cache_manager.register_concept_cache("by_name", ListCache(), lambda obj: obj.name, True)
+        cache_manager.register_concept_cache("by_name2", ListIfNeededCache(), lambda obj: obj.name, True)
+
+        cache_manager.add_concept(one)
+        cache_manager.add_concept(two)
+        cache_manager.add_concept(two_2)
+        cache_manager.add_concept(three)
+
+        assert len(cache_manager.caches) == 3
+        assert cache_manager.caches["by_id"].cache._cache == {
+            "1001": one,
+            "1002": two,
+            "1003": two_2,
+            "1004": three,
+        }
+        assert cache_manager.caches["by_name"].cache._cache == {
+            "one": [one],
+            "two": [two, two_2],
+            "three": [three]
+        }
+        assert cache_manager.caches["by_name2"].cache._cache == {
+            "one": one,
+            "two": [two, two_2],
+            "three": three
+        }
+
+        assert cache_manager.get("by_id", "1002") == two
+        assert cache_manager.get("by_name", "two") == [two, two_2]
+        assert cache_manager.get("by_name2", "two") == [two, two_2]
+
+    def test_default_for_dictionary_cache(self):
+        cache = DictionaryCache(default={"key": "value", "key2": "value2"})
+
+        assert cache.get("key") == "value"
+        assert "key2" in cache
+        assert len(cache) == 2
+
+        cache.clear()
+        assert cache.get("key3") is None
+        assert len(cache) == 2
+        assert "key" in cache
+        assert "key2" in cache
+
+        # default is not modified
+        cache._cache["key"] = "another value"  # operation that is normally not possible
+        cache.clear()
+        assert cache.get("key") == "value"
+
+    def test_default_callable_for_dictionary_cache(self):
+        cache = DictionaryCache(default=lambda k: {"key": "value", "key2": "value2"})
+
+        assert cache.get("key") == "value"
+        assert "key2" in cache
+        assert len(cache) == 2
+
+        cache.clear()
+        assert cache.get("key3") is None
+        assert len(cache) == 2
+        assert "key" in cache
+        assert "key2" in cache
+
+    def test_dictionary_cache_cannot_be_null(self):
+        cache = DictionaryCache(default=lambda k: None)
+        assert cache.get("key") is None
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=None)
+        assert cache.get("key") is None
+        assert cache._cache == {}
+
+    @pytest.mark.parametrize("cache, default, new_value, expected", [
+        (ListCache(), lambda k: None, "value", ["value"]),
+        (ListCache(), lambda k: ["value"], "value", ["value", "value"]),
+        (ListIfNeededCache(), lambda k: None, "value", "value"),
+        (ListIfNeededCache(), lambda k: "value", "value1", ["value", "value1"]),
+        (ListIfNeededCache(), lambda k: ["value1", "value2"], "value1", ["value1", "value2", "value1"]),
+        (SetCache(), lambda k: None, "value", {"value"}),
+        (SetCache(), lambda k: {"value"}, "value", {"value"}),
+        (SetCache(), lambda k: {"value1"}, "value2", {"value1", "value2"}),
+    ])
+    def test_default_is_called_before_put_to_keep_in_sync(self, cache, default, new_value, expected):
+        cache.configure(default=default)
+        cache.put("key", new_value)
+
+        assert cache.get("key") == expected
+
+    def test_default_is_called_before_updating_simple_cache(self):
+        cache = Cache(default=lambda k: None)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = Cache(default=lambda k: "old_value")
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("new_key") == "new_value"
+
+    def test_default_is_called_before_updating_list_cache(self):
+        cache = ListCache(default=lambda k: None)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"])
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == ["new_value", "other old value"]
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else None)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == ["other old value"]
+        assert cache.get("new_key") == ["new_value"]
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else ["other new"])
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == ["other old value"]
+        assert cache.get("new_key") == ["other new", "new_value"]
+
+    def test_default_is_called_before_updating_list_if_needed_cache(self):
+        cache = ListIfNeededCache(default=lambda k: None)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = ListIfNeededCache(default=lambda k: "old_value")
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == "new_value"
+
+        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"])
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == ["new_value", "other old value"]
+
+        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else None)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == ["other old value"]
+        assert cache.get("new_key") == "new_value"
+
+    def test_default_is_called_before_updating_set_cache(self):
+        cache = SetCache(default=lambda k: None)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"})
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == {"new_value", "other old value"}
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else None)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == {"other old value"}
+        assert cache.get("new_key") == {"new_value"}
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else {"other new"})
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == {"other old value"}
+        assert cache.get("new_key") == {"other new", "new_value"}
+
+    def test_i_can_delete_an_entry_from_cache(self):
+        cache = Cache()
+        cache.put("key", "value")
+
+        assert cache.get("key") == "value"
+        cache.delete("key")
+        assert cache.get("value") is None
+        assert cache.to_remove == {"key"}