Implemented SheerkaOntology

tests/BaseTest.py

@@ -1,17 +1,101 @@
 import ast
+from dataclasses import dataclass, field
 
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
-from core.rule import Rule
+from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, freeze_concept_attrs
+from core.rule import Rule, ACTION_TYPE_PRINT
 from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.Sheerka import Sheerka
 from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
 from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.BnfNodeParser import StrMatch
 from sdp.sheerkaDataProvider import Event
 
 
+@dataclass
+class InitTestHelper:
+    sheerka: Sheerka
+    context: ExecutionContext
+    items: list = field(default_factory=list)
+
+    def push(self, *items):
+        self.items.extend(items)
+
+    def unpack(self):
+        return self.sheerka, self.context, *self.items
+
+    def with_concepts(self, *concepts, **kwargs):
+        create_new = kwargs.get("create_new", False)
+
+        for c in concepts:
+            if isinstance(c, str):
+                c = Concept(c)
+
+            if c.get_metadata().definition and c.get_metadata().definition_type != DEFINITION_TYPE_DEF:
+                desc = f"Resolving BNF {c.get_metadata().definition}"
+                with self.context.push(BuiltinConcepts.INIT_BNF,
+                                       c,
+                                       obj=c,
+                                       desc=desc) as sub_context:
+
+                    bnf_parser = BnfDefinitionParser()
+                    res = bnf_parser.parse(sub_context, c.get_metadata().definition)
+                    if res.status:
+                        c.set_bnf(res.value.value)
+                        c.get_metadata().definition_type = DEFINITION_TYPE_BNF
+                    else:
+                        raise Exception(f"Error in bnf definition '{c.get_metadata().definition}'",
+                                        self.sheerka.get_error(res))
+
+            if create_new:
+                self.sheerka.create_new_concept(self.context, c)
+            else:
+                c.init_key()
+                self.sheerka.set_id_if_needed(c, False)
+                self.sheerka.test_only_add_in_cache(c)
+                freeze_concept_attrs(c)
+
+            self.items.append(c)
+
+        return self
+
+    def with_rules(self, *rules, **kwargs):
+        create_new = kwargs.get("create_new", True)
+        compile_rule = kwargs.get("compile_rule", True)
+
+        for rule_template in rules:
+            if isinstance(rule_template, tuple):
+                if len(rule_template) == 3:
+                    rule = Rule(ACTION_TYPE_PRINT, rule_template[0], rule_template[1], rule_template[2])
+                else:
+                    rule = Rule(ACTION_TYPE_PRINT, None, rule_template[0], rule_template[1])
+            else:
+                rule = rule_template
+
+            is_enabled = rule.metadata.is_enabled  # remember the value...
+
+            if compile_rule:
+                self.sheerka.services[SheerkaRuleManager.NAME].init_rule(self.context, rule)
+            else:
+                rule.metadata.is_compiled = True
+
+            if create_new:
+                res = self.sheerka.create_new_rule(self.context, rule)
+                if not res.status:
+                    raise Exception(f"Error in rule definition '{res.body}'",
+                                    self.sheerka.get_error(res))
+                self.items.append(res.body.body)
+            else:
+                self.items.append(rule)
+
+            if is_enabled is not None:  # ...and back the value if it was not None
+                rule.metadata.is_enabled = is_enabled
+
+        return self
+
+
 class BaseTest:
-    def get_sheerka(self, **kwargs):
+    def get_sheerka(self, **kwargs) -> Sheerka:
         pass
 
     def get_context(self, sheerka=None, eval_body=False, eval_where=False):
@@ -23,6 +107,20 @@ class BaseTest:
 
         return context
 
+    @staticmethod
+    def get_init_test_args(**kwargs):
+        return {k: v for k, v in kwargs.items() if k in ["cache_only", "ontology", "eval_body", "eval_where"]}
+
+    @staticmethod
+    def get_with_concepts_args(**kwargs):
+        return {k: v for k, v in kwargs.items() if k in ["create_new"]}
+
+    def init_test(self, cache_only=None, ontology=None, eval_body=False, eval_where=False):
+        sheerka = self.get_sheerka(cache_only=cache_only, ontology=ontology)
+        context = self.get_context(sheerka=sheerka, eval_body=eval_body, eval_where=eval_where)
+
+        return InitTestHelper(sheerka, context)
+
     def get_default_concept(self):
         concept = Concept(
             name="a + b",
@@ -47,77 +145,12 @@ class BaseTest:
         return [t.repr_value for t in tokens]
 
     def init_concepts(self, *concepts, **kwargs):
-        sheerka = self.get_sheerka(**kwargs)
+        init_test_args = self.get_init_test_args(**kwargs)
+        with_concepts_args = self.get_with_concepts_args(**kwargs)
+        return self.init_test(**init_test_args).with_concepts(*concepts, **with_concepts_args).unpack()
 
-        context_args = dict([(k, v) for k, v in kwargs.items() if k in ["sheerka", "eval_body", "eval_where"]])
-        context = self.get_context(sheerka, **context_args)
-        create_new = kwargs.get("create_new", None)
-
-        result = []
-        for c in concepts:
-            if isinstance(c, str):
-                c = Concept(c)
-
-            if c.get_metadata().definition and c.get_metadata().definition_type != DEFINITION_TYPE_DEF:
-                desc = f"Resolving BNF {c.get_metadata().definition}"
-                with context.push(BuiltinConcepts.INIT_BNF,
-                                  c,
-                                  obj=c,
-                                  desc=desc) as sub_context:
-
-                    bnf_parser = BnfDefinitionParser()
-                    res = bnf_parser.parse(sub_context, c.get_metadata().definition)
-                    if res.status:
-                        c.set_bnf(res.value.value)
-                        c.get_metadata().definition_type = DEFINITION_TYPE_BNF
-                    else:
-                        raise Exception(f"Error in bnf definition '{c.get_metadata().definition}'",
-                                        sheerka.get_error(res))
-
-            if create_new:
-                sheerka.create_new_concept(context, c)
-            else:
-                c.init_key()
-                sheerka.set_id_if_needed(c, False)
-                sheerka.test_only_add_in_cache(c)
-
-            result.append(c)
-
-        return sheerka, context, *result
-
-    def init_format_rules(self, *rules, create_new=True, compile_rule=True, concepts=None, **kwargs):
-        if concepts:
-            sheerka, context, *concepts = self.init_concepts(*concepts, **kwargs)
-        else:
-            sheerka, context, *concepts = self.init_concepts(**kwargs)
-
-        if create_new:
-            sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache.clear()
-            sheerka.cache_manager.delete(sheerka.OBJECTS_IDS_ENTRY, SheerkaRuleManager.RULE_IDS)
-            with sheerka.sdp.get_transaction(context.event.get_digest()) as transaction:
-                transaction.clear(SheerkaRuleManager.FORMAT_RULE_ENTRY)
-
-        initialized = []
-        for rule_blue_print in rules:
-            if isinstance(rule_blue_print, tuple):
-                rule = Rule("print", None, rule_blue_print[0], rule_blue_print[1])
-                if not compile_rule:
-                    rule.metadata.is_compiled = True
-            else:
-                rule = rule_blue_print
-
-            is_enabled = rule.metadata.is_enabled
-            sheerka.services[SheerkaRuleManager.NAME].init_rule(context, rule)
-            if create_new:
-                res = sheerka.create_new_rule(context, rule)
-                initialized.append(res.body.body)
-            else:
-                initialized.append(rule)
-
-            if is_enabled is not None:
-                rule.metadata.is_enabled = is_enabled
-
-        return sheerka, context, *initialized
+    def init_format_rules(self, *rules, **kwargs):
+        return self.init_test(**kwargs).with_rules(*rules, **kwargs).unpack()
 
     @staticmethod
     def get_concept_instance(sheerka, concept, **kwargs):
@@ -179,6 +212,7 @@ class BaseTest:
         concept.init_key()
         sheerka.set_id_if_needed(concept, False)
         sheerka.test_only_add_in_cache(concept)
+        freeze_concept_attrs(concept)
         return concept
 
     @staticmethod

tests/TestUsingFileBasedSheerka.py

@@ -1,43 +1,37 @@
-import os
-import shutil
-from os import path
-
-import pytest
-from core.concept import ALL_ATTRIBUTES
+from conftest import SHEERKA_TEST_FOLDER
 from core.sheerka.Sheerka import Sheerka
+from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager
 
 from tests.BaseTest import BaseTest
 
 
 class TestUsingFileBasedSheerka(BaseTest):
-    tests_root = path.abspath("../../build/tests")
-    root_folder = "init_folder"
+    sheerka = None
+    context = None
+    root_ontology_name = SheerkaOntologyManager.ROOT_ONTOLOGY_NAME
 
-    @pytest.fixture(autouse=True)
-    def init_test(self):
-        if path.exists(self.tests_root):
-            shutil.rmtree(self.tests_root)
-
-        if not path.exists(self.tests_root):
-            os.makedirs(self.tests_root)
-        current_pwd = os.getcwd()
-        os.chdir(self.tests_root)
-
-        yield None
-
-        os.chdir(current_pwd)
-
-    def get_sheerka(self, **kwargs):
-        reset_attrs = kwargs.get("reset_attrs", True)
-        if reset_attrs:
-            ALL_ATTRIBUTES.clear()
-
-        use_dict = kwargs.get("use_dict", False)
-        # use dictionary based io instead of file
-        # If you do so, information between two different instances of sheerka
-        #   won't be shared
-        root = "mem://" if use_dict else self.root_folder
-        sheerka = Sheerka()
-        sheerka.initialize(root, save_execution_context=False, enable_process_return_values=False)
+    def teardown_method(self, method):
+        # to do after the test
+        if TestUsingFileBasedSheerka.sheerka:
+            while TestUsingFileBasedSheerka.sheerka.om.current_ontology().name != self.root_ontology_name:
+                ontology = TestUsingFileBasedSheerka.sheerka.pop_ontology().body.body
+                ontology.cache_manager.sdp.test_only_destroy_refs()
 
+    @staticmethod
+    def new_sheerka_instance(cache_only):
+        sheerka = Sheerka(cache_only=cache_only)
+        sheerka.initialize(SHEERKA_TEST_FOLDER,
+                           save_execution_context=False,
+                           enable_process_return_values=False)
         return sheerka
+
+    def get_sheerka(self, **kwargs) -> Sheerka:
+        cache_only = kwargs.get("cache_only", False)
+        ontology_name = kwargs.get("ontology", "#unit_test#") or "#unit_test#"
+
+        if TestUsingFileBasedSheerka.sheerka is None:
+            TestUsingFileBasedSheerka.sheerka = self.new_sheerka_instance(False)
+            TestUsingFileBasedSheerka.context = self.get_context(TestUsingFileBasedSheerka.sheerka)
+
+        self.sheerka.push_ontology(self.context, ontology_name, cache_only=cache_only)
+        return TestUsingFileBasedSheerka.sheerka

tests/TestUsingMemoryBasedSheerka.py

@@ -1,42 +1,33 @@
-from core.concept import ALL_ATTRIBUTES
 from core.sheerka.Sheerka import Sheerka
+from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager
 
 from tests.BaseTest import BaseTest
 
 
 class TestUsingMemoryBasedSheerka(BaseTest):
-    singleton_instance = None
-    dump = None
+    sheerka = None
+    context = None
+    root_ontology_name = SheerkaOntologyManager.ROOT_ONTOLOGY_NAME
+
+    def teardown_method(self, method):
+        # to do after the test
+        if TestUsingMemoryBasedSheerka.sheerka:
+            while TestUsingMemoryBasedSheerka.sheerka.om.current_ontology().name != self.root_ontology_name:
+                TestUsingMemoryBasedSheerka.sheerka.pop_ontology()
 
     @staticmethod
-    def _inner_get_sheerka(cache_only):
-        ALL_ATTRIBUTES.clear()
+    def new_sheerka_instance(cache_only):
         sheerka = Sheerka(cache_only=cache_only)
         sheerka.initialize("mem://", save_execution_context=False, enable_process_return_values=False)
         return sheerka
 
-    def get_sheerka(self, **kwargs):
+    def get_sheerka(self, **kwargs) -> Sheerka:
         cache_only = kwargs.get("cache_only", True)
-        use_singleton = kwargs.get("singleton", True)
-        reset_attrs = kwargs.get("reset_attrs", True)
+        ontology_name = kwargs.get("ontology", "#unit_test#") or "#unit_test#"
 
-        sheerka = kwargs.get("sheerka", None)
-        if sheerka:
-            return sheerka
+        if TestUsingMemoryBasedSheerka.sheerka is None:
+            TestUsingMemoryBasedSheerka.sheerka = self.new_sheerka_instance(False)
+            TestUsingMemoryBasedSheerka.context = self.get_context(TestUsingMemoryBasedSheerka.sheerka)
 
-        if reset_attrs:
-            ALL_ATTRIBUTES.clear()
-
-        if use_singleton:
-            singleton_instance = TestUsingMemoryBasedSheerka.singleton_instance
-            if singleton_instance:
-                singleton_instance.reset(cache_only)
-                singleton_instance.cache_manager.init_from_dump(TestUsingMemoryBasedSheerka.dump)
-                return singleton_instance
-            else:
-                new_instance = self._inner_get_sheerka(cache_only)
-                TestUsingMemoryBasedSheerka.dump = new_instance.cache_manager.dump()
-                TestUsingMemoryBasedSheerka.singleton_instance = new_instance
-                return TestUsingMemoryBasedSheerka.singleton_instance
-
-        return self._inner_get_sheerka(cache_only)
+        self.sheerka.push_ontology(self.context, ontology_name, cache_only=cache_only)
+        return self.sheerka

tests/cache/__init__.py

@@ -0,0 +1,18 @@
+class FakeSdp:
+    def __init__(self, /, get_value=None, extend_exists=None, get_alt_value=None, populate=None):
+        self.get_value = get_value
+        self.extend_exists = extend_exists
+        self.populate_function = populate
+        self.get_alt_value = get_alt_value
+
+    def get(self, cache_name, key):
+        return self.get_value(cache_name, key)
+
+    def exists(self, cache_name, key):
+        return self.extend_exists(cache_name, key)
+
+    def alt_get(self, cache_name, key):
+        return self.get_alt_value(cache_name, key)
+
+    def populate(self):
+        return self.populate_function() if callable(self.populate_function) else self.populate_function

tests/cache/test_CacheManager.py

@@ -0,0 +1,224 @@
+import pytest
+from cache.Cache import Cache
+from cache.CacheManager import CacheManager, ConceptNotFound
+from cache.DictionaryCache import DictionaryCache
+from cache.ListCache import ListCache
+from cache.ListIfNeededCache import ListIfNeededCache
+from core.concept import Concept
+from core.global_symbols import NotFound
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.cache import FakeSdp
+
+
+class TestCacheManager(TestUsingMemoryBasedSheerka):
+    def test_i_do_not_push_into_sdp_when_cache_only_is_true(self):
+        sheerka, context = self.init_test().unpack()
+
+        cache_manager = CacheManager(True)
+        cache_manager.register_cache("test", Cache(), persist=True)
+        cache_manager.put("test", "key", "value")
+
+        cache_manager.commit(context)
+        assert not sheerka.om.current_cache_manager().sdp.exists("test", "key")
+
+    def test_i_do_not_get_value_from_sdp_when_cache_only_is_true(self):
+        sheerka, context = self.init_test().unpack()
+
+        sdp = sheerka.om.get_sdp()
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        cache = Cache(default=lambda k: sdp.get("test", k))
+
+        cache_manager = CacheManager(True)
+        cache_manager.register_cache("test", cache, persist=True)
+
+        assert cache_manager.get("test", "key") is NotFound
+
+    def test_i_do_not_get_value_from_inner_sdp_when_cache_only_is_true(self):
+        sheerka, context = self.init_test().unpack()
+
+        cache_manager = CacheManager(True, sdp=sheerka.om.get_sdp("test"))
+        cache = Cache(default=lambda _sdp, k: _sdp.get("test", k))
+        cache_manager.register_cache("test", cache, persist=True)
+
+        with cache_manager.sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        assert cache_manager.get("test", "key") is NotFound
+
+    def test_i_can_get_value_from_sdp_when_cache_only_is_false(self):
+        sheerka, context = self.init_test().unpack()
+
+        sdp = sheerka.om.get_sdp()
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        cache_manager = CacheManager(False)
+        cache = Cache(default=lambda k: sdp.get("test", k))
+        cache_manager.register_cache("test", cache, persist=True)
+
+        assert cache_manager.get("test", "key") == "value"
+
+    def test_i_can_get_value_from_inner_sdp_when_cache_only_is_false(self):
+        sheerka, context = self.init_test().unpack()
+
+        cache_manager = CacheManager(False, sdp=sheerka.om.get_sdp("test"))
+        cache = Cache(default=lambda _sdp, k: _sdp.get("test", k))
+        cache_manager.register_cache("test", cache, persist=True)
+
+        with cache_manager.sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        assert cache_manager.get("test", "key") == "value"
+
+    def test_i_can_get_value_from_alt_sdp_when_cache_only_is_true(self):
+        sheerka, context = self.init_test().unpack()
+
+        cache_manager = CacheManager(cache_only=True, sdp=sheerka.om.get_sdp("test"))
+        cache_manager.register_cache("test", Cache().auto_configure("test"), persist=True)
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !")
+        assert cache_manager.get("test", "key", alt_sdp=alt_sdp) is "value found !"
+
+    def test_i_can_commit_simple_cache(self):
+        sheerka, context = self.init_test().unpack()
+
+        cache_manager = CacheManager(False, sheerka.om.get_sdp("test"))
+        cache_manager.register_cache("test", Cache(), persist=True)
+        cache = cache_manager.caches["test"].cache
+
+        cache_manager.put("test", "key", "value")
+
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") == "value"
+
+        cache.update("key", "value", "key", "another_value")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") == "another_value"
+
+        cache.update("key", "another_value", "key2", "another_value")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") is NotFound
+        assert cache_manager.sdp.get("test", "key2") == "another_value"
+
+        # sanity check
+        # sdp 'test' has value, but sdp '__default__' does not
+        assert cache_manager.sdp.name == "test"
+        assert cache_manager.sdp.state.data == {'test': {'key2': 'another_value'}}
+
+        default_sdp = sheerka.om.ontologies[-1].cache_manager.sdp
+        assert default_sdp.name == "__default__"
+        assert "test" not in default_sdp.state.data
+
+    def test_i_can_commit_list_cache(self):
+        sheerka, context = self.init_test().unpack()
+
+        cache_manager = CacheManager(False, sheerka.om.get_sdp("test"))
+        cache_manager.register_cache("test", ListCache(), persist=True)
+        cache = cache_manager.caches["test"].cache
+
+        cache.put("key", "value")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") == ["value"]
+
+        cache.put("key", "value2")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") == ["value", "value2"]
+
+        cache.update("key", "value2", "key2", "value2")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") == ["value"]
+        assert cache_manager.sdp.get("test", "key2") == ["value2"]
+
+        cache.update("key2", "value2", "key3", "value2")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") == ["value"]
+        assert cache_manager.sdp.get("test", "key2") is NotFound
+        assert cache_manager.sdp.get("test", "key3") == ["value2"]
+
+    def test_i_can_commit_dictionary_cache(self):
+        sheerka, context = self.init_test().unpack()
+
+        cache_manager = CacheManager(False, sheerka.om.get_sdp("test"))
+        cache_manager.register_cache("test", DictionaryCache(), persist=True)
+        cache = cache_manager.caches["test"].cache
+
+        cache.put(False, {"key": "value", "key2": "value2"})
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test") == {"key": "value", "key2": "value2"}
+        assert cache_manager.sdp.get("test", "key") == "value"
+
+        cache.put(False, {"key": "value", "key2": "value2", "key3": "value3"})
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test") == {"key": "value", "key2": "value2", "key3": "value3"}
+
+    def test_i_can_get_value_from_sdp_when_dictionary_cache(self):
+        sheerka, context = self.init_test().unpack()
+
+        cache_manager = CacheManager(False, sheerka.om.get_sdp("cache_name"))
+        cache_manager.register_cache("cache_name", DictionaryCache().auto_configure("cache_name"), persist=True)
+
+        with cache_manager.sdp.get_transaction(context.event) as transaction:
+            transaction.add("cache_name", None, {"key1": "value1", "key2": "value2"})
+
+        assert cache_manager.get("cache_name", "key3") is NotFound
+
+        # make sure that the first call retrieves the whole remote repository
+        assert cache_manager.caches["cache_name"].cache.copy() == {"key1": "value1", "key2": "value2"}
+
+        assert cache_manager.get("cache_name", "key1") == "value1"
+        assert cache_manager.get("cache_name", "key2") == "value2"
+
+    def test_i_can_remove_a_concept_from_concepts_caches(self):
+        cache_manager = CacheManager(True)
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
+
+        sheerka, context, one, two, three, two_bis = self.init_concepts("one", "two", "three", Concept("two", body="2"))
+
+        for concept in [one, two, three, two_bis]:
+            cache_manager.add_concept(concept)
+
+        # sanity check before removing
+        cache_def = cache_manager.caches["id"]
+        assert cache_def.cache.copy() == {one.id: one, two.id: two, three.id: three, two_bis.id: two_bis}
+        cache_def = cache_manager.caches["key"]
+        assert cache_def.cache.copy() == {one.key: one, two.key: [two, two_bis], three.key: three}
+
+        for cache_name in cache_manager.concept_caches:
+            cache_manager.caches[cache_name].cache.reset_events()
+
+        cache_manager.remove_concept(sheerka.new(("two", two_bis.id)))
+
+        cache_def = cache_manager.caches["id"]
+        assert cache_def.cache.copy() == {one.id: one, two.id: two, three.id: three}
+        assert cache_def.cache.to_remove == {two_bis.id}
+        assert cache_def.cache.to_add == set()
+        assert len(cache_def.cache) == 3
+
+        cache_def = cache_manager.caches["key"]
+        assert cache_def.cache.copy() == {one.key: one, two.key: two, three.key: three}
+        assert cache_def.cache.to_remove == set()
+        assert cache_def.cache.to_add == {"two"}
+        assert len(cache_def.cache) == 3
+
+    def test_i_cannot_remove_a_concept_that_does_not_exists(self):
+        cache_manager = CacheManager(True)
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
+
+        with pytest.raises(ConceptNotFound) as ex:
+            cache_manager.remove_concept(Concept("foo", id="1001"))
+
+        assert ex.value.concept == Concept("foo", id="1001")
+
+    def test_i_can_configure_a_cache_with_internal_sdp(self):
+        cache_manager = CacheManager(cache_only=False,
+                                     sdp=FakeSdp(get_value=lambda cache_name, key: key + "_not_found"))
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key))
+        cache_manager.register_cache("test", cache)
+
+        assert cache.get("key") == "key_not_found"
+        assert cache_manager.get("test", "key") == "key_not_found"

tests/cache/test_DictionaryCache.py

@@ -0,0 +1,162 @@
+import pytest
+
+from cache.DictionaryCache import DictionaryCache
+from core.global_symbols import NotFound
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.cache import FakeSdp
+
+
+class TestDictionaryCache(TestUsingMemoryBasedSheerka):
+    def test_i_can_put_and_retrieve_value_from_dictionary_cache(self):
+        cache = DictionaryCache()
+
+        # key must be None
+        with pytest.raises(KeyError):
+            cache.put("key", None)
+
+        # value must be a dictionary
+        with pytest.raises(ValueError):
+            cache.put(True, "value")
+
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(False, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) == id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+        # I can append values
+        cache.put(True, {"key": "another_value", "key3": "value3"})
+        assert len(cache) == 4
+        assert cache.get("key") == "another_value"
+        assert cache.get("key2") == ["value21", "value22"]
+        assert cache.get("key3") == "value3"
+
+        # I can reset
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(False, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) == id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+        assert cache.copy() == {'key': 'value', 'key2': ['value21', 'value22']}
+
+    @pytest.mark.parametrize("key", [
+        None,
+        "something"
+    ])
+    def test_keys_have_constraints_when_dictionary_cache(self, key):
+        cache = DictionaryCache()
+        with pytest.raises(KeyError):
+            cache.put(key, None)
+
+    def test_i_can_sync_with_remote_repository(self):
+        cache = DictionaryCache()
+
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(False, entry)
+
+        assert len(cache) == 3
+        assert id(cache._cache) == id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+    def test_i_can_get_a_value_that_does_not_exist_without_compromising_the_cache(self):
+        cache = DictionaryCache()
+        cache.put(False, {"key": "value"})
+
+        assert cache.get("key2") is NotFound
+        assert cache.copy() == {"key": "value"}
+
+    @pytest.mark.parametrize("value", [
+        None,
+        "something"
+    ])
+    def test_values_have_constraints_when_dictionary_cache(self, value):
+        cache = DictionaryCache()
+        with pytest.raises(ValueError):
+            cache.put(True, value)
+
+    def test_i_can_append_to_a_dictionary_cache_even_if_it_is_new(self):
+        cache = DictionaryCache()
+
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(True, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) != id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+    def test_exists_in_dictionary_cache(self):
+        cache = DictionaryCache()
+        assert not cache.exists("key")
+
+        cache.put(True, {"key": "value"})
+        assert cache.exists("key")
+
+    def test_default_for_dictionary_cache(self):
+        cache = DictionaryCache(default={"key": "value", "key2": "value2"})
+
+        # cache is fully set when the value is found
+        assert cache.get("key") == "value"
+        assert cache.copy() == {"key": "value", "key2": "value2"}
+
+        # cache is fully set when the value is not found
+        cache.test_only_reset()
+        assert cache.get("key3") is NotFound
+        assert cache.copy() == {"key": "value", "key2": "value2"}
+
+        # cache is not corrupted when value is found
+        cache.put(True, {"key3": "value3", "key4": "value4"})
+        assert cache.get("key3") == "value3"
+        assert cache.copy() == {"key": "value", "key2": "value2", "key3": "value3", "key4": "value4"}
+
+        # cache is not corrupted when value is not found
+        cache._cache["key"] = "another value"  # operation that is normally not possible
+        assert cache.get("key5") is NotFound
+        assert cache.copy() == {"key": "value", "key2": "value2", "key3": "value3", "key4": "value4"}
+
+    def test_default_callable_for_dictionary_cache(self):
+        cache = DictionaryCache(default=lambda k: {"key": "value", "key2": "value2"})
+
+        assert cache.get("key") == "value"
+        assert "key2" in cache
+        assert len(cache) == 2
+
+        cache.clear()
+        assert cache.get("key3") is NotFound
+        assert len(cache) == 2
+        assert "key" in cache
+        assert "key2" in cache
+
+    def test_default_callable_with_internal_sdp_for_dictionary_cache(self):
+        cache = DictionaryCache(default=lambda sdp, key: sdp.get("cache_name", key),
+                                sdp=FakeSdp(lambda entry, k: {"key": "value", "key2": "value2"}))
+
+        assert cache.get("key") == "value"
+        assert "key2" in cache
+        assert len(cache) == 2
+
+        cache.clear()
+        assert cache.get("key3") is NotFound
+        assert len(cache) == 2
+        assert "key" in cache
+        assert "key2" in cache
+
+    def test_dictionary_cache_cannot_be_null(self):
+        cache = DictionaryCache(default=lambda k: NotFound)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=NotFound)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=lambda k: None)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=None)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}

tests/cache/test_FastCache.py

@@ -1,4 +1,5 @@
 from cache.FastCache import FastCache
+from core.global_symbols import NotFound
 
 
 def test_i_can_put_an_retrieve_values():
@@ -47,7 +48,7 @@ def test_i_can_put_the_same_key_several_times():
 
 def test_none_is_returned_when_not_found():
     cache = FastCache()
-    assert cache.get("foo") is None
+    assert cache.get("foo") is NotFound
 
 
 def test_i_can_evict_by_key():
@@ -65,3 +66,14 @@ def test_i_can_evict_by_key():
                            "to_keep3": "to_keep_value3"}
 
     assert cache.lru == ["to_keep1", "to_keep2", "to_keep3"]
+
+
+def test_i_can_get_default_value():
+    cache = FastCache(max_size=3, default=lambda key: key + 1)
+
+    assert cache.get(1) == 2
+    assert cache.get(2) == 3
+    assert cache.get(3) == 4
+    assert cache.get(4) == 5
+
+    assert cache.cache == {2: 3, 3: 4, 4: 5}  # only 3 values

tests/cache/test_IncCache.py

@@ -0,0 +1,84 @@
+from cache.IncCache import IncCache
+from core.global_symbols import NotFound, Removed
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.cache import FakeSdp
+
+
+class FakeIncSdp:
+    def __init__(self, init_value1, init_value2):
+        self.internals = [IncCache(), IncCache()]
+        if init_value1:
+            self.internals[0].put("key", init_value1)
+
+        if init_value2:
+            self.internals[1].put("key", init_value2)
+
+    def alt_get(self, cache_name, key):
+        for cache in self.internals:
+            value = cache.alt_get(key)
+            if value is not NotFound:
+                return value
+
+        return NotFound
+
+
+class TestIncCache(TestUsingMemoryBasedSheerka):
+    def test_i_can_put_and_retrieve_values_from_inc_cache(self):
+        cache = IncCache()
+
+        assert cache.get("key") == 1
+        assert cache.get("key") == 2
+        assert cache.get("key") == 3
+        assert cache.get("key2") == 1
+        assert cache.get("key2") == 2
+
+        cache.put("key", 100)
+        assert cache.get("key") == 101
+
+        assert cache.copy() == {'key': 101, 'key2': 2}
+
+    def test_i_can_alt_get(self):
+        cache = IncCache()
+
+        assert cache.get("key") == 1
+        assert cache.get("key") == 2
+        assert cache.alt_get("key") == 2
+        assert cache.alt_get("key") == 2
+        assert cache.get("key") == 3
+
+    def test_current_cache_takes_precedence_over_alt_sdp(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        assert cache.get("key") == 1
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: 10)
+        assert cache.get("key", alt_sdp=alt_sdp) == 2
+
+    def test_remote_repository_takes_precedence_over_alt_sdp(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: 5)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: 10)
+        assert cache.get("key", alt_sdp=alt_sdp) == 6
+        assert cache.get("key", alt_sdp=alt_sdp) == 7  # then we use the value from the cache
+
+    def test_i_can_take_value_from_alt_sdp(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeIncSdp(10, NotFound)
+        assert cache.get("key", alt_sdp=alt_sdp) == 11
+        assert cache.get("key", alt_sdp=alt_sdp) == 12  # then we use the value from the cache
+
+    def test_i_can_get_when_alt_sdp_and_cache_is_cleared(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        alt_sdp = FakeIncSdp(10, NotFound)
+        assert cache.get("key", alt_sdp=alt_sdp) == 1
+        assert cache.get("key", alt_sdp=alt_sdp) == 2  # then we use the value from the cache
+
+    def test_i_can_manage_when_the_value_from_alt_sdp_is_removed(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeIncSdp(Removed, 10)
+        assert cache.get("key", alt_sdp=alt_sdp) == 1
+        assert cache.get("key", alt_sdp=alt_sdp) == 2  # then we use the value from the cache

tests/cache/test_ListCache.py

@@ -0,0 +1,274 @@
+import pytest
+
+from cache.ListCache import ListCache
+from core.global_symbols import NotFound, Removed
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.cache import FakeSdp
+
+
+class TestListIfNeededCache(TestUsingMemoryBasedSheerka):
+    def test_i_can_put_and_retrieve_value_from_list_cache(self):
+        cache = ListCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == ["value"]
+        assert len(cache) == 1
+
+        cache.put("key", "value2")  # we can append to this list
+        assert cache.get("key") == ["value", "value2"]
+        assert len(cache) == 2
+
+        cache.put("key2", "value")
+        assert cache.get("key2") == ["value"]
+        assert len(cache) == 3
+
+        # duplicates are allowed
+        cache.put("key", "value")
+        assert cache.get("key") == ["value", "value2", "value"]
+        assert len(cache) == 4
+
+        assert cache.copy() == {'key': ['value', 'value2', 'value'], 'key2': ['value']}
+
+    def test_i_can_put_in_list_cache_when_alt_sdp_returns_values(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]))
+        assert cache.get("key") == ["value1", "value2"]
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == ["value1"]
+
+    def test_i_can_put_in_list_cache_when_alt_sdp_returns_values_and_cache_is_cleared(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]))
+        assert cache.get("key") == ["value2"]
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == ["value1"]
+
+    def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_list_cache(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value1")
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_list_cache(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: ["value1"])).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_i_can_update_from_list_cache(self):
+        cache = ListCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "another value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value2", "value"]  # only the first one is affected
+
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key2") == ["value2"]
+
+        cache.update("key2", "value2", "key3", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key3") == ["value2"]
+        assert cache.get("key2") is NotFound
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self):
+        cache = ListCache(default=lambda sdp, key: sdp.get("cache_name", key),
+                          extend_exists=lambda sdp, key: sdp.exists("cache_name", key),
+                          sdp=FakeSdp(get_value=lambda cache_name, key: NotFound))
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == ["new_value"]
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self):
+        # There is nothing in cache or remote repository.
+        # We must ust the value from alt_sdp
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        previous_value = ["old_1", "old_2", "value"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: previous_value)
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == ["old_1", "old_2", "new_value"]
+        assert previous_value == ["old_1", "old_2", "value"]
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self):
+        # keys are different
+        # make sure that current cache take precedence over alt_sdp
+        # In this test, the values from alt_sdp are never used
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: ["xxx1"] if key == "key1" else NotFound)
+
+        # one values in 'key1'
+        cache.put("key1", "old_1")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2"]
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self):
+        # keys are different
+        # make sure that current repo take precedence over alt_sdp
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1"] if key == "key1" else NotFound)
+        cache = ListCache(sdp=remote).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: ["xxx1"] if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2"] if key == "key1" else NotFound)
+        cache = ListCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2"]
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self):
+        # keys are different
+        # No value found in cache or remote repository,
+        # Will use values from alt_sdp
+        # The old key is the same, so it has to be marked as Removed
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one values in 'key1'
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: ["old_1"] if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.test_only_reset()
+        old_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2"]
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == ["old_1", "old_2"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: ["xxx2"] if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.put("key2", "old_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        remote_repo = FakeSdp(get_value=lambda cache_name, key: ["old_value"] if key == "key2" else NotFound)
+        cache = ListCache(sdp=remote_repo).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: ["xxx2"] if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key1", "source_value")
+        previous_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ["old_1", "old_2", 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+        assert previous_values == ["old_1", "old_2"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])
+        cache.clear()
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp)
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp)
+
+    def test_default_is_called_before_updating_list_cache(self):
+        cache = ListCache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"])
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == ["new_value", "other old value"]
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else NotFound)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == ["other old value"]
+        assert cache.get("new_key") == ["new_value"]
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else ["other new"])
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == ["other old value"]
+        assert cache.get("new_key") == ["other new", "new_value"]

tests/cache/test_ListIfNeededCache.py

@@ -0,0 +1,569 @@
+import pytest
+
+from cache.ListIfNeededCache import ListIfNeededCache
+from core.global_symbols import NotFound, Removed
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.cache import FakeSdp
+
+
+class TestListIfNeededCache(TestUsingMemoryBasedSheerka):
+    def test_i_can_put_and_retrieve_value_from_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == "value"
+
+        # second time with the same key creates a list
+        cache.put("key", "value2")
+        assert cache.get("key") == ["value", "value2"]
+        assert len(cache) == 2
+
+        # third time, we now have a list
+        cache.put("key", "value3")
+        assert cache.get("key") == ["value", "value2", "value3"]
+        assert len(cache) == 3
+
+        # other keys are not affected
+        cache.put("key2", "value")
+        assert cache.get("key2") == "value"
+        assert len(cache) == 4
+
+        # duplicates are allowed
+        cache.put("key", "value")
+        assert cache.get("key") == ["value", "value2", "value3", "value"]
+        assert len(cache) == 5
+
+    def test_i_can_put_in_list_if_need_cache_when_alt_sdp_returns_values(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "value1"))
+        assert cache.get("key") == ["value1", "value2"]
+
+        cache.put("key2", "value3", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2"]))
+        assert cache.get("key2") == ["value1", "value2", "value3"]
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == "value1"
+
+    def test_i_can_put_in_list_if_need__cache_when_alt_sdp_returns_values_and_cache_is_cleared(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "value1"))
+        assert cache.get("key") == "value2"
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == "value1"
+
+    def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_list_if_needed_cache(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value1")
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_list_if_needed_cache(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: "value1")).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_i_can_update_from_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value")
+
+        # only the first 'value' is affected
+        cache.update("key", "value", "key", "another value")
+        assert len(cache._cache) == 1
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value2", "value"]
+
+        # change the key
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key2") == "value2"
+
+        # rename the newly created key
+        cache.update("key2", "value2", "key3", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key3") == "value2"
+        assert cache.get("key2") is NotFound
+
+        # from list to single item and vice versa
+        cache.update("key", "value", "key3", "value")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == "another value"  # 'key' is no longer a list
+        assert cache.get("key3") == ["value2", "value"]  # 'key3' is now a list
+        assert cache.get("key2") is NotFound
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == "new_value"
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one value in alt_sdp
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: "old_value")
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == "new_value"
+
+        # multiple values in alt_sdp
+        cache.test_only_reset()
+        previous_value = ["old_1", "old_2", "value"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: previous_value)
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == ["old_1", "old_2", "new_value"]
+        assert previous_value == ["old_1", "old_2", "value"]
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self):
+        # keys are different
+        # make sure that current cache take precedence over alt_sdp
+        # In this test, the values from alt_sdp are never used
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: "xxx1" if key == "key1" else NotFound)
+
+        # one values in 'key1'
+        cache.put("key1", "old_1")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == "old_2"
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # List of values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+        cache.put("key1", "old_3")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2", "old_3"]
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self):
+        # keys are different
+        # make sure that current repo take precedence over alt_sdp
+        remote = FakeSdp(get_value=lambda cache_name, key: "old_1" if key == "key1" else NotFound)
+        cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: "xxx1" if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2"] if key == "key1" else NotFound)
+        cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == "old_2"
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # List of values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2", "old_3"] if key == "key1" else NotFound)
+        cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2", "old_3"]
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self):
+        # keys are different
+        # No value found in cache or remote repository,
+        # Will use values from alt_sdp
+        # The old key is the same, so it has to be marked as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one values in 'key1'
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: "old_1" if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.test_only_reset()
+        old_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == "old_2"
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == ["old_1", "old_2"]  # not modified
+
+        # List of values in 'key1'
+        cache.test_only_reset()
+        old_values = ["old_1", "old_2", "old_3"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2", "old_3"]
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == ["old_1", "old_2", "old_3"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.put("key2", "old_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        remote_repo = FakeSdp(get_value=lambda cache_name, key: "old_value" if key == "key2" else NotFound)
+        cache = ListIfNeededCache(sdp=remote_repo).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        # one value in 'key2'
+        cache.put("key1", "source_value")
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "old_value" if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+        # Multiple values in 'key2'
+        cache.test_only_reset()
+        cache.put("key1", "source_value")
+        previous_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ["old_1", "old_2", 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+        assert previous_values == ["old_1", "old_2"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1")
+        cache.clear()
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp)
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp)
+
+    def test_default_is_called_before_updating_list_if_needed_cache(self):
+        cache = ListIfNeededCache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = ListIfNeededCache(default=lambda k: "old_value")
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == "new_value"
+
+        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"])
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == ["new_value", "other old value"]
+
+        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else NotFound)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == "other old value"
+        assert cache.get("new_key") == "new_value"
+
+    def test_i_can_delete_key_and_values(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+        cache.put("key", "value11")
+        cache.put("key2", "value2")
+        cache.put("key2", "value22")
+        cache.put("key2", "value222")
+        cache.put("key3", "value3")
+        cache.put("key3", "value33")
+        cache.put("key4", "value4")
+        cache.reset_events()
+
+        assert len(cache) == 8
+
+        # I can remove a whole key
+        cache.delete("key")
+        assert cache.get("key") is NotFound
+        assert len(cache) == 6
+        assert cache.to_remove == {"key"}
+        assert cache.to_add == set()
+
+        # I can remove an element while a list is remaining
+        cache.reset_events()
+        cache.delete("key2", "value22")
+        assert cache.get("key2") == ["value2", "value222"]
+        assert len(cache) == 5
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while a single element is remaining
+        cache.reset_events()
+        cache.delete("key3", "value33")
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 4
+        assert cache.to_add == {"key3"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while nothing remains
+        cache.reset_events()
+        cache.delete("key4", "value4")
+        assert cache.get("key4") is NotFound
+        assert len(cache) == 3
+        assert cache.to_remove == {"key4"}
+        assert cache.to_add == set()
+
+        # I do not remove when the value is not the same
+        cache.reset_events()
+        cache.delete("key3", "value33")  # value33 was already remove
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 3
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+        cache.put("key", "value2")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": "value2"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: "value")).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": "value2"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1, value2"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1",
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_one_value_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, one value remains in the cache
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": "value2"}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_multiple_values_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, one value remains in the cache
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2", "value3"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ["value2", "value3"]}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_an_already_removed_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # But the alternate sdp returns Removed, which means that previous value was deleted
+        # It's like there is nothing to delete
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_deleting_an_entry_that_does_not_exist_is_not_an_error(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+
+        cache.reset_events()
+        cache.delete("key3")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key3", "value")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value2")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value",
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.clear()
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()

tests/cache/test_SetCache.py

@@ -0,0 +1,477 @@
+import pytest
+
+from cache.SetCache import SetCache
+from core.global_symbols import NotFound, Removed
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.cache import FakeSdp
+
+
+class TestSetCache(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_put_and_retrieve_values_from_set_cache(self):
+        cache = SetCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == {"value"}
+        assert len(cache) == 1
+
+        # we can add to this set
+        cache.put("key", "value2")
+        assert cache.get("key") == {"value", "value2"}
+        assert len(cache) == 2
+
+        # other keys are not affected
+        cache.put("key2", "value")
+        assert cache.get("key2") == {"value"}
+        assert len(cache) == 3
+
+        # duplicates are removed
+        cache.put("key", "value")
+        assert cache.get("key") == {"value", "value2"}
+        assert len(cache) == 3
+
+        assert cache.copy() == {'key': {'value', 'value2'}, 'key2': {'value'}}
+
+    def test_i_can_put_in_set_cache_when_alt_sdp_returns_values(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: {"value1"}))
+        assert cache.get("key") == {"value1", "value2"}
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == {"value1"}
+
+    def test_i_can_put_in_set_cache_when_alt_sdp_returns_values_and_cache_is_cleared(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: {"value1"}))
+        assert cache.get("key") == {"value2"}
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == {"value1"}
+
+    def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_set_cache(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value1")
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == {"value1", "value2"}
+
+    def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_set_cache(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: {"value1"})).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == {"value1", "value2"}
+
+    def test_i_can_update_from_set_cache(self):
+        cache = SetCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.update("key", "value", "key", "another value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 2
+        assert cache.get("key") == {"another value", "value2"}
+
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 2
+        assert cache.get("key") == {"another value"}
+        assert cache.get("key2") == {"value2"}
+
+        cache.update("key", "another value", "key3", "another value")
+        assert len(cache._cache) == 2
+        assert len(cache) == 2
+        assert cache.get("key") is NotFound
+        assert cache.get("key2") == {"value2"}
+        assert cache.get("key3") == {"another value"}
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == {"new_value"}
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self):
+        # There is nothing in cache or remote repository.
+        # We must ust the value from alt_sdp
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        previous_value = {"old_1", "old_2", "value"}
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: previous_value)
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == {"old_1", "old_2", "new_value"}
+        assert previous_value == {"old_1", "old_2", "value"}
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self):
+        # keys are different
+        # make sure that current cache take precedence over alt_sdp
+        # In this test, the values from alt_sdp are never used
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: {"xxx1"} if key == "key1" else NotFound)
+
+        # one values in 'key1'
+        cache.put("key1", "old_1")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == {"old_2"}
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self):
+        # keys are different
+        # make sure that current repo take precedence over alt_sdp
+        remote = FakeSdp(get_value=lambda cache_name, key: {"old_1"} if key == "key1" else NotFound)
+        cache = SetCache(sdp=remote).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: {"xxx1"} if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: {"old_1", "old_2"} if key == "key1" else NotFound)
+        cache = SetCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == {"old_2"}
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self):
+        # keys are different
+        # No value found in cache or remote repository,
+        # Will use values from alt_sdp
+        # The old key is the same, so it has to be marked as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one values in 'key1'
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: {"old_1"} if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.test_only_reset()
+        old_values = {"old_1", "old_2"}
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == {"old_2"}
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == {"old_1", "old_2"}  # not modified
+
+    def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: {"xxx2"} if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.put("key2", "old_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == {'old_value', 'new_value'}
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        remote_repo = FakeSdp(get_value=lambda cache_name, key: {"old_value"} if key == "key2" else NotFound)
+        cache = SetCache(sdp=remote_repo).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == {'old_value', 'new_value'}
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key1", "source_value")
+        previous_values = {"old_1", "old_2"}
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == {"old_1", "old_2", 'new_value'}
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+        assert previous_values == {"old_1", "old_2"}  # not modified
+
+    def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])
+        cache.clear()
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp)
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp)
+
+    def test_default_is_called_before_updating_set_cache(self):
+        cache = SetCache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"})
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == {"new_value", "other old value"}
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else NotFound)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == {"other old value"}
+        assert cache.get("new_key") == {"new_value"}
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else {"other new"})
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == {"other old value"}
+        assert cache.get("new_key") == {"other new", "new_value"}
+
+    def test_i_can_delete_values_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+        cache.reset_events()
+
+        cache.delete("key", "fake_value")
+        assert cache.get("key") == {"value1", "value2"}
+        assert len(cache) == 2
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value1")
+        assert cache.get("key") == {"value2"}
+        assert cache.to_add == {"key"}
+        assert len(cache) == 1
+
+        cache.delete("key", "value2")
+        assert cache.get("key") is NotFound
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_key_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+
+        cache.delete("key")
+        assert cache.get("key") is NotFound
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_a_key_that_does_not_exists(self):
+        cache = SetCache()
+        cache.delete("key")
+
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But there is a value in the current cache after deletion
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+
+        cache.delete("key", value="value1", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": {"value2"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value"})).auto_configure("cache_name")
+
+        cache.delete("key", value="value", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But there is a value in the current cache after deletion
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name")
+
+        cache.delete("key", value="value1", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": {"value2"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1, value2"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_one_value_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1", "value2"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": {"value2"}}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_key_that_does_not_exist_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=(lambda cache_name, key: {"value1", "value2"} if key == "key" else NotFound),
+                          extend_exists=lambda cache_name, key: key == "key")
+
+        cache.delete("key2", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_that_does_not_exist_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1", "value2"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value4", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_an_already_removed_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # But the alternate sdp returns Removed, which means that previous value was deleted
+        # It's like there is nothing to delete
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.clear()
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()

tests/cache/test_cache.py

@@ -8,11 +8,48 @@ from cache.ListCache import ListCache
 from cache.ListIfNeededCache import ListIfNeededCache
 from cache.SetCache import SetCache
 from core.concept import Concept
+from core.global_symbols import NotFound, Removed
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.cache import FakeSdp
 
 
 class TestCache(TestUsingMemoryBasedSheerka):
+    def test_i_can_configure(self):
+        cache = Cache()
+        cache.configure(max_size=256,
+                        default="default_delegate",
+                        extend_exists="extend_exists_delegate",
+                        alt_sdp_get="alt_sdp_delegate",
+                        sdp=FakeSdp())
+
+        # Caution, in this test, I initialize default, extend_exists and alt_get_delegate with string
+        # to simplify the test, but it real usage, they are lambda
+        # default = lambda sdp, key: sdp.get(cache_name, key) or lambda key: func(key)
+        # extend_exists = lambda sdp, key: sdp.exists(cache_name, key) or lambda key: func(key)
+        # alt_sdp_get = lambda sdp, key: sdp.alt_get(cache_name, key)
+
+        assert cache._max_size == 256
+        assert cache._default == "default_delegate"
+        assert cache._extend_exists == "extend_exists_delegate"
+        assert cache._alt_sdp_get == "alt_sdp_delegate"
+        assert cache._sdp is not None
+
+    def test_i_can_auto_configure(self):
+        sdp = FakeSdp(get_value=lambda cache_name, key: key + 1 if cache_name == "cache_name" else NotFound,
+                      extend_exists=lambda cache_name, key: True if cache_name == "cache_name" else False,
+                      get_alt_value=lambda cache_name, key: key + 2 if cache_name == "cache_name" else NotFound)
+
+        cache = Cache(sdp=sdp).auto_configure("cache_name")
+        assert cache._default(cache._sdp, 10) == 11
+        assert cache._extend_exists(cache._sdp, 10) == True
+        assert cache._alt_sdp_get(cache._sdp, 10) == 12
+
+        cache = Cache(sdp=sdp).auto_configure("another_cache")
+        assert cache._default(cache._sdp, 10) == NotFound
+        assert cache._extend_exists(cache._sdp, 10) == False
+        assert cache._alt_sdp_get(cache._sdp, 10) == NotFound
+
     def test_i_can_get_an_retrieve_value_from_cache(self):
         cache = Cache()
         cache.put("key", "value")
@@ -44,9 +81,26 @@ class TestCache(TestUsingMemoryBasedSheerka):
             assert len(cache) == maxsize
             assert not cache.has(key - maxsize)
 
+    def test_i_can_get_a_value_from_alt_sdp(self):
+        cache = Cache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !")
+        assert cache.get("key", alt_sdp=alt_sdp) == "value found !"
+
+        # The value is now in cache
+        assert cache.copy() == {'key': 'value found !'}
+
+    def test_i_cannot_get_a_value_from_alt_sdp_when_cache_is_cleared(self):
+        cache = Cache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !")
+        assert cache.get("key", alt_sdp=alt_sdp) is NotFound
+        assert cache.copy() == {}
+
     def test_i_can_get_default_value_from_simple_cache(self):
         cache = Cache()
-        assert cache.get("key") is None
+        assert cache.get("key") is NotFound
 
         cache = Cache(default=10)
         assert cache.get("key") == 10
@@ -56,7 +110,12 @@ class TestCache(TestUsingMemoryBasedSheerka):
         assert cache.get("key") == "key_not_found"
         assert "key" in cache  # default callable are put in cache
 
-    def test_i_dont_ask_the_remote_repository_twice(self):
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda entry, key: key + "_not_found"))
+        assert cache.get("key") == "key_not_found"
+        assert "key" in cache  # default callable are put in cache
+
+    def test_i_do_not_ask_the_remote_repository_twice(self):
         nb_request = []
 
         cache = Cache(default=lambda key: nb_request.append("requested"))
@@ -64,155 +123,6 @@ class TestCache(TestUsingMemoryBasedSheerka):
         assert cache.get("key") is None
         assert len(nb_request) == 1
 
-    def test_i_can_put_and_retrieve_value_from_list_cache(self):
-        cache = ListCache()
-
-        cache.put("key", "value")
-        assert cache.get("key") == ["value"]
-        assert len(cache) == 1
-
-        cache.put("key", "value2")  # we can append to this list
-        assert cache.get("key") == ["value", "value2"]
-        assert len(cache) == 2
-
-        cache.put("key2", "value")
-        assert cache.get("key2") == ["value"]
-        assert len(cache) == 3
-
-        # duplicates are allowed
-        cache.put("key", "value")
-        assert cache.get("key") == ["value", "value2", "value"]
-        assert len(cache) == 4
-
-        assert cache.copy() == {'key': ['value', 'value2', 'value'], 'key2': ['value']}
-
-    def test_i_can_put_and_retrieve_value_from_list_if_needed_cache(self):
-        cache = ListIfNeededCache()
-
-        cache.put("key", "value")
-        assert cache.get("key") == "value"
-
-        # second time with the same key creates a list
-        cache.put("key", "value2")
-        assert cache.get("key") == ["value", "value2"]
-        assert len(cache) == 2
-
-        # third time, we now have a list
-        cache.put("key", "value3")
-        assert cache.get("key") == ["value", "value2", "value3"]
-        assert len(cache) == 3
-
-        # other keys are not affected
-        cache.put("key2", "value")
-        assert cache.get("key2") == "value"
-        assert len(cache) == 4
-
-        # duplicates are allowed
-        cache.put("key", "value")
-        assert cache.get("key") == ["value", "value2", "value3", "value"]
-        assert len(cache) == 5
-
-    def test_i_can_put_and_retrieve_values_from_set_cache(self):
-        cache = SetCache()
-
-        cache.put("key", "value")
-        assert cache.get("key") == {"value"}
-        assert len(cache) == 1
-
-        # we can add to this set
-        cache.put("key", "value2")
-        assert cache.get("key") == {"value", "value2"}
-        assert len(cache) == 2
-
-        # other keys are not affected
-        cache.put("key2", "value")
-        assert cache.get("key2") == {"value"}
-        assert len(cache) == 3
-
-        # duplicates are removed
-        cache.put("key", "value")
-        assert cache.get("key") == {"value", "value2"}
-        assert len(cache) == 3
-
-        assert cache.copy() == {'key': {'value', 'value2'}, 'key2': {'value'}}
-
-    def test_i_can_put_and_retrieve_value_from_dictionary_cache(self):
-        cache = DictionaryCache()
-
-        # # key must be None
-        # with pytest.raises(KeyError):
-        #     cache.put("key", None)
-        #
-        # # value must be a dictionary
-        # with pytest.raises(ValueError):
-        #     cache.put(True, "value")
-
-        entry = {"key": "value", "key2": ["value21", "value22"]}
-        cache.put(False, entry)
-        assert len(cache) == 3
-        assert id(cache._cache) == id(entry)
-        assert cache.get("key") == "value"
-        assert cache.get("key2") == ["value21", "value22"]
-
-        # I can append values
-        cache.put(True, {"key": "another_value", "key3": "value3"})
-        assert len(cache) == 4
-        assert cache.get("key") == "another_value"
-        assert cache.get("key2") == ["value21", "value22"]
-        assert cache.get("key3") == "value3"
-
-        # I can reset
-        entry = {"key": "value", "key2": ["value21", "value22"]}
-        cache.put(False, entry)
-        assert len(cache) == 3
-        assert id(cache._cache) == id(entry)
-        assert cache.get("key") == "value"
-        assert cache.get("key2") == ["value21", "value22"]
-
-        assert cache.copy() == {'key': 'value', 'key2': ['value21', 'value22']}
-
-    def test_i_can_put_and_retrieve_values_from_inc_cache(self):
-        cache = IncCache()
-
-        assert cache.get("key") == 1
-        assert cache.get("key") == 2
-        assert cache.get("key") == 3
-        assert cache.get("key2") == 1
-        assert cache.get("key2") == 2
-
-        cache.put("key", 100)
-        assert cache.get("key") == 101
-
-        assert cache.copy() == {'key': 101, 'key2': 2}
-
-    @pytest.mark.parametrize("key", [
-        None,
-        "something"
-    ])
-    def test_keys_have_constraints_when_dictionary_cache(self, key):
-        cache = DictionaryCache()
-        with pytest.raises(KeyError):
-            cache.put(key, None)
-
-    @pytest.mark.parametrize("value", [
-        None,
-        "something"
-    ])
-    def test_values_have_constraints_when_dictionary_cache(self, value):
-        cache = DictionaryCache()
-        with pytest.raises(ValueError):
-            cache.put(True, value)
-
-    def test_i_can_append_to_a_dictionary_cache_even_if_it_s_new(self):
-        cache = DictionaryCache()
-
-        entry = {"key": "value", "key2": ["value21", "value22"]}
-        cache.put(True, entry)
-        assert len(cache) == 3
-        assert id(cache._cache) != id(entry)
-        assert cache.get("key") == "value"
-        assert cache.get("key2") == ["value21", "value22"]
-
     def test_i_can_update_from_simple_cache(self):
         cache = Cache()
         cache.put("key", "value")
@@ -221,98 +131,43 @@ class TestCache(TestUsingMemoryBasedSheerka):
         assert len(cache._cache) == 1
         assert len(cache) == 1
         assert cache.get("key") == "new_value"
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
 
+        cache.reset_events()
         cache.update("key", "new_value", "another_key", "another_value")
         assert len(cache._cache) == 1
         assert len(cache) == 1
-        assert cache.get("key") is None
+        assert cache.get("key") is NotFound
         assert cache.get("another_key") == "another_value"
+        assert cache.to_add == {"another_key"}
+        assert cache.to_remove == {"key"}
 
         with pytest.raises(KeyError):
             cache.update("wrong key", "value", "key", "value")
 
-    def test_i_can_update_from_list_cache(self):
-        cache = ListCache()
+    def test_i_can_update_when_alt_sdp_same_keys(self):
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key),
+                      extend_exists=lambda sdp, key: sdp.exists("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda cache_name, key: NotFound))
 
         cache.put("key", "value")
-        cache.put("key", "value2")
-        cache.put("key", "value")
-        cache.update("key", "value", "key", "another value")
+        cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True))
 
-        assert len(cache._cache) == 1
-        assert len(cache) == 3
-        assert cache.get("key") == ["another value", "value2", "value"]  # only the first one is affected
+        assert cache.get("key") == "new_value"
 
-        cache.update("key", "value2", "key2", "value2")
-        assert len(cache._cache) == 2
-        assert len(cache) == 3
-        assert cache.get("key") == ["another value", "value"]
-        assert cache.get("key2") == ["value2"]
-
-        cache.update("key2", "value2", "key3", "value2")
-        assert len(cache._cache) == 2
-        assert len(cache) == 3
-        assert cache.get("key") == ["another value", "value"]
-        assert cache.get("key3") == ["value2"]
-        assert cache.get("key2") is None
-
-        with pytest.raises(KeyError):
-            cache.update("wrong key", "value", "key", "value")
-
-    def test_i_can_update_from_list_if_needed_cache(self):
-        cache = ListIfNeededCache()
+    def test_i_can_update_when_alt_sdp_different_keys(self):
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key),
+                      extend_exists=lambda sdp, key: sdp.exists("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda cache_name, key: NotFound))
 
         cache.put("key", "value")
-        cache.put("key", "value2")
-        cache.put("key", "value")
-        cache.update("key", "value", "key", "another value")
+        cache.update("key", "value", "key2", "value2", FakeSdp(extend_exists=lambda cache_name, key: True))
 
-        assert len(cache._cache) == 1
-        assert len(cache) == 3
-        assert cache.get("key") == ["another value", "value2", "value"]  # only the first one is affected
-
-        cache.update("key", "value2", "key2", "value2")
-        assert len(cache._cache) == 2
-        assert len(cache) == 3
-        assert cache.get("key") == ["another value", "value"]
         assert cache.get("key2") == "value2"
-
-        cache.update("key2", "value2", "key3", "value2")
-        assert len(cache._cache) == 2
-        assert len(cache) == 3
-        assert cache.get("key") == ["another value", "value"]
-        assert cache.get("key3") == "value2"
-        assert cache.get("key2") is None
-
-        with pytest.raises(KeyError):
-            cache.update("wrong key", "value", "key", "value")
-
-    def test_i_can_update_from_set_cache(self):
-        cache = SetCache()
-
-        cache.put("key", "value")
-        cache.put("key", "value2")
-        cache.update("key", "value", "key", "another value")
-
-        assert len(cache._cache) == 1
-        assert len(cache) == 2
-        assert cache.get("key") == {"another value", "value2"}
-
-        cache.update("key", "value2", "key2", "value2")
-        assert len(cache._cache) == 2
-        assert len(cache) == 2
-        assert cache.get("key") == {"another value"}
-        assert cache.get("key2") == {"value2"}
-
-        cache.update("key", "another value", "key3", "another value")
-        assert len(cache._cache) == 2
-        assert len(cache) == 2
-        assert cache.get("key") is None
-        assert cache.get("key2") == {"value2"}
-        assert cache.get("key3") == {"another value"}
-
-        with pytest.raises(KeyError):
-            cache.update("wrong key", "value", "key", "value")
+        assert cache.get("key") == Removed
+        assert cache.to_add == {"key", "key2"}
+        assert cache.to_remove == set()
 
     @pytest.mark.parametrize("cache", [
         Cache(), ListCache(), ListIfNeededCache(), SetCache(), IncCache()
@@ -365,21 +220,19 @@ class TestCache(TestUsingMemoryBasedSheerka):
         cache.put("key", "value")
         assert cache.exists("key")
 
-    def test_exists_in_dictionary_cache(self):
-        cache = DictionaryCache()
-        assert not cache.exists("key")
-
-        cache.put(True, {"key": "value"})
-        assert cache.exists("key")
-
     def test_exists_extend(self):
         cache = Cache(extend_exists=lambda k: True if k == "special_key" else False)
         assert not cache.exists("key")
         assert cache.exists("special_key")
 
+    def test_i_can_extend_exists_when_internal_sdp(self):
+        cache = Cache(extend_exists=lambda sdp, k: True if k == "special_key" else False, sdp=FakeSdp)
+        assert not cache.exists("key")
+        assert cache.exists("special_key")
+
     def test_add_concept_fills_all_dependent_caches(self):
         sheerka, context, one, two, two_2, three = self.init_concepts("one", "two", Concept("two"), "three")
-        cache_manager = CacheManager(None)
+        cache_manager = CacheManager(cache_only=True, sdp=None)
 
         cache_manager.register_concept_cache("by_id", Cache(), lambda obj: obj.id, True)
         cache_manager.register_concept_cache("by_name", ListCache(), lambda obj: obj.name, True)
@@ -412,53 +265,13 @@ class TestCache(TestUsingMemoryBasedSheerka):
         assert cache_manager.get("by_name", "two") == [two, two_2]
         assert cache_manager.get("by_name2", "two") == [two, two_2]
 
-    def test_default_for_dictionary_cache(self):
-        cache = DictionaryCache(default={"key": "value", "key2": "value2"})
-
-        assert cache.get("key") == "value"
-        assert "key2" in cache
-        assert len(cache) == 2
-
-        cache.clear()
-        assert cache.get("key3") is None
-        assert len(cache) == 2
-        assert "key" in cache
-        assert "key2" in cache
-
-        # default is not modified
-        cache._cache["key"] = "another value"  # operation that is normally not possible
-        cache.clear()
-        assert cache.get("key") == "value"
-
-    def test_default_callable_for_dictionary_cache(self):
-        cache = DictionaryCache(default=lambda k: {"key": "value", "key2": "value2"})
-
-        assert cache.get("key") == "value"
-        assert "key2" in cache
-        assert len(cache) == 2
-
-        cache.clear()
-        assert cache.get("key3") is None
-        assert len(cache) == 2
-        assert "key" in cache
-        assert "key2" in cache
-
-    def test_dictionary_cache_cannot_be_null(self):
-        cache = DictionaryCache(default=lambda k: None)
-        assert cache.get("key") is None
-        assert cache._cache == {}
-
-        cache = DictionaryCache(default=None)
-        assert cache.get("key") is None
-        assert cache._cache == {}
-
     @pytest.mark.parametrize("cache, default, new_value, expected", [
-        (ListCache(), lambda k: None, "value", ["value"]),
+        (ListCache(), lambda k: NotFound, "value", ["value"]),
         (ListCache(), lambda k: ["value"], "value", ["value", "value"]),
-        (ListIfNeededCache(), lambda k: None, "value", "value"),
+        (ListIfNeededCache(), lambda k: NotFound, "value", "value"),
         (ListIfNeededCache(), lambda k: "value", "value1", ["value", "value1"]),
         (ListIfNeededCache(), lambda k: ["value1", "value2"], "value1", ["value1", "value2", "value1"]),
-        (SetCache(), lambda k: None, "value", {"value"}),
+        (SetCache(), lambda k: NotFound, "value", {"value"}),
         (SetCache(), lambda k: {"value"}, "value", {"value"}),
         (SetCache(), lambda k: {"value1"}, "value2", {"value1", "value2"}),
     ])
@@ -469,7 +282,7 @@ class TestCache(TestUsingMemoryBasedSheerka):
         assert cache.get("key") == expected
 
     def test_default_is_called_before_updating_simple_cache(self):
-        cache = Cache(default=lambda k: None)
+        cache = Cache(default=lambda k: NotFound)
         with pytest.raises(KeyError):
             cache.update("old_key", "old_value", "new_key", "new_value")
 
@@ -477,180 +290,70 @@ class TestCache(TestUsingMemoryBasedSheerka):
         cache.update("old_key", "old_value", "new_key", "new_value")
         assert cache.get("new_key") == "new_value"
 
-    def test_default_is_called_before_updating_list_cache(self):
-        cache = ListCache(default=lambda k: None)
-        with pytest.raises(KeyError):
-            cache.update("old_key", "old_value", "new_key", "new_value")
-
-        cache = ListCache(default=lambda k: ["old_value", "other old value"])
-        cache.update("old_key", "old_value", "old_key", "new_value")
-        assert cache.get("old_key") == ["new_value", "other old value"]
-
-        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else None)
-        cache.update("old_key", "old_value", "new_key", "new_value")
-        assert cache.get("old_key") == ["other old value"]
-        assert cache.get("new_key") == ["new_value"]
-
-        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else ["other new"])
-        cache.update("old_key", "old_value", "new_key", "new_value")
-        assert cache.get("old_key") == ["other old value"]
-        assert cache.get("new_key") == ["other new", "new_value"]
-
-    def test_default_is_called_before_updating_list_if_needed_cache(self):
-        cache = ListIfNeededCache(default=lambda k: None)
-        with pytest.raises(KeyError):
-            cache.update("old_key", "old_value", "new_key", "new_value")
-
-        cache = ListIfNeededCache(default=lambda k: "old_value")
-        cache.update("old_key", "old_value", "old_key", "new_value")
-        assert cache.get("old_key") == "new_value"
-
-        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"])
-        cache.update("old_key", "old_value", "old_key", "new_value")
-        assert cache.get("old_key") == ["new_value", "other old value"]
-
-        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else None)
-        cache.update("old_key", "old_value", "new_key", "new_value")
-        assert cache.get("old_key") == ["other old value"]
-        assert cache.get("new_key") == "new_value"
-
-    def test_default_is_called_before_updating_set_cache(self):
-        cache = SetCache(default=lambda k: None)
-        with pytest.raises(KeyError):
-            cache.update("old_key", "old_value", "new_key", "new_value")
-
-        cache = SetCache(default=lambda k: {"old_value", "other old value"})
-        cache.update("old_key", "old_value", "old_key", "new_value")
-        assert cache.get("old_key") == {"new_value", "other old value"}
-
-        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else None)
-        cache.update("old_key", "old_value", "new_key", "new_value")
-        assert cache.get("old_key") == {"other old value"}
-        assert cache.get("new_key") == {"new_value"}
-
-        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else {"other new"})
-        cache.update("old_key", "old_value", "new_key", "new_value")
-        assert cache.get("old_key") == {"other old value"}
-        assert cache.get("new_key") == {"other new", "new_value"}
-
     def test_i_can_delete_an_entry_from_cache(self):
         cache = Cache()
         cache.put("key", "value")
 
         assert cache.get("key") == "value"
         cache.delete("key")
-        assert cache.get("key") is None
+        assert cache.get("key") is NotFound
         assert cache.to_remove == {"key"}
 
-    def test_i_can_delete_values_from_set_cache(self):
-        cache = SetCache()
-        cache.put("key", "value1")
-        cache.put("key", "value2")
-        cache.reset_events()
+    def test_i_can_delete_when_entry_is_only_in_db(self):
+        cache = Cache(default=lambda k: "value" if k == 'key' else NotFound)
 
-        cache.delete("key", "fake_value")
-        assert cache.get("key") == {"value1", "value2"}
-        assert len(cache) == 2
+        cache.delete("another_key")
+        assert cache.copy() == {}
         assert cache.to_add == set()
         assert cache.to_remove == set()
 
-        cache.delete("key", "value1")
-        assert cache.get("key") == {"value2"}
+        cache.delete("key")
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == {"key"}
+
+    def test_i_can_delete_an_entry_from_cache_when_alt_sdp_and_value_in_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = Cache(extend_exists=lambda sdp, k: sdp.exists("cache_name", k))
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
         assert cache.to_add == {"key"}
-        assert len(cache) == 1
-
-        cache.delete("key", "value2")
-        assert cache.get("key") is None
-        assert cache.to_remove == {"key"}
-        assert len(cache) == 0
-
-    def test_i_can_delete_key_from_set_cache(self):
-        cache = SetCache()
-        cache.put("key", "value1")
-        cache.put("key", "value2")
-
-        cache.delete("key")
-        assert cache.get("key") is None
-        assert cache.to_remove == {"key"}
-        assert len(cache) == 0
-
-    def test_i_can_delete_a_key_that_does_not_exists(self):
-        cache = SetCache()
-        cache.delete("key")
-
-        assert cache.to_add == set()
         assert cache.to_remove == set()
 
-    def test_i_can_delete_from_a_key_from_list_id_needed(self):
-        cache = ListIfNeededCache()
-        cache.put("key", "value1")
-        cache.put("key", "value11")
-        cache.put("key2", "value2")
-        cache.put("key2", "value22")
-        cache.put("key2", "value222")
-        cache.put("key3", "value3")
-        cache.put("key3", "value33")
-        cache.put("key4", "value4")
-        cache.reset_events()
+    def test_i_can_delete_an_entry_from_cache_when_alt_sdp_when_in_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = Cache(default=lambda k: "value", extend_exists=lambda sdp, k: sdp.exists("cache_name", k))
 
-        assert len(cache) == 8
-
-        # I can remove a whole key
-        cache.delete("key")
-        assert cache.get("key") is None
-        assert len(cache) == 6
-        assert cache.to_remove == {"key"}
-        assert cache.to_add == set()
-
-        # I can remove an element while a list is remaining
-        cache.reset_events()
-        cache.delete("key2", "value22")
-        assert cache.get("key2") == ["value2", "value222"]
-        assert len(cache) == 5
-        assert cache.to_add == {"key2"}
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
         assert cache.to_remove == set()
 
-        # I can remove an element while a single element is remaining
-        cache.reset_events()
-        cache.delete("key3", "value33")
-        assert cache.get("key3") == "value3"
-        assert len(cache) == 4
-        assert cache.to_add == {"key3"}
+    def test_i_can_delete_an_entry_from_cache_when_alt_sdp_and_no_value_in_cache_or_remote_repository(self):
+        # alt_cache_manager is used when no value found
+        cache = Cache(default=lambda sdp, k: sdp.get("cache_name", k),
+                      extend_exists=lambda sdp, k: sdp.exists("cache_name", k),
+                      sdp=FakeSdp(get_value=lambda entry, k: NotFound))
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
         assert cache.to_remove == set()
 
-        # I can remove an element while nothing remains
-        cache.reset_events()
-        cache.delete("key4", "value4")
-        assert cache.get("key4") is None
-        assert len(cache) == 3
-        assert cache.to_remove == {"key4"}
-        assert cache.to_add == set()
+    def test_no_error_when_deleting_a_key_that_does_not_exists_when_alt_sdp(self):
+        # alt_cache_manager is used when no value found
+        cache = Cache(default=lambda sdp, k: sdp.get("cache_name", k),
+                      extend_exists=lambda sdp, k: sdp.exists("cache_name", k),
+                      sdp=FakeSdp(get_value=lambda entry, k: NotFound))
 
-        # I do not remove when the value is not the same
-        cache.reset_events()
-        cache.delete("key3", "value33")  # value33 was already remove
-        assert cache.get("key3") == "value3"
-        assert len(cache) == 3
-        assert cache.to_add == set()
-        assert cache.to_remove == set()
-
-    def test_deleting_a_list_if_need_entry_that_does_not_exist_is_not_an_error(self):
-        cache = ListIfNeededCache()
-        cache.put("key", "value1")
-
-        cache.reset_events()
-        cache.delete("key3")
-        assert len(cache) == 1
-        assert cache.to_add == set()
-        assert cache.to_remove == set()
-
-        cache.delete("key3", "value")
-        assert len(cache) == 1
-        assert cache.to_add == set()
-        assert cache.to_remove == set()
-
-        cache.delete("key", "value2")
-        assert len(cache) == 1
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: False))
+        assert cache.copy() == {}
         assert cache.to_add == set()
         assert cache.to_remove == set()
 
@@ -689,6 +392,38 @@ class TestCache(TestUsingMemoryBasedSheerka):
         assert cache.get("2") == ("2", "2")
         assert cache.get("3") == ("3", "3")
 
+        assert cache.to_add == {"1", "2", "3"}
+        assert cache.to_remove == set()
+
+    def test_i_can_populate_using_internal_sdp(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache(sdp=FakeSdp(populate=items))
+
+        cache.populate(lambda sdp: sdp.populate(), lambda item: item[0])
+
+        assert len(cache) == 3
+        assert cache.get("1") == ("1", "1")
+        assert cache.get("2") == ("2", "2")
+        assert cache.get("3") == ("3", "3")
+
+        assert cache.to_add == {"1", "2", "3"}
+        assert cache.to_remove == set()
+
+    def test_i_can_reset_the_event_after_populate(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache()
+        cache.to_add = {"some_value"}
+        cache.to_remove = {"some_other_value"}
+
+        cache.populate(lambda: items, lambda item: item[0], reset_events=True)
+
+        assert len(cache) == 3
+        assert cache.copy() == {"1": ("1", "1"),
+                                "2": ("2", "2"),
+                                "3": ("3", "3")}
+        assert cache.to_add == {"some_value"}
+        assert cache.to_remove == {"some_other_value"}
+
     def test_max_size_is_respected_when_populate(self):
         items = [("1", "1"), ("2", "2"), ("3", "3"), ("4", "4"), ("5", "5")]
         cache = Cache(max_size=3)
@@ -709,3 +444,61 @@ class TestCache(TestUsingMemoryBasedSheerka):
         res = cache.get_all()
         assert len(res) == 3
         assert list(res) == [('1', '1'), ('2', '2'), ('3', '3')]
+
+    def test_i_can_clone_cache(self):
+        cache = Cache(max_size=256,
+                      default=lambda sdp, key: sdp.get("cache_name", key),
+                      extend_exists=False,
+                      alt_sdp_get=lambda sdp, key: sdp.alt_get("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda entry, key: key + "_not_found"))
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+
+        clone = cache.clone()
+        assert type(cache) == type(clone)
+        assert clone._max_size == cache._max_size
+        assert clone._default == cache._default
+        assert clone._extend_exists == cache._extend_exists
+        assert clone._alt_sdp_get == cache._alt_sdp_get
+        assert clone._sdp == cache._sdp
+        assert clone._cache == {}  # value are not copied
+        assert clone._initialized_keys == set()
+        assert clone._current_size == 0
+        assert clone.to_add == set()
+        assert clone.to_remove == set()
+
+        clone.configure(sdp=FakeSdp(lambda entry, key: key + " found !"))
+
+        assert cache.get("key3") == "key3_not_found"
+        assert clone.get("key3") == "key3 found !"
+
+    @pytest.mark.parametrize("cache", [
+        Cache(),
+        DictionaryCache(),
+        IncCache(),
+        ListCache(),
+        ListIfNeededCache()
+    ])
+    def test_i_can_clone_all_caches(self, cache):
+        clone = cache.clone()
+        assert type(clone) == type(cache)
+
+    def test_sanity_check_on_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+        clone = cache.clone()
+
+        clone.put("key", "value1")
+        clone.put("key", "value2")
+
+        assert clone.get("key") == ["value1", "value2"]
+
+    def test_i_can_clear_when_alt_sdp(self):
+        cache = Cache().auto_configure("cache_name")
+
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+
+        cache.clear()
+
+        assert cache.copy() == {}
+        assert cache._is_cleared

tests/cache/test_cache_manager.py

@@ -1,157 +0,0 @@
-import pytest
-from cache.Cache import Cache
-from cache.CacheManager import CacheManager, ConceptNotFound
-from cache.DictionaryCache import DictionaryCache
-from cache.ListCache import ListCache
-from cache.ListIfNeededCache import ListIfNeededCache
-from core.concept import Concept
-
-from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-
-
-class TestCacheManager(TestUsingMemoryBasedSheerka):
-    def test_i_do_not_push_into_sdp_when_cache_only(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-
-        cache_manager = CacheManager(True)
-        cache_manager.register_cache("test", Cache(), persist=True)
-        cache_manager.put("test", "key", "value")
-
-        cache_manager.commit(context)
-        assert not sheerka.sdp.exists("test", "key")
-
-    def test_i_do_not_get_value_from_sdp_when_cache_only_is_true(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-        with sheerka.sdp.get_transaction(context.event) as transaction:
-            transaction.add("test", "key", "value")
-
-        cache = Cache(default=lambda k: sheerka.sdp.get("test", k))
-
-        cache_manager = CacheManager(True)
-        cache_manager.register_cache("test", cache, persist=True)
-
-        assert cache_manager.get("test", "key") is None
-
-    def test_i_can_get_value_from_sdp_when_cache_only_is_false(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-        with sheerka.sdp.get_transaction(context.event) as transaction:
-            transaction.add("test", "key", "value")
-
-        cache = Cache(default=lambda k: sheerka.sdp.get("test", k))
-
-        cache_manager = CacheManager(False)
-        cache_manager.register_cache("test", cache, persist=True)
-
-        assert cache_manager.get("test", "key") == "value"
-
-    def test_i_can_commit_simple_cache(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-
-        cache_manager = CacheManager(False)
-        cache_manager.register_cache("test", Cache(), persist=True)
-        cache = cache_manager.caches["test"].cache
-
-        cache_manager.put("test", "key", "value")
-
-        cache_manager.commit(context)
-        assert sheerka.sdp.get("test", "key") == "value"
-
-        cache.update("key", "value", "key", "another_value")
-        cache_manager.commit(context)
-        assert sheerka.sdp.get("test", "key") == "another_value"
-
-        cache.update("key", "another_value", "key2", "another_value")
-        cache_manager.commit(context)
-        assert sheerka.sdp.get("test", "key") is None
-        assert sheerka.sdp.get("test", "key2") == "another_value"
-
-    def test_i_can_commit_list_cache(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-
-        cache_manager = CacheManager(False)
-        cache_manager.register_cache("test", ListCache(), persist=True)
-        cache = cache_manager.caches["test"].cache
-
-        cache.put("key", "value")
-        cache_manager.commit(context)
-        assert sheerka.sdp.get("test", "key") == ["value"]
-
-        cache.put("key", "value2")
-        cache_manager.commit(context)
-        assert sheerka.sdp.get("test", "key") == ["value", "value2"]
-
-        cache.update("key", "value2", "key2", "value2")
-        cache_manager.commit(context)
-        assert sheerka.sdp.get("test", "key") == ["value"]
-        assert sheerka.sdp.get("test", "key2") == ["value2"]
-
-        cache.update("key2", "value2", "key3", "value2")
-        cache_manager.commit(context)
-        assert sheerka.sdp.get("test", "key") == ["value"]
-        assert sheerka.sdp.get("test", "key2") is None
-        assert sheerka.sdp.get("test", "key3") == ["value2"]
-
-    def test_i_can_commit_dictionary_cache(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-
-        cache_manager = CacheManager(False)
-        cache_manager.register_cache("test", DictionaryCache(), persist=True)
-        cache = cache_manager.caches["test"].cache
-
-        cache.put(False, {"key": "value", "key2": "value2"})
-        cache_manager.commit(context)
-        assert sheerka.sdp.get("test") == {"key": "value", "key2": "value2"}
-        assert sheerka.sdp.get("test", "key") == "value"
-
-        cache.put(False, {"key": "value", "key2": "value2", "key3": "value3"})
-        cache_manager.commit(context)
-        assert sheerka.sdp.get("test") == {"key": "value", "key2": "value2", "key3": "value3"}
-
-    def test_i_can_remove_a_concept_from_concepts_caches(self):
-        cache_manager = CacheManager(True)
-        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
-        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
-
-        sheerka, context, one, two, three, two_bis = self.init_concepts("one", "two", "three", Concept("two", body="2"))
-
-        for concept in [one, two, three, two_bis]:
-            cache_manager.add_concept(concept)
-
-        # sanity check
-        cache_def = cache_manager.caches["id"]
-        assert cache_def.cache.copy() == {one.id: one, two.id: two, three.id: three, two_bis.id: two_bis}
-        cache_def = cache_manager.caches["key"]
-        assert cache_def.cache.copy() == {one.key: one, two.key: [two, two_bis], three.key: three}
-
-        for cache_name in cache_manager.concept_caches:
-            cache_manager.caches[cache_name].cache.reset_events()
-
-        cache_manager.remove_concept(sheerka.new(("two", two_bis.id)))
-
-        cache_def = cache_manager.caches["id"]
-        assert cache_def.cache.copy() == {one.id: one, two.id: two, three.id: three}
-        assert cache_def.cache.to_remove == {two_bis.id}
-        assert cache_def.cache.to_add == set()
-        assert len(cache_def.cache) == 3
-
-        cache_def = cache_manager.caches["key"]
-        assert cache_def.cache.copy() == {one.key: one, two.key: two, three.key: three}
-        assert cache_def.cache.to_remove == set()
-        assert cache_def.cache.to_add == {"two"}
-        assert len(cache_def.cache) == 3
-
-    def test_i_cannot_remove_a_concept_that_does_not_exists(self):
-        cache_manager = CacheManager(True)
-        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
-        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
-
-        with pytest.raises(ConceptNotFound) as ex:
-            cache_manager.remove_concept(Concept("foo", id="1001"))
-
-        assert ex.value.concept == Concept("foo", id="1001")

tests/core/test_SheerkaAdmin.py

@@ -1,9 +1,17 @@
+from core.builtin_concepts_ids import BuiltinConcepts
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
 class TestSheerkaAdmin(TestUsingMemoryBasedSheerka):
-    def test_i_can_get_last_ret(self):
-        pass
+    def test_i_can_get_concepts(self):
+        sheerka = self.get_sheerka(cache_only=False, singleton=False)
+
+        res = sheerka.concepts()
+        concepts = list(res.body)
+
+        assert sheerka.isinstance(res, BuiltinConcepts.TO_LIST)
+        assert concepts[0] == sheerka
+        assert concepts[1].id == "2"
 
     # def test_i_can_get_last_error_ret(self):
     #     sheerka, context = self.init_concepts()

tests/core/test_SheerkaComparisonManager.py

@@ -1,7 +1,8 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, EVENT_RULE_PRECEDENCE_MODIFIED, \
+from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, \
+    EVENT_RULE_PRECEDENCE_MODIFIED, \
     RULE_COMPARISON_CONTEXT
 from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager, ComparisonObj
 
@@ -33,36 +34,37 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
-        in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        in_cache = sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#")]
 
-        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        weighted = sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
         assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2}
 
         # I can commit
-        sheerka.cache_manager.commit(context)
-        in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        sheerka.om.commit(context)
+        in_db = sheerka.om.current_sdp().get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#")]
 
     def test_i_can_add_is_greater_than_for_rules(self):
-        sheerka, context, r1, r2 = self.init_format_rules(("True", "true"), ("False", "false"),
-                                                          cache_only=False,
-                                                          compile_rule=False)
+        sheerka, context, r1, r2 = self.init_test(cache_only=False).with_rules(("True", "true"),
+                                                                               ("False", "false"),
+                                                                               compile_rule=False).unpack()
+
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
         res = service.set_is_greater_than(context, "prop_name", r2, r1)
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
-        in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        in_cache = sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", r2.str_id, r1.str_id, ">", "#")]
 
-        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
-        assert weighted == {"r:|1:": 1, "r:|2:": 2}
+        weighted = sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        assert weighted == {r1.str_id: 1, r2.str_id: 2}
 
         # I can commit
-        sheerka.cache_manager.commit(context)
-        in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        sheerka.om.commit(context)
+        in_db = sheerka.om.current_sdp().get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", r2.str_id, r1.str_id, ">", "#")]
 
     def test_i_can_add_a_is_less_than(self):
@@ -73,36 +75,38 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
-        in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        in_cache = sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", one.str_id, two.str_id, "<", "#")]
 
-        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        weighted = sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
         assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2}
 
         # I can commit
-        sheerka.cache_manager.commit(context)
-        in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        sheerka.om.commit(context)
+        in_db = sheerka.om.current_sdp().get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", one.str_id, two.str_id, "<", "#")]
 
     def test_i_can_add_is_less_than_for_rules(self):
-        sheerka, context, r1, r2 = self.init_format_rules(("True", "true"), ("False", "false"),
-                                                          cache_only=False,
-                                                          compile_rule=False)
+        sheerka, context, r1, r2 = self.init_test(cache_only=False).with_rules(("True", "true"),
+                                                                               ("False", "false"),
+                                                                               compile_rule=False,
+                                                                               create_new=True).unpack()
+
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
         res = service.set_is_less_than(context, "prop_name", r1, r2)
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
-        in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        in_cache = sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", r1.str_id, r2.str_id, "<", "#")]
 
-        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
-        assert weighted == {"r:|1:": 1, "r:|2:": 2}
+        weighted = sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        assert weighted == {r1.str_id: 1, r2.str_id: 2}
 
         # I can commit
-        sheerka.cache_manager.commit(context)
-        in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        sheerka.om.commit(context)
+        in_db = sheerka.om.current_sdp().get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", r1.str_id, r2.str_id, "<", "#")]
 
     def test_i_can_add_multiples_constraints(self):
@@ -112,31 +116,31 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         service.set_is_greater_than(context, "prop_name", two, one)
         service.set_is_greater_than(context, "prop_name", three, two)
 
-        in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        in_cache = sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [
             ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"),
             ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#")
         ]
 
         # I can commit
-        sheerka.cache_manager.commit(context)
-        in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        sheerka.om.commit(context)
+        in_db = sheerka.om.current_sdp().get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_db == [
             ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"),
             ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#")
         ]
 
-        sheerka.cache_manager.clear(SheerkaComparisonManager.COMPARISON_ENTRY)  # reset the cache
+        sheerka.om.clear(SheerkaComparisonManager.COMPARISON_ENTRY)  # reset the cache
 
         service.set_is_greater_than(context, "prop_name", four, three)
-        in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        in_cache = sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [
             ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"),
             ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#"),
             ComparisonObj(context.event.get_digest(), "prop_name", four.str_id, three.str_id, ">", "#"),
         ]
 
-        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        weighted = sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
         assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3, "c:four|1004:": 4}
 
     def test_i_can_add_multiple_constraints_2(self):
@@ -146,23 +150,54 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         service.set_is_greater_than(context, "prop_name", two, one)
         service.set_is_greater_than(context, "prop_name", three, two)
 
-        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        weighted = sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
         assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3}
 
         service.set_is_greater_than(context, "prop_name", three, one)  # should not change order
-        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        weighted = sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
         assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3}
 
-    def test_i_lesser_than_and_opposite_greater_than(self):
+    def test_i_can_add_lesser_than_and_opposite_greater_than(self):
         sheerka, context, one, two = self.init_concepts("one", "two")
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
         service.set_is_greater_than(context, "prop_name", two, one)
         service.set_is_less_than(context, "prop_name", one, two)
 
-        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        weighted = sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
         assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2}
 
+    def test_i_can_support_multiple_ontology_layers(self):
+        sheerka, context, one, two, three = self.init_concepts("one", "two", "three", cache_only=False)
+        service = sheerka.services[SheerkaComparisonManager.NAME]
+
+        service.set_is_greater_than(context, "prop_name", two, one)
+
+        # sanity check
+        expected_in_cache = [ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#")]
+        assert sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") == expected_in_cache
+        expected_weights = {"c:one|1001:": 1, "c:two|1002:": 2}
+        assert sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") == expected_weights
+
+        # I still can access to the previous values
+        sheerka.push_ontology(context, "new ontology")
+        assert sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") == expected_in_cache
+        assert sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") == expected_weights
+
+        # I can modify
+        service.set_is_greater_than(context, "prop_name", three, two)
+        expected_in_cache2 = [
+            ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"),
+            ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#")]
+        assert sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") == expected_in_cache2
+        expected_weights2 = {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3}
+        assert sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") == expected_weights2
+
+        # I can retrieve the previous values
+        sheerka.pop_ontology()
+        assert sheerka.om.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#") == expected_in_cache
+        assert sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#") == expected_weights
+
     @pytest.mark.parametrize("entries, expected", [
         (["two > one"], {'c:one|1001:': 1, 'c:two|1002:': 2}),
         (["one < two"], {'c:one|1001:': 1, 'c:two|1002:': 2}),
@@ -192,7 +227,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
         service.set_is_lesser(context, "prop_name", one)
-        sheerka.cache_manager.clear(service.RESOLVED_COMPARISON_ENTRY)
+        sheerka.om.clear(service.RESOLVED_COMPARISON_ENTRY)
 
         assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0}
 
@@ -244,7 +279,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         service.set_is_greater_than(context, "prop_name", two, one)
         service.set_is_less_than(context, "prop_name", one, two)
 
-        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        weighted = sheerka.om.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
         assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2}
 
     def test_methods_are_correctly_bound(self):
@@ -263,8 +298,8 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0, "c:two|1002:": 1, "c:three|1003:": 2}
 
         # I can commit
-        sheerka.cache_manager.commit(context)
-        in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        sheerka.om.commit(context)
+        in_db = sheerka.om.current_sdp().get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_db == [
             ComparisonObj(context.event.get_digest(), "prop_name", one.str_id, None, "<<", "#"),
             ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#")
@@ -458,14 +493,12 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         assert event_received
 
     def test_an_event_is_fired_when_modifying_rule_precedence(self):
-        sheerka, context, r1, r2 = self.init_format_rules(
-            ("True", "True"),
-            ("False", "False"),
-            compile_rule=False,
-        )
+        sheerka, context, r1, r2 = self.init_test(cache_only=False).with_rules(("True", "true"),
+                                                                               ("False", "false"),
+                                                                               compile_rule=False).unpack()
         foo = Concept("foo")
         event_received = False
-        sheerka.cache_manager.clear(SheerkaComparisonManager.COMPARISON_ENTRY)
+        sheerka.om.clear(SheerkaComparisonManager.COMPARISON_ENTRY)
 
         def receive_event(c):
             nonlocal event_received

tests/core/test_SheerkaConceptAlgebra.py

@@ -61,15 +61,15 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka):
                                             BuiltinConcepts.HASA: {hasa2}, }
 
     def test_i_can_recognize_myself_when_using_sdp_repository(self):
-        sheerka, context, foo, isa1, hasa1, = self.init_concepts("foo", "isa1", "has1",
-                                                                 cache_only=False,
-                                                                 create_new=True)
-        sheerka.cache_manager.commit(context)
+        sheerka, context, foo, isa1, hasa1, = self.init_test(cache_only=False). \
+            with_concepts("foo", "isa1", "has1", create_new=True). \
+            unpack()
+        sheerka.om.commit(context)
 
         new_foo = sheerka.new("foo")
         sheerka.set_isa(context, new_foo, isa1)
         sheerka.set_hasa(context, new_foo, hasa1)
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
         assert sheerka.recognize(new_foo, all_scores=True) == [ConceptScore(1, new_foo, new_foo)]
 

tests/core/test_SheerkaConceptManager.py

@@ -1,8 +1,9 @@
 import pytest
 from cache.CacheManager import ConceptNotFound
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs, NotInit, \
+from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs, \
     DEFINITION_TYPE_BNF
+from core.global_symbols import NotInit, NotFound
 from core.sheerka.Sheerka import Sheerka
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \
     UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced
@@ -21,7 +22,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaConceptManager.NAME]
 
         res = sheerka.create_new_concept(context, concept)
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
         assert res.status
         assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
@@ -47,15 +48,15 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
 
         # I can get by the first entry
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
-        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
+        assert sheerka.om.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
+        assert sheerka.om.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
 
         # saved in sdp
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
-        assert sheerka.sdp.exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
+        assert sheerka.om.current_sdp().exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
 
     def test_i_cannot_create_a_bnf_concept_that_references_a_concept_that_cannot_be_resolved(self):
         sheerka, context, one_1, one_1_0 = self.init_concepts(Concept("one", body="1"), Concept("one", body="1.0"))
@@ -74,7 +75,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaConceptManager.NAME]
 
         res = sheerka.create_new_concept(self.get_context(sheerka), concept)
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
         assert res.status
         assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
@@ -99,11 +100,11 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
 
         # saved in sdp
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
-        assert sheerka.sdp.exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello")
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
+        assert sheerka.om.current_sdp().exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello")
 
     def test_i_cannot_add_the_same_concept_twice(self):
         """
@@ -184,55 +185,55 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert res.status
 
         # I can get by the first entry
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
-        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
+        assert sheerka.om.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
+        assert sheerka.om.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
 
     @pytest.mark.parametrize("expression", [
         "--'filter' ('one' | 'two') ",
         "'--filter' ('one' | 'two') ",
     ])
     def test_i_can_get_first_token_when_bnf_concept_and_not_a_letter(self, expression):
-        sheerka, context, bnf_concept = self.init_concepts(
+        sheerka, context, bnf_concept = self.init_test().with_concepts(
             Concept("foo", definition=expression),
-            create_new=True)
+            create_new=True).unpack()
 
         # I can get by the first entry
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
-        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
+        assert sheerka.om.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
+        assert sheerka.om.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
 
     def test_concept_references_are_updated_1(self):
-        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
+        sheerka, context, one, two, number, twenty, twenties = self.init_test().with_concepts(
             "one",
             "two",
             "number",
             "twenty",
             Concept("twenties", definition="twenty one | two 'hundred'"),
             create_new=True
-        )
+        ).unpack()
         service = sheerka.services[SheerkaConceptManager.NAME]
 
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) == {twenties.id}
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) == {twenties.id}
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) is None
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) == {twenties.id}
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) == {twenties.id}
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) is NotFound
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is NotFound
 
     def test_concept_references_are_updated_2(self):
-        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
+        sheerka, context, one, two, number, twenty, twenties = self.init_test().with_concepts(
             "one",
             "two",
             "number",
             "twenty",
             Concept("twenties", definition="twenty number"),
             create_new=True
-        )
+        ).unpack()
         service = sheerka.services[SheerkaConceptManager.NAME]
 
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) is None
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) is None
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) == {twenties.id}
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
-        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) is NotFound
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) is NotFound
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) == {twenties.id}
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
+        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is NotFound
 
     @pytest.mark.parametrize("attr", [
         "name",
@@ -357,16 +358,18 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.get_by_hash(new_concept.get_definition_hash()).get_metadata().body == "metadata value"
 
         # sdp is updated
-        sheerka.cache_manager.commit(context)
-        from_sdp = sheerka.sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
+        sheerka.om.commit(context)
+        from_sdp = sheerka.om.current_sdp().get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
         assert from_sdp.get_metadata().body == "metadata value"
         assert from_sdp.get_metadata().variables == [("var_name", "default value")]
         assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}
 
-        assert sheerka.sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).get_metadata().body == "metadata value"
-        assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).get_metadata().body == "metadata value"
-        assert sheerka.sdp.get(service.CONCEPTS_BY_HASH_ENTRY,
-                               new_concept.get_definition_hash()).get_metadata().body == "metadata value"
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_NAME_ENTRY,
+                                            new_concept.name).get_metadata().body == "metadata value"
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY,
+                                            new_concept.key).get_metadata().body == "metadata value"
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_HASH_ENTRY,
+                                            new_concept.get_definition_hash()).get_metadata().body == "metadata value"
 
     def test_caches_are_update_when_i_modify_the_name(self):
         sheerka, context, foo = self.init_concepts("foo", cache_only=False)
@@ -391,14 +394,15 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert not sheerka.is_known(sheerka.get_by_key(foo.key))
         assert not sheerka.is_known(sheerka.get_by_hash(foo.get_definition_hash()))
 
-        sheerka.cache_manager.commit(context)
-        assert sheerka.sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id).name == "bar"
-        assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).name == "bar"
-        assert sheerka.sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).name == "bar"
-        assert sheerka.sdp.get(service.CONCEPTS_BY_HASH_ENTRY, new_concept.get_definition_hash()).name == "bar"
-        assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, foo.key) is None
-        assert sheerka.sdp.get(service.CONCEPTS_BY_NAME_ENTRY, foo.name) is None
-        assert sheerka.sdp.get(service.CONCEPTS_BY_HASH_ENTRY, foo.get_definition_hash()) is None
+        sheerka.om.commit(context)
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id).name == "bar"
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).name == "bar"
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).name == "bar"
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_HASH_ENTRY,
+                                            new_concept.get_definition_hash()).name == "bar"
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY, foo.key) is NotFound
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_NAME_ENTRY, foo.name) is NotFound
+        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_HASH_ENTRY, foo.get_definition_hash()) is NotFound
 
     def test_i_can_modify_a_concept_from_a_list_of_concepts(self):
         sheerka, context, foo1, foo2 = self.init_concepts(
@@ -467,12 +471,12 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert new_concept.key == "__var__0 bar __var__1 __var__2"
 
     def test_bnf_is_modified_when_modifying_the_definition(self):
-        sheerka, context, one, two, foo = self.init_concepts(
+        sheerka, context, one, two, foo = self.init_test().with_concepts(
             "one",
             "two",
             Concept(name="foo", definition="'twenty' one"),
             create_new=True
-        )
+        ).unpack()
 
         to_add = {"meta": {"definition": "'twenty' two"}}
 
@@ -484,17 +488,17 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert new_concept.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(two, rule_name='two'))
 
     def test_concept_by_first_keyword_is_updated_after_concept_modification(self):
-        sheerka, context, foo, bar, baz = self.init_concepts(
+        sheerka, context, foo, bar, baz = self.init_test().with_concepts(
             Concept("foo"),
             Concept("bar"),
             Concept("baz", definition="foo"),
-            create_new=True)
+            create_new=True).unpack()
 
-        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             "foo": ["1001"],
             "bar": ["1002"],
             'c:|1001:': ['1003']}
-        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             'foo': ['1001', '1003'],
             'bar': ['1002']}
 
@@ -502,18 +506,18 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         res = sheerka.modify_concept(context, foo, to_add)
 
         assert res.status
-        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             "bar": ["1002", "1001"],
             'c:|1001:': ['1003']}
-        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             'bar': ['1002', '1001', '1003']}
 
     def test_references_are_updated_after_concept_modification(self):
-        sheerka, context, one, twenty_one = self.init_concepts(
+        sheerka, context, one, twenty_one = self.init_test().with_concepts(
             "onz",
             Concept("twenty one", definition="'twenty' onz"),
             create_new=True
-        )
+        ).unpack()
 
         assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(one, rule_name='onz'))
 
@@ -530,6 +534,35 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         BaseNodeParser.ensure_bnf(context, twenty_one)
         assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(modified, rule_name='one'))
 
+    def test_i_can_modify_on_top_of_a_new_ontology_layer(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b"), cache_only=False)
+
+        sheerka.push_ontology(context, "new ontology")
+
+        to_add = {
+            "meta": {"body": "a body"},
+            "props": {BuiltinConcepts.ISA: "bar"},
+            "variables": {"c": "value"}
+        }
+        to_remove = {
+            "variables": ["b"]
+        }
+
+        res = sheerka.modify_concept(context, foo, to_add=to_add, to_remove=to_remove)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.get_metadata().body == "a body"
+        assert res.body.body.get_metadata().variables == [("a", None), ("c", "value")]
+        assert res.body.body.get_metadata().props == {BuiltinConcepts.ISA: {"bar"}}
+
+        # and correctly set in cache
+        updated = sheerka.get_by_id(foo.id)
+        assert updated.get_metadata().body == "a body"
+        assert updated.get_metadata().variables == [("a", None), ("c", "value")]
+        assert updated.get_metadata().props == {BuiltinConcepts.ISA: {"bar"}}
+
+        sheerka.pop_ontology()
+
     def test_i_cannot_modify_without_any_modification(self):
         sheerka, context, foo = self.init_concepts("foo")
         service = sheerka.services[SheerkaConceptManager.NAME]
@@ -627,10 +660,10 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.get_attr(foo, prop) == bar
 
     def test_i_cannot_remove_a_concept_which_has_reference(self):
-        sheerka, context, one, twenty_one = self.init_concepts(
+        sheerka, context, one, twenty_one = self.init_test().with_concepts(
             Concept("one"),
             Concept("twenty one", definition="'twenty' one"),
-            create_new=True)
+            create_new=True).unpack()
 
         res = sheerka.remove_concept(context, one)
 
@@ -639,9 +672,9 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert res.body.body == ConceptIsReferenced([twenty_one])
 
     def test_i_can_remove_a_concept(self):
-        sheerka, context, one = self.init_concepts(
+        sheerka, context, one = self.init_test().with_concepts(
             Concept("one"),
-            create_new=True)
+            create_new=True).unpack()
 
         # sanity check
         assert sheerka.get_by_id(one.id) == one
@@ -669,6 +702,34 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == ConceptNotFound(one)
 
+    def test_i_can_create_concepts_in_multiple_ontology_layers(self):
+        sheerka, context = self.init_concepts(cache_only=False)
+
+        res = sheerka.create_new_concept(context, Concept("foo"))
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+
+        sheerka.push_ontology(context, "new ontology")
+        res = sheerka.create_new_concept(context, Concept("bar"))
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+
+        # I cannot defined foo again, even if it's not the same layer
+        res = sheerka.create_new_concept(context, Concept("foo"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+
+        # I cannot define bar again in this layer
+        res = sheerka.create_new_concept(context, Concept("bar"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+
+        sheerka.pop_ontology()
+        # But I can if I remove the layer
+        res = sheerka.create_new_concept(context, Concept("bar"))
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+
 
 class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
     def test_i_can_add_several_concepts(self):
@@ -678,42 +739,40 @@ class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
         hello = Concept("Hello world a").def_var("a")
         res = sheerka.create_new_concept(context, hello)
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
         assert res.status
 
         sheerka = self.get_sheerka()  # another instance
         context = self.get_context(sheerka)
         greeting = Concept("Greeting a").def_var("a")
         res = sheerka.create_new_concept(context, greeting)
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
         assert res.status
 
         sheerka = self.get_sheerka()  # another instance again
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, hello.key)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, greeting.key)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, hello.id)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, greeting.id)
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, "Hello world a")
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, "Greeting a")
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, hello.get_definition_hash())
-        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, greeting.get_definition_hash())
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, hello.key)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, greeting.key)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, hello.id)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, greeting.id)
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, "Hello world a")
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, "Greeting a")
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, hello.get_definition_hash())
+        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, greeting.get_definition_hash())
 
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello")
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting")
+        assert sheerka.om.current_sdp().exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello")
+        assert sheerka.om.current_sdp().exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting")
 
     def test_i_cannot_add_the_same_concept_twice_using_sdp(self):
         """
         Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
         :return:
         """
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = self.get_default_concept()
+        sheerka, context, concept = self.init_concepts("foo")
 
         sheerka.create_new_concept(context, concept)
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
-        sheerka.cache_manager.clear()
+        sheerka.om.current_cache_manager().clear(set_is_cleared=False)
         res = sheerka.create_new_concept(context, concept)
 
         assert not res.status
@@ -727,13 +786,13 @@ class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
         sheerka.create_new_concept(context, Concept("foo", body="1"))
         sheerka.create_new_concept(context, Concept("foo", body="2"))
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
-        assert len(sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 2
+        assert len(sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 2
 
         sheerka = self.get_sheerka()  # new instance
         context = self.get_context(sheerka)
         sheerka.create_new_concept(context, Concept("foo", body="3"))
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
-        assert len(sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 3
+        assert len(sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 3

tests/core/test_SheerkaDebugManager.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, NotInit
+from core.concept import Concept
+from core.global_symbols import NotInit, NotFound
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugItem, ConceptDebugObj
 from parsers.PythonParser import PythonNode
@@ -830,7 +831,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
         dummy = DummyObj(foo, "value")
         res = sheerka.inspect(context, dummy, "#type#", "fake", "a", "b")
         assert res.body == {'#type#': 'DummyObj',
-                            'fake': "** Not Found **",
+                            'fake': NotFound,
                             'a': foo,
                             'b': 'value'}
 
@@ -850,19 +851,19 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         res = sheerka.inspect(context, 0)
 
-        assert res.body == {'#type#': 'NotFound',
+        assert res.body == {'#type#': 'NotFoundConcept',
                             'id': sheerka.concepts_ids[BuiltinConcepts.NOT_FOUND],
                             'key': '__NOT_FOUND',
                             'name': '__NOT_FOUND',
                             'body': 'no digest'}
 
     def test_i_can_inspect_values(self):
-        sheerka, context, table, how, little = self.init_concepts(
+        sheerka, context, table, how, little = self.init_test().with_concepts(
             "table",
             Concept("how is x").def_var("x"),
             Concept("little x").def_var("x"),
             create_new=True
-        )
+        ).unpack()
 
         return_values = sheerka.evaluate_user_input("how is little table")
 
@@ -911,3 +912,64 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         assert str(ConceptDebugObj(foo)) == \
                "(:foo|1001:meta.x='x_meta', meta.y='y_meta', compiled.x=(:bar|1002:meta.a='a_meta', value.a='a_value'), value.x='x_value', value.z='extra_value')"
+
+    def test_i_can_save_and_restore_state_to_default_state(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        sheerka.push_ontology(context, "new ontology")
+
+        service.set_debug(context)
+        service.set_explicit(context)
+        service.debug_var(context, "var_service.var_method.var_name", "1+", 1)
+        service.debug_rule(context, "rule_service.rule_method.rule_name", "2+", 2)
+        service.debug_concept(context, "concept_service.concept_method.concept_name", "3+", 3)
+
+        # sanity check
+        assert service.activated
+        assert service.explicit
+        assert service.debug_vars_settings != []
+        assert service.debug_rules_settings != []
+        assert service.debug_concepts_settings != []
+
+        sheerka.pop_ontology()
+        assert not service.activated
+        assert not service.explicit
+        assert service.context_cache == set()
+        assert service.variable_cache == set()
+        assert service.debug_vars_settings == []
+        assert service.debug_rules_settings == []
+        assert service.debug_concepts_settings == []
+
+    def test_i_can_save_and_restore_state_to_specific_state(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        service.set_debug(context)
+        service.set_explicit(context)
+        service.debug_var(context, "v_service.v_method.v_name", "1+", 1)
+        service.debug_rule(context, "r_service.r_method.r_name", "2+", 2)
+        service.debug_concept(context, "c_serv.c_method.c_name", "3+", 3)
+
+        sheerka.push_ontology(context, "new ontology")
+
+        # modify the state
+        service.set_debug(context, False)
+        service.set_explicit(context, False)
+        service.debug_var(context, "var_service2.var_method2.var_name2", "11+", 11)
+        service.debug_rule(context, "rule_service2.rule_method2.rule_name2", "22+", 22)
+        service.debug_concept(context, "concept_service2.concept_method2.concept_name2", "33+", 33)
+
+        # sanity
+        assert not service.activated
+        assert not service.explicit
+        assert len(service.debug_vars_settings) == 2
+        assert len(service.debug_rules_settings) == 2
+        assert len(service.debug_concepts_settings) == 2
+
+        sheerka.pop_ontology()
+        assert service.activated
+        assert service.explicit
+        assert service.debug_vars_settings == [DebugItem("v_name", "v_service", "v_method", 1, True, 1, False, True)]
+        assert service.debug_rules_settings == [DebugItem("r_name", "r_service", "r_method", 2, True, 2, False, True)]
+        assert service.debug_concepts_settings == [DebugItem("c_name", "c_serv", "c_method", 3, True, 3, False, True)]

tests/core/test_SheerkaEvaluateConcept.py

@@ -2,8 +2,9 @@ from dataclasses import dataclass
 
 import pytest
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, CB, NotInit, \
+from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, CB, \
     concept_part_value, DEFINITION_TYPE_DEF
+from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from parsers.BaseParser import BaseParser
@@ -13,7 +14,6 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, python_ret_val
 
 
-
 class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("body, expected", [
@@ -27,7 +27,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         ("1 > 2", False),
     ])
     def test_i_can_evaluate_a_concept_with_simple_body(self, body, expected):
-        sheerka, context, concept = self.init_concepts(Concept("foo", body=body), eval_body=True)
+        sheerka, context, concept = self.init_test(eval_body=True).with_concepts(Concept("foo", body=body)).unpack()
 
         evaluated = sheerka.evaluate_concept(context, concept)
 
@@ -99,7 +99,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
     def test_i_can_evaluate_when_the_body_is_the_name_of_the_concept(self):
         # to prove that I can distinguish from a string
-        sheerka, context, concept = self.init_concepts(Concept("foo", body="'foo'"), eval_body=True, create_new=True)
+        sheerka, context, concept = self.init_test(eval_body=True).with_concepts(
+            Concept("foo", body="'foo'"),
+            create_new=True).unpack()
 
         evaluated = sheerka.evaluate_concept(context, concept)
 

tests/core/test_SheerkaEvaluateRules.py

@@ -9,7 +9,7 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
     def test_i_can_evaluate_python_rules(self):
-        sheerka, context, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_format_rules(
+        sheerka, context, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_test().with_rules(
             Rule(predicate="a == 1", action="", priority=1),  # r1
             Rule(predicate="a == 2", action="", priority=1),  # r2
             Rule(predicate="a == 3", action="", priority=0),  # r3
@@ -19,7 +19,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
             Rule(predicate="a == 7", action="", priority=1, is_enabled=False),  # r7
             Rule(predicate="a == 8", action="", priority=1),  # r8
             Rule(predicate="a == 9", action="", priority=2),  # r9
-        )
+        ).unpack()
         service = sheerka.services[SheerkaEvaluateRules.NAME]
         rules = sorted([r1, r2, r3, r4, r5, r6, r7, r8, r9], key=operator.attrgetter('priority'), reverse=True)
 
@@ -33,7 +33,8 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         }
 
     def test_i_can_evaluate_concept_rules(self):
-        sheerka, context, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_format_rules(
+        sheerka, context, concept, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_test().with_concepts(
+            Concept("x equals y", body="x == y").def_var("x").def_var("y"), create_new=True).with_rules(
             Rule(predicate="a equals 1", action="", priority=1),  # r1
             Rule(predicate="a equals 2", action="", priority=1),  # r2
             Rule(predicate="a equals 3", action="", priority=0),  # r3
@@ -43,8 +44,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
             Rule(predicate="a equals 7", action="", priority=1, is_enabled=False),  # r7
             Rule(predicate="a equals 8", action="", priority=1),  # r8
             Rule(predicate="a equals 9", action="", priority=2),  # r9
-            concepts=[Concept("x equals y", body="x == y").def_var("x").def_var("y")],
-        )
+        ).unpack()
 
         service = sheerka.services[SheerkaEvaluateRules.NAME]
         rules = sorted([r1, r2, r3, r4, r5, r6, r7, r8, r9], key=operator.attrgetter('priority'), reverse=True)

tests/core/test_SheerkaEventManager.py

@@ -1,3 +1,5 @@
+import pytest
+
 from core.sheerka.services.SheerkaEventManager import SheerkaEventManager
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -44,7 +46,7 @@ example_of_class_method. event=xxx
 """
 
         service = sheerka.services[SheerkaEventManager.NAME]
-        service.reset_topic(topic)
+        service.test_only_reset_topic(topic)
 
     def test_i_can_subscribe_and_publish_with_data(self, capsys):
         sheerka, context = self.init_concepts()
@@ -63,4 +65,21 @@ example_of_class_method. event=xxx, data='42'
 """
 
         service = sheerka.services[SheerkaEventManager.NAME]
-        service.reset_topic(topic)
+        service.test_only_reset_topic(topic)
+
+    def test_i_can_save_and_reset_state(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaEventManager.NAME]
+
+        sheerka.subscribe("my first topic", self.example_of_class_method_with_data)
+
+        sheerka.push_ontology(context, "new ontology")
+        sheerka.subscribe("my second topic", self.example_of_class_method_with_data)
+
+        # I can access to the topic
+        assert "my first topic" in service.subscribers
+        assert "my second topic" in service.subscribers
+
+        sheerka.pop_ontology()
+        assert "my first topic" in service.subscribers
+        assert "my second topic" not in service.subscribers

tests/core/test_SheerkaFunctionsParametersHistory.py

@@ -1,5 +1,8 @@
+import pytest
+
 from core.sheerka.services.SheerkaFunctionsParametersHistory import SheerkaFunctionsParametersHistory, \
     FunctionParametersObj
+from core.utils import sheerka_deepcopy
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -7,13 +10,15 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 class TestSheerkaFunctionsParametersHistory(TestUsingMemoryBasedSheerka):
     def test_i_can_add_a_parameter_value(self):
         sheerka, context = self.init_concepts(cache_only=False)
-        service = SheerkaFunctionsParametersHistory(sheerka).initialize()
+        sheerka.om.test_only_unfreeze()
+        service = SheerkaFunctionsParametersHistory(sheerka).initialize()  # since service is no longer auto init'ed
+        sheerka.om.freeze()
 
         service.record_function_parameter(context, "function", 1, "10")
         service.record_function_parameter(context, "function", 2, "True")
         service.record_function_parameter(context, "function", 3, "'string value'")
 
-        assert service.cache.copy() == {"function": FunctionParametersObj(
+        assert sheerka.om.copy(service.FUNCTIONS_PARAMETERS_ENTRY) == {"function": FunctionParametersObj(
             context.event.get_digest(),
             "function",
             {
@@ -23,8 +28,8 @@ class TestSheerkaFunctionsParametersHistory(TestUsingMemoryBasedSheerka):
             })}
 
         # and i can serialize
-        sheerka.cache_manager.commit(context)
-        from_db = sheerka.sdp.get(SheerkaFunctionsParametersHistory.FUNCTIONS_PARAMETERS_ENTRY, "function")
+        sheerka.om.commit(context)
+        from_db = sheerka.om.current_sdp().get(service.FUNCTIONS_PARAMETERS_ENTRY, "function")
         assert from_db.event_id == context.event.get_digest()
         assert from_db.name == "function"
         assert from_db.params == {
@@ -35,14 +40,16 @@ class TestSheerkaFunctionsParametersHistory(TestUsingMemoryBasedSheerka):
 
     def test_i_can_add_the_same_value_multiple_times(self):
         sheerka, context = self.init_concepts(cache_only=True)
-        service = SheerkaFunctionsParametersHistory(sheerka)
+        sheerka.om.test_only_unfreeze()
+        service = SheerkaFunctionsParametersHistory(sheerka).initialize()  # since service is no longer auto init'ed
+        sheerka.om.freeze()
 
         service.record_function_parameter(context, "function", 1, "10")
         service.record_function_parameter(context, "function", 1, "20")
         service.record_function_parameter(context, "function", 2, "True")
         service.record_function_parameter(context, "function", 1, "20")
 
-        assert service.cache.copy() == {"function": FunctionParametersObj(
+        assert sheerka.om.copy(service.FUNCTIONS_PARAMETERS_ENTRY) == {"function": FunctionParametersObj(
             context.event.get_digest(),
             "function",
             {
@@ -52,12 +59,14 @@ class TestSheerkaFunctionsParametersHistory(TestUsingMemoryBasedSheerka):
 
     def test_i_can_specify_parameter_in_any_order(self):
         sheerka, context = self.init_concepts()
-        service = SheerkaFunctionsParametersHistory(sheerka)
+        sheerka.om.test_only_unfreeze()
+        service = SheerkaFunctionsParametersHistory(sheerka).initialize()  # since service is no longer auto init'ed
+        sheerka.om.freeze()
 
         service.record_function_parameter(context, "function", 3, "'string value'")
         service.record_function_parameter(context, "function", 2, "True")
 
-        assert service.cache.copy() == {"function": FunctionParametersObj(
+        assert sheerka.om.copy(service.FUNCTIONS_PARAMETERS_ENTRY) == {"function": FunctionParametersObj(
             context.event.get_digest(),
             "function",
             {
@@ -67,7 +76,9 @@ class TestSheerkaFunctionsParametersHistory(TestUsingMemoryBasedSheerka):
 
     def test_no_value_is_managed(self):
         sheerka, context = self.init_concepts()
-        service = SheerkaFunctionsParametersHistory(sheerka)
+        sheerka.om.test_only_unfreeze()
+        service = SheerkaFunctionsParametersHistory(sheerka).initialize()  # since service is no longer auto init'ed
+        sheerka.om.freeze()
 
         # no entry for the function
         assert service.get_function_parameters("function", 2) == []
@@ -78,10 +89,45 @@ class TestSheerkaFunctionsParametersHistory(TestUsingMemoryBasedSheerka):
 
     def test_i_can_get_sorted_parameters(self):
         sheerka, context = self.init_concepts()
-        service = SheerkaFunctionsParametersHistory(sheerka)
+        sheerka.om.test_only_unfreeze()
+        service = SheerkaFunctionsParametersHistory(sheerka).initialize()  # since service is no longer auto init'ed
+        sheerka.om.freeze()
 
         service.record_function_parameter(context, "function", 2, "'string value'")
         service.record_function_parameter(context, "function", 2, "True")
         service.record_function_parameter(context, "function", 2, "True")
 
         assert service.get_function_parameters("function", 2) == ["True", "'string value'"]
+
+    def test_i_can_add_and_retrieve_parameters_when_multiple_ontology_layers(self):
+        sheerka, context = self.init_concepts(cache_only=False)
+        sheerka.om.test_only_unfreeze()
+        service = SheerkaFunctionsParametersHistory(sheerka).initialize()  # since service is no longer auto init'ed
+        sheerka.om.freeze()
+
+        service.record_function_parameter(context, "function", 1, "10")
+        service.record_function_parameter(context, "function", 2, "True")
+        service.record_function_parameter(context, "function", 3, "'string value'")
+
+        sheerka.push_ontology(context, "new ontology")
+        service.record_function_parameter(context, "function", 1, "20")
+        service.record_function_parameter(context, "function", 2, "True")
+
+        assert sheerka.om.copy(service.FUNCTIONS_PARAMETERS_ENTRY) == {"function": FunctionParametersObj(
+            context.event.get_digest(),
+            "function",
+            {
+                1: [('10', 1), ("20", 1)],
+                2: [('True', 2)],
+                3: [("'string value'", 1)]
+            })}
+
+        sheerka.pop_ontology()
+        assert sheerka.om.copy(service.FUNCTIONS_PARAMETERS_ENTRY) == {"function": FunctionParametersObj(
+            context.event.get_digest(),
+            "function",
+            {
+                1: [('10', 1)],
+                2: [('True', 1)],
+                3: [("'string value'", 1)]
+            })}

tests/core/test_SheerkaHistoryManager.py

@@ -1,10 +1,10 @@
 from core.sheerka.services.SheerkaHistoryManager import hist
-from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 
 
-class TestSheerkaHistoryManager(TestUsingMemoryBasedSheerka):
+class TestSheerkaHistoryManager(TestUsingFileBasedSheerka):
     def test_i_can_retrieve_history(self):
-        sheerka = self.get_sheerka(singleton=False)
+        sheerka = self.get_sheerka()
         sheerka.save_execution_context = True
 
         sheerka.evaluate_user_input("def concept one as 1")

tests/core/test_SheerkaIsAManager.py

@@ -1,12 +1,14 @@
+import pytest
+
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.sheerka.services.SheerkaSetsManager import SheerkaSetsManager
+from core.sheerka.services.SheerkaIsAManager import SheerkaIsAManager
 
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
-class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
+class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka):
 
     def test_i_can_add_a_concept_to_a_set(self):
         sheerka, context, foo, group = self.init_concepts(
@@ -14,14 +16,18 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
             Concept("group"),
             cache_only=False
         )
-        assert sheerka.add_concept_to_set(context, foo, group).status
 
-        group_elements = sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY, group.id)
+        res = sheerka.add_concept_to_set(context, foo, group)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        group_elements = sheerka.om.get(SheerkaIsAManager.CONCEPTS_GROUPS_ENTRY, group.id)
         assert group_elements == {foo.id}
 
         # it can be persisted
-        sheerka.cache_manager.commit(context)
-        assert sheerka.sdp.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY, group.id) == {foo.id}
+        sheerka.om.commit(context)
+        assert sheerka.om.current_sdp().get(SheerkaIsAManager.CONCEPTS_GROUPS_ENTRY, group.id) == {foo.id}
 
     def test_i_cannot_add_the_same_concept_twice_in_a_set(self):
         sheerka, context, foo, group = self.init_concepts(Concept("foo"), Concept("group"))
@@ -35,7 +41,7 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
         assert res.body.body == foo
         assert res.body.concept_set == group
 
-        all_entries = sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY, group.id)
+        all_entries = sheerka.om.get(SheerkaIsAManager.CONCEPTS_GROUPS_ENTRY, group.id)
         assert all_entries == {foo.id}
 
     def test_i_can_have_multiple_groups(self):
@@ -53,12 +59,12 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
         assert sheerka.add_concept_to_set(context, bar, group2).status
         assert sheerka.add_concept_to_set(context, baz, group2).status
 
-        assert sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY, group1.id) == {foo.id, bar.id}
-        assert sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY, group2.id) == {baz.id, bar.id}
+        assert sheerka.om.get(SheerkaIsAManager.CONCEPTS_GROUPS_ENTRY, group1.id) == {foo.id, bar.id}
+        assert sheerka.om.get(SheerkaIsAManager.CONCEPTS_GROUPS_ENTRY, group2.id) == {baz.id, bar.id}
 
         # I can save in db
-        sheerka.cache_manager.commit(context)
-        assert sheerka.sdp.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY) == {
+        sheerka.om.commit(context)
+        assert sheerka.om.current_sdp().get(SheerkaIsAManager.CONCEPTS_GROUPS_ENTRY) == {
             '1004': {'1001', '1002'}, '1005': {'1002', '1003'}
         }
 
@@ -96,7 +102,7 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
         sheerka, context, foo, bar, group1, group2 = self.init_concepts(
             "foo", "bar", "group1", Concept("group2", body="group1"))
 
-        service = sheerka.services[SheerkaSetsManager.NAME]
+        service = sheerka.services[SheerkaIsAManager.NAME]
         service.add_concepts_to_set(context, [foo, bar], group1)
 
         assert sheerka.isaset(context, group2)
@@ -112,7 +118,7 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
             Concept("number"),
             Concept("sub_number", body="number", where="number < 4")
         )
-        service = sheerka.services[SheerkaSetsManager.NAME]
+        service = sheerka.services[SheerkaIsAManager.NAME]
         service.add_concepts_to_set(context, [one, two, three, four, five], number)
 
         assert sheerka.isaset(context, sub_number)
@@ -129,7 +135,7 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
             Concept("sub_number", body="number", where="number < 4"),
             Concept("sub_sub_number", body="sub_number", where="sub_number > 2")
         )
-        service = sheerka.services[SheerkaSetsManager.NAME]
+        service = sheerka.services[SheerkaIsAManager.NAME]
         service.add_concepts_to_set(context, [one, two, three, four, five], number)
 
         assert sheerka.isaset(context, sub_sub_number)
@@ -153,7 +159,7 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
             Concept("number"),
             Concept("sub_number", body="number", where="number < 4")
         )
-        service = sheerka.services[SheerkaSetsManager.NAME]
+        service = sheerka.services[SheerkaIsAManager.NAME]
         service.add_concepts_to_set(context, [one, two, three, four, five], number)
 
         assert sheerka.isaset(context, sub_number)
@@ -175,7 +181,7 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
             Concept("number"),
             Concept("sub_number", body="number", where="number >= 20")
         )
-        service = sheerka.services[SheerkaSetsManager.NAME]
+        service = sheerka.services[SheerkaIsAManager.NAME]
         service.add_concepts_to_set(context, [one, two, twenty, twenties], number)
 
         assert sheerka.isaset(context, sub_number)
@@ -192,15 +198,14 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
         :return:
         """
 
-        sheerka, context, one, two, twenty, twenties, number = self.init_concepts(
+        sheerka, context, one, two, twenty, twenties, number = self.init_test().with_concepts(
             Concept("one", body="1"),
             Concept("two", body="2"),
             Concept("twenty", body="20"),
             Concept("twenties", definition="twenty (one|two)=unit", body="twenty + unit").def_var("unit"),
             Concept("number"),
-            create_new=True
-        )
-        service = sheerka.services[SheerkaSetsManager.NAME]
+            create_new=True).unpack()
+        service = sheerka.services[SheerkaIsAManager.NAME]
         service.add_concepts_to_set(context, [one, two, twenty, twenties], number)
         assert sheerka.isinset(twenties, number)
 
@@ -210,12 +215,11 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinset(res[0].body, number)
 
     def test_a_concept_can_be_in_multiple_sets(self):
-        sheerka, context, foo, all_foo, all_bar = self.init_concepts(
+        sheerka, context, foo, all_foo, all_bar = self.init_test().with_concepts(
             Concept("foo"),
             Concept("all_foo"),
             Concept("all_bar"),
-            create_new=True
-        )
+            create_new=True).unpack()
 
         foo = sheerka.new(foo.key)  # new instance
         sheerka.set_isa(context, foo, all_foo)
@@ -270,7 +274,7 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
         elements = sheerka.get_set_elements(context, number)
         assert [c.id for c in elements] == [one.id]
 
-        concepts_in_cache = sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_IN_GROUPS_ENTRY, number.id)
+        concepts_in_cache = sheerka.om.get(SheerkaIsAManager.CONCEPTS_IN_GROUPS_ENTRY, number.id)
         assert [c.id for c in concepts_in_cache] == [one.id]
 
         # pretend that number has been updated in sheerka.concepts_grammar
@@ -281,23 +285,63 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
         elements = sheerka.get_set_elements(context, number)
         assert {c.id for c in elements} == {one.id, two.id}
 
-        concepts_in_cache = sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_IN_GROUPS_ENTRY, number.id)
+        concepts_in_cache = sheerka.om.get(SheerkaIsAManager.CONCEPTS_IN_GROUPS_ENTRY, number.id)
         assert {c.id for c in concepts_in_cache} == {one.id, two.id}
 
         # make sure the bnf definition is also updated
         assert number.id not in sheerka.concepts_grammars
 
+    def test_i_can_get_and_set_isa_when_multiple_ontology_layers(self):
+        sheerka, context, foo, group1, group2 = self.init_concepts(
+            Concept("foo"),
+            Concept("group1"),
+            Concept("group2"),
+            cache_only=False
+        )
+
+        sheerka.set_isa(context, foo, group1)
+
+        assert sheerka.isaset(context, group1)
+        assert sheerka.isinset(foo, group1)
+        assert sheerka.isa(foo, group1)
+
+        sheerka.push_ontology(context, "new ontology")
+        assert sheerka.isaset(context, group1)
+        assert sheerka.isinset(foo, group1)
+        assert sheerka.isa(foo, group1)
+        assert not sheerka.isaset(context, group2)
+        assert not sheerka.isinset(foo, group2)
+        assert not sheerka.isa(foo, group2)
+
+        sheerka.set_isa(context, foo, group2)
+        assert sheerka.isaset(context, group1)
+        assert sheerka.isinset(foo, group1)
+        assert sheerka.isa(foo, group1)
+        assert sheerka.isaset(context, group2)
+        assert sheerka.isinset(foo, group2)
+        assert sheerka.isa(foo, group2)
+
+        # I can revert back
+        sheerka.pop_ontology()
+        assert sheerka.isaset(context, group1)
+        assert sheerka.isinset(foo, group1)
+        assert sheerka.isa(foo, group1)
+        assert not sheerka.isaset(context, group2)
+        assert not sheerka.isinset(foo, group2)
+
+        foo = sheerka.get_by_id(foo.id)
+        assert not sheerka.isa(foo, group2)
+
 
 class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
     def test_i_can_add_concept_to_set_and_retrieve_it_in_another_session(self):
-        sheerka, context, foo, bar, group = self.init_concepts(
-            Concept("foo"),
-            Concept("bar"),
-            Concept("group"),
-            create_new=True)
+        sheerka, context, foo, bar, group = self.init_test().with_concepts(Concept("foo"),
+                                                                           Concept("bar"),
+                                                                           Concept("group"),
+                                                                           create_new=True).unpack()
 
         assert sheerka.add_concept_to_set(context, foo, group).status
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
         sheerka = self.get_sheerka(reset_attrs=False)  # another session
         context = self.get_context(sheerka)
@@ -306,8 +350,8 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
         # I can get the elements
         assert set(sheerka.get_set_elements(context, group)) == {foo, bar}
 
-        sheerka.cache_manager.commit(context)  # save in db
-        all_entries = sheerka.sdp.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY)  # check the db
+        sheerka.om.commit(context)  # save in db
+        all_entries = sheerka.om.current_sdp().get(SheerkaIsAManager.CONCEPTS_GROUPS_ENTRY)  # check the db
         assert all_entries == {
             group.id: {foo.id, bar.id}
         }
@@ -320,33 +364,31 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
         for c in [foo3, foo4]:
             sheerka.create_new_concept(context, c)
 
-        sets_handler = sheerka.services[SheerkaSetsManager.NAME]
+        sets_handler = sheerka.services[SheerkaIsAManager.NAME]
         res = sets_handler.add_concepts_to_set(context, (foo3, foo4), group)
         assert res.status
 
         # I can get the elements
         assert set(sheerka.get_set_elements(context, group)) == {foo, bar, foo3, foo4}
 
-        sheerka.cache_manager.commit(context)  # save in db
-        all_entries = sheerka.sdp.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY)  # check the db
+        sheerka.om.commit(context)  # save in db
+        all_entries = sheerka.om.current_sdp().get(SheerkaIsAManager.CONCEPTS_GROUPS_ENTRY)  # check the db
         assert all_entries == {
             group.id: {foo.id, bar.id, foo3.id, foo4.id}
         }
 
     def test_i_can_set_isa(self):
-        sheerka, context, foo, bar, group = self.init_concepts(
-            "foo",
-            "bar",
-            "group",
-            create_new=True,  # needed by modify
-        )
+        sheerka, context, foo, bar, group = self.init_test().with_concepts("foo",
+                                                                           "bar",
+                                                                           "group",
+                                                                           ).unpack()
 
         # nothing was previously in ISA
         foo = sheerka.new(foo.key)
         assert BuiltinConcepts.ISA not in foo.get_metadata().props
         res = sheerka.set_isa(context, foo, group)
         assert res.status
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
         sheerka = self.get_sheerka(reset_attrs=False)
         assert foo.get_prop(BuiltinConcepts.ISA) == {group}
@@ -363,11 +405,11 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
         assert sheerka.isinset(bar, group)
         assert sheerka.isaset(context, group)
 
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
         # they are both in the same group
         sheerka = self.get_sheerka(reset_attrs=False)
-        all_entries = sheerka.sdp.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY)
+        all_entries = sheerka.om.current_sdp().get(SheerkaIsAManager.CONCEPTS_GROUPS_ENTRY)
         assert all_entries == {
             group.id: {foo.id, bar.id}
         }

tests/core/test_SheerkaMemory.py

@@ -1,5 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
+from core.global_symbols import NotFound
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.services.SheerkaMemory import SheerkaMemory, MemoryObject
 
@@ -19,7 +20,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo)
 
     def test_i_can_add_context_short_term_memory(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
         service = sheerka.services[SheerkaMemory.NAME]
 
         foo = Concept("foo")
@@ -28,10 +29,10 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         context_id = ExecutionContext.ids[context.event.get_digest()]
         assert service.short_term_objects.copy() == {context_id: {'a': foo}}
         assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
-        assert sheerka.get_from_short_term_memory(None, "a") is None
+        assert sheerka.get_from_short_term_memory(None, "a") is NotFound
 
     def test_i_can_add_many(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
         bag = {"a": "foo", "b": "bar", }
         context_id = ExecutionContext.ids[context.event.get_digest()]
         service = sheerka.services[SheerkaMemory.NAME]
@@ -40,7 +41,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         assert service.short_term_objects.copy() == {context_id: bag}
 
     def test_i_can_get_obj_from_parents(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
         foo = Concept("foo")
 
         sheerka.add_to_short_term_memory(None, "a", foo)
@@ -53,54 +54,54 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
 
             assert id(sheerka.get_from_short_term_memory(sub_context, "b")) == id(foo)
             assert id(sheerka.get_from_short_term_memory(context, "b")) == id(foo)
-            assert sheerka.get_from_short_term_memory(None, "b") is None
+            assert sheerka.get_from_short_term_memory(None, "b") is NotFound
 
     def test_short_term_memory_entries_are_removed_on_context_exit(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         with context.push(BuiltinConcepts.TESTING, None) as sub_context:
             foo = Concept("foo")
             sheerka.add_to_short_term_memory(sub_context, "a", foo)
             assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
 
-        assert sheerka.get_from_short_term_memory(sub_context, "a") is None
+        assert sheerka.get_from_short_term_memory(sub_context, "a") is NotFound
 
     def test_short_term_memory_entries_are_removed_on_context_exit_2(self):
         # this time we test the bulk insert
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         with context.push(BuiltinConcepts.TESTING, None) as sub_context:
             foo = Concept("foo")
             sheerka.add_many_to_short_term_memory(sub_context, {"a": foo})
             assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
 
-        assert sheerka.get_from_short_term_memory(sub_context, "a") is None
+        assert sheerka.get_from_short_term_memory(sub_context, "a") is NotFound
 
     def test_i_can_add_and_retrieve_from_memory(self):
-        sheerka, context = self.init_concepts()
-        service = sheerka.services[SheerkaMemory.NAME]
+        sheerka, context = self.init_test().unpack()
 
-        assert sheerka.get_from_memory(context, "a") is None
+        assert sheerka.get_from_memory(context, "a") is NotFound
 
         foo = Concept("foo")
         sheerka.add_to_memory(context, "a", foo)
 
-        assert service.memory_objects.copy() == {"a": MemoryObject(context.event.get_digest(), foo)}
+        assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(), foo)}
         assert id(sheerka.get_from_memory(context, "a").obj) == id(foo)
 
     def test_i_can_use_memory_to_get_the_list_of_all_objects(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test(cache_only=False).unpack()
         foo = Concept("foo")
         bar = Concept("bar")
 
         sheerka.add_to_memory(context, "foo", 'value that will not appear')
         sheerka.add_to_memory(context, "foo", foo)
         sheerka.add_to_memory(context, "bar", bar)
+        sheerka.om.commit(context)
 
         assert sheerka.memory(context) == {"foo": foo, "bar": bar}
 
     def test_i_can_use_memory_with_a_string(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         foo = Concept("foo")
         sheerka.add_to_memory(context, "foo", foo)
@@ -108,7 +109,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         assert sheerka.memory(context, "foo") == foo
 
     def test_i_can_use_memory_with_a_concept(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         foo = Concept("foo")
         sheerka.add_to_memory(context, "foo", foo)
@@ -116,7 +117,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         assert sheerka.memory(context, Concept("foo")) == foo
 
     def test_concept_not_found_is_return_when_not_found(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         res = sheerka.memory(context, "foo")
 
@@ -124,7 +125,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         assert res.body == {"#name": "foo"}
 
     def test_memory_only_returns_the_last_object(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         foo = Concept("foo")
         bar = Concept("bar")
@@ -135,17 +136,17 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         assert sheerka.memory(context, "item") == bar
 
     def test_object_are_not_added_in_memory_during_the_initialisation(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         assert len(sheerka.memory(context)) == 0
 
 
 class TestSheerkaMemoryUsingFileBase(TestUsingFileBasedSheerka):
     def test_i_can_record_memory_objects(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         sheerka.add_to_memory(context, "item", Concept("foo"))
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
         sheerka = self.get_sheerka()
         context = self.get_context(sheerka)

tests/core/test_SheerkaRuleManager.py

@@ -3,8 +3,8 @@ import ast
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, CMV
-from core.global_symbols import RULE_COMPARISON_CONTEXT
-from core.rule import Rule
+from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound
+from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
 from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleParser, \
     FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \
     FormatRuleSyntaxError, FormatAstList, UnexpectedEof, FormatAstColor, RulePredicate, FormatAstDict, FormatAstMulti
@@ -28,13 +28,12 @@ CONCEPT_EVALUATOR_NAME = "Concept"
 class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("action_type, cache_entry", [
-        ("print", SheerkaRuleManager.FORMAT_RULE_ENTRY),
-        ("exec", SheerkaRuleManager.EXEC_RULE_ENTRY),
+        (ACTION_TYPE_PRINT, SheerkaRuleManager.FORMAT_RULE_ENTRY),
+        (ACTION_TYPE_EXEC, SheerkaRuleManager.EXEC_RULE_ENTRY),
     ])
     def test_i_can_create_a_new_rule(self, action_type, cache_entry):
         sheerka, context = self.init_concepts(cache_only=False)
-        previous_rules_number = sheerka.cache_manager.caches[sheerka.OBJECTS_IDS_ENTRY].cache.copy()[
-            SheerkaRuleManager.RULE_IDS]
+        previous_rules_number = sheerka.om.get_all(sheerka.OBJECTS_IDS_ENTRY)[SheerkaRuleManager.RULE_IDS]
 
         rule = Rule(action_type, "name", "True", "Hello world")
 
@@ -52,33 +51,50 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         assert created_rule.metadata.action == "Hello world"
 
         # saved in cache
-        assert len(sheerka.cache_manager.caches[cache_entry].cache) > 0
-        from_cache = sheerka.cache_manager.get(cache_entry, expected_id)
+        assert len(sheerka.om.current_cache_manager().caches[cache_entry].cache) > 0
+        from_cache = sheerka.om.get(cache_entry, expected_id)
         assert from_cache.metadata.id == expected_id
         assert from_cache.metadata.name == "name"
         assert from_cache.metadata.predicate == "True"
         assert from_cache.metadata.action_type == action_type
         assert from_cache.metadata.action == "Hello world"
 
-        sheerka.cache_manager.commit(context)
+        # the rule is also saved by name
+        by_name = sheerka.get_rule_by_name("name")
+        assert by_name.metadata.id == expected_id
+        assert by_name.metadata.name == "name"
+        assert by_name.metadata.predicate == "True"
+        assert by_name.metadata.action_type == action_type
+        assert by_name.metadata.action == "Hello world"
+
+        sheerka.om.commit(context)
 
         # saved in sdp
-        from_sdp = sheerka.sdp.get(cache_entry, expected_id)
+        from_sdp = sheerka.om.current_sdp().get(cache_entry, expected_id)
         assert from_sdp.metadata.id == expected_id
         assert from_sdp.metadata.name == "name"
         assert from_sdp.metadata.predicate == "True"
         assert from_sdp.metadata.action_type == action_type
         assert from_sdp.metadata.action == "Hello world"
 
+        by_name = sheerka.om.current_sdp().get(SheerkaRuleManager.RULES_BY_NAME_ENTRY, "name")
+        assert by_name.metadata.id == expected_id
+        assert by_name.metadata.name == "name"
+        assert by_name.metadata.predicate == "True"
+        assert by_name.metadata.action_type == action_type
+        assert by_name.metadata.action == "Hello world"
+
     def test_i_can_create_multiple_rules(self):
         sheerka, context = self.init_concepts(cache_only=False)
-        previous_rules_number = len(sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache)
+        previous_rules_number = len(
+            sheerka.om.current_cache_manager().caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache)
 
         sheerka.create_new_rule(context, Rule("print", "name1", "True", "Hello world"))
         sheerka.create_new_rule(context, Rule("print", "name2", "value() is __EXPLANATION", "list(value())"))
 
         assert len(
-            sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache) == 2 + previous_rules_number
+            sheerka.om.current_cache_manager().caches[
+                SheerkaRuleManager.FORMAT_RULE_ENTRY].cache) == 2 + previous_rules_number
 
     @pytest.mark.parametrize("text, expected", [
         ("", FormatAstRawText("")),
@@ -186,7 +202,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         assert res[0].concept is None
 
     def test_i_can_compile_predicate_when_python_and_concept(self):
-        sheerka, context, *concepts = self.init_concepts(Concept("foo bar"), create_new=True)
+        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack()
         service = sheerka.services[SheerkaRuleManager.NAME]
         text = "foo bar == 5"
         ast_ = ast.parse("__C__foo0bar__1001__C__ == 5", "<source>", 'eval')
@@ -207,12 +223,12 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         ("cat is an b", ["cat", "b"]),
     ])
     def test_i_can_compile_predicate_when_exact_concept(self, text, expected_variables):
-        sheerka, context, *concepts = self.init_concepts(
+        sheerka, context, *concepts = self.init_test().with_concepts(
             Concept("x is an y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
             Concept("cat"),
             Concept("animal"),
             create_new=True
-        )
+        ).unpack()
         service = sheerka.services[SheerkaRuleManager.NAME]
 
         expected = concepts[0]
@@ -232,12 +248,12 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         ("a cat is an b", ["a cat", "b"]),
     ])
     def test_i_can_compile_predicate_when_sya_node_parser(self, text, expected_variables):
-        sheerka, context, *concepts = self.init_concepts(
+        sheerka, context, *concepts = self.init_test().with_concepts(
             Concept("x is an y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
             Concept("a cat"),
             Concept("animal"),
             create_new=True
-        )
+        ).unpack()
         service = sheerka.services[SheerkaRuleManager.NAME]
         expected = CMV(concepts[0], x=expected_variables[0], y=expected_variables[1])
 
@@ -251,11 +267,11 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         assert res[0].concept == expected
 
     def test_i_can_compile_predicate_when_bnf_node_parser(self):
-        sheerka, context, *concepts = self.init_concepts(
+        sheerka, context, *concepts = self.init_test().with_concepts(
             Concept("animal"),
             Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
             create_new=True
-        )
+        ).unpack()
         service = sheerka.services[SheerkaRuleManager.NAME]
         expected = concepts[1]
 
@@ -269,11 +285,11 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         assert res[0].concept == expected
 
     def test_i_can_compile_predicate_when_multiple_choices(self):
-        sheerka, context, *concepts = self.init_concepts(
+        sheerka, context, *concepts = self.init_test().with_concepts(
             Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
             Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
             create_new=True
-        )
+        ).unpack()
         service = sheerka.services[SheerkaRuleManager.NAME]
 
         res = service.compile_when(context, "test", "a is a b")
@@ -291,6 +307,34 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         assert sheerka.objvalue(res[1].predicate)[0].concept == CMV(concepts[1], x="a", y="b")
         assert res[1].concept == CMV(concepts[1], x="a", y="b")
 
+    def test_i_can_get_rule_priorities(self):
+        sheerka, context, rule_true, rule_false = self.init_test().with_rules(("True", "True"),
+                                                                              ("False", "False")).unpack()
+
+        sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE,
+                                    rule_true,
+                                    rule_false,
+                                    RULE_COMPARISON_CONTEXT)
+
+        rules_from_cache = sheerka.om.get_all(SheerkaRuleManager.FORMAT_RULE_ENTRY)
+
+        assert rules_from_cache[rule_true.id].priority == 2
+        assert rules_from_cache[rule_false.id].priority == 1
+
+    def test_i_can_get_and_retrieve_rules_when_multiple_ontology_layers(self):
+        sheerka, context, rule_true = self.init_test().with_rules(("true", "True", "True")).unpack()
+
+        sheerka.push_ontology(context, "new ontology")
+        rule_false = sheerka.create_new_rule(context, Rule(ACTION_TYPE_EXEC, "false", "False", "False")).body.body
+
+        # All rules are visible
+        assert sheerka.get_rule_by_id(rule_true.id) == rule_true
+        assert sheerka.get_rule_by_id(rule_false.id) == rule_false
+
+        sheerka.pop_ontology()
+        assert sheerka.get_rule_by_id(rule_true.id) == rule_true
+        assert not sheerka.is_known(sheerka.get_rule_by_id(rule_false.id))
+
     # @pytest.mark.skip
     # @pytest.mark.parametrize("text, expected", [
     #     ("cat is an animal", set()),
@@ -337,24 +381,58 @@ class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
                                     rules[1],
                                     RULE_COMPARISON_CONTEXT)
 
-        sheerka.cache_manager.commit(context)
-        assert len(sheerka.cache_manager.copy(SheerkaRuleManager.FORMAT_RULE_ENTRY)) == len(rules)
+        sheerka.om.commit(context)
+        expected_rules_by_id = sheerka.om.get_all(SheerkaRuleManager.FORMAT_RULE_ENTRY)
 
-        sheerka = self.get_sheerka()  # new instance
-        assert len(sheerka.cache_manager.copy(SheerkaRuleManager.FORMAT_RULE_ENTRY)) == len(rules)
+        sheerka = self.new_sheerka_instance(False)  # new instance
+        rules_by_id = sheerka.om.get_all(SheerkaRuleManager.FORMAT_RULE_ENTRY)
 
-        # manually update the rules (I need their new priorities)
-        service = sheerka.services[SheerkaRuleManager.NAME]
-        rules = [service.format_rule_cache.get(rule_id) for rule_id in service.format_rule_cache]
+        assert len(rules_by_id) == len(expected_rules_by_id)
 
-        # check if the rules are correctly initialized
-        rules_as_map = {rule.id: rule for rule in rules}
-        for rule_id in service.format_rule_cache:
-            actual = service.format_rule_cache.get(rule_id)
-            expected = rules_as_map[rule_id]
-            assert actual.metadata.is_compiled == expected.metadata.is_compiled
-            assert actual.metadata.is_enabled == expected.metadata.is_enabled
-            assert actual.compiled_action == expected.compiled_action
-            assert actual.compiled_predicate == expected.compiled_predicate
-            assert actual.priority is not None
-            assert actual.priority == expected.priority
+        for rule_id, rule in rules_by_id.items():
+            expected = expected_rules_by_id[rule_id]
+
+            assert rule.metadata.action_type == expected.metadata.action_type
+            assert rule.metadata.name == expected.metadata.name
+            assert rule.metadata.predicate == expected.metadata.predicate
+            assert rule.metadata.action == expected.metadata.action
+            assert rule.metadata.id == expected.metadata.id
+            assert rule.metadata.is_compiled == expected.metadata.is_compiled
+            assert rule.metadata.is_enabled == expected.metadata.is_enabled
+            assert rule.compiled_action == expected.compiled_action
+            assert rule.compiled_predicate == expected.compiled_predicate
+            assert rule.priority is not None
+            assert rule.priority == expected.priority
+
+    def test_rules_are_still_accessible_after_a_new_ontology_layer(self):
+        sheerka, context, *rules = self.init_format_rules(
+            Rule("print", "name1", "True", "Hello world"),
+            Rule("print", "name2", "value() is __EXPLANATION", "list(value())")
+        )
+        sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE,
+                                    rules[0],
+                                    rules[1],
+                                    RULE_COMPARISON_CONTEXT)
+
+        sheerka.om.commit(context)
+        expected_rules_by_id = sheerka.om.get_all(SheerkaRuleManager.FORMAT_RULE_ENTRY)
+
+        sheerka.push_ontology(context, "new ontology")
+        rules_by_id = sheerka.om.get_all(SheerkaRuleManager.FORMAT_RULE_ENTRY)
+
+        assert len(rules_by_id) == len(expected_rules_by_id)
+
+        for rule_id, rule in rules_by_id.items():
+            expected = expected_rules_by_id[rule_id]
+
+            assert rule.metadata.action_type == expected.metadata.action_type
+            assert rule.metadata.name == expected.metadata.name
+            assert rule.metadata.predicate == expected.metadata.predicate
+            assert rule.metadata.action == expected.metadata.action
+            assert rule.metadata.id == expected.metadata.id
+            assert rule.metadata.is_compiled == expected.metadata.is_compiled
+            assert rule.metadata.is_enabled == expected.metadata.is_enabled
+            assert rule.compiled_action == expected.compiled_action
+            assert rule.compiled_predicate == expected.compiled_predicate
+            assert rule.priority is not None
+            assert rule.priority == expected.priority

tests/core/test_SheerkaVariableManager.py

@@ -1,4 +1,7 @@
+import pytest
+
 from core.concept import Concept, ConceptParts
+from core.global_symbols import NotFound
 from core.sheerka.services.SheerkaVariableManager import SheerkaVariableManager
 
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
@@ -15,10 +18,11 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
         assert res == 1
 
         # I can persist in db
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
-        assert sheerka.sdp.exists(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable|my_variable")
-        loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable|my_variable")
+        assert sheerka.om.current_sdp().exists(SheerkaVariableManager.VARIABLES_ENTRY,
+                                               "TestSheerkaVariable|my_variable")
+        loaded = sheerka.om.current_sdp().get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable|my_variable")
         assert loaded.event_id == context.event.get_digest()
         assert loaded.key == "my_variable"
         assert loaded.value == 1
@@ -45,10 +49,10 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
         concept = Concept("foo")
 
         sheerka.record_var(context, "TestSheerkaVariable", "my_variable", concept)
-        assert sheerka.load_var("TestSheerkaVariable", "my_variable") is not None
+        assert sheerka.load_var("TestSheerkaVariable", "my_variable") == concept
 
         sheerka.delete_var(context, "TestSheerkaVariable", "my_variable")
-        assert sheerka.load_var("TestSheerkaVariable", "my_variable") is None
+        assert sheerka.load_var("TestSheerkaVariable", "my_variable") is NotFound
 
     def test_i_can_set_and_get_a_value(self):
         sheerka = self.get_sheerka(cache_only=False)
@@ -60,35 +64,101 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
         assert res == "my value"
 
         # I can persist in db
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
-        assert sheerka.sdp.exists(SheerkaVariableManager.VARIABLES_ENTRY, "Test_user|my_variable")
-        loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "Test_user|my_variable")
+        assert sheerka.om.current_sdp().exists(SheerkaVariableManager.VARIABLES_ENTRY, "Test_user|my_variable")
+        loaded = sheerka.om.current_sdp().get(SheerkaVariableManager.VARIABLES_ENTRY, "Test_user|my_variable")
         assert loaded.event_id == context.event.get_digest()
         assert loaded.key == "my_variable"
         assert loaded.value == "my value"
         assert loaded.who == "Test_user"
         assert loaded.parents is None
 
+    def test_i_can_record_and_load_internal_variables(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+
+        sheerka.record_internal_var(context, "TestSheerkaVariable", "my_variable", 1)
+        res = sheerka.load_internal_var("TestSheerkaVariable", "my_variable")
+        assert res == 1
+
+        sheerka.record_internal_var(context, "TestSheerkaVariable", "lambda", lambda x: x + 1)
+        res = sheerka.load_internal_var("TestSheerkaVariable", "lambda")
+        assert res(10) == 11
+
+    def test_i_can_record_and_load_in_multiple_ontology_layers(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+
+        sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1)
+        sheerka.record_internal_var(context, "TestSheerkaVariable", "lambda", lambda x: x + 1)
+        sheerka.om.commit(context)
+
+        sheerka.push_ontology(context, "new ontology")
+        sheerka.push_ontology(context, "another ontology")
+
+        # I can still access to the previous values
+        assert sheerka.load_var("TestSheerkaVariable", "my_variable") == 1
+        assert sheerka.load_internal_var("TestSheerkaVariable", "lambda")(10) == 11
+
+        # I can modify the current values
+        sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 2)
+        sheerka.record_internal_var(context, "TestSheerkaVariable", "lambda", lambda x: x + 2)
+        sheerka.om.commit(context)
+
+        assert sheerka.load_var("TestSheerkaVariable", "my_variable") == 2
+        assert sheerka.load_internal_var("TestSheerkaVariable", "lambda")(10) == 12
+
+        # I can revert back
+        sheerka.pop_ontology()
+        assert sheerka.load_var("TestSheerkaVariable", "my_variable") == 1
+        assert sheerka.load_internal_var("TestSheerkaVariable", "lambda")(10) == 11
+
+        sheerka.pop_ontology()
+        assert sheerka.load_var("TestSheerkaVariable", "my_variable") == 1
+        assert sheerka.load_internal_var("TestSheerkaVariable", "lambda")(10) == 11
+
 
 class TestSheerkaVariableUsingFileBasedSdp(TestUsingFileBasedSheerka):
 
     def test_i_can_bound_variables_to_sheerka(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         old_value = sheerka.enable_process_return_values
         new_value = not old_value
-        sheerka.record_var(context, "TestSheerkaVariable", "sheerka.enable_process_return_values", new_value)
-        sheerka.cache_manager.commit(context)
+        sheerka.record_var(context, sheerka.name, "enable_process_return_values", new_value)
+        sheerka.om.commit(context)
 
         assert sheerka.enable_process_return_values == new_value
 
         # the modification is persisted upon new Sheerka creation
-        sheerka = self.get_sheerka()
+        sheerka = self.new_sheerka_instance(False)
+        assert sheerka.load_var(sheerka.name, "enable_process_return_values") == new_value
         assert sheerka.enable_process_return_values == new_value
 
         # reset old value
-        sheerka.enable_process_return_values = old_value
+        sheerka.record_var(context, sheerka.name, "enable_process_return_values", old_value)
+
+    def test_i_can_bound_variables_to_sheerka_when_ontology_layer(self):
+        sheerka, context = self.init_test().unpack()
+
+        old_value = sheerka.enable_process_return_values
+        new_value = not old_value
+        sheerka.record_var(context, sheerka.name, "enable_process_return_values", new_value)
+        sheerka.om.commit(context)
+
+        # sanity
+        assert sheerka.enable_process_return_values == new_value
+        assert sheerka.load_var(sheerka.name, "enable_process_return_values") == new_value
+
+        # the modification is seen even when there is an ontology layer
+        sheerka.push_ontology(context, "new ontology")
+        sheerka = self.new_sheerka_instance(False)
+        assert sheerka.load_var(sheerka.name, "enable_process_return_values") == new_value
+        assert sheerka.enable_process_return_values == new_value
+
+        # reset old value
+        sheerka.record_var(context, sheerka.name, "enable_process_return_values", old_value)
 
     # def test_i_can_get_the_parent_when_modified(self):
     #     sheerka = self.get_sheerka()
@@ -99,14 +169,14 @@ class TestSheerkaVariableUsingFileBasedSdp(TestUsingFileBasedSheerka):
     #     res = sheerka.load_var("TestSheerkaVariable", "my_variable")
     #     assert res == 2
     #
-    #     loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
+    #     loaded = sheerka.om.current_sdp().get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
     #     assert loaded.event_id == context.event.get_digest()
     #     assert loaded.key == "my_variable"
     #     assert loaded.value == 2
     #     assert loaded.who == "TestSheerkaVariable"
     #     assert loaded.parents == ['8c9ada7bf488d84229f6539f76042431638f16d600fe3b7ec7e7161043a40d59']
     #
-    #     parent = sheerka.sdp.load_obj(loaded.parents[0])
+    #     parent = sheerka.om.current_sdp().load_obj(loaded.parents[0])
     #     assert parent.event_id == context.event.get_digest()
     #     assert parent.key == "my_variable"
     #     assert parent.value == 1
@@ -120,7 +190,7 @@ class TestSheerkaVariableUsingFileBasedSdp(TestUsingFileBasedSheerka):
     #     sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1)
     #     sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1)
     #
-    #     loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
+    #     loaded = sheerka.om.current_sdp().get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
     #     assert loaded.event_id == context.event.get_digest()
     #     assert loaded.key == "my_variable"
     #     assert loaded.value == 1

tests/core/test_sheerka.py

@@ -1,10 +1,14 @@
 import os
 
 import pytest
+
+from conftest import SHEERKA_TEST_FOLDER
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
 from core.builtin_concepts_ids import AllBuiltinConcepts
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, get_concept_attrs
+from core.global_symbols import NotInit
 from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS
+from core.sheerka.SheerkaOntologyManager import OntologyAlreadyExists
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.tokenizer import Token, TokenKind
 
@@ -107,7 +111,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert ret.body == "fake_concept"
 
     def test_i_can_instantiate_a_concept(self):
-        sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
+        sheerka, context, concept = self.init_test().with_concepts(self.get_default_concept(), create_new=True).unpack()
 
         new = sheerka.new(concept.key, a=10, b="value")
 
@@ -119,10 +123,10 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert new.get_value("b") == "value"
 
     def test_i_can_instantiate_multiple_when_same_key(self):
-        sheerka, context, *concepts = self.init_concepts(
+        sheerka, context, *concepts = self.init_test().with_concepts(
             Concept("foo", body="foo1"),
             Concept("foo", body="foo2"),
-            create_new=True)
+            create_new=True).unpack()
 
         # when no id, i get two instances
         concepts = sheerka.new("foo")
@@ -141,7 +145,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert foo2.get_metadata().body == "foo2"
 
     def test_instances_are_different_when_asking_for_new(self):
-        sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
+        sheerka, context, concept = self.init_test().with_concepts(self.get_default_concept(), create_new=True).unpack()
 
         new1 = sheerka.new(concept.key, a=10, b="value")
         new2 = sheerka.new(concept.key, a=10, b="value")
@@ -150,7 +154,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert id(new1) != id(new2)
 
     def test_new_instance_does_not_impact_each_others(self):
-        sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
+        sheerka, context, foo, bar = self.init_test().with_concepts("foo", "bar", create_new=True).unpack()
 
         new_foo = sheerka.new("foo")
         new_foo.get_metadata().body = "metadata value"  # modify metadata
@@ -167,7 +171,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert new_foo.get_compiled() != foo.get_compiled()
 
     def test_i_get_the_same_instance_when_is_unique_is_true(self):
-        sheerka, context, concept = self.init_concepts(Concept(name="unique", is_unique=True), create_new=True)
+        sheerka, context, concept = self.init_test(). \
+            with_concepts(Concept(name="unique", is_unique=True), create_new=True).unpack()
 
         new1 = sheerka.new(concept.key)
         new2 = sheerka.new(concept.key, a=10, b="value")  # not that variables are simply discareded
@@ -176,10 +181,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert id(new1) == id(new2)
 
     def test_values_are_reset_when_asking_for_a_new_instance(self):
-        sheerka, context, template = self.init_concepts(
-            Concept("foo", body="'foo body'"),
-            create_new=True,
-            eval_body=True)
+        sheerka, context, template = self.init_test(eval_body=True).with_concepts(Concept("foo", body="'foo body'"),
+                                                                                  create_new=True).unpack()
 
         sheerka.evaluate_concept(context, sheerka.get_by_id(template.id))
         assert template.get_metadata().is_evaluated
@@ -202,10 +205,9 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert new.body == ('key', 'fake_concept')
 
     def test_i_cannot_instantiate_with_invalid_id(self):
-        sheerka, context, *concepts = self.init_concepts(
-            Concept("foo", body="foo1"),
-            Concept("foo", body="foo2"),
-            create_new=True)
+        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
+                                                                     Concept("foo", body="foo2"),
+                                                                     create_new=True).unpack()
 
         new = sheerka.new(("foo", "invalid_id"))
 
@@ -213,10 +215,9 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert new.body == [('key', 'foo'), ('id', 'invalid_id')]
 
     def test_i_cannot_instantiate_with_invalid_key(self):
-        sheerka, context, *concepts = self.init_concepts(
-            Concept("foo", body="foo1"),
-            Concept("foo", body="foo2"),
-            create_new=True)
+        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
+                                                                     Concept("foo", body="foo2"),
+                                                                     create_new=True).unpack()
 
         new = sheerka.new(("invalid_key", "1001"))
 
@@ -224,9 +225,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert new.body == [('key', 'invalid_key'), ('id', '1001')]
 
     def test_concept_id_is_irrelevant_when_only_one_concept(self):
-        sheerka, context, *concepts = self.init_concepts(
-            Concept("foo", body="foo1"),
-            create_new=True)
+        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
+                                                                     create_new=True).unpack()
 
         new = sheerka.new(("foo", "invalid_id"))
 
@@ -234,7 +234,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert new.get_metadata().body == "foo1"
 
     def test_i_cannot_instantiate_when_properties_are_not_recognized(self):
-        sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
+        sheerka, context, concept = self.init_test().with_concepts(self.get_default_concept(),
+                                                                   create_new=True).unpack()
 
         new = sheerka.new(concept.key, a=10, c="value")
 
@@ -326,46 +327,138 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert sheerka.resolve(concept) == real_expected
 
     def test_i_can_resolve_when_searching_by_definition(self):
-        sheerka, context, plus = self.init_concepts(
-            self.from_def_concept("plus", "a plus b", ["a", "b"]),
-            create_new=True
-        )
+        sheerka, context, plus = self.init_test().with_concepts(self.from_def_concept("plus", "a plus b", ["a", "b"]),
+                                                                create_new=True).unpack()
 
         assert sheerka.resolve("a plus b") == plus
 
+    def test_i_can_reset_global_concept_definition_when_adding_and_removing_layers(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b"))
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+        sheerka.push_ontology(context, "another ontology")
+        sheerka.modify_concept(context, foo, to_add={"variables": {"c": None}}, to_remove={"variables": ["b"]})
+        assert get_concept_attrs(foo) == ["a", "c"]
+
+        sheerka.pop_ontology()
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+    def test_i_can_manage_concepts_ids_on_multiple_ontology_layers(self):
+        sheerka, context = self.init_concepts()
+
+        res = sheerka.create_new_concept(context, Concept("foo"))
+        assert res.body.body.id == "1001"
+
+        sheerka.push_ontology(context, "new ontology")
+        res = sheerka.create_new_concept(context, Concept("bar"))
+        assert res.body.body.id == "1002"
+
+        res = sheerka.create_new_concept(context, Concept("baz"))
+        assert res.body.body.id == "1003"
+
+        sheerka.pop_ontology()
+        res = sheerka.create_new_concept(context, Concept("baz"))
+        assert res.body.body.id == "1002"
+
+    def test_i_can_add_ontology(self):
+        sheerka, context = self.init_test().unpack()
+
+        # Create an ontology an set some values
+        res = sheerka.push_ontology(context, "new ontology")
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        foo = sheerka.create_new_concept(context, Concept("foo").def_var("a").def_var("b")).body.body
+        sheerka.locals = {"key1": "value1"}
+
+        # sanity check
+        assert sheerka.get_by_name("foo") == foo
+        assert not sheerka.is_known(sheerka.get_by_name("bar"))
+        assert sheerka.locals == {"key1": "value1"}
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+        # record the ontology
+        ontology = sheerka.get_ontology(context)
+        sheerka.pop_ontology()
+
+        # Create another ontology with some other values
+        sheerka.push_ontology(context, "another ontology")
+        foo2 = sheerka.create_new_concept(context, Concept("foo").def_var("a").def_var("b").def_var("c")).body.body
+        bar = sheerka.create_new_concept(context, Concept("bar")).body.body
+        sheerka.locals = {"key2": "value2"}
+
+        # sanity check
+        assert sheerka.get_by_name("foo") == foo2
+        assert sheerka.get_by_name("bar") == bar
+        assert sheerka.locals == {"key2": "value2"}
+        assert get_concept_attrs(foo) == ["a", "b", "c"]
+
+        # put pack the previous ontology
+        sheerka.add_ontology(context, ontology)
+
+        assert sheerka.get_by_name("foo") == foo  # not [foo, foo2], foo2 is not seen !!!
+        assert sheerka.get_by_name("bar") == bar
+        assert sheerka.locals == {"key1": "value1"}
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+        # sanity check
+        sheerka.pop_ontology()
+        assert sheerka.get_by_name("foo") == foo2
+        assert sheerka.get_by_name("bar") == bar
+        assert sheerka.locals == {"key2": "value2"}
+        assert get_concept_attrs(foo) == ["a", "b", "c"]
+
+    def test_adding_the_same_ontology_twice_has_no_effect(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka.push_ontology(context, "#unit_test#")
+
+        assert len(sheerka.om.ontologies) == 2
+
+    def test_i_cannot_add_an_ontology_that_already_exists(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka.push_ontology(context, "new ontology")
+
+        res = sheerka.push_ontology(context, "#unit_test#")
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED)
+
 
 class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
     def test_root_folder_is_created_after_initialization(self):
-        return_value = Sheerka().initialize(self.root_folder)
+        return_value = Sheerka().initialize(SHEERKA_TEST_FOLDER)
         assert return_value.status, "initialisation should be successful"
-        assert os.path.exists(self.root_folder), "init folder should be created"
+        assert os.path.exists(SHEERKA_TEST_FOLDER), "init folder should be created"
 
     def test_builtin_concepts_are_initialized(self):
-        sheerka = self.get_sheerka()
+        sheerka, context = self.init_test().unpack()
         service = sheerka.services[SheerkaConceptManager.NAME]
         for concept_name in AllBuiltinConcepts:
-            assert service.has_key(str(concept_name))
-            assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, str(concept_name)) is not None
+            # check that the concept is already in cache
+            assert sheerka.om.ontologies[-1].cache_manager.has(service.CONCEPTS_BY_KEY_ENTRY, concept_name)
+
+            # check that we can access it
+            assert sheerka.om.get(service.CONCEPTS_BY_KEY_ENTRY, concept_name) is not None
 
         # I can get back data from the sdp when the cache is empty
-        sheerka.cache_manager.clear()
+        sheerka.push_ontology(context, 'new ontology')
 
         # caches are empty
         assert not service.has_id("1")
-        assert not service.has_key(str(BuiltinConcepts.SHEERKA))
+        assert not service.has_key(BuiltinConcepts.SHEERKA)
 
         assert sheerka.get_by_id("1") == sheerka  # use sdp
 
-    #   assert sheerka.has_key(str(BuiltinConcepts.SHEERKA))  # auto update the other caches
-
     def test_builtin_concepts_can_be_updated(self):
         sheerka = self.get_sheerka()
         service = sheerka.services[SheerkaConceptManager.NAME]
         before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
         before_parsing.get_metadata().desc = "I have a description"
         before_parsing.get_metadata().full_serialization = True
-        with sheerka.sdp.get_transaction("Test") as transac:
+        with sheerka.om.current_sdp().get_transaction("Test") as transac:
             transac.add(service.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True)
 
         sheerka = self.get_sheerka()  # another fresh new instance
@@ -374,8 +467,8 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
         assert before_parsing.get_metadata().desc == "I have a description"
 
     def test_i_first_look_in_local_cache(self):
-        sheerka, context, concept = self.init_concepts("foo", create_new=True)
-        sheerka.cache_manager.commit(context)
+        sheerka, context, concept = self.init_test().with_concepts("foo", create_new=True).unpack()
+        sheerka.om.commit(context)
 
         sheerka.get_by_key(concept.key).new_property = "I have modified the concept in cache"
 
@@ -385,7 +478,7 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
         assert from_cache.new_property == "I have modified the concept in cache"
 
         # sdp instance is not modified
-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
         from_sdp = sheerka.get_by_key(concept.key)
         assert from_sdp is not None
         assert from_sdp.key == concept.key
@@ -394,42 +487,42 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
     def test_i_can_retrieve_from_sdp_when_cache_is_reset(self):
         sheerka, context, concept = self.init_concepts(Concept("foo", body="1"))
         service = sheerka.services[SheerkaConceptManager.NAME]
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
         sheerka.get_by_key("foo")
         assert service.has_key("foo")
 
         # It's also updated when sdp returns more than one element
         concept2 = Concept("foo", body="2")
         sheerka.create_new_concept(context, concept2)
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)
 
-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
         assert len(sheerka.get_by_key("foo")) == 2
         assert service.has_key("foo")
 
         # updated when by_id
-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
         assert sheerka.get_by_id("1001") == concept
         assert service.has_id("1001")
 
-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
         assert sheerka.get_by_name("foo") == [concept, concept2]
         assert service.has_name("foo")
 
-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
         assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
         assert service.has_hash(concept.get_definition_hash())
 
     def test_get_by_key_retrieve_all_elements(self):
-        sheerka, context, *concepts = self.init_concepts(
+        sheerka, context, *concepts = self.init_test().with_concepts(
             Concept("foo", body="1"),
             Concept("foo", body="2"),
-            create_new=True)
-        sheerka.cache_manager.commit(context)
+            create_new=True).unpack()
+        sheerka.om.commit(context)
 
-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
         sheerka.get_by_key("foo", "1001")  # I ask only for the one with id = "1001"
 
         # but the two keys are returned
@@ -439,28 +532,55 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
         assert concepts[1].id == "1002"
 
     def test_concept_node_parsing_is_initialized_at_startup(self):
-        sheerka, context, foo, bar, baz = self.init_concepts(
+        sheerka, context, foo, bar, baz = self.init_test().with_concepts(
             "foo",
             "bar",
             Concept("baz", definition="foo"),
-            create_new=True)
-        sheerka.cache_manager.commit(context)
+            create_new=True).unpack()
+        sheerka.om.commit(context)
 
-        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             'bar': ['1002'],
             'c:|1001:': ['1003'],
             'foo': ['1001']}
-        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             'bar': ['1002'],
             'foo': ['1001', '1003']
         }
 
         sheerka = self.get_sheerka()  # another instance
-        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             'bar': ['1002'],
             'c:|1001:': ['1003'],
             'foo': ['1001']}
-        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             'bar': ['1002'],
             'foo': ['1001', '1003']
         }
+
+    def test_i_can_remember_ontologies_when_sheerka_is_recreated(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka.push_ontology(context, "to remove")
+        sheerka.pop_ontology()
+        sheerka.push_ontology(context, "new ontology")
+        sheerka.push_ontology(context, "another ontology")
+
+        # sanity check
+        ontologies_names = [o.name for o in sheerka.om.ontologies]
+        assert ontologies_names == ['another ontology', 'new ontology', '#unit_test#', '__default__']
+
+        # get new instance
+        sheerka = self.new_sheerka_instance(False)
+
+        # make sure that ontologies are recreated
+        ontologies_names = [o.name for o in sheerka.om.ontologies]
+        assert ontologies_names == ['another ontology', 'new ontology', '#unit_test#', '__default__']
+
+    def test_adding_the_same_ontology_twice_has_no_effect(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka = self.new_sheerka_instance(False)  # new instance that remembers the top layer ontology
+        sheerka.push_ontology(context, "#unit_test#")
+
+        assert len(sheerka.om.ontologies) == 2

tests/core/test_sheerkaResultManager.py

@@ -3,6 +3,7 @@ import types
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager
 from core.sheerka.services.SheerkaResultManager import SheerkaResultConcept
 from sdp.sheerkaDataProvider import Event
 
@@ -10,26 +11,27 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
 class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
-    io_cache = None
 
     @classmethod
     def setup_class(cls):
-        sheerka = cls().get_sheerka(cache_only=False)
+        sheerka = cls().get_sheerka(cache_only=False, ontology="#TestSheerkaResultManager#")
         sheerka.save_execution_context = True
+        cls.root_ontology_name = "#TestSheerkaResultManager#"
 
     @classmethod
     def teardown_class(cls):
-        sheerka = cls().get_sheerka()
-        sheerka.save_execution_context = False
+        cls.sheerka.pop_ontology()
+        cls.root_ontology_name = SheerkaOntologyManager.ROOT_ONTOLOGY_NAME
 
-    def init_test(self):
-        sheerka, context = self.init_concepts()
+    def init_service(self):
+        sheerka, context = self.init_test().unpack()
         service = sheerka.services[SheerkaResultConcept.NAME]
 
         return sheerka, context, service
 
     def test_i_can_record_execution_contexts(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
+        service.test_only_reset()
 
         sheerka.evaluate_user_input("foo")
 
@@ -40,7 +42,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
 
         assert execution_context.desc == "Evaluating 'foo'"
 
-        executions_contexts_in_db = sheerka.sdp.load_result(event_id)
+        executions_contexts_in_db = sheerka.om.current_sdp().load_result(event_id)
         assert executions_contexts_in_db is not None
         assert executions_contexts_in_db.desc == "Evaluating 'foo'"
 
@@ -48,7 +50,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert service.last_execution.desc == "Evaluating 'foo'"
 
     def test_i_can_get_the_result_by_digest_using_cache(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
 
         digest = service.last_execution.event.get_digest()
@@ -77,10 +79,10 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert list(res.body) == previous_results
 
     def test_i_can_get_result_by_digest_using_db(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
         digest = service.last_execution.event.get_digest()
-        service.reset()
+        service.test_only_reset()
 
         res = sheerka.get_results_by_digest(context, digest)
 
@@ -106,18 +108,18 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert list(res.body) == previous_results
 
     def test_i_cannot_get_result_by_digest_if_the_digest_does_not_exist(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         res = sheerka.get_results_by_digest(context, "fake digest")
         assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
         assert res.body == {'digest': 'fake digest'}
 
     def test_i_cannot_get_results_if_no_previous_digest(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
         assert sheerka.get_results(context) is None
 
     def test_i_can_get_the_result_by_command_name(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
 
         digest = service.last_execution.event.get_digest()
@@ -130,16 +132,16 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert isinstance(res.body, types.GeneratorType)
 
     def test_i_can_get_the_result_by_command_name_using_db(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
         sheerka.evaluate_user_input("one")  # another command
-        service.reset()
+        service.test_only_reset()
 
         res = sheerka.get_results_by_command(context, "def concept")
         assert res.command == "def concept one as 1"
 
     def test_i_can_get_the_result_by_command_when_not_in_the_same_page_size(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
 
         sheerka.evaluate_user_input("one")
@@ -152,14 +154,17 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.command == "def concept one as 1"
 
     def test_i_cannot_get_results_from_command_if_the_command_does_not_exist(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context, service = self.init_service()
+        service.test_only_reset()
 
         res = sheerka.get_results_by_command(context, "def concept")
         assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
         assert res.body == {'command': 'def concept'}
 
     def test_i_cannot_get_result_from_command_if_the_command_does_not_exists_multiple_pages(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
+        service.test_only_reset()
+
         sheerka.evaluate_user_input("def concept one as 1")
 
         sheerka.evaluate_user_input("one")
@@ -172,7 +177,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.body == {'command': 'fake command'}
 
     def test_i_can_get_last_results(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
 
         sheerka.evaluate_user_input("one")
@@ -182,21 +187,21 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.command == "one"
 
     def test_i_can_get_last_results_using_db(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
         sheerka.evaluate_user_input("one")
-        service.reset()
+        service.test_only_reset()
 
         res = sheerka.get_last_results(context)
         assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
         assert res.command == "one"
 
     def test_i_can_get_last_results_when_event_with_no_result(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
-        sheerka.sdp.save_event(Event("event 1"))
-        sheerka.sdp.save_event(Event("event 2"))
-        sheerka.sdp.save_event(Event("event 3"))
+        sheerka.om.save_event(Event("event 1"))
+        sheerka.om.save_event(Event("event 2"))
+        sheerka.om.save_event(Event("event 3"))
         sheerka.evaluate_user_input("def concept one as 1")
 
         res = sheerka.get_last_results(context)
@@ -204,18 +209,20 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.command == "def concept one as 1"
 
     def test_i_cannot_get_last_results_when_no_result(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context, service = self.init_service()
+        service.test_only_reset()
 
         res = sheerka.get_last_results(context)
         assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
         assert res.body == {'query': 'last'}
 
     def test_i_cannot_get_last_results_when_only_events(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context, service = self.init_service()
+        service.test_only_reset()
 
-        sheerka.sdp.save_event(Event("event 1"))
-        sheerka.sdp.save_event(Event("event 2"))
-        sheerka.sdp.save_event(Event("event 3"))
+        sheerka.om.save_event(Event("event 1"))
+        sheerka.om.save_event(Event("event 2"))
+        sheerka.om.save_event(Event("event 3"))
 
         res = sheerka.get_last_results(context)
         assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
@@ -227,7 +234,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         {"desc": "Evaluating 'def concept one as 1'", "id": 0}
     ])
     def test_i_can_get_last_results_using_kwarg(self, kwargs):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
 
@@ -244,7 +251,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
     ])
     def test_i_can_get_last_results_using_filter(self, predicate, expected):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
 
@@ -261,7 +268,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
     ])
     def test_i_can_get_last_results_using_the_first_argument_to_filter(self, predicate, expected):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
 
@@ -278,7 +285,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         {"desc": "Evaluating 'def concept one as 1'", "id": 0}
     ])
     def test_i_can_get_results_using_kwarg(self, kwargs):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
         sheerka.get_last_results(context)
@@ -296,7 +303,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
     ])
     def test_i_can_get_results_using_filter(self, predicate, expected):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
         sheerka.get_last_results(context)
@@ -314,7 +321,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
     ])
     def test_i_can_get_results_using_the_first_argument_to_filter(self, predicate, expected):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         sheerka.evaluate_user_input("def concept one as 1")
         ExecutionContext.ids.clear()
         sheerka.get_last_results(context)
@@ -332,33 +339,32 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
             SheerkaResultConcept.get_predicate(**predicate)
 
     def test_i_can_get_last_return_value(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
 
         sheerka.evaluate_user_input("def concept one as 1")
-        ret = sheerka.last_ret(context)
+        ret = sheerka.get_last_return_value(context)
         assert sheerka.isinstance(ret[0].body, BuiltinConcepts.NEW_CONCEPT)
 
         sheerka.evaluate_user_input("eval one")
-        ret = sheerka.last_ret(context)
+        ret = sheerka.get_last_return_value(context)
         assert ret[0].body == 1
 
     def test_i_can_track_new_concept(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
 
         res = sheerka.evaluate_user_input("def concept one as 1")
         new_concept = res[0].body.body
 
-        assert sheerka.last_created_concept(context) == new_concept
+        assert sheerka.get_last_created_concept(context) == new_concept
         assert service.last_created_concept_id == new_concept.id
 
     def test_last_created_concept_is_recorded(self):
-        sheerka, context, service = self.init_test()
+        sheerka, context, service = self.init_service()
         res = sheerka.evaluate_user_input("def concept one as 1")
         new_concept = res[0].body.body
-        service.reset()
+        service.test_only_reset()
 
         service.initialize_deferred(context, False)
 
         assert service.last_created_concept_id == new_concept.id
-        assert sheerka.last_created_concept(context) == new_concept
-
+        assert sheerka.get_last_created_concept(context) == new_concept

tests/core/test_sheerka_call_evaluators.py

@@ -252,7 +252,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
     def teardown_class(cls):
         # At the end of the tests, sheerka singleton instance will be corrupted
         # Ask for a new one
-        TestUsingMemoryBasedSheerka.singleton_instance = None
+        TestUsingMemoryBasedSheerka.sheerka = None
 
     def test_i_can_get_evaluators_when_context_is_not_altered(self):
         sheerka, context = self.init_concepts()

tests/core/test_sheerka_call_parsers.py

@@ -140,7 +140,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
     def teardown_class(cls):
         # At the end of the tests, sheerka singleton instance will be corrupted
         # Ask for a new one
-        TestUsingMemoryBasedSheerka.singleton_instance = None
+        TestUsingMemoryBasedSheerka.sheerka = None
 
     def test_i_can_get_parser_when_context_is_not_altered(self):
         sheerka, context = self.init_concepts()

tests/core/test_sheerka_printer.py

@@ -2,7 +2,8 @@ from dataclasses import dataclass
 
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts
+from core.concept import Concept, ConceptParts, freeze_concept_attrs
+from core.simple_debug import my_debug
 from printer.Formatter import Formatter, BraceToken
 from printer.SheerkaPrinter import FormatInstructions
 
@@ -154,10 +155,13 @@ class TestSheerkaPrinter(TestUsingMemoryBasedSheerka):
         # for all obj of a given type
 
         sheerka = self.get_sheerka()
+        sheerka.printer_handler.reset()
+
         foo = Concept("foo a b").def_var("a").def_var("b").init_key()
         foo.set_value("a", "value a").set_value("b", "value b")
         foo.set_value(ConceptParts.BODY, "body")
         sheerka.set_id_if_needed(foo, False)
+        freeze_concept_attrs(foo)
 
         sheerka.printer_handler.register_format_l(foo, "DEFAULT:{id}-{name}-{key}-{body}-{a}-{b}")
 
@@ -170,10 +174,13 @@ class TestSheerkaPrinter(TestUsingMemoryBasedSheerka):
         # for all obj of a given type in the current print call
 
         sheerka = self.get_sheerka()
+        sheerka.printer_handler.reset()
+
         foo = Concept("foo a b").def_var("a").def_var("b").init_key()
         foo.set_value("a", "value a").set_value("b", "value b")
         foo.set_value(ConceptParts.BODY, "body")
         sheerka.set_id_if_needed(foo, False)
+        freeze_concept_attrs(foo)
 
         sheerka.printer_handler.register_format_l(foo, "DEFAULT:{id}-{name}-{key}-{body}-{a}-{b}")
         context_instructions = FormatInstructions().set_format_l(foo, "CONTEXT:{id}-{name}-{key}-{body}-{a}-{b}")
@@ -187,10 +194,13 @@ class TestSheerkaPrinter(TestUsingMemoryBasedSheerka):
         # for the item only
 
         sheerka = self.get_sheerka()
+        sheerka.printer_handler.reset()
+
         foo = Concept("foo a b").def_var("a").def_var("b").init_key()
         foo.set_value("a", "value a").set_value("b", "value b")
         foo.set_value(ConceptParts.BODY, "body")
         sheerka.set_id_if_needed(foo, False)
+        freeze_concept_attrs(foo)
 
         sheerka.printer_handler.register_format_l(foo, "{id}-{name}-{key}-{body}-{a}-{b}")
         context_instructions = FormatInstructions().set_format_l(foo, "CONTEXT:{id}-{name}-{key}-{body}-{a}-{b}")
@@ -242,6 +252,8 @@ class TestSheerkaPrinter(TestUsingMemoryBasedSheerka):
 
     def test_i_can_format_d_concepts_using_default_definition(self, capsys):
         sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b"))
+        sheerka.printer_handler.reset()
+        
         foo_1 = sheerka.new(foo.key, a="value a", b="value b")
         foo_2 = sheerka.new(foo.key, a="value c", b="value d")
         lst = [foo_1, foo_2]
@@ -259,6 +271,8 @@ b: DEFAULT:'value d'
 
     def test_i_can_format_d_concepts_using_context_definition(self, capsys):
         sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b"))
+        sheerka.printer_handler.reset()
+
         foo_1 = sheerka.new(foo.key, a="value a", b="value b")
         foo_2 = sheerka.new(foo.key, a="value c", b="value d")
         lst = [foo_1, foo_2]
@@ -277,6 +291,8 @@ b: CONTEXT:'value d'
 
     def test_i_can_format_d_concepts_using_item_definition(self, capsys):
         sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b"))
+        sheerka.printer_handler.reset()
+
         item_instructions = FormatInstructions().set_format_d(foo, {"a": "ITEM:{a}", "b": "ITEM:{b}"})
         foo.set_format_instructions(item_instructions)
         foo_1 = sheerka.new(foo.key, a="value a", b="value b")
@@ -371,6 +387,8 @@ self: (1001)foo a b
 
     def test_i_can_format_d_when_dictionary(self, capsys):
         sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b"))
+        sheerka.printer_handler.reset()
+
         dict_value = {
             "a": "value a",
             "beta": {"b1": 10, "b2": Obj("10", 15), "b3": ["items", "in", "a", "list"]},
@@ -436,6 +454,8 @@ bar: *name 'bar' is not defined*
     ])
     def test_i_can_concat_print_instruction_and_register_format(self, capsys, template, expected):
         sheerka = self.get_sheerka()
+        sheerka.printer_handler.reset()
+
         foo = Obj("value a", "value b")
 
         sheerka.printer_handler.register_format_l("tests.core.test_sheerka_printer.Obj", "{a}")

tests/core/test_utils.py

@@ -3,7 +3,9 @@ from dataclasses import dataclass
 import core.utils
 import pytest
 from core.builtin_concepts import BuiltinConcepts
+from core.builtin_helpers import evaluate_expression
 from core.concept import Concept
+from core.global_symbols import NotFound, NotInit, Removed
 from core.tokenizer import Token, TokenKind, Tokenizer, Keywords
 
 
@@ -306,7 +308,7 @@ def test_i_can_make_unique():
     ("a.prop2.bar", {"a": Obj2(None, {'foo': 'dict-first', 'bar': 'dict-second'})}, 'dict-second'),
 ])
 def test_i_can_evaluate_expression(expression, bag, expected):
-    assert core.utils.evaluate_expression(expression, bag) == expected
+    assert evaluate_expression(expression, bag) == expected
 
 
 @pytest.mark.parametrize("expression, bag, expected_error, prop_name", [
@@ -315,7 +317,7 @@ def test_i_can_evaluate_expression(expression, bag, expected):
 ])
 def test_i_cannot_evaluate_expression(expression, bag, expected_error, prop_name):
     with pytest.raises(expected_error) as e:
-        core.utils.evaluate_expression(expression, bag)
+        evaluate_expression(expression, bag)
 
     assert e.value.args == (prop_name,)
 
@@ -417,3 +419,9 @@ def test_i_can_deep_copy_a_concept():
     concept_props = sorted(list(concept.get_prop(BuiltinConcepts.ISA)), key=lambda o: o.id)
     for copied_prop, concept_prop in zip(copied_props, concept_props):
         check_are_the_same(copied_prop, concept_prop)
+
+
+def test_i_can_deep_copy_a_custom_type():
+    assert core.utils.sheerka_deepcopy(NotInit) is NotInit
+    assert core.utils.sheerka_deepcopy(NotFound) is NotFound
+    assert core.utils.sheerka_deepcopy(Removed) is Removed

tests/evaluators/test_AddConceptInSetEvaluator.py

@@ -51,7 +51,7 @@ class TestAddConceptInSetEvaluator(TestUsingMemoryBasedSheerka):
         assert res.value.body == "bar"
 
     def test_i_can_add_concept_to_a_set_of_concept(self):
-        sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
+        sheerka, context, foo, bar = self.init_test().with_concepts("foo", "bar", create_new=True).unpack()
         ret_val = get_isa_ret_val("foo", "bar")
 
         res = AddConceptInSetEvaluator().eval(context, ret_val)
@@ -72,12 +72,12 @@ class TestAddConceptInSetEvaluator(TestUsingMemoryBasedSheerka):
         So 'foo' cannot be put is set
         :return:
         """
-        sheerka, context, one, two, foo, bar = self.init_concepts(
+        sheerka, context, one, two, foo, bar = self.init_test().with_concepts(
             "one",
             "two",
             Concept("foo", definition="(one|two)=a 'plus' (one|two)=b", body="a + b").def_var("a").def_var("b"),
             "bar",
-            create_new=True)
+            create_new=True).unpack()
 
         ret_val = get_isa_ret_val("foo", "bar")
         res = AddConceptInSetEvaluator().eval(context, ret_val)
@@ -107,7 +107,7 @@ class TestAddConceptInSetEvaluator(TestUsingMemoryBasedSheerka):
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
 
     def test_i_cannot_add_the_same_concept_twice(self):
-        sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
+        sheerka, context, foo, bar = self.init_test().with_concepts("foo", "bar", create_new=True).unpack()
 
         ret_val = get_isa_ret_val("foo", "bar")
         AddConceptInSetEvaluator().eval(context, ret_val)

tests/evaluators/test_DefConceptEvaluator.py

@@ -158,7 +158,7 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert created_concept.get_metadata().variables == [("x", None), ("y", None)]
 
     def test_other_concepts_are_not_variables(self):
-        sheerka, context, *concepts = self.init_concepts("little", "size", create_new=True)
+        sheerka, context, *concepts = self.init_test().with_concepts("little", "size", create_new=True).unpack()
 
         def_concept_node = self.get_def_concept_node_from_name_only("little x")
         name_to_use = DefConceptEvaluator.get_name_to_use(def_concept_node)

tests/evaluators/test_EvalEvaluator.py

@@ -1,7 +1,7 @@
 import pytest
 from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
 from core.concept import Concept
-from core.sheerka.services.SheerkaSetsManager import SheerkaSetsManager
+from core.sheerka.services.SheerkaIsAManager import SheerkaIsAManager
 from evaluators.ReturnBodyEvaluator import ReturnBodyEvaluator
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -89,7 +89,7 @@ class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
             Concept("bar"),
             Concept("baz"),
             Concept("number"))
-        sets_handler = sheerka.services[SheerkaSetsManager.NAME]
+        sets_handler = sheerka.services[SheerkaIsAManager.NAME]
         sets_handler.add_concepts_to_set(context, [foo, bar, baz], number)
 
         evaluated = ReturnBodyEvaluator().eval(context, [retval(number)])

tests/evaluators/test_LexerNodeEvaluator.py

@@ -55,7 +55,7 @@ class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka):
         assert LexerNodeEvaluator().matches(context, ret_val) == expected
 
     def test_concept_is_returned_when_only_one_in_the_list(self):
-        sheerka, context, foo = self.init_concepts(self.bnf_concept("foo"), singleton=True)
+        sheerka, context, foo = self.init_concepts(self.bnf_concept("foo"))
         ret_val = self.init_from_concepts(context, [foo], "foo")
 
         evaluator = LexerNodeEvaluator()

tests/evaluators/test_PythonEvaluator.py

@@ -68,8 +68,8 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert PythonEvaluator().matches(context, ret_val) == expected
 
     @pytest.mark.parametrize("text, expected", [
-        # ("1 + 1", 2),
-        # ("test()", "I have access to Sheerka !"),
+        ("1 + 1", 2),
+        ("test()", "I have access to Sheerka !"),
         ("sheerka.test()", "I have access to Sheerka !"),
         ("a=10\na", 10),
         ("Concept('foo')", Concept('foo')),
@@ -131,7 +131,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.value == "I have access to Sheerka ! param='value for param', event='xxx'."
 
     def test_i_can_eval_using_context_when_self_is_not_sheerka(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
         parsed = PythonParser().parse(context, ParserInput("create_new_concept(Concept('foo'))"))
 
         evaluated = PythonEvaluator().eval(context, parsed)
@@ -230,7 +230,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
                                                                             'c:__var__0 mult __var__1|1002:': 2}
 
     def test_i_can_define_variables(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
         parsed = PythonParser().parse(context, ParserInput("a=10"))
         python_evaluator = PythonEvaluator()
 
@@ -293,7 +293,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert error1.concepts == {'foo': 'string'}
 
     def test_i_can_use_sheerka_locals(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
 
         def func(i):
             return i + 1
@@ -318,7 +318,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.value == "I have access to Sheerka ! param=(1001)one, event='xxx'."
 
     def test_i_can_eval_rules_from_python_parser(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test().unpack()
         parsed_ret_val = PythonParser().parse(context, ParserInput("r:|1:.id"))
 
         assert parsed_ret_val.status

tests/non_reg/test_sheerka_display.py

@@ -2,11 +2,6 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
 class TestSheerkaNonRegDisplay(TestUsingMemoryBasedSheerka):
-    @classmethod
-    def teardown_class(cls):
-        # At the end of the tests, sheerka singleton instance will be corrupted
-        # Ask for a new one
-        TestSheerkaNonRegDisplay.singleton_instance = None
 
     def test_i_can_display_results_when_return_values_processing_is_on(self, capsys):
         init = [
@@ -27,7 +22,7 @@ class TestSheerkaNonRegDisplay(TestUsingMemoryBasedSheerka):
         ]
         sheerka = self.init_scenario(init)
         capsys.readouterr()
-        
+
         sheerka.enable_process_return_values = True
         sheerka.evaluate_user_input("desc(foo)")
 
@@ -72,4 +67,24 @@ props     : {}
         captured = capsys.readouterr()
         assert captured.out == """ReturnValue(who=evaluators.Concept, status=True, value=(1001)foo)
 ReturnValue(who=evaluators.Concept, status=True, value=(1002)foo)
+"""
+
+    def test_i_can_list_ontologies(self, capsys):
+        init = [
+            "push_ontology('test 1')",
+            "push_ontology('test 2')",
+            "pop_ontology()",
+            "push_ontology('test 3')",
+        ]
+        sheerka = self.init_scenario(init)
+        capsys.readouterr()
+
+        sheerka.enable_process_return_values = True
+        sheerka.evaluate_user_input("ontologies()")
+
+        captured = capsys.readouterr()
+        assert captured.out == """test 3
+test 1
+#unit_test#
+__default__
 """

tests/non_reg/test_sheerka_non_reg.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit, CC
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, CC
+from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
@@ -115,10 +116,10 @@ as:
         assert service.has_id(concept_saved.id)
         assert service.has_name(concept_saved.name)
         assert service.has_hash(concept_saved.get_definition_hash())
-        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {'+': ['1001']}
+        assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {'+': ['1001']}
 
         # sdp is up to date
-        assert sheerka.sdp.exists(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, expected.key)
+        assert sheerka.om.current_sdp().exists(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, expected.key)
 
     def test_i_can_evaluate_def_concept_part_when_one_part_is_a_ref_of_another_concept(self):
         """
@@ -182,10 +183,10 @@ as:
         assert sheerka.isinstance(res[0].value, BuiltinConcepts.NOP)
 
     def test_i_can_recognize_concept_with_variable(self):
-        sheerka, context, concept_foo, concept_hello = self.init_concepts(
+        sheerka, context, concept_foo, concept_hello = self.init_test().with_concepts(
             "foo",
             Concept(name="hello a").def_var("a"),
-            create_new=True)
+            create_new=True).unpack()
 
         res = sheerka.evaluate_user_input("hello foo")
         return_value = res[0].value
@@ -787,14 +788,10 @@ as:
             "def concept plus from a plus b as a + b",
             "def concept mult from a mult b as a * b",
             "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit",
+            "set_is_greater_than(BuiltinConcepts.PRECEDENCE, mult, plus, 'Sya')"
         ]
 
         sheerka = self.init_scenario(definitions)
-        context = self.get_context(sheerka)
-        sheerka.test_only_force_sya_def(context, [
-            (sheerka.get_by_name("mult").id, 20, SyaAssociativity.Right),
-            (sheerka.get_by_name("plus").id, 10, SyaAssociativity.Right),
-        ])
 
         res = sheerka.evaluate_user_input("eval one plus two mult three")
         assert len(res) == 1
@@ -952,8 +949,8 @@ as:
         sheerka = self.init_scenario(init)
 
         # simulate that sheerka was stopped and restarted
-        sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
-        sheerka.cache_manager.get(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, "twenties").set_compiled({})
+        sheerka.om.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
+        sheerka.om.get(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, "twenties").set_compiled({})
 
         res = sheerka.evaluate_user_input("eval twenty one")
         assert res[0].status
@@ -1109,12 +1106,10 @@ as:
         assert len(res) == 1
         assert res[0].status
 
-        sheerka = self.init_scenario(init)
         res = sheerka.evaluate_user_input("desc(bar)")
         assert len(res) == 1
         assert res[0].status
 
-        sheerka = self.init_scenario(init)
         res = sheerka.evaluate_user_input("desc(baz)")
         assert len(res) == 1
         assert res[0].status
@@ -1208,6 +1203,32 @@ as:
         assert res[0].status
         assert res[0].body == 2
 
+    def test_i_can_parse_when_multiple_ontology_layers(self):
+        init = [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept a plus b as a + b",  # sya node
+            "def concept twenties from bnf 'twenty' (one | two)=unit as 20 + unit",  # bnf node
+        ]
+        sheerka = self.init_scenario(init)
+
+        sheerka.push_ontology(self.get_context(sheerka), "new ontology")
+
+        res = sheerka.evaluate_user_input("eval one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 1
+
+        res = sheerka.evaluate_user_input("eval one plus two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 3
+
+        res = sheerka.evaluate_user_input("eval twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 21
+
 
 class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
     def test_i_can_def_several_concepts(self):
@@ -1231,7 +1252,7 @@ class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
         assert res[0].status
         assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
 
-        saved_concept = sheerka.sdp.get(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, "plus")
+        saved_concept = sheerka.om.current_sdp().get(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, "plus")
         assert saved_concept.key == "plus"
         assert saved_concept.get_metadata().definition == "a ('plus' plus)?"
         assert "a" in saved_concept.values()
@@ -1295,3 +1316,15 @@ class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
 
         assert sheerka.evaluate_user_input("eval twenty one")[0].body == 21
         assert sheerka.evaluate_user_input("eval thirty one")[0].body == 31
+
+    def test_i_can_pop_ontology_after_restart(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluate_user_input("push_ontology('test')")
+
+        sheerka = self.new_sheerka_instance(False)
+
+        res = sheerka.evaluate_user_input("pop_ontology()")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.ONTOLOGY_REMOVED)

tests/out/test_SheerkaOut.py

@@ -4,16 +4,19 @@ import pytest
 from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
 from core.concept import Concept
 from core.rule import Rule
+from core.sheerka.Sheerka import Sheerka
+from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager
+from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
 from core.sheerka.services.SheerkaOut import SheerkaOut
 from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, FormatAstVariable, FormatAstSequence, \
-    FormatAstColor, FormatAstVariableNotFound, FormatAstList, FormatAstDict
+    FormatAstColor, FormatAstVariableNotFound, FormatAstList, FormatAstDict, SheerkaRuleManager
 from core.utils import flatten_all_children
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
-seq = FormatAstSequence
-raw = FormatAstRawText
-var = FormatAstVariable
+
+def seq(*args, **kwargs):
+    return FormatAstSequence(*args, **kwargs)
 
 
 @dataclass
@@ -24,6 +27,20 @@ class DummyObj:
 
 class TestSheerkaOut(TestUsingMemoryBasedSheerka):
 
+    @classmethod
+    def setup_class(cls):
+        sheerka = cls().get_sheerka(cache_only=False, ontology="#TestSheerkaOut#")
+        sheerka.om.clear(SheerkaRuleManager.FORMAT_RULE_ENTRY)
+        sheerka.om.clear(SheerkaComparisonManager.COMPARISON_ENTRY)
+        sheerka.om.clear(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY)
+        sheerka.om.delete(Sheerka.OBJECTS_IDS_ENTRY, SheerkaRuleManager.RULE_IDS)
+        cls.root_ontology_name = "#TestSheerkaOut#"
+
+    @classmethod
+    def teardown_class(cls):
+        cls.sheerka.pop_ontology()
+        cls.root_ontology_name = SheerkaOntologyManager.ROOT_ONTOLOGY_NAME
+
     def init_service_with_rules(self, *rules, **kwargs):
         sheerka, context, *rules = self.init_format_rules(*rules, **kwargs)
         service = sheerka.services[SheerkaOut.NAME]

tests/parsers/test_BaseNodeParser.py

@@ -24,7 +24,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
 
         sheerka, context, *updated = self.init_concepts(concept)
 
-        res = BaseNodeParser.get_concepts_by_first_token(context, updated)
+        res = BaseNodeParser.compute_concepts_by_first_token(context, updated)
 
         assert res.status
         assert res.body == expected
@@ -54,7 +54,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         concept.set_bnf(bnf)
         sheerka.set_id_if_needed(concept, False)
 
-        res = BaseNodeParser.get_concepts_by_first_token(context, [concept])
+        res = BaseNodeParser.compute_concepts_by_first_token(context, [concept])
 
         assert res.status
         assert res.body == expected
@@ -75,7 +75,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux")))
         sheerka.set_id_if_needed(foo, False)
 
-        res = BaseNodeParser.get_concepts_by_first_token(context, [bar, baz, foo])
+        res = BaseNodeParser.compute_concepts_by_first_token(context, [bar, baz, foo])
 
         assert res.status
         assert res.body == {
@@ -87,12 +87,12 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         }
 
     def test_i_can_get_concepts_by_first_keyword_using_sheerka(self):
-        sheerka, context, *updated = self.init_concepts(
+        sheerka, context, *updated = self.init_test().with_concepts(
             "one",
             "two",
             Concept("twenty", definition="'twenty' (one|two)"),
             create_new=True
-        )
+        ).unpack()
 
         bar = Concept("bar").init_key()
         sheerka.set_id_if_needed(bar, False)
@@ -102,7 +102,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux")))
         sheerka.set_id_if_needed(foo, False)
 
-        res = BaseNodeParser.get_concepts_by_first_token(context, [bar, foo], use_sheerka=True)
+        res = BaseNodeParser.compute_concepts_by_first_token(context, [bar, foo], use_sheerka=True)
 
         assert res.status
         assert res.body == {
@@ -117,7 +117,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
 
     def test_i_cannot_get_concept_by_first_keyword_when_no_first_keyword(self):
         sheerka, context, foo = self.init_concepts(Concept("x y", body="x y").def_var("x").def_var("y"))
-        res = BaseNodeParser.get_concepts_by_first_token(context, [foo])
+        res = BaseNodeParser.compute_concepts_by_first_token(context, [foo])
 
         assert not res.status
         assert res.body == NoFirstTokenError(foo, foo.key)
@@ -126,9 +126,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, *updated = self.init_concepts(
             "one",
             Concept("two", definition="one"),
-            Concept("three", definition="two"),
-            create_new=False
-        )
+            Concept("three", definition="two"))
 
         concepts_by_first_keywords = {
             "one": ["1001"],
@@ -152,7 +150,6 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
             "hundred",
             Concept("twenties", definition="twenty number"),
             Concept("hundreds", definition="number hundred"),
-            create_new=True  # mandatory because set_isa() needs it
         )
 
         sheerka.set_isa(context, sheerka.new("one"), number)
@@ -166,7 +163,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         sheerka.concepts_grammars.clear()  # reset all the grammar to simulate Sheerka restart
 
         # cbft : concept_by_first_token (I usually don't  use abbreviation)
-        cbft = BaseNodeParser.get_concepts_by_first_token(context, [number] + concepts).body
+        cbft = BaseNodeParser.compute_concepts_by_first_token(context, [number] + concepts).body
         resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, cbft)
 
         assert resolved_ret_val.status
@@ -188,7 +185,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
             ConceptExpression("foo"),
             ConceptExpression("bar")))
 
-        concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(
+        concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(
             context, [good, foo, bar, baz]).body
 
         resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
@@ -204,7 +201,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         a = self.create_and_add_in_cache_concept(sheerka, "a", bnf=Sequence("one", "two"))
         b = self.create_and_add_in_cache_concept(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two"))
 
-        concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(
+        concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(
             context, [a, b]).body
 
         resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
@@ -220,7 +217,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
             self.bnf_concept("bar", ConceptExpression("foo")),
         )
 
-        concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(context, [good, foo, bar]).body
+        concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(context, [good, foo, bar]).body
         resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
         assert resolved_ret_val.status
         assert resolved_ret_val.body == {
@@ -237,7 +234,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
             self.bnf_concept("three", ConceptExpression("two")),
         )
 
-        concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(context, [good, one, two, three]).body
+        concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(context, [good, one, two, three]).body
         resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
         assert resolved_ret_val.status
         assert resolved_ret_val.body == {
@@ -255,7 +252,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
             self.bnf_concept("one", ConceptExpression("three")),
         )
 
-        concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(context, [good, one, two, three]).body
+        concepts_by_first_keywords = BaseNodeParser.compute_concepts_by_first_token(context, [good, one, two, three]).body
         resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
         assert resolved_ret_val.status
         assert resolved_ret_val.body == {

tests/parsers/test_BnfNodeParser.py

@@ -1,13 +1,14 @@
 import pytest
+
+import tests.parsers.parsers_utils
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF, NotInit
+from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF
+from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.BaseNodeParser import CNC, UTN, CN
 from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
     Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser
-
-import tests.parsers.parsers_utils
 from tests.BaseTest import BaseTest
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -82,28 +83,26 @@ def compute_expected_array(my_concepts_map, expression, expected, exclude_body=F
 
 
 class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
-    sheerka = None
+    shared_ontology = None
 
     @classmethod
     def setup_class(cls):
-        t = cls()
-        TestBnfNodeParser.sheerka, context, _ = t.init_parser(
-            cmap,
-            singleton=False,
-            create_new=True,
-            init_from_sheerka=True)
+        init_test_helper = cls().init_test(cache_only=False, ontology="#TestBnfNodeParser#")
+        sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
+        for i, concept_name in enumerate(cmap):
+            cmap[concept_name] = updated[i]
 
         # end of initialisation
         sheerka = TestBnfNodeParser.sheerka
-        sheerka.set_isa(context, sheerka.new("one"), sheerka.new("number"))
-        sheerka.set_isa(context, sheerka.new("two"), sheerka.new("number"))
-        sheerka.set_isa(context, sheerka.new("three"), sheerka.new("number"))
-        sheerka.set_isa(context, sheerka.new("four"), sheerka.new("number"))
-        sheerka.set_isa(context, sheerka.new("thirty"), sheerka.new("number"))
-        sheerka.set_isa(context, sheerka.new("forty"), sheerka.new("number"))
-        sheerka.set_isa(context, sheerka.new("fifty"), sheerka.new("number"))
-        sheerka.set_isa(context, sheerka.new("one hundred"), sheerka.new("number"))
-        sheerka.set_isa(context, sheerka.new("hundreds"), sheerka.new("number"))
+        sheerka.set_isa(context, cmap["one"], cmap["number"])
+        sheerka.set_isa(context, cmap["two"], cmap["number"])
+        sheerka.set_isa(context, cmap["three"], cmap["number"])
+        sheerka.set_isa(context, cmap["four"], cmap["number"])
+        sheerka.set_isa(context, cmap["thirty"], cmap["number"])
+        sheerka.set_isa(context, cmap["forty"], cmap["number"])
+        sheerka.set_isa(context, cmap["fifty"], cmap["number"])
+        sheerka.set_isa(context, cmap["one hundred"], cmap["number"])
+        sheerka.set_isa(context, cmap["hundreds"], cmap["number"])
 
         # Pay attention. 'twenties (t1 and t2) are not set as number
 
@@ -135,6 +134,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         cmap["thousands"] = sheerka.create_new_concept(context, thousands).body.body
         sheerka.set_isa(context, sheerka.new("thousands"), sheerka.new("number"))
 
+        cls.shared_ontology = sheerka.get_ontology(context)
+        sheerka.pop_ontology()
+
     @staticmethod
     def update_bnf(context, concept):
         bnf_parser = BnfDefinitionParser()
@@ -147,19 +149,38 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         return concept
 
     def init_parser(self, my_concepts_map=None, init_from_sheerka=False, **kwargs):
-        if my_concepts_map is not None:
-            sheerka, context, *updated = self.init_concepts(*my_concepts_map.values(), **kwargs)
+        if my_concepts_map is None:
+            sheerka, context = self.init_test().unpack()
+            sheerka.add_ontology(context, self.shared_ontology)
+        else:
+            sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
             for i, pair in enumerate(my_concepts_map):
                 my_concepts_map[pair] = updated[i]
-        else:
-            sheerka = TestBnfNodeParser.sheerka
-            context = self.get_context(sheerka)
 
         parser = BnfNodeParser(sheerka=sheerka) if init_from_sheerka else BnfNodeParser()
         return sheerka, context, parser
 
-    def exec_get_concepts_sequences(self, my_map, text, expected, multiple_result=False, post_init_concepts=None):
-        sheerka, context, *updated = self.init_concepts(*my_map.values(), create_new=False, singleton=True)
+    def validate_get_concepts_sequences(self, my_map, text, expected, multiple_result=False, post_init_concepts=None):
+        sheerka, context, *updated = self.init_test().with_concepts(*my_map.values(), create_new=False).unpack()
+        sequences = self.exec_get_concepts_sequences(context,
+                                                     my_map,
+                                                     text,
+                                                     expected,
+                                                     multiple_result,
+                                                     post_init_concepts,
+                                                     *updated)
+        return sequences
+
+    @staticmethod
+    def exec_get_concepts_sequences(context,
+                                    my_map,
+                                    text,
+                                    expected,
+                                    multiple_result=False,
+                                    post_init_concepts=None,
+                                    *concepts):
+        sheerka = context.sheerka
+
         if not multiple_result:
             expected_array = [compute_expected_array(my_map, text, expected)]
         else:
@@ -169,7 +190,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             post_init_concepts(sheerka, context)
 
         parser = BnfNodeParser()
-        parser.init_from_concepts(context, updated)
+        parser.init_from_concepts(context, concepts)
         parser.reset_parser(context, ParserInput(text))
 
         bnf_parsers_helpers = parser.get_concepts_sequences(context)
@@ -179,15 +200,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             assert parser_helper.sequence == expected_sequence
 
         if len(bnf_parsers_helpers) == 1:
-            return sheerka, context, bnf_parsers_helpers[0].sequence
+            return bnf_parsers_helpers[0].sequence
         else:
-            return sheerka, context, [pe.sequence for pe in bnf_parsers_helpers]
-
-    def validate_get_concepts_sequences(self, my_map, text, expected, multiple_result=False, post_init_concepts=None):
-        sheerka, context, sequences = self.exec_get_concepts_sequences(
-            my_map, text, expected, multiple_result, post_init_concepts
-        )
-        return sequences
+            return [pe.sequence for pe in bnf_parsers_helpers]
 
     def test_i_cannot_parse_empty_strings(self):
         sheerka, context, parser = self.init_parser({}, singleton=True)
@@ -706,10 +721,11 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
                 ConceptExpression("foo"),
                 OrderedChoice(StrMatch("one"), StrMatch("two")))),
         }
+        sheerka, context, *concepts = self.init_test().with_concepts(*my_map.values(), create_new=False).unpack()
 
         text = "twenty two"
         expected = [CN("bar", source="twenty two")]
-        sheerka, context, sequences = self.exec_get_concepts_sequences(my_map, text, expected)
+        sequences = self.exec_get_concepts_sequences(context, my_map, text, expected, False, None, *concepts)
 
         concept_bar = sequences[0].concept
         assert concept_bar.get_compiled() == {
@@ -720,7 +736,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         text = "thirty one"
         expected = [CN("bar", source="thirty one")]
-        sequences = self.validate_get_concepts_sequences(my_map, text, expected)
+        sequences = self.exec_get_concepts_sequences(context, my_map, text, expected, False, None, *concepts)
         concept_bar = sequences[0].concept
         assert concept_bar.get_compiled() == {
             ConceptParts.BODY: DoNotResolve("thirty one"),
@@ -817,7 +833,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         # every obvious cyclic recursion are removed from concept_by_first_keyword dict
         parser.init_from_concepts(context, my_map.values())
-        assert parser.concepts_by_first_keyword == expected
+        assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == expected
 
         # get_parsing_expression() also returns CHICKEN_AND_EGG
         parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
@@ -842,7 +858,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         # every obvious cyclic recursion are removed from concept_by_first_keyword dict
         parser.init_from_concepts(context, my_map.values())
-        assert parser.concepts_by_first_keyword == {}
+        assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
 
         parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
         assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
@@ -868,7 +884,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         # every obvious cyclic recursion are removed from concept_by_first_keyword dict
         parser.init_from_concepts(context, my_map.values())
-        assert parser.concepts_by_first_keyword == {}
+        assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
 
         parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
         assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
@@ -1008,8 +1024,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             ConceptExpression(my_map["one"], rule_name="one"))
 
     @pytest.mark.parametrize("expr, text, expected", [
-        (ZeroOrMore(StrMatch("one"), sep=","), "one,", [CNC("foo", source="one"), UTN(",")]),
-        (StrMatch("one"), "one two", [CNC("foo", source="one"), UTN(" two")]),
+        # (ZeroOrMore(StrMatch("one"), sep=","), "one,", [CNC("foo", source="one"), UTN(",")]),
+        # (StrMatch("one"), "one two", [CNC("foo", source="one"), UTN(" two")]),
         (StrMatch("one"), "two one", [UTN("two "), CNC("foo", source="one")]),
     ])
     def test_i_can_recognize_unknown_concepts(self, expr, text, expected):
@@ -1442,6 +1458,22 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_get_expression_from_concept_name(self, name, expected):
         assert BnfNodeParser.get_expression_from_concept_name(name) == expected
 
+    def test_i_can_parse_when_multiple_layers(self):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+
+        # sanity
+        text = "thirty one"
+        res = parser.parse(context, ParserInput(text))
+        assert res.status
+        assert res.value.value == compute_expected_array(cmap, text, [CN("thirties", source=text)])
+
+        # add a layer, I still can parse the text
+        sheerka.push_ontology(context, "new layer")
+        parser = BnfNodeParser(sheerka=sheerka)
+        res = parser.parse(context, ParserInput(text))
+        assert res.status
+        assert res.value.value == compute_expected_array(cmap, text, [CN("thirties", source=text)])
+
     # @pytest.mark.parametrize("parser_input, expected", [
     #     ("one", [
     #         (True, [CNC("bnf_one", source="one", one="one", body="one")]),

tests/parsers/test_DefFormatRuleParser.py

@@ -18,19 +18,26 @@ cmap = {
 
 
 class TestDefFormatRuleParser(TestUsingMemoryBasedSheerka):
-    sheerka = None
+    shared_ontology = None
 
     @classmethod
     def setup_class(cls):
-        t = cls()
-        cls.sheerka, context, _ = t.init_parser(cmap)
+        init_test_helper = cls().init_test(cache_only=False, ontology="#TestDefFormatRuleParser#")
+        sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
+        for i, concept_name in enumerate(cmap):
+            cmap[concept_name] = updated[i]
 
-    def init_parser(self, concepts_map=None):
-        if concepts_map is not None:
-            sheerka, context, *concepts = self.init_concepts(*concepts_map.values(), create_new=True)
+        cls.shared_ontology = sheerka.get_ontology(context)
+        sheerka.pop_ontology()
+
+    def init_parser(self, my_concepts_map=None, **kwargs):
+        if my_concepts_map is None:
+            sheerka, context = self.init_test().unpack()
+            sheerka.add_ontology(context, self.shared_ontology)
         else:
-            sheerka = TestDefFormatRuleParser.sheerka
-            context = self.get_context(sheerka)
+            sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
+            for i, pair in enumerate(my_concepts_map):
+                my_concepts_map[pair] = updated[i]
 
         parser = DefFormatRuleParser()
         return sheerka, context, parser
@@ -134,4 +141,3 @@ class TestDefFormatRuleParser(TestUsingMemoryBasedSheerka):
 
         assert res.status
         assert format_ast == expected
-

tests/parsers/test_FunctionParser.py

@@ -18,19 +18,26 @@ cmap = {
 
 
 class TestFunctionParser(TestUsingMemoryBasedSheerka):
-    sheerka = None
+    shared_ontology = None
 
     @classmethod
     def setup_class(cls):
-        t = cls()
-        cls.sheerka, context, _ = t.init_parser(cmap)
+        init_test_helper = cls().init_test(cache_only=False, ontology="#TestFunctionParser#")
+        sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
+        for i, concept_name in enumerate(cmap):
+            cmap[concept_name] = updated[i]
 
-    def init_parser(self, concepts_map=None):
-        if concepts_map is not None:
-            sheerka, context, *concepts = self.init_concepts(*concepts_map.values(), create_new=True)
+        cls.shared_ontology = sheerka.get_ontology(context)
+        sheerka.pop_ontology()
+
+    def init_parser(self, my_concepts_map=None, **kwargs):
+        if my_concepts_map is None:
+            sheerka, context = self.init_test().unpack()
+            sheerka.add_ontology(context, self.shared_ontology)
         else:
-            sheerka = TestFunctionParser.sheerka
-            context = self.get_context(sheerka)
+            sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
+            for i, pair in enumerate(my_concepts_map):
+                my_concepts_map[pair] = updated[i]
 
         parser = FunctionParser()
         return sheerka, context, parser
@@ -244,4 +251,3 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
         assert isinstance(concept.get_compiled()["b"], list)
         for item in concept.get_compiled()["b"]:
             assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)
-

tests/parsers/test_RuleParser.py

@@ -5,23 +5,26 @@ from parsers.RuleParser import RuleParser, RuleNotFoundError
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
-my_rules = {("__rets", "list(__rets)")}
+my_rules = [("__rets", "list(__rets)")]
 
 
 class TestRuleParser(TestUsingMemoryBasedSheerka):
-    sheerka = None
+    shared_ontology = None
 
     @classmethod
     def setup_class(cls):
-        t = cls()
-        cls.sheerka, context, _ = t.init_parser(my_rules)
+        init_test_helper = cls().init_test(cache_only=False, ontology="#TestRuleParser#")
+        sheerka, context, *updated = init_test_helper.with_rules(*my_rules).unpack()
 
-    def init_parser(self, rules=None):
-        if rules is not None:
-            sheerka, context, *concepts = self.init_format_rules(*rules)
+        cls.shared_ontology = sheerka.get_ontology(context)
+        sheerka.pop_ontology()
+
+    def init_parser(self, rules=None, **kwargs):
+        if rules is None:
+            sheerka, context = self.init_test().unpack()
+            sheerka.add_ontology(context, self.shared_ontology)
         else:
-            sheerka = TestRuleParser.sheerka
-            context = self.get_context(sheerka)
+            sheerka, context, *updated = self.init_test().with_rules(*rules, **kwargs).unpack()
 
         parser = RuleParser()
         return sheerka, context, parser

tests/parsers/test_SequenceNodeParser.py

@@ -11,10 +11,9 @@ from tests.parsers.parsers_utils import compute_expected_array
 
 class TestAtomsParser(TestUsingMemoryBasedSheerka):
     def init_parser(self, my_map, create_new=False, singleton=True, use_sheerka=False):
-        sheerka, context, *updated_concepts = self.init_concepts(
+        sheerka, context, *updated_concepts = self.init_test().with_concepts(
             *my_map.values(),
-            create_new=create_new,
-            singleton=singleton)
+            create_new=create_new).unpack()
 
         if use_sheerka:
             parser = SequenceNodeParser(sheerka=sheerka)

tests/parsers/test_SyaNodeParser.py

@@ -41,16 +41,14 @@ cmap = {
 
 
 class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
-    sheerka = None
+    shared_ontology = None
 
     @classmethod
     def setup_class(cls):
-        t = TestSyaNodeParser()
-        TestSyaNodeParser.sheerka, context, _ = t.init_parser(
-            cmap,
-            singleton=False,
-            create_new=True,
-            init_from_sheerka=True)
+        init_test_helper = cls().init_test(cache_only=False, ontology="#TestSyaNodeParser#")
+        sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
+        for i, concept_name in enumerate(cmap):
+            cmap[concept_name] = updated[i]
 
         cmap["plus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
         cmap["mult"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
@@ -66,35 +64,24 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
                                                       cmap["minus"],
                                                       CONCEPT_COMPARISON_CONTEXT)
 
-        # TestSyaNodeParser.sheerka.test_only_force_sya_def(context, [
-        #     (cmap["plus"].id, 5, SyaAssociativity.Right),
-        #     (cmap["mult"].id, 10, SyaAssociativity.Right),
-        #     (cmap["minus"].id, 5, SyaAssociativity.Right)])
+        cls.shared_ontology = sheerka.get_ontology(context)
+        sheerka.pop_ontology()
 
     def init_parser(self,
                     my_concepts_map=None,
                     sya_def=None,
                     post_init_concepts=None,
                     **kwargs):
-
-        if my_concepts_map is not None:
-            # a new concept map is given
-            # use it but
-            #   do not instantiate a new sheerka
-            #   do not update / init from sheerka
-            if 'singleton' not in kwargs:
-                kwargs["singleton"] = True
-            init_from_sheerka = kwargs.get("init_from_sheerka", False)
-            sheerka, context, *concepts = self.init_concepts(*my_concepts_map.values(), **kwargs)
-        else:
-            # No custom concept map is given -> Use the global cmap
-            # Sheerka is already initialized (the class instance)
-            # Use it to initialize the parser
-            init_from_sheerka = kwargs.get("init_from_sheerka", True)
-            sheerka = TestSyaNodeParser.sheerka
-            context = self.get_context(sheerka)
+        if my_concepts_map is None:
+            sheerka, context = self.init_test().unpack()
+            sheerka.add_ontology(context, self.shared_ontology)
             concepts = cmap.values()
-            ALL_ATTRIBUTES.clear()
+            init_from_sheerka = kwargs.get("init_from_sheerka", True)
+        else:
+            sheerka, context, *concepts = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
+            for i, pair in enumerate(my_concepts_map):
+                my_concepts_map[pair] = concepts[i]
+            init_from_sheerka = kwargs.get("init_from_sheerka", False)
 
         if post_init_concepts:
             post_init_concepts(sheerka, context)
@@ -112,7 +99,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
             parser = SyaNodeParser()
             if my_concepts_map:
                 parser.init_from_concepts(context, concepts, sya=sya_def_to_use)
-
         return sheerka, context, parser
 
     @pytest.mark.parametrize("expression, expected_sequences", [
@@ -1346,3 +1332,27 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
                 assert actual.function == resolved_function_name[0]
             else:
                 assert actual.function is None
+
+    def test_i_can_parse_when_multiple_ontologies(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "suffixed 1 + 1"
+        res = parser.parse(context, ParserInput(text))
+        wrapper = res.body
+        lexer_nodes = res.body.body
+
+        assert res.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
+
+        # add an ontology layer and make sure will still can parse
+        sheerka.push_ontology(context, "new ontology")
+        parser = SyaNodeParser(sheerka=sheerka)
+
+        res = parser.parse(context, ParserInput(text))
+        wrapper = res.body
+        lexer_nodes = res.body.body
+
+        assert res.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]

tests/parsers/test_UnrecognizedNodeParser.py

@@ -70,25 +70,31 @@ concepts_map = {
 
 
 class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
-    sheerka = None
+    shared_ontology = None
 
     @classmethod
     def setup_class(cls):
-        t = TestUnrecognizedNodeParser()
-        TestUnrecognizedNodeParser.sheerka, context, _ = t.init_parser(concepts_map, create_new=True)
-        TestUnrecognizedNodeParser.sheerka.test_only_force_sya_def(context, [
-            (concepts_map["mult"].id, 20, SyaAssociativity.Right),
-            (concepts_map["plus"].id, 10, SyaAssociativity.Right),
-        ])
+        init_test_helper = cls().init_test(cache_only=False, ontology="#TestUnrecognizedNodeParser#")
+        sheerka, context, *updated = init_test_helper.with_concepts(*concepts_map.values(), create_new=True).unpack()
+        for i, concept_name in enumerate(concepts_map):
+            concepts_map[concept_name] = updated[i]
+
+        sheerka.set_is_greater_than(context,
+                                    BuiltinConcepts.PRECEDENCE,
+                                    concepts_map["mult"],
+                                    concepts_map["plus"], 'Sya')
+
+        cls.shared_ontology = sheerka.get_ontology(context)
+        sheerka.pop_ontology()
 
     def init_parser(self, my_concepts_map=None, **kwargs):
-        if my_concepts_map:
-            sheerka, context, *updated_concepts = self.init_concepts(*my_concepts_map.values(), **kwargs)
-            for i, pair in enumerate(my_concepts_map):
-                my_concepts_map[pair] = updated_concepts[i]
+        if my_concepts_map is None:
+            sheerka, context = self.init_test().unpack()
+            sheerka.add_ontology(context, self.shared_ontology)
         else:
-            sheerka = TestUnrecognizedNodeParser.sheerka
-            context = self.get_context(sheerka)
+            sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
+            for i, pair in enumerate(my_concepts_map):
+                my_concepts_map[pair] = updated[i]
 
         parser = UnrecognizedNodeParser()
         return sheerka, context, parser
@@ -186,12 +192,14 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         assert res.body.concept.get_compiled()["a"][0].body.source == "1 "
 
         assert res.body.concept.get_compiled()["b"] == concepts_map["mult"]
-        assert sheerka.isinstance(res.body.concept.get_compiled()["b"].get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(res.body.concept.get_compiled()["b"].get_compiled()["a"][0],
+                                  BuiltinConcepts.RETURN_VALUE)
         assert res.body.concept.get_compiled()["b"].get_compiled()["a"][0].status
         assert res.body.concept.get_compiled()["b"].get_compiled()["a"][0].who == "parsers.Python"
         assert res.body.concept.get_compiled()["b"].get_compiled()["a"][0].body.source == " 2 "
 
-        assert sheerka.isinstance(res.body.concept.get_compiled()["b"].get_compiled()["b"][0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(res.body.concept.get_compiled()["b"].get_compiled()["b"][0],
+                                  BuiltinConcepts.RETURN_VALUE)
         assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].status
         assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].who == "parsers.Bnf"
         expected_nodes = compute_expected_array(
@@ -339,7 +347,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert parser_result.source == expression
         assert len(actual_nodes) == 1
-        assert actual_nodes[0].nodes[0].concept.get_metadata().is_evaluated  # 'a plus b' is recognized as concept definition
+        assert actual_nodes[0].nodes[
+            0].concept.get_metadata().is_evaluated  # 'a plus b' is recognized as concept definition
 
     def test_i_can_parse_unrecognized_source_code_with_concept_node_when_var_in_short_term_memory(self):
         sheerka, context, parser = self.init_parser()

tests/sdp/test_sheerkaDataProvider.py

@@ -5,6 +5,8 @@ from datetime import date, datetime
 from os import path
 
 import pytest
+
+from core.global_symbols import NotFound
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 from sdp.sheerkaSerializer import PickleSerializer
 
@@ -116,6 +118,28 @@ def test_i_can_save_and_load_an_event(root):
     evt = sdp.load_event()
     assert evt.message == "hello world"
 
+    # check that the last event is updated
+    last_event_file = path.join(sdp.io.root, SheerkaDataProvider.LastEventFile)
+    assert sdp.io.exists(last_event_file)
+    assert sdp.io.read_text(last_event_file) == evt_digest
+
+
+def test_i_can_save_and_load_events_with_multiple_sdp():
+    root = ".sheerka"
+    sdp1 = SheerkaDataProvider(root)
+    sdp1.save_event(Event("event 1", date=date(year=2007, month=9, day=10), user_id="kodjo"))
+    sdp1.save_event(Event("event 2", date=date(year=2007, month=9, day=10), user_id="kodjo"))
+
+    sdp2 = SheerkaDataProvider(root, "Another sdp")
+    sdp2.save_event(Event("event 3", date=date(year=2007, month=9, day=10), user_id="kodjo"))
+    sdp2.save_event(Event("event 4", date=date(year=2007, month=9, day=10), user_id="kodjo"))
+
+    events_from_1 = list(sdp1.load_events(-1))
+    events_from_2 = list(sdp2.load_events(-1))
+
+    assert [e.message for e in events_from_1] == ['event 4', 'event 3', 'event 2', 'event 1']
+    assert [e.message for e in events_from_2] == ['event 4', 'event 3', 'event 2', 'event 1']
+
 
 @pytest.mark.parametrize("root", [
     ".sheerka",
@@ -219,7 +243,7 @@ def test_i_can_add_and_reload_one_item(root):
     }
 
     assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
-    assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.HeadFile))
+    assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.RefFolder, sdp.name, SheerkaDataProvider.HeadFile))
 
     assert state.date is not None
     assert state.parents == []
@@ -227,7 +251,8 @@ def test_i_can_add_and_reload_one_item(root):
     assert state.data == {"entry": {'key': 'foo => bar', 'key2': ObjNoKey("a", "b")},
                           'entry2': {'key': 'value2'}}
 
-    assert sdp.io.read_text(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) == last_commit
+    assert sdp.io.read_text(
+        path.join(sdp.io.root, SheerkaDataProvider.RefFolder, sdp.name, SheerkaDataProvider.HeadFile)) == last_commit
 
 
 @pytest.mark.parametrize("root", [
@@ -419,7 +444,7 @@ def test_i_can_remove_elements(root):
     with sdp.get_transaction(evt_digest) as transaction:
         transaction.remove("entry", "key")
 
-    assert sdp.get("entry", "key") is None
+    assert sdp.get("entry", "key") is NotFound
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {
@@ -456,6 +481,25 @@ def test_i_can_keep_state_history(root):
     assert state.parents == []
 
 
+@pytest.mark.parametrize("root", [
+    ".sheerka",
+    "mem://"
+])
+def test_i_can_save_and_load_ontologies_names(root):
+    sdp = SheerkaDataProvider(root)
+
+    ontologies = ['new ontology', '#unit_test#', '__default__']
+    sdp.save_ontologies(ontologies)
+    assert sdp.load_ontologies() == ontologies
+
+    # extra
+    ontologies_files = path.join(sdp.io.root, SheerkaDataProvider.OntologiesFiles)
+    assert sdp.io.exists(ontologies_files)
+    assert sdp.io.read_text(ontologies_files) == """new ontology
+#unit_test#
+__default__"""
+
+
 def test_i_can_remove_even_if_not_exist():
     sdp = SheerkaDataProvider("mem://")
     with sdp.get_transaction(evt_digest) as transaction:
@@ -478,3 +522,14 @@ def test_exists():
     assert not sdp.exists("entry2")
     assert not sdp.exists("entry", "key2")
     assert sdp.exists("entry", "key")
+
+
+def test_not_found_is_returned_when_an_entry_is_not_found():
+    sdp = SheerkaDataProvider("mem://")
+
+    with sdp.get_transaction(evt_digest) as transaction:
+        transaction.add("entry", "key", "value")
+
+    assert sdp.get("entry", "key") == "value"
+    assert sdp.get("entry", "key2") == NotFound
+    assert sdp.get("entry2") == NotFound

tests/sdp/test_sheerkaSerializer.py

@@ -1,8 +1,9 @@
 import pytest
 from dataclasses import dataclass
 
+from core.global_symbols import NotInit, NotFound, Removed
 from sdp.sheerkaDataProvider import Event
-from sdp.sheerkaSerializer import Serializer, JsonSerializer, SerializerContext
+from sdp.sheerkaSerializer import Serializer, JsonSerializer, SerializerContext, CustomTypeSerializer
 from datetime import datetime
 import core.utils
 
@@ -50,6 +51,20 @@ def test_i_can_serialize_an_object():
     assert loaded.prop1 == "value"
 
 
+@pytest.mark.parametrize("custom_type", [
+    NotInit, NotFound, Removed
+])
+def test_i_can_serialize_custom_type(custom_type):
+    serializer = Serializer()
+    serializer.register(CustomTypeSerializer())
+    context = SerializerContext("kodjo", "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b")
+
+    stream = serializer.serialize(custom_type, context)
+    loaded = serializer.deserialize(stream, context)
+
+    assert loaded == custom_type
+
+
 @pytest.mark.parametrize("obj, expected", [
     (Obj("10", "value"), "tests.sdp.test_sheerkaSerializer.Obj")
 ])

tests/sheerkapickle/test_SheerkaPickler.py

@@ -2,6 +2,7 @@ import logging
 
 import pytest
 from core.concept import Concept
+from core.global_symbols import NotInit, NotFound, Removed
 from core.tokenizer import Keywords
 from sheerkapickle import tags
 from sheerkapickle.SheerkaPickler import SheerkaPickler
@@ -55,6 +56,19 @@ class TestSheerkaPickler(TestUsingMemoryBasedSheerka):
         decoded = SheerkaUnpickler(sheerka).restore(flatten)
         assert decoded == obj
 
+    @pytest.mark.parametrize("obj, expected", [
+        (NotInit, {tags.CUSTOM: NotInit.value}),
+        (NotFound, {tags.CUSTOM: NotFound.value}),
+        (Removed, {tags.CUSTOM: Removed.value}),
+    ])
+    def test_i_can_flatten_and_restore_custom_types(self, obj, expected):
+        sheerka = self.get_sheerka()
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == expected
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
     def test_i_can_flatten_and_restore_instances(self):
         sheerka = self.get_sheerka()
 
@@ -175,3 +189,18 @@ class TestSheerkaPickler(TestUsingMemoryBasedSheerka):
         flatten = SheerkaPickler(sheerka).flatten(obj)
         decoded = SheerkaUnpickler(sheerka).restore(flatten)
         assert decoded == Obj("foo", None, {"a": None, "b": None})
+
+    def test_ontology_are_not_serialized(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka.push_ontology(context, "new ontology")
+        ontology = sheerka.pop_ontology().body.body
+        obj = sheerka.ret(sheerka.name, True, ontology)
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == {
+            '_sheerka/obj': 'core.builtin_concepts.ReturnValueConcept',
+            'concept/id': ('__RETURN_VALUE', '43'),
+            'status': True,
+            'value': 'new ontology',
+            'who': '__SHEERKA'}

tests/sheerkapickle/test_sheerka_handlers.py

@@ -69,7 +69,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, concept)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == concept
-        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.variables": [["a", "value_a"], ["b", "value_b"]], "values": [["a", null], ["b", null]]}'
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.variables": [["a", "value_a"], ["b", "value_b"]], "values": [["a", {"_sheerka/custom": "**NotInit**"}], ["b", {"_sheerka/id": 1}]]}'
 
         concept = Concept("foo").init_key()
         sheerka.create_new_concept(self.get_context(sheerka), concept)