Implemented ConceptManager with concept creation, modification and deletion

tests/BaseTest.py

@@ -79,7 +79,7 @@ class BaseTest:
             else:
                 c.init_key()
                 sheerka.set_id_if_needed(c, False)
-                sheerka.add_in_cache(c)
+                sheerka.test_only_add_in_cache(c)
 
             result.append(c)
 
@@ -93,7 +93,7 @@ class BaseTest:
 
         if create_new:
             sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache.clear()
-            sheerka.cache_manager.delete(sheerka.CONCEPTS_KEYS_ENTRY, SheerkaRuleManager.RULE_IDS)
+            sheerka.cache_manager.delete(sheerka.OBJECTS_IDS_ENTRY, SheerkaRuleManager.RULE_IDS)
             with sheerka.sdp.get_transaction(context.event.get_digest()) as transaction:
                 transaction.clear(SheerkaRuleManager.FORMAT_RULE_ENTRY)
 
@@ -146,8 +146,6 @@ class BaseTest:
         """True ret_val + add concept in cache"""
         if isinstance(obj, Concept):
             obj.init_key()
-            if sheerka.has_key(obj.key):
-                sheerka.add_in_cache(obj)
         return sheerka.ret(who, True, obj)
 
     @staticmethod
@@ -180,7 +178,7 @@ class BaseTest:
             concept.get_metadata().definition_type = DEFINITION_TYPE_BNF
         concept.init_key()
         sheerka.set_id_if_needed(concept, False)
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
         return concept
 
     @staticmethod

tests/cache/test_cache.py

@@ -539,9 +539,121 @@ class TestCache(TestUsingMemoryBasedSheerka):
 
         assert cache.get("key") == "value"
         cache.delete("key")
-        assert cache.get("value") is None
+        assert cache.get("key") is None
         assert cache.to_remove == {"key"}
 
+    def test_i_can_delete_values_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+        cache.reset_events()
+
+        cache.delete("key", "fake_value")
+        assert cache.get("key") == {"value1", "value2"}
+        assert len(cache) == 2
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value1")
+        assert cache.get("key") == {"value2"}
+        assert cache.to_add == {"key"}
+        assert len(cache) == 1
+
+        cache.delete("key", "value2")
+        assert cache.get("key") is None
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_key_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+
+        cache.delete("key")
+        assert cache.get("key") is None
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_a_key_that_does_not_exists(self):
+        cache = SetCache()
+        cache.delete("key")
+
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_from_a_key_from_list_id_needed(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+        cache.put("key", "value11")
+        cache.put("key2", "value2")
+        cache.put("key2", "value22")
+        cache.put("key2", "value222")
+        cache.put("key3", "value3")
+        cache.put("key3", "value33")
+        cache.put("key4", "value4")
+        cache.reset_events()
+
+        assert len(cache) == 8
+
+        # I can remove a whole key
+        cache.delete("key")
+        assert cache.get("key") is None
+        assert len(cache) == 6
+        assert cache.to_remove == {"key"}
+        assert cache.to_add == set()
+
+        # I can remove an element while a list is remaining
+        cache.reset_events()
+        cache.delete("key2", "value22")
+        assert cache.get("key2") == ["value2", "value222"]
+        assert len(cache) == 5
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while a single element is remaining
+        cache.reset_events()
+        cache.delete("key3", "value33")
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 4
+        assert cache.to_add == {"key3"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while nothing remains
+        cache.reset_events()
+        cache.delete("key4", "value4")
+        assert cache.get("key4") is None
+        assert len(cache) == 3
+        assert cache.to_remove == {"key4"}
+        assert cache.to_add == set()
+
+        # I do not remove when the value is not the same
+        cache.reset_events()
+        cache.delete("key3", "value33")  # value33 was already remove
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 3
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_deleting_a_list_if_need_entry_that_does_not_exist_is_not_an_error(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+
+        cache.reset_events()
+        cache.delete("key3")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key3", "value")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value2")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
     def test_initialized_key_is_removed_when_the_entry_is_found(self):
         caches = [Cache(), ListCache(), ListIfNeededCache(), SetCache()]
 

tests/cache/test_cache_manager.py

@@ -1,7 +1,10 @@
+import pytest
 from cache.Cache import Cache
-from cache.CacheManager import CacheManager
+from cache.CacheManager import CacheManager, ConceptNotFound
 from cache.DictionaryCache import DictionaryCache
 from cache.ListCache import ListCache
+from cache.ListIfNeededCache import ListIfNeededCache
+from core.concept import Concept
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -109,3 +112,46 @@ class TestCacheManager(TestUsingMemoryBasedSheerka):
         cache.put(False, {"key": "value", "key2": "value2", "key3": "value3"})
         cache_manager.commit(context)
         assert sheerka.sdp.get("test") == {"key": "value", "key2": "value2", "key3": "value3"}
+
+    def test_i_can_remove_a_concept_from_concepts_caches(self):
+        cache_manager = CacheManager(True)
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
+
+        sheerka, context, one, two, three, two_bis = self.init_concepts("one", "two", "three", Concept("two", body="2"))
+
+        for concept in [one, two, three, two_bis]:
+            cache_manager.add_concept(concept)
+
+        # sanity check
+        cache_def = cache_manager.caches["id"]
+        assert cache_def.cache.copy() == {one.id: one, two.id: two, three.id: three, two_bis.id: two_bis}
+        cache_def = cache_manager.caches["key"]
+        assert cache_def.cache.copy() == {one.key: one, two.key: [two, two_bis], three.key: three}
+
+        for cache_name in cache_manager.concept_caches:
+            cache_manager.caches[cache_name].cache.reset_events()
+
+        cache_manager.remove_concept(sheerka.new(("two", two_bis.id)))
+
+        cache_def = cache_manager.caches["id"]
+        assert cache_def.cache.copy() == {one.id: one, two.id: two, three.id: three}
+        assert cache_def.cache.to_remove == {two_bis.id}
+        assert cache_def.cache.to_add == set()
+        assert len(cache_def.cache) == 3
+
+        cache_def = cache_manager.caches["key"]
+        assert cache_def.cache.copy() == {one.key: one, two.key: two, three.key: three}
+        assert cache_def.cache.to_remove == set()
+        assert cache_def.cache.to_add == {"two"}
+        assert len(cache_def.cache) == 3
+
+    def test_i_cannot_remove_a_concept_that_does_not_exists(self):
+        cache_manager = CacheManager(True)
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
+
+        with pytest.raises(ConceptNotFound) as ex:
+            cache_manager.remove_concept(Concept("foo", id="1001"))
+
+        assert ex.value.concept == Concept("foo", id="1001")

tests/core/test_SheerkaConceptAlgebra.py

@@ -20,7 +20,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka):
 
         assert isinstance(res, Concept)
         assert res.get_metadata().props == {BuiltinConcepts.ISA: {male, human},
-                                      BuiltinConcepts.HASA: {car, licence}, }
+                                            BuiltinConcepts.HASA: {car, licence}, }
 
     def test_can_add_concepts_when_property_already_exist(self):
         sheerka, context, man, human, king, male = self.init_concepts(
@@ -58,7 +58,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka):
         res = sheerka.csub(context, new_foo, new_bar)
         assert isinstance(res, Concept)
         assert res.get_metadata().props == {BuiltinConcepts.ISA: {isa2},
-                                      BuiltinConcepts.HASA: {hasa2}, }
+                                            BuiltinConcepts.HASA: {hasa2}, }
 
     def test_i_can_recognize_myself_when_using_sdp_repository(self):
         sheerka, context, foo, isa1, hasa1, = self.init_concepts("foo", "isa1", "has1",

tests/core/test_SheerkaConceptManager.py

@@ -0,0 +1,739 @@
+import pytest
+from cache.CacheManager import ConceptNotFound
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs, NotInit, \
+    DEFINITION_TYPE_BNF
+from core.sheerka.Sheerka import Sheerka
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \
+    UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced
+from parsers.BaseNodeParser import BaseNodeParser
+from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
+    def test_i_can_create_a_concept(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = self.get_default_concept()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        res = sheerka.create_new_concept(context, concept)
+        sheerka.cache_manager.commit(context)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_found = res.value.body
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
+
+        assert concept_found.key == "__var__0 + __var__1"
+        assert concept_found.id == "1001"
+        assert get_concept_attrs(concept) == ['a', 'b']
+
+        # saved in cache
+        assert service.has_id(concept.id)
+        assert service.has_key(concept.key)
+        assert service.has_name(concept.name)
+        assert service.has_hash(concept.get_definition_hash())
+
+        # I can get the concept using various index
+        assert sheerka.get_by_id(concept.id) == concept
+        assert sheerka.get_by_key(concept.key) == concept
+        assert sheerka.get_by_name(concept.name) == concept
+        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
+
+        # I can get by the first entry
+        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
+        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
+
+        # saved in sdp
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
+        assert sheerka.sdp.exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
+
+    def test_i_cannot_create_a_bnf_concept_that_references_a_concept_that_cannot_be_resolved(self):
+        sheerka, context, one_1, one_1_0 = self.init_concepts(Concept("one", body="1"), Concept("one", body="1.0"))
+        twenty_one = Concept("twenty one", definition="'twenty' one", definition_type=DEFINITION_TYPE_BNF)
+
+        res = sheerka.create_new_concept(context, twenty_one)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
+        assert res.value.body == ("key", "one")
+
+    def test_i_can_add_a_concept_when_name_differs_from_the_key(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a")
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+        sheerka.cache_manager.commit(context)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_found = res.value.body
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
+
+        assert concept_found.key == "hello __var__0"
+        assert concept_found.id == "1001"
+
+        # saved in cache
+        assert service.has_id(concept.id)
+        assert service.has_key(concept.key)
+        assert service.has_name(concept.name)
+        assert service.has_hash(concept.get_definition_hash())
+
+        # I can get the concept using various index
+        assert sheerka.get_by_id(concept.id) == concept
+        assert sheerka.get_by_key(concept.key) == concept
+        assert sheerka.get_by_name(concept.name) == concept
+        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
+
+        # saved in sdp
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
+        assert sheerka.sdp.exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello")
+
+    def test_i_cannot_add_the_same_concept_twice(self):
+        """
+        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+        assert res.value.body == concept
+
+    def test_i_can_get_a_newly_created_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        from_cache = sheerka.get_by_key(concept.key)
+        assert from_cache is not None
+        assert from_cache == concept
+
+        from_cache = sheerka.get_by_id(concept.id)
+        assert from_cache is not None
+        assert from_cache == concept
+
+    def test_i_can_get_list_of_concept_when_same_key_using_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.get_metadata().body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get_by_key(concept1.key)
+        assert len(result) == 2
+        assert result[0] == concept1
+        assert result[1] == concept2
+
+    def test_concept_that_references_itself_is_correctly_created(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo", body="foo")
+
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert res.status
+
+    def test_i_can_get_by_name_when_created_with_def_definition(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = self.from_def_concept("plus", "a plus b", ["a", "b"])
+
+        res = sheerka.create_new_concept(context, concept)
+
+        assert res.status
+        assert sheerka.get_by_name(concept.name) == concept
+        assert sheerka.get_by_name(concept.get_metadata().definition) == concept
+
+        concept = Concept(name="foo", definition="foo", definition_type=DEFINITION_TYPE_DEF)
+        res = sheerka.create_new_concept(context, concept)
+
+        assert res.status
+        assert sheerka.get_by_name(concept.name) == concept  # it's not a list, ie the entry is not duplicated
+
+    def test_i_can_get_first_token_when_not_a_letter(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = Concept("--filter a").def_var("a")
+
+        res = sheerka.create_new_concept(context, concept)
+        assert res.status
+
+        # I can get by the first entry
+        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
+        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
+
+    @pytest.mark.parametrize("expression", [
+        "--'filter' ('one' | 'two') ",
+        "'--filter' ('one' | 'two') ",
+    ])
+    def test_i_can_get_first_token_when_bnf_concept_and_not_a_letter(self, expression):
+        sheerka, context, bnf_concept = self.init_concepts(
+            Concept("foo", definition=expression),
+            create_new=True)
+
+        # I can get by the first entry
+        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
+        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
+
+    def test_concept_references_are_updated_1(self):
+        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
+            "one",
+            "two",
+            "number",
+            "twenty",
+            Concept("twenties", definition="twenty one | two 'hundred'"),
+            create_new=True
+        )
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) is None
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
+
+    def test_concept_references_are_updated_2(self):
+        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
+            "one",
+            "two",
+            "number",
+            "twenty",
+            Concept("twenties", definition="twenty number"),
+            create_new=True
+        )
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) is None
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) is None
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
+
+    @pytest.mark.parametrize("attr", [
+        "name",
+        "is_unique",
+        "body",
+        "where",
+        "pre",
+        "post",
+        "ret",
+        "definition",
+        "definition_type",
+        "desc",
+        "is_evaluated",
+        "need_validation",
+        "full_serialization",
+    ])
+    def test_i_can_modify_a_metadata_attribute(self, attr):
+        sheerka, context, foo = self.init_concepts("foo")
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {attr: "new value"}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert getattr(res.body.body.get_metadata(), attr) == "new value"
+
+    def test_i_can_modify_a_concept_when_at_least_one_attr_is_different(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo", body="a body"))
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"name": "foo", "body": "a body", "pre": "new pre"}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert getattr(res.body.body.get_metadata(), "name") == "foo"
+        assert getattr(res.body.body.get_metadata(), "body") == "a body"
+        assert getattr(res.body.body.get_metadata(), "pre") == "new pre"
+
+    def test_i_can_modify_add_a_property(self):
+        sheerka, context, one, foo = self.init_concepts("one", Concept("foo", props={BuiltinConcepts.ISA: {"value"}}))
+
+        res = sheerka.modify_concept(context, foo, to_add={"props": {BuiltinConcepts.ISA: "value2",
+                                                                     BuiltinConcepts.HASA: one}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.get_prop(BuiltinConcepts.ISA) == {"value", "value2"}
+        assert res.body.body.get_prop(BuiltinConcepts.HASA) == {sheerka.new("one")}
+
+    def test_i_can_modify_remove_a_property(self):
+        sheerka, context, foo = self.init_concepts(
+            Concept("foo", props={"a": {"value1", "value2", "value3"},
+                                  "b": {"value4"}}))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"props": {"a": {"value2", "value3"},
+                                                                        "b": "value4"}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.get_prop("a") == {"value1"}
+        assert res.body.body.get_prop("b") is None
+
+    def test_i_can_modify_add_variables(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a", "value"))
+
+        res = sheerka.modify_concept(context, foo, to_add={"variables": {"b": "some_value",
+                                                                         "a": "new_value",
+                                                                         "c": None}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.get_metadata().variables == [("a", "new_value"), ("b", "some_value"), ("c", None)]
+        assert res.body.body.values() == {"a": NotInit, "b": NotInit, "c": NotInit}
+
+    def test_i_can_modify_remove_variables(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b", "value").def_var("c"))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"variables": ["a", "c"]})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.get_metadata().variables == [("b", "value")]
+        assert res.body.body.values() == {"b": NotInit}
+
+    def test_i_can_modify_the_concept_source(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar")
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"body": "new value"}})
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert getattr(foo.get_metadata(), "body") is None
+
+        res = sheerka.modify_concept(context, bar, to_add={"meta": {"body": "new value"}}, modify_source=True)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert getattr(bar.get_metadata(), "body") == "new value"
+
+    def test_caches_are_updated_when_i_modify_the_properties_and_the_variables(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar", cache_only=False)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        to_add = {"meta": {"body": "metadata value"},
+                  "variables": {"var_name": "default value"},
+                  "props": {BuiltinConcepts.ISA: bar}}
+
+        res = sheerka.modify_concept(context, foo, to_add)
+        new_concept = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert new_concept.get_metadata().body == "metadata value"
+        assert new_concept.get_metadata().variables == [("var_name", "default value")]
+        assert new_concept.get_prop(BuiltinConcepts.ISA) == {bar}
+
+        # test that object
+        foo_from_sheerka = sheerka.get_by_key(new_concept.key)
+        assert foo_from_sheerka.get_metadata().body == "metadata value"
+        assert foo_from_sheerka.get_metadata().variables == [("var_name", "default value")]
+        assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == {bar}
+
+        # other caches are also updated
+        assert sheerka.get_by_id(new_concept.id).get_metadata().body == "metadata value"
+        assert sheerka.get_by_name(new_concept.name).get_metadata().body == "metadata value"
+        assert sheerka.get_by_hash(new_concept.get_definition_hash()).get_metadata().body == "metadata value"
+
+        # sdp is updated
+        sheerka.cache_manager.commit(context)
+        from_sdp = sheerka.sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
+        assert from_sdp.get_metadata().body == "metadata value"
+        assert from_sdp.get_metadata().variables == [("var_name", "default value")]
+        assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}
+
+        assert sheerka.sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).get_metadata().body == "metadata value"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).get_metadata().body == "metadata value"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_HASH_ENTRY,
+                               new_concept.get_definition_hash()).get_metadata().body == "metadata value"
+
+    def test_caches_are_update_when_i_modify_the_name(self):
+        sheerka, context, foo = self.init_concepts("foo", cache_only=False)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        sheerka.is_known(sheerka.get_by_name(foo.name))
+        sheerka.is_known(sheerka.get_by_key(foo.key))
+        sheerka.get_by_hash(foo.get_definition_hash())
+
+        to_add = {"meta": {"name": "bar"}}
+
+        res = sheerka.modify_concept(context, foo, to_add)
+        new_concept = res.body.body
+
+        assert new_concept.name == "bar"
+        assert sheerka.get_by_id(new_concept.id).name == "bar"
+        assert sheerka.get_by_key(new_concept.key).name == "bar"
+        assert sheerka.get_by_name(new_concept.name).name == "bar"
+        assert sheerka.get_by_hash(new_concept.get_definition_hash()).name == "bar"
+
+        assert not sheerka.is_known(sheerka.get_by_name(foo.name))
+        assert not sheerka.is_known(sheerka.get_by_key(foo.key))
+        assert not sheerka.is_known(sheerka.get_by_hash(foo.get_definition_hash()))
+
+        sheerka.cache_manager.commit(context)
+        assert sheerka.sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id).name == "bar"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).name == "bar"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).name == "bar"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_HASH_ENTRY, new_concept.get_definition_hash()).name == "bar"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, foo.key) is None
+        assert sheerka.sdp.get(service.CONCEPTS_BY_NAME_ENTRY, foo.name) is None
+        assert sheerka.sdp.get(service.CONCEPTS_BY_HASH_ENTRY, foo.get_definition_hash()) is None
+
+    def test_i_can_modify_a_concept_from_a_list_of_concepts(self):
+        sheerka, context, foo1, foo2 = self.init_concepts(
+            Concept("foo", body="1"),
+            Concept("foo", body="2"))
+
+        to_add = {"meta": {"body": "new_value"}}
+
+        res = sheerka.modify_concept(context, foo1, to_add)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+
+        new_concept = res.body.body
+
+        assert new_concept.id == foo1.id
+        assert res.body.body.get_metadata().body == "new_value"
+
+        assert sheerka.get_by_id(foo1.id).get_metadata().body == "new_value"
+        assert sheerka.get_by_id(foo2.id).get_metadata().body == "2"
+
+    def test_values_are_modified_when_variables_are_added_or_removed(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b"))
+
+        to_add = {"meta": {"body": "metadata value"},
+                  "variables": {"c": "default value"}}
+
+        to_remove = {"variables": ["a"]}
+
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+        res = sheerka.modify_concept(context, foo, to_add, to_remove)
+        new_concept = res.body.body
+
+        assert res.status
+        assert get_concept_attrs(foo) == ["b", "c"]
+        assert get_concept_attrs(new_concept) == ["b", "c"]
+
+    def test_key_is_modified_when_modifying_name_or_variables(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b"))
+
+        to_add = {"meta": {"name": "b bar c d"},
+                  "variables": {"c": None, "d": None}}
+
+        to_remove = {"variables": ["a"]}
+
+        res = sheerka.modify_concept(context, foo, to_add, to_remove)
+        new_concept = res.body.body
+
+        assert res.status
+        assert new_concept.key == "__var__0 bar __var__1 __var__2"
+
+    def test_key_is_modified_when_modifying_the_definition(self):
+        sheerka, context, foo = self.init_concepts(
+            Concept(name="foo", definition="foo a b", definition_type=DEFINITION_TYPE_DEF).def_var("a").def_var("b"))
+
+        to_add = {"meta": {"definition": "b bar c d"},
+                  "variables": {"c": None, "d": None}}
+
+        to_remove = {"variables": ["a"]}
+
+        res = sheerka.modify_concept(context, foo, to_add, to_remove)
+        new_concept = res.body.body
+
+        assert res.status
+        assert new_concept.key == "__var__0 bar __var__1 __var__2"
+
+    def test_bnf_is_modified_when_modifying_the_definition(self):
+        sheerka, context, one, two, foo = self.init_concepts(
+            "one",
+            "two",
+            Concept(name="foo", definition="'twenty' one"),
+            create_new=True
+        )
+
+        to_add = {"meta": {"definition": "'twenty' two"}}
+
+        res = sheerka.modify_concept(context, foo, to_add)
+        new_concept = res.body.body
+
+        assert res.status
+        assert new_concept.get_metadata().definition == "'twenty' two"
+        assert new_concept.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(two, rule_name='two'))
+
+    def test_concept_by_first_keyword_is_updated_after_concept_modification(self):
+        sheerka, context, foo, bar, baz = self.init_concepts(
+            Concept("foo"),
+            Concept("bar"),
+            Concept("baz", definition="foo"),
+            create_new=True)
+
+        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "foo": ["1001"],
+            "bar": ["1002"],
+            'c:|1001:': ['1003']}
+        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            'foo': ['1001', '1003'],
+            'bar': ['1002']}
+
+        to_add = {"meta": {"name": "bar"}}
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert res.status
+        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "bar": ["1002", "1001"],
+            'c:|1001:': ['1003']}
+        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            'bar': ['1002', '1001', '1003']}
+
+    def test_references_are_updated_after_concept_modification(self):
+        sheerka, context, one, twenty_one = self.init_concepts(
+            "onz",
+            Concept("twenty one", definition="'twenty' onz"),
+            create_new=True
+        )
+
+        assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(one, rule_name='onz'))
+
+        to_add = {"meta": {"name": "one"}}
+        res = sheerka.modify_concept(context, one, to_add)
+        modified = res.body.body
+
+        assert res.status
+
+        twenty_one = sheerka.get_by_name("twenty one")
+        assert twenty_one.get_metadata().definition == "'twenty' one"
+        assert twenty_one.get_bnf() is None
+
+        BaseNodeParser.ensure_bnf(context, twenty_one)
+        assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(modified, rule_name='one'))
+
+    def test_i_cannot_modify_without_any_modification(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        res = service.modify_concept(context, foo)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == NoModificationFound(foo)
+
+    def test_i_cannot_modify_forbidden_attributes(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        for attr in service.forbidden_meta:
+            res = service.modify_concept(context, foo, to_add={"meta": {attr: "new value"}})
+            assert not res.status
+            assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+            assert res.body.body == ForbiddenAttribute(attr)
+
+    def test_i_cannot_modify_unknown_attributes(self):
+        sheerka, context, foo = self.init_concepts("foo")
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"dummy": "new value"}})
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == UnknownAttribute("dummy")
+
+    def test_i_cannot_modify_if_all_new_values_are_the_same(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo", body="a body"))
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"name": "foo", "body": "a body"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == NoModificationFound(foo, {"name": "foo", "body": "a body"})
+
+    def test_i_cannot_remove_meta_attributes(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo"))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"meta": {"any_value": "foo"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == CannotRemoveMeta({"any_value": "foo"})
+
+    def test_i_cannot_remove_props_that_does_not_exists(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo"))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"props": {"any_value": "foo"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == UnknownAttribute("any_value")
+
+    def test_i_cannot_remove_props_value_that_does_not_exists(self):
+        # Need to returns an error, otherwise, we will save a concept that is not modified
+        sheerka, context, foo = self.init_concepts(Concept("foo", props={"a": {"value"}}))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"props": {"a": "dummy"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == ValueNotFound("a", "dummy")
+
+    def test_i_cannot_remove_variable_that_does_not_exists(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a"))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"variables": ["b"]})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == UnknownAttribute("b")
+
+    def test_i_cannot_modify_a_concept_that_is_not_known(self):
+        sheerka, context = self.init_concepts()
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"body": "new value"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT)
+
+    def test_i_can_get_and_set_attribute(self):
+        sheerka, context = self.init_concepts()
+        foo = Concept("foo")
+        prop = Concept("property")
+        bar = Concept("bar")
+
+        res = sheerka.set_attr(foo, prop, bar)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        assert sheerka.get_attr(foo, prop) == bar
+
+    def test_i_cannot_remove_a_concept_which_has_reference(self):
+        sheerka, context, one, twenty_one = self.init_concepts(
+            Concept("one"),
+            Concept("twenty one", definition="'twenty' one"),
+            create_new=True)
+
+        res = sheerka.remove_concept(context, one)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == ConceptIsReferenced([twenty_one])
+
+    def test_i_can_remove_a_concept(self):
+        sheerka, context, one = self.init_concepts(
+            Concept("one"),
+            create_new=True)
+
+        # sanity check
+        assert sheerka.get_by_id(one.id) == one
+        assert sheerka.get_by_name(one.name) == one
+        assert sheerka.get_by_key(one.key) == one
+        assert sheerka.get_by_hash(one.get_definition_hash()) == one
+
+        res = sheerka.remove_concept(context, one)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        assert sheerka.isinstance(sheerka.get_by_id(one.id), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_name(one.name), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)
+
+    def test_i_cannot_remove_a_concept_that_does_not_exist(self):
+        sheerka, context = self.init_concepts()
+        one = Concept("one", id="1001")
+
+        res = sheerka.remove_concept(context, one)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == ConceptNotFound(one)
+
+
+class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
+    def test_i_can_add_several_concepts(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        hello = Concept("Hello world a").def_var("a")
+        res = sheerka.create_new_concept(context, hello)
+        sheerka.cache_manager.commit(context)
+        assert res.status
+
+        sheerka = self.get_sheerka()  # another instance
+        context = self.get_context(sheerka)
+        greeting = Concept("Greeting a").def_var("a")
+        res = sheerka.create_new_concept(context, greeting)
+        sheerka.cache_manager.commit(context)
+        assert res.status
+
+        sheerka = self.get_sheerka()  # another instance again
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, hello.key)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, greeting.key)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, hello.id)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, greeting.id)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, "Hello world a")
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, "Greeting a")
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, hello.get_definition_hash())
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, greeting.get_definition_hash())
+
+        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello")
+        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting")
+
+    def test_i_cannot_add_the_same_concept_twice_using_sdp(self):
+        """
+        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
+        :return:
+        """
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(context, concept)
+        sheerka.cache_manager.commit(context)
+
+        sheerka.cache_manager.clear()
+        res = sheerka.create_new_concept(context, concept)
+
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+        assert res.value.body == concept
+
+    def test_new_entry_does_not_override_the_previous_ones(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        sheerka.create_new_concept(context, Concept("foo", body="1"))
+        sheerka.create_new_concept(context, Concept("foo", body="2"))
+        sheerka.cache_manager.commit(context)
+
+        assert len(sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 2
+
+        sheerka = self.get_sheerka()  # new instance
+        context = self.get_context(sheerka)
+        sheerka.create_new_concept(context, Concept("foo", body="3"))
+        sheerka.cache_manager.commit(context)
+
+        assert len(sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 3

tests/core/test_SheerkaCreateNewConcept.py

@@ -1,281 +0,0 @@
-import pytest
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs
-from core.sheerka.Sheerka import Sheerka
-
-from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
-from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-
-
-class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
-
-    def test_i_can_create_a_concept(self):
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = self.get_default_concept()
-
-        res = sheerka.create_new_concept(context, concept)
-        sheerka.cache_manager.commit(context)
-
-        assert res.status
-        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
-
-        concept_found = res.value.body
-        for prop in PROPERTIES_TO_SERIALIZE:
-            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
-
-        assert concept_found.key == "__var__0 + __var__1"
-        assert concept_found.id == "1001"
-        assert get_concept_attrs(concept) == ['a', 'b']
-
-        # saved in cache
-        assert sheerka.has_id(concept.id)
-        assert sheerka.has_key(concept.key)
-        assert sheerka.has_name(concept.name)
-        assert sheerka.has_hash(concept.get_definition_hash())
-
-        # I can get the concept using various index
-        assert sheerka.get_by_id(concept.id) == concept
-        assert sheerka.get_by_key(concept.key) == concept
-        assert sheerka.get_by_name(concept.name) == concept
-        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
-
-        # I can get by the first entry
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
-        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
-
-        # saved in sdp
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_KEY_ENTRY, concept.key)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
-
-    def test_i_can_add_a_concept_when_name_differs_from_the_key(self):
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a")
-
-        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
-        sheerka.cache_manager.commit(context)
-
-        assert res.status
-        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
-
-        concept_found = res.value.body
-        for prop in PROPERTIES_TO_SERIALIZE:
-            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
-
-        assert concept_found.key == "hello __var__0"
-        assert concept_found.id == "1001"
-
-        # saved in cache
-        assert sheerka.has_id(concept.id)
-        assert sheerka.has_key(concept.key)
-        assert sheerka.has_name(concept.name)
-        assert sheerka.has_hash(concept.get_definition_hash())
-
-        # I can get the concept using various index
-        assert sheerka.get_by_id(concept.id) == concept
-        assert sheerka.get_by_key(concept.key) == concept
-        assert sheerka.get_by_name(concept.name) == concept
-        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
-
-        # saved in sdp
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_KEY_ENTRY, concept.key)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello")
-
-    def test_i_cannot_add_the_same_concept_twice(self):
-        """
-        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
-        :return:
-        """
-        sheerka = self.get_sheerka()
-        concept = self.get_default_concept()
-
-        sheerka.create_new_concept(self.get_context(sheerka), concept)
-        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
-
-        assert not res.status
-        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
-        assert res.value.body == concept
-
-    def test_i_can_get_a_newly_created_concept(self):
-        sheerka = self.get_sheerka()
-        concept = self.get_default_concept()
-
-        sheerka.create_new_concept(self.get_context(sheerka), concept)
-
-        from_cache = sheerka.get_by_key(concept.key)
-        assert from_cache is not None
-        assert from_cache == concept
-
-        from_cache = sheerka.get_by_id(concept.id)
-        assert from_cache is not None
-        assert from_cache == concept
-
-    def test_i_can_get_list_of_concept_when_same_key_using_cache(self):
-        sheerka = self.get_sheerka()
-        concept1 = self.get_default_concept()
-        concept2 = self.get_default_concept()
-        concept2.get_metadata().body = "a+b"
-
-        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
-        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
-
-        assert res1.value.body.key == res2.value.body.key  # same key
-
-        result = sheerka.get_by_key(concept1.key)
-        assert len(result) == 2
-        assert result[0] == concept1
-        assert result[1] == concept2
-
-    def test_concept_that_references_itself_is_correctly_created(self):
-        sheerka = self.get_sheerka()
-        concept = Concept("foo", body="foo")
-
-        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
-
-        assert res.status
-
-    def test_i_can_get_by_name_when_created_with_def_definition(self):
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = self.from_def_concept("plus", "a plus b", ["a", "b"])
-
-        res = sheerka.create_new_concept(context, concept)
-
-        assert res.status
-        assert sheerka.get_by_name(concept.name) == concept
-        assert sheerka.get_by_name(concept.get_metadata().definition) == concept
-
-        concept = Concept(name="foo", definition="foo", definition_type=DEFINITION_TYPE_DEF)
-        res = sheerka.create_new_concept(context, concept)
-
-        assert res.status
-        assert sheerka.get_by_name(concept.name) == concept  # it's not a list, ie the entry is not duplicated
-
-    def test_i_can_get_first_token_when_not_a_letter(self):
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = Concept("--filter a").def_var("a")
-
-        res = sheerka.create_new_concept(context, concept)
-        assert res.status
-
-        # I can get by the first entry
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
-        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
-
-    @pytest.mark.parametrize("expression", [
-        "--'filter' ('one' | 'two') ",
-        "'--filter' ('one' | 'two') ",
-    ])
-    def test_i_can_get_first_token_when_bnf_concept_and_not_a_letter(self, expression):
-        sheerka, context, bnf_concept = self.init_concepts(
-            Concept("foo", definition=expression),
-            create_new=True)
-
-        # I can get by the first entry
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
-        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
-
-    def test_concept_references_are_updated_1(self):
-        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
-            "one",
-            "two",
-            "number",
-            "twenty",
-            Concept("twenties", definition="twenty one | two 'hundred'"),
-            create_new=True
-        )
-
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, one.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, two.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, number.id) is None
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
-
-    def test_concept_references_are_updated_2(self):
-        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
-            "one",
-            "two",
-            "number",
-            "twenty",
-            Concept("twenties", definition="twenty number"),
-            create_new=True
-        )
-
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, one.id) is None
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, two.id) is None
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, number.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
-
-
-class TestSheerkaCreateNewConceptFileBased(TestUsingFileBasedSheerka):
-    def test_i_can_add_several_concepts(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-        hello = Concept("Hello world a").def_var("a")
-        res = sheerka.create_new_concept(context, hello)
-        sheerka.cache_manager.commit(context)
-        assert res.status
-
-        sheerka = self.get_sheerka()  # another instance
-        context = self.get_context(sheerka)
-        greeting = Concept("Greeting a").def_var("a")
-        res = sheerka.create_new_concept(context, greeting)
-        sheerka.cache_manager.commit(context)
-        assert res.status
-
-        sheerka = self.get_sheerka()  # another instance again
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_KEY_ENTRY, hello.key)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_KEY_ENTRY, greeting.key)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, hello.id)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, greeting.id)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, "Hello world a")
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, "Greeting a")
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, hello.get_definition_hash())
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, greeting.get_definition_hash())
-
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello")
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting")
-
-    def test_i_cannot_add_the_same_concept_twice_using_sdp(self):
-        """
-        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
-        :return:
-        """
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = self.get_default_concept()
-
-        sheerka.create_new_concept(context, concept)
-        sheerka.cache_manager.commit(context)
-
-        sheerka.cache_manager.clear()
-        res = sheerka.create_new_concept(context, concept)
-
-        assert not res.status
-        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
-        assert res.value.body == concept
-
-    def test_new_entry_does_not_override_the_previous_ones(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-        sheerka.create_new_concept(context, Concept("foo", body="1"))
-        sheerka.create_new_concept(context, Concept("foo", body="2"))
-        sheerka.cache_manager.commit(context)
-
-        assert len(sheerka.sdp.get(Sheerka.CONCEPTS_BY_KEY_ENTRY, "foo")) == 2
-
-        sheerka = self.get_sheerka()  # new instance
-        context = self.get_context(sheerka)
-        sheerka.create_new_concept(context, Concept("foo", body="3"))
-        sheerka.cache_manager.commit(context)
-
-        assert len(sheerka.sdp.get(Sheerka.CONCEPTS_BY_KEY_ENTRY, "foo")) == 3

tests/core/test_SheerkaEvaluateConcept.py

@@ -375,6 +375,26 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.get_metadata().is_evaluated
         assert sheerka.objvalue(evaluated) == 2
 
+    def test_i_can_evaluate_a_concept_that_references_another_concept_twice(self):
+        """
+        Test that a new instance of concept is return when the metadata refers to a concept
+        :return:
+        """
+        sheerka, context, predicate, foo = self.init_concepts(
+            Concept("Sometimes True", body="in_context('a')"),
+            Concept("foo", pre="c:Sometimes True:"))
+
+        foo1 = sheerka.new("foo")
+        foo1 = sheerka.evaluate_concept(context, foo1)  # 'a' is not in context, so it fails
+
+        context2 = self.get_context(sheerka)
+        context2.add_to_protected_hints('a')
+        foo2 = sheerka.new("foo")
+        foo2 = sheerka.evaluate_concept(context2, foo2)  # 'a' in context + new instance of 'Sometimes True'
+
+        assert sheerka.isinstance(foo1, BuiltinConcepts.CONDITION_FAILED)
+        assert sheerka.isinstance(foo2, "foo")
+
     def test_i_can_reference_sheerka(self):
         sheerka = self.get_sheerka()
 
@@ -624,8 +644,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
             eval_body=True
         )
 
-        sheerka.add_in_cache(one_str)
-        sheerka.add_in_cache(one_digit)
+        sheerka.test_only_add_in_cache(one_str)
+        sheerka.test_only_add_in_cache(one_digit)
 
         evaluated = sheerka.evaluate_concept(context, one_digit)
         assert evaluated.key == one_digit.key

tests/core/test_SheerkaModifyConcept.py

@@ -1,121 +0,0 @@
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts, get_concept_attrs
-
-from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
-from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-
-
-class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka):
-
-    def test_i_can_modify_a_concept(self):
-        sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True, cache_only=False)
-
-        assert get_concept_attrs(foo) == []
-
-        foo_instance = sheerka.new("foo")
-        foo_instance.get_metadata().body = "metadata value"  # modify metadata
-        foo_instance.def_var("var_name", "default value")  # modify definition of variables
-        foo_instance.add_prop(BuiltinConcepts.ISA, bar)  # modify property
-        foo_instance.set_value(ConceptParts.BODY, "body value")  # modify value
-        foo_instance.set_value("var_name", "var value")  # modify value
-        res = sheerka.modify_concept(context, foo_instance)
-
-        assert res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
-        assert res.body.body.get_metadata().body == "metadata value"
-        assert res.body.body.get_metadata().variables == [("var_name", "default value")]
-        assert res.body.body.get_prop(BuiltinConcepts.ISA) == {bar}
-        assert res.body.body.body == "body value"
-        assert res.body.body.get_value("var_name") == "var value"
-        assert get_concept_attrs(foo) == ["var_name"]
-
-        # test that object
-        foo_from_sheerka = sheerka.get_by_key("foo")
-        assert foo_from_sheerka.get_metadata().body == "metadata value"
-        assert foo_from_sheerka.get_metadata().variables == [("var_name", "default value")]
-        assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == {bar}
-        assert foo_from_sheerka.body == "body value"
-        assert foo_from_sheerka.get_value("var_name") == "var value"
-
-        # other caches are also updated
-        assert sheerka.get_by_id(foo.id).get_metadata().body == "metadata value"
-        assert sheerka.get_by_name(foo.name).get_metadata().body == "metadata value"
-        assert sheerka.get_by_hash(foo_instance.get_definition_hash()).get_metadata().body == "metadata value"
-
-        # sdp can be updated
-        sheerka.cache_manager.commit(context)
-        from_sdp = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, foo.id)
-        assert from_sdp.get_metadata().body == "metadata value"
-        assert from_sdp.get_metadata().variables == [("var_name", "default value")]
-        assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}
-        assert from_sdp.body == "body value"
-        assert from_sdp.get_value("var_name") == "var value"
-
-    def test_i_cannot_modify_a_concept_that_does_not_exists(self):
-        sheerka, context = self.init_concepts()
-
-        foo = Concept("foo").init_key()
-        sheerka.set_id_if_needed(foo, False)
-
-        res = sheerka.modify_concept(context, foo)
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert res.body.body == ("id", foo.id)
-
-    def test_i_cannot_modify_a_concept_that_returns_an_error(self):
-        sheerka, context = self.init_concepts()
-
-        foo = Concept("foo").init_key()
-        res = sheerka.modify_concept(context, foo)
-
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
-
-    def test_i_cannot_modify_if_the_concept_has_not_changed(self):
-        sheerka, context, foo = self.init_concepts("foo", create_new=True)
-        res = sheerka.modify_concept(context, foo)
-
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
-
-    def test_i_can_modify_a_concept_that_is_in_a_list(self):
-        sheerka, context, foo1, foo2 = self.init_concepts(
-            Concept("foo", body="1"),
-            Concept("foo", body="2"), create_new=True)
-
-        foo2_instance = sheerka.new("foo")[1]
-        foo2_instance.get_metadata().body = "value"
-
-        res = sheerka.modify_concept(context, foo2_instance)
-        assert res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
-        assert res.body.body.get_metadata().body == "value"
-
-        foo_from_sheerka = sheerka.new("foo")
-        assert foo_from_sheerka[0].get_metadata().body == "1"
-        assert foo_from_sheerka[1].get_metadata().body == "value"
-
-    def test_i_can_get_and_set_attribute(self):
-        sheerka, context = self.init_concepts()
-        foo = Concept("foo")
-        prop = Concept("property")
-        bar = Concept("bar")
-
-        res = sheerka.set_attr(foo, prop, bar)
-        assert res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
-
-        assert sheerka.get_attr(foo, prop) == bar
-
-
-class TestSheerkaModifyConceptUsingFile(TestUsingFileBasedSheerka):
-
-    def test_i_can_modify_a_concept_from_a_new_sheerka(self):
-        sheerka, context, foo = self.init_concepts("foo", create_new=True)
-        sheerka.cache_manager.commit(context)
-
-        sheerka = self.get_sheerka()
-        foo.add_prop("a", "b")
-        res = sheerka.modify_concept(context, foo)
-
-        assert res.status

tests/core/test_SheerkaRuleManager.py

@@ -33,7 +33,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_create_a_new_rule(self, action_type, cache_entry):
         sheerka, context = self.init_concepts(cache_only=False)
-        previous_rules_number = sheerka.cache_manager.caches[sheerka.CONCEPTS_KEYS_ENTRY].cache.copy()[
+        previous_rules_number = sheerka.cache_manager.caches[sheerka.OBJECTS_IDS_ENTRY].cache.copy()[
             SheerkaRuleManager.RULE_IDS]
 
         rule = Rule(action_type, "name", "True", "Hello world")

tests/core/test_sheerka.py

@@ -4,6 +4,7 @@ import pytest
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept, AllBuiltinConcepts
 from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit
 from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.tokenizer import Token, TokenKind
 
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
@@ -342,16 +343,17 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
     def test_builtin_concepts_are_initialized(self):
         sheerka = self.get_sheerka()
+        service = sheerka.services[SheerkaConceptManager.NAME]
         for concept_name in AllBuiltinConcepts:
-            assert sheerka.has_key(str(concept_name))
-            assert sheerka.sdp.get(sheerka.CONCEPTS_BY_KEY_ENTRY, str(concept_name)) is not None
+            assert service.has_key(str(concept_name))
+            assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, str(concept_name)) is not None
 
         # I can get back data from the sdp when the cache is empty
         sheerka.cache_manager.clear()
 
         # caches are empty
-        assert not sheerka.has_id("1")
-        assert not sheerka.has_key(str(BuiltinConcepts.SHEERKA))
+        assert not service.has_id("1")
+        assert not service.has_key(str(BuiltinConcepts.SHEERKA))
 
         assert sheerka.get_by_id("1") == sheerka  # use sdp
 
@@ -359,11 +361,12 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
     def test_builtin_concepts_can_be_updated(self):
         sheerka = self.get_sheerka()
+        service = sheerka.services[SheerkaConceptManager.NAME]
         before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
         before_parsing.get_metadata().desc = "I have a description"
         before_parsing.get_metadata().full_serialization = True
         with sheerka.sdp.get_transaction("Test") as transac:
-            transac.add(sheerka.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True)
+            transac.add(service.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True)
 
         sheerka = self.get_sheerka()  # another fresh new instance
         before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
@@ -390,11 +393,12 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
     def test_i_can_retrieve_from_sdp_when_cache_is_reset(self):
         sheerka, context, concept = self.init_concepts(Concept("foo", body="1"))
+        service = sheerka.services[SheerkaConceptManager.NAME]
         sheerka.cache_manager.commit(context)
 
         sheerka.cache_manager.clear()
         sheerka.get_by_key("foo")
-        assert sheerka.has_key("foo")
+        assert service.has_key("foo")
 
         # It's also updated when sdp returns more than one element
         concept2 = Concept("foo", body="2")
@@ -403,20 +407,20 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
         sheerka.cache_manager.clear()
         assert len(sheerka.get_by_key("foo")) == 2
-        assert sheerka.has_key("foo")
+        assert service.has_key("foo")
 
         # updated when by_id
         sheerka.cache_manager.clear()
         assert sheerka.get_by_id("1001") == concept
-        assert sheerka.has_id("1001")
+        assert service.has_id("1001")
 
         sheerka.cache_manager.clear()
         assert sheerka.get_by_name("foo") == [concept, concept2]
-        assert sheerka.has_name("foo")
+        assert service.has_name("foo")
 
         sheerka.cache_manager.clear()
         assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
-        assert sheerka.has_hash(concept.get_definition_hash())
+        assert service.has_hash(concept.get_definition_hash())
 
     def test_get_by_key_retrieve_all_elements(self):
         sheerka, context, *concepts = self.init_concepts(

tests/evaluators/test_AddConceptInSetEvaluator.py

@@ -41,7 +41,7 @@ class TestAddConceptInSetEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         foo = Concept("foo")
         context.sheerka.set_id_if_needed(foo, False)
-        context.sheerka.add_in_cache(foo)
+        context.sheerka.test_only_add_in_cache(foo)
 
         ret_val = get_isa_ret_val("foo", "bar")
         res = AddConceptInSetEvaluator().eval(context, ret_val)

tests/evaluators/test_ConceptEvaluator.py

@@ -93,10 +93,10 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
     def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(self):
         context = self.get_context()
         context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-        context.sheerka.add_in_cache(Concept(name="one").init_key())
-        concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b")
-                                                    .def_var("a", "one")
-                                                    .def_var("b", "two").init_key())
+        context.sheerka.test_only_add_in_cache(Concept(name="one").init_key())
+        concept_plus = context.sheerka.test_only_add_in_cache(Concept(name="a plus b")
+                                                              .def_var("a", "one")
+                                                              .def_var("b", "two").init_key())
 
         evaluator = ConceptEvaluator()
         item = self.pretval(concept_plus)

tests/evaluators/test_PythonEvaluator.py

@@ -3,6 +3,7 @@ import ast
 import pytest
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.concept import Concept, CB
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
 from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError, NamesWithAttributesVisitor
@@ -18,6 +19,10 @@ def get_obj_name(obj):
     return obj.name
 
 
+def return_return_value(status):
+    return ReturnValueConcept("who", status, f"the value is {status}")
+
+
 def get_source_code_node(source_code, concepts=None):
     if concepts:
         for concept_name, concept in sorted(concepts.items(), key=lambda kv: len(kv[0]), reverse=True):
@@ -132,8 +137,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         evaluated = PythonEvaluator().eval(context, parsed)
 
         assert evaluated.status
-        assert sheerka.has_key("foo")
-
+        assert sheerka.services[SheerkaConceptManager.NAME].has_key("foo")
 
     def test_i_can_eval_ast_expression_that_references_concepts(self):
         """
@@ -141,7 +145,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         :return:
         """
         context = self.get_context()
-        context.sheerka.add_in_cache(Concept("foo", body="1"))
+        context.sheerka.test_only_add_in_cache(Concept("foo", body="1"))
 
         parsed = PythonParser().parse(context, ParserInput("foo + 2"))
         evaluated = PythonEvaluator().eval(context, parsed)
@@ -155,7 +159,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         :return:
         """
         context = self.get_context()
-        context.sheerka.add_in_cache(Concept("foo"))
+        context.sheerka.test_only_add_in_cache(Concept("foo"))
 
         parsed = PythonParser().parse(context, ParserInput("def a(b):\n    return b\na(c:foo:)"))
         evaluated = PythonEvaluator().eval(context, parsed)
@@ -178,7 +182,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_eval_concept_token(self):
         context = self.get_context()
-        context.sheerka.add_in_cache(Concept("foo", body="2"))
+        context.sheerka.test_only_add_in_cache(Concept("foo", body="2"))
         context.add_to_short_term_memory("get_obj_name", get_obj_name)
 
         parsed = PythonParser().parse(context, ParserInput("get_obj_name(c:foo:)"))
@@ -190,7 +194,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_eval_when_expect_success(self):
         context = self.get_context()
-        context.sheerka.add_in_cache(Concept("foo", body="2"))
+        context.sheerka.test_only_add_in_cache(Concept("foo", body="2"))
 
         parsed = PythonParser().parse(context, ParserInput("foo==2"))
         python_evaluator = PythonEvaluator()
@@ -334,6 +338,23 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.value == "Print return values"
 
+    @pytest.mark.parametrize("method, expected_status", [
+        ("return_return_value(True)", True),
+        ("return_return_value(False)", False),
+    ])
+    def test_i_can_eval_a_function_that_returns_a_return_value(self, method, expected_status):
+        context = self.get_context()
+        context.add_to_short_term_memory("return_return_value", return_return_value)
+
+        parsed = FunctionParser().parse(context, ParserInput(method))
+        python_evaluator = PythonEvaluator()
+        evaluated = python_evaluator.eval(context, parsed)
+        ret_val = return_return_value(expected_status)
+
+        assert evaluated.status == expected_status
+        assert evaluated.value == ret_val.body
+        assert ret_val in evaluated.parents
+
     @pytest.mark.parametrize("text, expected", [
         ("foo.bar.baz", [["foo", "bar", "baz"]]),
         ("foo.bar.baz; one.two.three", [["foo", "bar", "baz"]]),

tests/non_reg/test_sheerka_non_reg.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit, CC
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
 from evaluators.PythonEvaluator import PythonEvalError
@@ -29,7 +30,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
     def test_i_can_recognize_concept_with_python_body(self):
         sheerka = self.get_sheerka()
         concept = Concept(name="one", body="1")
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
 
         text = "one"
         res = sheerka.evaluate_user_input(text)
@@ -45,8 +46,8 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
         sheerka = self.get_sheerka()
         concept_one = Concept(name="one")
         concept_un = Concept(name="un", body="one")
-        sheerka.add_in_cache(concept_one)
-        sheerka.add_in_cache(concept_un)
+        sheerka.test_only_add_in_cache(concept_one)
+        sheerka.test_only_add_in_cache(concept_un)
 
         res = sheerka.evaluate_user_input("un")
         return_value = res[0].value
@@ -61,7 +62,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
     def test_i_can_recognize_concept_with_no_body(self):
         sheerka = self.get_sheerka()
         concept = Concept(name="one")
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
 
         text = "one"
         res = sheerka.evaluate_user_input(text)
@@ -73,7 +74,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
     def test_is_unique_property_is_used_when_evaluating(self):
         sheerka = self.get_sheerka()
         concept = Concept(name="one", is_unique=True)
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
 
         text = "one"
         res = sheerka.evaluate_user_input(text)
@@ -113,14 +114,15 @@ as:
             assert getattr(concept_saved.get_metadata(), prop) == getattr(expected.get_metadata(), prop)
 
         # cache is up to date
-        assert sheerka.has_key(concept_saved.key)
-        assert sheerka.has_id(concept_saved.id)
-        assert sheerka.has_name(concept_saved.name)
-        assert sheerka.has_hash(concept_saved.get_definition_hash())
+        service = sheerka.services[SheerkaConceptManager.NAME]
+        assert service.has_key(concept_saved.key)
+        assert service.has_id(concept_saved.id)
+        assert service.has_name(concept_saved.name)
+        assert service.has_hash(concept_saved.get_definition_hash())
         assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {'+': ['1001']}
 
         # sdp is up to date
-        assert sheerka.sdp.exists(sheerka.CONCEPTS_BY_KEY_ENTRY, expected.key)
+        assert sheerka.sdp.exists(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, expected.key)
 
     def test_i_can_evaluate_def_concept_part_when_one_part_is_a_ref_of_another_concept(self):
         """
@@ -132,7 +134,7 @@ as:
 
         # concept 'a plus b' is known
         concept_a_plus_b = Concept(name="a plus b").def_var("a").def_var("b").init_key()
-        sheerka.add_in_cache(concept_a_plus_b)
+        sheerka.test_only_add_in_cache(concept_a_plus_b)
 
         res = sheerka.evaluate_user_input("def concept a xx b as a plus b")
         expected = Concept(name="a xx b", body="a plus b").def_var("a").def_var("b").init_key()
@@ -147,7 +149,7 @@ as:
         for prop in PROPERTIES_TO_SERIALIZE:
             assert getattr(concept_saved.get_metadata(), prop) == getattr(expected.get_metadata(), prop)
 
-        assert sheerka.has_key(concept_saved.key)
+        assert sheerka.services[SheerkaConceptManager.NAME].has_key(concept_saved.key)
 
     def test_i_cannot_evaluate_the_same_def_concept_twice(self):
         text = """
@@ -201,8 +203,8 @@ as:
 
     def test_i_can_recognize_concept_with_variable_and_python_as_body(self):
         sheerka = self.get_sheerka()
-        hello_a = sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_var("a"))
-        sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
+        hello_a = sheerka.test_only_add_in_cache(Concept(name="hello a", body="'hello ' + a").def_var("a"))
+        sheerka.test_only_add_in_cache(Concept(name="foo", body="'foo'"))
 
         res = sheerka.evaluate_user_input("hello foo")
         assert len(res) == 1
@@ -600,7 +602,7 @@ as:
     ])
     def test_i_can_manage_tokenizer_error(self, text):
         sheerka = self.get_sheerka()
-        sheerka.add_in_cache(Concept("foo"))
+        sheerka.test_only_add_in_cache(Concept("foo"))
 
         res = sheerka.evaluate_user_input(text)
 
@@ -610,7 +612,7 @@ as:
 
     def test_i_can_recognize_concept_from_string(self):
         sheerka = self.get_sheerka()
-        sheerka.add_in_cache(Concept("one", body="1"))
+        sheerka.test_only_add_in_cache(Concept("one", body="1"))
 
         res = sheerka.evaluate_user_input("'one'")
 
@@ -792,7 +794,7 @@ as:
 
         sheerka = self.init_scenario(definitions)
         context = self.get_context(sheerka)
-        sheerka.force_sya_def(context, [
+        sheerka.test_only_force_sya_def(context, [
             (sheerka.get_by_name("mult").id, 20, SyaAssociativity.Right),
             (sheerka.get_by_name("plus").id, 10, SyaAssociativity.Right),
         ])
@@ -954,7 +956,7 @@ as:
 
         # simulate that sheerka was stopped and restarted
         sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
-        sheerka.cache_manager.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "twenties").set_compiled({})
+        sheerka.cache_manager.get(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, "twenties").set_compiled({})
 
         res = sheerka.evaluate_user_input("eval twenty one")
         assert res[0].status
@@ -971,6 +973,7 @@ as:
         res = sheerka.evaluate_user_input("set_isa(last_created_concept(), number)")
 
         assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
         assert sheerka.isa(sheerka.new("one"), sheerka.new("number"))
 
     def test_i_can_evaluate_sya_and_ret_concepts(self):
@@ -1219,7 +1222,7 @@ class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
         assert res[0].status
         assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
 
-        saved_concept = sheerka.sdp.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "plus")
+        saved_concept = sheerka.sdp.get(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, "plus")
         assert saved_concept.key == "plus"
         assert saved_concept.get_metadata().definition == "a ('plus' plus)?"
         assert "a" in saved_concept.values()

tests/parsers/test_BaseNodeParser.py

@@ -48,7 +48,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
 
         bar = Concept("bar").init_key()
         sheerka.set_id_if_needed(bar, False)
-        sheerka.add_in_cache(bar)
+        sheerka.test_only_add_in_cache(bar)
 
         concept = Concept("foo").init_key()
         concept.set_bnf(bnf)
@@ -65,11 +65,11 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
 
         bar = Concept("bar").init_key()
         sheerka.set_id_if_needed(bar, False)
-        sheerka.add_in_cache(bar)
+        sheerka.test_only_add_in_cache(bar)
 
         baz = Concept("baz").init_key()
         sheerka.set_id_if_needed(baz, False)
-        sheerka.add_in_cache(baz)
+        sheerka.test_only_add_in_cache(baz)
 
         foo = Concept("foo").init_key()
         foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux")))
@@ -96,7 +96,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
 
         bar = Concept("bar").init_key()
         sheerka.set_id_if_needed(bar, False)
-        sheerka.add_in_cache(bar)
+        sheerka.test_only_add_in_cache(bar)
 
         foo = Concept("foo").init_key()
         foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux")))

tests/parsers/test_DefConceptParser.py

@@ -503,6 +503,15 @@ from give me the date !
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
         assert res.value.body == ("key", "unknown")
 
+    def test_i_cannot_parse_bnf_definition_referencing_multiple_concepts_sharing_the_same_name(self):
+        text = "def concept twenty one from bnf 'twenty' one"
+        sheerka, context, parser, *concepts = self.init_parser(Concept("one", body="1"), Concept("one", body="1.0"))
+        res = parser.parse(context, ParserInput(text))
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
+        assert res.value.body == ("key", "one")
+
     @pytest.mark.parametrize("text", [
         'def concept "def concept x"',
         'def concept "def concept x" as x',

tests/parsers/test_ExactConceptParser.py

@@ -210,7 +210,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
     # def test_i_can_detect_concept_from_tokens(self):
     #     context = self.get_context(self.get_sheerka(singleton=True))
     #     concept = get_concept("hello world", [])
-    #     context.sheerka.add_in_cache(concept)
+    #     context.sheerka.test_only_add_in_cache(concept)
     #
     #     source = "hello world"
     #     results = ExactConceptParser().parse(context, list(Tokenizer(source)))

tests/parsers/test_SyaNodeParser.py

@@ -66,7 +66,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
                                                       cmap["minus"],
                                                       CONCEPT_COMPARISON_CONTEXT)
 
-        # TestSyaNodeParser.sheerka.force_sya_def(context, [
+        # TestSyaNodeParser.sheerka.test_only_force_sya_def(context, [
         #     (cmap["plus"].id, 5, SyaAssociativity.Right),
         #     (cmap["mult"].id, 10, SyaAssociativity.Right),
         #     (cmap["minus"].id, 5, SyaAssociativity.Right)])

tests/parsers/test_UnrecognizedNodeParser.py

@@ -76,7 +76,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
     def setup_class(cls):
         t = TestUnrecognizedNodeParser()
         TestUnrecognizedNodeParser.sheerka, context, _ = t.init_parser(concepts_map, create_new=True)
-        TestUnrecognizedNodeParser.sheerka.force_sya_def(context, [
+        TestUnrecognizedNodeParser.sheerka.test_only_force_sya_def(context, [
             (concepts_map["mult"].id, 20, SyaAssociativity.Right),
             (concepts_map["plus"].id, 10, SyaAssociativity.Right),
         ])

tests/sheerkapickle/test_SheerkaPickler.py

@@ -138,7 +138,7 @@ class TestSheerkaPickler(TestUsingMemoryBasedSheerka):
 
         concept = Concept("foo").init_key()
         sheerka.set_id_if_needed(concept, False)
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
         obj = {concept: "a"}
         flatten = SheerkaPickler(sheerka).flatten(obj)
         assert flatten == {'c:foo|1001:': 'a'}