Implemented ConceptManager with concept creation, modification and deletion

_concepts_admin.txt

@@ -35,7 +35,6 @@ set_auto_eval(c:debug variable x:)
 def concept debug method x as debug_var(method=x)
 set_auto_eval(c:debug method x:)
 
-set_auto_eval(c:activate debug on x:)
 def concept deactivate debug on x as debug_var(x, enabled=False) where x
 set_auto_eval(c:deactivate debug on x:)
 

src/cache/Cache.py

@@ -27,5 +27,6 @@ class Cache(BaseCache):
 
     def _delete(self, key, value):
         del(self._cache[key])
+        self._current_size -= 1
         self._add_to_remove(key)
 

src/cache/CacheManager.py

@@ -6,14 +6,22 @@ from cache.BaseCache import BaseCache
 from core.concept import Concept
 
 
+@dataclass
 class MultipleEntryError(Exception):
     """
     Exception raised when trying to alter an entry with multiple element
     without giving the origin of the element
     """
 
-    def __init__(self, key):
+    key: str
-        self.key = key
+
+
+@dataclass
+class ConceptNotFound(Exception):
+    """
+    Thrown when you try to remove a concept that is not found
+    """
+    concept: object
 
 
 @dataclass
@@ -127,6 +135,29 @@ class CacheManager:
         #         pass
         #     self.is_dirty = True
 
+    def remove_concept(self, concept):
+        """
+        Remove a concept from all caches
+        :param concept:
+        :return:
+        """
+        with self._lock:
+            # the first concept cache must the one where all concept are unique
+            # eg it has to be the concept by id
+            ref_cache_def = self.caches[self.concept_caches[0]]
+            concept_id = ref_cache_def.get_key(concept)
+            ref_concept = ref_cache_def.cache.get(concept_id)
+
+            if ref_concept is None:
+                raise ConceptNotFound(concept)
+
+            for cache_name in self.concept_caches:
+                cache_def = self.caches[cache_name]
+                key = cache_def.get_key(ref_concept)
+                cache_def.cache.delete(key, ref_concept)
+
+            self.is_dirty = True
+
     def get(self, cache_name, key):
         """
         From concept cache, get an entry

src/cache/ListIfNeededCache.py

@@ -54,3 +54,23 @@ class ListIfNeededCache(BaseCache):
             else:
                 self._cache[new_key] = new_value
             self._add_to_add(new_key)
+
+    def _delete(self, key, value):
+        if value is None:
+            self._current_size -= len(self._cache[key])
+            del self._cache[key]
+            self._add_to_remove(key)
+        else:
+            previous = self._cache[key]
+            if isinstance(previous, list):
+                previous.remove(value)
+                if len(previous) == 1:
+                    self._cache[key] = previous[0]
+                self._current_size -= 1
+                self.to_add.add(key)
+            else:
+                if previous == value:
+                    del self._cache[key]
+                    self._current_size -= 1
+                    self.to_remove.add(key)
+

src/cache/SetCache.py

@@ -49,3 +49,17 @@ class SetCache(BaseCache):
             self._cache[new_key].remove(old_value)
             self._put(new_key, new_value)
             self._add_to_add(new_key)
+
+    def _delete(self, key, value):
+        if value is None:
+            self._current_size -= len(self._cache[key])
+            del self._cache[key]
+            self._add_to_remove(key)
+        else:
+            self._cache[key].remove(value)
+            self._current_size -= 1
+            if len(self._cache[key]) == 0:
+                del self._cache[key]
+                self._add_to_remove(key)
+            else:
+                self._add_to_add(key)

src/core/sheerka/Sheerka.py

@@ -8,10 +8,7 @@ from cache.Cache import Cache
 from cache.CacheManager import CacheManager
 from cache.DictionaryCache import DictionaryCache
 from cache.IncCache import IncCache
-from cache.ListIfNeededCache import ListIfNeededCache
+from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, UnknownConcept
-from cache.SetCache import SetCache
-from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
-    UnknownConcept, AllBuiltinConcepts
 from core.concept import Concept, ConceptParts, NotInit, get_concept_attrs
 from core.error import ErrorObj
 from core.global_symbols import EVENT_USER_INPUT_EVALUATED
@@ -49,12 +46,8 @@ class Sheerka(Concept):
     Main controller for the project
     """
 
-    CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"  # to store all the concepts
+    CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID"
-    CONCEPTS_BY_KEY_ENTRY = "Concepts_By_Key"
+    CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name"
-    CONCEPTS_BY_NAME_ENTRY = "Concepts_By_Name"
-    CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash"  # store hash of concepts definitions (not values)
-
-    CONCEPTS_REFERENCES_ENTRY = "Concepts_References"  # tracks references between concepts
 
     CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
     RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Resolved_Concepts_By_First_Keyword"
@@ -63,13 +56,10 @@ class Sheerka(Concept):
     CONCEPTS_GRAMMARS_ENTRY = "Concepts_Grammars"
     CHICKEN_AND_EGG_CONCEPTS_ENTRY = "Chicken_And_Egg_Concepts"
 
-    CONCEPTS_KEYS_ENTRY = "Concepts_Keys"
+    OBJECTS_IDS_ENTRY = "Objects_Ids"
     BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts"  # sequential key for builtin concepts
     USER_CONCEPTS_KEYS = "User_Concepts"  # sequential key for user defined concepts
 
-    MAX_EXECUTION_HISTORY = 100
-    MAX_RETURN_VALUES_HISTORY = 100
-
     ALL_ATTRIBUTES = []
 
     def __init__(self, cache_only=False, debug=False, loggers=None):
@@ -191,6 +181,8 @@ class Sheerka(Concept):
             initialize_pickle_handlers()
 
             self.sdp = SheerkaDataProvider(root_folder, self)
+            self.builtin_cache = self.get_builtins_classes_as_dict()
+
             self.initialize_caching()
             self.get_builtin_parsers()
             self.get_builtin_evaluators()
@@ -233,31 +225,8 @@ class Sheerka(Concept):
 
     def initialize_caching(self):
 
-        def params(cache_name):
+        cache = IncCache(default=lambda k: self.sdp.get(self.OBJECTS_IDS_ENTRY, k))
-            return {
+        self.cache_manager.register_cache(self.OBJECTS_IDS_ENTRY, cache)
-                'default': lambda k: self.sdp.get(cache_name, k),
-                'extend_exists': lambda k: self.sdp.exists(cache_name, k)
-            }
-
-        cache = IncCache(default=lambda k: self.sdp.get(self.CONCEPTS_KEYS_ENTRY, k))
-        self.cache_manager.register_cache(self.CONCEPTS_KEYS_ENTRY, cache)
-
-        register_concept_cache = self.cache_manager.register_concept_cache
-
-        cache = Cache(**params(self.CONCEPTS_BY_ID_ENTRY))
-        register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
-
-        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_KEY_ENTRY))
-        register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
-
-        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_NAME_ENTRY))
-        register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
-
-        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
-        register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
-
-        cache = SetCache(default=lambda k: self.sdp.get(self.CONCEPTS_REFERENCES_ENTRY, k))
-        self.cache_manager.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache)
 
         cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, k))
         self.cache_manager.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
@@ -307,8 +276,6 @@ class Sheerka(Concept):
                 service.initialize_deferred(context, is_first_time)
 
     def first_time_initialisation(self, context):
-
-        self.cache_manager.put(self.CONCEPTS_KEYS_ENTRY, self.USER_CONCEPTS_KEYS, 1000)
         self.record_var(context, self.name, "save_execution_context", True)
 
     def initialize_builtin_concepts(self):
@@ -317,38 +284,11 @@ class Sheerka(Concept):
         :return: None
         """
         # self.init_log.debug("Initializing builtin concepts")
-        builtins_classes = self.get_builtins_classes_as_dict()
+        from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
-
+        concept_service = self.services[SheerkaConceptManager.NAME]
-        # this all initialization of the builtins seems to be little bit complicated
+        concepts_ids = concept_service.initialize_builtin_concepts()
-        # why do we need to update it from DB ?
+        self.return_value_concept_id = concepts_ids[BuiltinConcepts.RETURN_VALUE]
-        for key in AllBuiltinConcepts:
+        self.error_concept_id = concepts_ids[BuiltinConcepts.ERROR]
-            concept = self if key == BuiltinConcepts.SHEERKA \
-                else builtins_classes[str(key)]() if str(key) in builtins_classes \
-                else Concept(key, True, False, key)
-
-            if key in BuiltinUnique:
-                concept._metadata.is_unique = True
-                concept._metadata.is_evaluated = True
-
-            if not concept._metadata.is_unique and str(key) in builtins_classes:
-                self.builtin_cache[key] = builtins_classes[str(key)]
-
-            from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept._metadata.key)
-            if from_db is None:
-                # self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
-                self.set_id_if_needed(concept, True)
-                self.cache_manager.add_concept(concept)
-
-                if key == BuiltinConcepts.RETURN_VALUE:
-                    self.return_value_concept_id = concept.id
-                elif key == BuiltinConcepts.ERROR:
-                    self.error_concept_id = concept.id
-
-            else:
-                # self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
-                concept.update_from(from_db)
-
-        return
 
     def get_builtin_parsers(self):
         """
@@ -479,114 +419,6 @@ class Sheerka(Concept):
         """
         self.printer_handler.print(result, instructions)
 
-    def set_id_if_needed(self, obj: Concept, is_builtin: bool):
-        """
-        Set the key for the concept if needed
-        :param obj:
-        :param is_builtin:
-        :return:
-        """
-        if obj._metadata.id is not None:
-            return
-
-        key = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
-        obj._metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key))
-        # self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
-
-    def force_sya_def(self, context, list_of_def):
-        """
-        Set the precedence and/or the associativity of a concept
-        FOR TESTS PURPOSE. TO REMOVE EVENTUALLY
-        :param context:
-        :param list_of_def list of tuple(concept_id, precedence (int), SyaAssociativity)
-        :return:
-        """
-
-        # validate the entries
-        # If one entry is an invalid concept, rollback everything
-        for concept_id, precedence, associativity in list_of_def:
-            if concept_id == BuiltinConcepts.UNKNOWN_CONCEPT:
-                return self.ret(self.name,
-                                False,
-                                self.new(BuiltinConcepts.ERROR, body=f"Concept {concept_id} is not known"))
-
-        sya_def = self.cache_manager.copy(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY) or {}
-
-        # update the definitions
-        for concept_id, precedence, associativity in list_of_def:
-            if precedence is None and associativity is None:
-                try:
-                    del self.sya_definitions[concept_id]
-                except KeyError:
-                    pass
-            else:
-                sya_def[concept_id] = (precedence, associativity)
-
-        # put in cache
-        self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, False, sya_def)
-
-        return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
-
-    def add_in_cache(self, concept: Concept):
-        """
-        Adds a concept template in cache.
-        The cache is used as a proxy before looking at sdp
-        :param concept:
-        :return:
-        """
-
-        # sanity check
-        if concept.key is None:
-            concept.init_key()
-
-        if concept.key is None:
-            raise KeyError()
-
-        self.cache_manager.add_concept(concept)
-
-        return concept
-
-    def get_by_key(self, concept_key, concept_id=None):
-        concept_key = str(concept_key) if isinstance(concept_key, BuiltinConcepts) else concept_key
-        return self.internal_get("key", concept_key, self.CONCEPTS_BY_KEY_ENTRY, concept_id)
-
-    def get_by_name(self, concept_name, concept_id=None):
-        return self.internal_get("name", concept_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
-
-    def get_by_hash(self, concept_hash, concept_id=None):
-        return self.internal_get("hash", concept_hash, self.CONCEPTS_BY_HASH_ENTRY, concept_id)
-
-    def get_by_id(self, concept_id):
-        return self.internal_get("id", concept_id, self.CONCEPTS_BY_ID_ENTRY, None)
-
-    def internal_get(self, index_name, key, cache_name, concept_id=None):
-        """
-        Tries to find an entry
-        :param index_name: name of the index (ex by_id, by_key...)
-        :param key: index value
-        :param cache_name: name of the cache (ex Concepts_By_ID...)
-        :param concept_id: id of the concept if none, in case where there are multiple results
-        :return:
-        """
-
-        if key is None:
-            return ErrorConcept(f"Concept '{key}' is undefined.")
-
-        concepts = self.cache_manager.get(cache_name, key)
-        if concepts:
-            if concept_id is None:
-                return concepts
-
-            if not hasattr(concepts, "__iter__"):
-                return concepts
-
-            for c in concepts:
-                if c.id == concept_id:
-                    return c
-
-        metadata = [(index_name, key), ("id", concept_id)] if concept_id else (index_name, key)
-        return self.get_unknown(metadata)
-
     def resolve(self, concept):
         """
         Try to find a concept by its name, id, or c:: definition
@@ -655,12 +487,19 @@ class Sheerka(Concept):
         if isinstance(key, Token):
             if key.type == TokenKind.RULE:  # do not recognize rules !!!
                 return None
-
-            if key.value[1]:
-                concept = self.cache_manager.get(self.CONCEPTS_BY_ID_ENTRY, key.value[1])
             else:
-                concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key.value[0])
+                key = key.value
+        elif isinstance(key, str) and key.startswith("c:"):
+            key = core.utils.unstr_concept(key)
 
+        if isinstance(key, tuple):
+            if key == (None, None):
+                return None
+
+            if key[1]:
+                concept = self.cache_manager.get(self.CONCEPTS_BY_ID_ENTRY, key[1])
+            else:
+                concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key[0])
         else:
             concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key)
 
@@ -668,44 +507,6 @@ class Sheerka(Concept):
             return None
         return new_instances(concept) if return_new else concept
 
-    def has_id(self, concept_id):
-        """
-        Returns True if a concept with this id exists in cache
-        It does not search in the remote repository
-        :param concept_id:
-        :return:
-        """
-        if concept_id is None:
-            return False
-        return self.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
-
-    def has_key(self, concept_key):
-        """
-        Returns True if concept(s) with this key exist in cache
-        It does not search in the remote repository
-        :param concept_key:
-        :return:
-        """
-        return self.cache_manager.has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
-
-    def has_name(self, concept_name):
-        """
-        Returns True if concept(s) with this name exist in cache
-        It does not search in the remote repository
-        :param concept_name:
-        :return:
-        """
-        return self.cache_manager.has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
-
-    def has_hash(self, concept_hash):
-        """
-        Returns True if concept(s) with this hash exist in cache
-        It does not search in the remote repository
-        :param concept_hash:
-        :return:
-        """
-        return self.cache_manager.has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
-
     def new(self, concept_key, **kwargs):
         """
         Returns an instance of a new concept
@@ -816,21 +617,6 @@ class Sheerka(Concept):
 
         return (self.objvalue(obj) for obj in objs.body)
 
-    def value_by_concept(self, obj, concept):
-        if obj is None:
-            return None
-
-        if not isinstance(obj, Concept):
-            return None
-
-        if isinstance(concept, tuple) and obj.key in [str(key) for key in concept]:
-            return obj
-
-        if obj.key == str(concept):
-            return obj
-
-        return self.value_by_concept(obj.body, concept)
-
     def get_error(self, obj):
         if isinstance(obj, Concept) and obj._metadata.is_builtin and obj.key in BuiltinErrors:
             return obj
@@ -863,16 +649,6 @@ class Sheerka(Concept):
 
         return self.parsers_prefix + name
 
-    def test(self):
-        return f"I have access to Sheerka !"
-
-    def test_using_context(self, context, param):
-        event = context.event.get_digest()
-        return f"I have access to Sheerka ! {param=}, {event=}."
-
-    def test_error(self):
-        raise Exception("I can raise an error")
-
     @staticmethod
     def is_success(obj):
         if isinstance(obj, bool):  # quick win
@@ -1014,6 +790,69 @@ class Sheerka(Concept):
         }
         return self.new(BuiltinConcepts.TO_DICT, body=bag)
 
+    def test(self):
+        return f"I have access to Sheerka !"
+
+    def test_using_context(self, context, param):
+        event = context.event.get_digest()
+        return f"I have access to Sheerka ! {param=}, {event=}."
+
+    def test_error(self):
+        raise Exception("I can raise an error")
+
+    def test_only_force_sya_def(self, context, list_of_def):
+        """
+        Set the precedence and/or the associativity of a concept
+        FOR TESTS PURPOSE. TO REMOVE EVENTUALLY
+        :param context:
+        :param list_of_def list of tuple(concept_id, precedence (int), SyaAssociativity)
+        :return:
+        """
+
+        # validate the entries
+        # If one entry is an invalid concept, rollback everything
+        for concept_id, precedence, associativity in list_of_def:
+            if concept_id == BuiltinConcepts.UNKNOWN_CONCEPT:
+                return self.ret(self.name,
+                                False,
+                                self.new(BuiltinConcepts.ERROR, body=f"Concept {concept_id} is not known"))
+
+        sya_def = self.cache_manager.copy(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY) or {}
+
+        # update the definitions
+        for concept_id, precedence, associativity in list_of_def:
+            if precedence is None and associativity is None:
+                try:
+                    del self.sya_definitions[concept_id]
+                except KeyError:
+                    pass
+            else:
+                sya_def[concept_id] = (precedence, associativity)
+
+        # put in cache
+        self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, False, sya_def)
+
+        return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
+
+    def test_only_add_in_cache(self, concept: Concept):
+        """
+        Adds a concept template in cache.
+        The cache is used as a proxy before looking at sdp
+        :param concept:
+        :return:
+        """
+
+        # sanity check
+        if concept.key is None:
+            concept.init_key()
+
+        if concept.key is None:
+            raise KeyError()
+
+        self.cache_manager.add_concept(concept)
+
+        return concept
+
 
 def to_profile():
     sheerka = Sheerka()

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -172,7 +172,7 @@ class SheerkaComparisonManager(BaseService):
 
         cycles = self.detect_cycles(new)
         if cycles:
-            concepts_in_cycle = [self.sheerka.resolve(c) for c in cycles]
+            concepts_in_cycle = [self.sheerka.fast_resolve(c) for c in cycles]
             chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle)
             return self.sheerka.ret(self.NAME, False, chicken_an_egg)
 

src/core/sheerka/services/SheerkaConceptManager.py

@@ -0,0 +1,632 @@
+from dataclasses import dataclass
+
+import core.utils
+from cache.Cache import Cache
+from cache.CacheManager import ConceptNotFound
+from cache.ListIfNeededCache import ListIfNeededCache
+from cache.SetCache import SetCache
+from core.builtin_concepts import BuiltinConcepts, ErrorConcept, AllBuiltinConcepts, BuiltinUnique
+from core.builtin_helpers import ensure_concept
+from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, NotInit, \
+    ConceptMetadata
+from core.error import ErrorObj
+from core.global_symbols import EVENT_CONCEPT_CREATED
+from core.sheerka.services.sheerka_service import BaseService
+from core.tokenizer import Tokenizer, TokenKind
+from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
+
+BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
+
+
+@dataclass
+class NoModificationFound(ErrorObj):
+    """
+    Trying to modify a concept without modifying it
+    """
+    concept: Concept
+    attrs: dict = None
+
+
+@dataclass
+class ForbiddenAttribute(ErrorObj):
+    """
+    When trying to modify an attribute that must not be modified
+    """
+    attr: str
+
+
+@dataclass
+class UnknownAttribute(ErrorObj):
+    """
+    When trying to modify an attribute that does not exist
+    """
+    attr: str
+
+
+@dataclass
+class CannotRemoveMeta(ErrorObj):
+    """
+    When trying to remove a metadata attribute (ConceptMeta is a class, you cannot remove attr form it)
+    """
+    attrs: dict
+
+
+@dataclass
+class ValueNotFound(ErrorObj):
+    """
+    When trying to remove a value that does not exists (but the props/variable exists)
+    """
+    item: str
+    value: object
+
+
+@dataclass
+class ConceptIsReferenced(ErrorObj):
+    """
+    When trying to remove a concept that is referenced by other concept(s)
+    """
+    references: list
+
+
+class SheerkaConceptManager(BaseService):
+    NAME = "ConceptManager"
+
+    BUILTIN_CONCEPTS_IDS = "Builtins_Concepts_IDs"  # sequential key for builtin concepts
+    USER_CONCEPTS_IDS = "User_Concepts_IDs"  # sequential key for user defined concepts
+
+    CONCEPTS_IDS_ENTRY = "ConceptManager:Concepts_IDs"
+
+    CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID"  # to store all the concepts
+    CONCEPTS_BY_KEY_ENTRY = "ConceptManager:Concepts_By_Key"
+    CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name"
+    CONCEPTS_BY_HASH_ENTRY = "ConceptManager:Concepts_By_Hash"  # store hash of concepts definitions (not values)
+    CONCEPTS_REFERENCES_ENTRY = "ConceptManager:Concepts_References"  # tracks references between concepts
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS)  # BaseNodeParser
+        self.forbidden_meta = {"is_builtin", "key", "id", "props", "variables"}
+        self.allowed_meta = {attr for attr in vars(ConceptMetadata) if
+                             not attr.startswith("_") and attr not in self.forbidden_meta}
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.create_new_concept, True)
+        self.sheerka.bind_service_method(self.modify_concept, True)
+        self.sheerka.bind_service_method(self.remove_concept, True)
+        self.sheerka.bind_service_method(self.set_id_if_needed, True)
+        self.sheerka.bind_service_method(self.set_attr, True)
+        self.sheerka.bind_service_method(self.get_attr, False)
+        self.sheerka.bind_service_method(self.get_by_key, False, visible=False)
+        self.sheerka.bind_service_method(self.get_by_name, False, visible=False)
+        self.sheerka.bind_service_method(self.get_by_hash, False, visible=False)
+        self.sheerka.bind_service_method(self.get_by_id, False, visible=False)
+        self.sheerka.bind_service_method(self.not_is_variable, False, visible=False)
+
+        def params(cache_name):
+            return {
+                'default': lambda k: self.sheerka.sdp.get(cache_name, k),
+                'extend_exists': lambda k: self.sheerka.sdp.exists(cache_name, k)
+            }
+
+        register_concept_cache = self.sheerka.cache_manager.register_concept_cache
+
+        cache = Cache(**params(self.CONCEPTS_BY_ID_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
+
+        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_KEY_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
+
+        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_NAME_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
+
+        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
+
+        cache = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_REFERENCES_ENTRY, k))
+        self.sheerka.cache_manager.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache)
+
+    def initialize_deferred(self, context, is_first_time):
+        if is_first_time:
+            self.sheerka.cache_manager.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
+
+    def initialize_builtin_concepts(self):
+        """
+        Initializes the builtin concepts
+        :return: None
+        """
+        builtin_concepts_ids = {}
+
+        for key in AllBuiltinConcepts:
+            concept = self.sheerka if key == BuiltinConcepts.SHEERKA \
+                else self.sheerka.builtin_cache[str(key)]() if str(key) in self.sheerka.builtin_cache \
+                else Concept(key, True, False, key)
+
+            if key in BuiltinUnique:
+                concept.get_metadata().is_unique = True
+                concept.get_metadata().is_evaluated = True
+
+            from_db = self.sheerka.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key)
+            if from_db is None:
+                # self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
+                self.set_id_if_needed(concept, True)
+                self.sheerka.cache_manager.add_concept(concept)
+            else:
+                # self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
+                concept.update_from(from_db)
+
+            builtin_concepts_ids[key] = concept.id
+
+        return builtin_concepts_ids
+
+    def create_new_concept(self, context, concept: Concept):
+        """
+        Adds a new concept to the system
+        :param context:
+        :param concept:  DefConceptNode
+        :return: digest of the new concept
+        """
+        ensure_concept(concept)
+
+        sheerka = self.sheerka
+
+        concept.init_key()
+        init_bnf_ret_value = None
+
+        cache_manager = sheerka.cache_manager
+
+        if cache_manager.exists(self.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
+            error = SheerkaDataProviderDuplicateKeyError(self.CONCEPTS_BY_KEY_ENTRY + "." + concept.key, concept)
+            return sheerka.ret(
+                self.NAME,
+                False,
+                sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
+                error.args[0])
+
+        # set id before saving in db
+        sheerka.set_id_if_needed(concept, False)
+
+        # freeze attributes
+        freeze_concept_attrs(concept)
+
+        # check if the bnf definition is correctly computed
+        try:
+            self.bnp.ensure_bnf(context, concept)
+        except Exception as ex:
+            return sheerka.ret(self.NAME, False, ex.args[0])
+
+        # compute new concepts_by_first_keyword
+        init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True)
+        if not init_ret_value.status:
+            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
+        concepts_by_first_keyword = init_ret_value.body
+
+        # computes resolved concepts_by_first_keyword
+        init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
+        if not init_ret_value.status:
+            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
+        resolved_concepts_by_first_keyword = init_ret_value.body
+
+        # if everything is fine
+        concept.freeze_definition_hash()
+        cache_manager.add_concept(concept)
+        cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
+        cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
+
+        if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
+            # allow search by definition when definition relevant
+            cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept)
+
+        # update references
+        for ref in self.compute_references(concept):
+            cache_manager.put(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
+
+        # TODO : this line seems to be useless
+        # The grammar is never reset
+        if concept.get_bnf() and init_bnf_ret_value is not None and init_bnf_ret_value.status:
+            sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
+
+        # publish the new concept
+        sheerka.publish(context, EVENT_CONCEPT_CREATED, concept)
+
+        # process the return if needed
+        ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
+        return ret
+
+    def modify_concept(self, context, concept, to_add=None, to_remove=None, modify_source=False):
+        """
+        Modify the definition of a concept
+        :param context:
+        :param concept: concept to modify
+        :param to_add: meta, props or variables to add/update
+        :param to_remove: props or variables to remove
+        :param modify_source: update or not the concept given in parameter
+        :return:
+        """
+
+        # to_add is a dictionary
+        # to_add = {
+        #           'meta'     : {<key, value>} of metadata to add/update,
+        #           'props'    : {<key, value>} of properties to add/update,
+        #           'variables': {<key, value>} of variables to add/update,
+        #           }
+        # if the <key> already exists, the entry is updated, otherwise a new value is created
+        # for props, if the <key> already exists, a new entry is added to the set
+        #
+        # to_remove = {
+        #               'props'    : {<key, [value]>} entries to remove. 'value' can be a list or a single entry
+        #               'variables': [<key>] list of keys to remove
+        #              }
+        #
+        sheerka = self.sheerka
+        cache_manager = self.sheerka.cache_manager
+
+        if not to_add and not to_remove:
+            return sheerka.ret(self.NAME, False, sheerka.err(NoModificationFound(concept)))
+
+        if not sheerka.cache_manager.exists(self.CONCEPTS_BY_ID_ENTRY, concept.id):
+            return sheerka.ret(self.NAME, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept))
+
+        # modify the metadata. Almost all ConceptMetadata attributes except variables and props
+        new_concept = sheerka.new_from_template(concept, concept.key)  # reload from cache or database ?
+
+        res = self._update_concept(context, new_concept, to_add, to_remove)
+        if res is not None:
+            return res
+
+        freeze_concept_attrs(new_concept)
+
+        # To update concept by first keyword
+        # first remove the old references
+        keywords = self.bnp.get_first_tokens(sheerka, concept)  # keyword of the old concept
+        concepts_by_first_keyword = cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
+        for keyword in keywords:
+            try:
+                concepts_by_first_keyword[keyword].remove(concept.id)
+                if len(concepts_by_first_keyword[keyword]) == 0:
+                    del concepts_by_first_keyword[keyword]
+            except KeyError:  # only occurs in unit tests when concepts are created without create_new()
+                pass
+
+        # and then update
+        init_ret_value = self.bnp.get_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
+        if not init_ret_value.status:
+            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
+        concepts_by_first_keyword = init_ret_value.body
+
+        # computes resolved concepts_by_first_keyword
+        init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context,
+                                                                    concepts_by_first_keyword,
+                                                                    {new_concept.id: new_concept})
+        if not init_ret_value.status:
+            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
+        resolved_concepts_by_first_keyword = init_ret_value.body
+
+        # update concept that referenced the old concept and clear old references
+        self.update_references(context, concept, new_concept, to_add)
+        for ref in self.compute_references(concept):
+            cache_manager.delete(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
+
+        # compute new references
+        for ref in self.compute_references(new_concept):
+            cache_manager.put(self.CONCEPTS_REFERENCES_ENTRY, ref, new_concept.id)
+
+        cache_manager.update_concept(concept, new_concept)
+        cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
+        cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False,
+                          resolved_concepts_by_first_keyword)
+
+        # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
+        # TODO : Update concepts grammars : have a look at update_references() below
+        if modify_source:
+            self._update_concept(context, concept, to_add, to_remove)
+
+        ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=new_concept))
+        return ret
+
+    def remove_concept(self, context, concept):
+        """
+        Remove a concept
+        :param context:
+        :param concept:
+        :return:
+        """
+        sheerka = context.sheerka
+        refs = self.sheerka.cache_manager.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
+        if refs:
+            refs_instances = [sheerka.new_from_template(c, c.key) for c in [self.get_by_id(ref) for ref in refs]]
+            return sheerka.ret(self.NAME, False, sheerka.err(ConceptIsReferenced(refs_instances)))
+
+        try:
+            sheerka.cache_manager.remove_concept(concept)
+            return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS))
+        except ConceptNotFound as ex:
+            return sheerka.ret(self.NAME, False, sheerka.err(ex))
+
+    def set_attr(self, concept, attribute, value):
+        """
+        Modifies an attribute of a concept (concept.values)
+        :param context:
+        :param concept:
+        :param attribute:
+        :param value:
+        :return:
+        """
+        ensure_concept(concept)
+
+        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
+        concept.set_value(attr, value)
+        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def get_attr(self, concept, attribute):
+        """
+        Returns the attribute of a concept
+        :param context:
+        :param concept:
+        :param attribute:
+        :return:
+        """
+        ensure_concept()
+        if not self.sheerka.is_success(concept):
+            return concept
+
+        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
+
+        if (value := concept.get_value(attr)) == NotInit:
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute})
+        return value
+
+    def set_id_if_needed(self, obj: Concept, is_builtin: bool):
+        """
+        Set the key for the concept if needed
+        :param obj:
+        :param is_builtin:
+        :return:
+        """
+        if obj.get_metadata().id is not None:
+            return
+
+        entry_key = self.BUILTIN_CONCEPTS_IDS if is_builtin else self.USER_CONCEPTS_IDS
+        obj.get_metadata().id = str(self.sheerka.cache_manager.get(self.sheerka.OBJECTS_IDS_ENTRY, entry_key))
+        # self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
+
+    def get_by_key(self, concept_key, concept_id=None):
+        concept_key = str(concept_key) if isinstance(concept_key, BuiltinConcepts) else concept_key
+        return self.internal_get("key", concept_key, self.CONCEPTS_BY_KEY_ENTRY, concept_id)
+
+    def get_by_name(self, concept_name, concept_id=None):
+        return self.internal_get("name", concept_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
+
+    def get_by_hash(self, concept_hash, concept_id=None):
+        return self.internal_get("hash", concept_hash, self.CONCEPTS_BY_HASH_ENTRY, concept_id)
+
+    def get_by_id(self, concept_id):
+        return self.internal_get("id", concept_id, self.CONCEPTS_BY_ID_ENTRY, None)
+
+    def has_id(self, concept_id):
+        """
+        Returns True if a concept with this id exists in cache
+        It does not search in the remote repository
+        :param concept_id:
+        :return:
+        """
+        if concept_id is None:
+            return False
+        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
+
+    def has_key(self, concept_key):
+        """
+        Returns True if concept(s) with this key exist in cache
+        It does not search in the remote repository
+        :param concept_key:
+        :return:
+        """
+        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
+
+    def has_name(self, concept_name):
+        """
+        Returns True if concept(s) with this name exist in cache
+        It does not search in the remote repository
+        :param concept_name:
+        :return:
+        """
+        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
+
+    def has_hash(self, concept_hash):
+        """
+        Returns True if concept(s) with this hash exist in cache
+        It does not search in the remote repository
+        :param concept_hash:
+        :return:
+        """
+        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
+
+    def internal_get(self, index_name, key, cache_name, concept_id=None):
+        """
+        Tries to find an entry
+        :param index_name: name of the index (ex by_id, by_key...)
+        :param key: index value
+        :param cache_name: name of the cache (ex Concepts_By_ID...)
+        :param concept_id: id of the concept if none, in case where there are multiple results
+        :return:
+        """
+
+        if key is None:
+            return ErrorConcept(f"Concept '{key}' is undefined.")
+
+        concepts = self.sheerka.cache_manager.get(cache_name, key)
+        if concepts:
+            if concept_id is None:
+                return concepts
+
+            if not hasattr(concepts, "__iter__"):
+                return concepts
+
+            for c in concepts:
+                if c.id == concept_id:
+                    return c
+
+        metadata = [(index_name, key), ("id", concept_id)] if concept_id else (index_name, key)
+        return self.sheerka.get_unknown(metadata)
+
+    def update_references(self, context, concept, modified_concept=None, modifications=None):
+        """
+        Updates all the concepts that reference concept
+        :param context:
+        :param concept:
+        :param modified_concept:
+        :param modifications:
+        :return:
+        """
+
+        refs = self.sheerka.cache_manager.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
+        if not refs:
+            return
+
+        for concept_id in refs:
+            # remove the grammar entry so that it can be recreated
+            self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
+
+            # reset the bnf definition if needed
+            if modified_concept:
+                if self.has_id(concept_id):
+                    to_update = self.get_by_id(concept_id)
+                    metadata = to_update.get_metadata()
+                    if metadata.definition_type == DEFINITION_TYPE_BNF and self._name_has_changed(modifications):
+                        tokens = list(Tokenizer(metadata.definition))
+                        modified = False
+                        for i, token in enumerate(tokens):
+                            if token.type == TokenKind.IDENTIFIER and token.value == concept.name:
+                                clone = token.clone()
+                                clone.value = modified_concept.name
+                                tokens[i] = clone
+                                modified = True
+
+                        if modified:
+                            to_update.get_metadata().definition = core.utils.get_text_from_tokens(tokens)
+                            to_update.set_bnf(None)
+
+    def compute_references(self, concept):
+        """
+        We need to keep a track of all concepts used by the current concept
+        So that if one of these are modified, we can modify the current concept accordingly
+        :param concept:
+        :return:
+        """
+        refs = set()
+
+        if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
+            from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
+            other_concepts_visitor = BnfNodeConceptExpressionVisitor()
+            other_concepts_visitor.visit(concept.get_bnf())
+
+            for concept in other_concepts_visitor.references:
+                if isinstance(concept, str):
+                    concept = self.sheerka.get_by_key(concept)
+                refs.add(concept.id)
+
+        return refs
+
+    def not_is_variable(self, name):
+        """
+        Given a name tells if it refers to a variable name
+        :param name:
+        :return:
+        """
+        return not self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name)
+
+    @staticmethod
+    def _name_has_changed(to_add):
+        if to_add is None or "meta" not in to_add:
+            return False
+
+        return "name" in to_add["meta"]
+
+    @staticmethod
+    def _definition_has_changed(to_add):
+        if to_add is None or "meta" not in to_add:
+            return False
+
+        return "definition" in to_add["meta"]
+
+    def _update_concept(self, context, concept, to_add, to_remove):
+        sheerka = context.sheerka
+        same_values = {}
+
+        if to_add:
+            if "meta" in to_add:
+                # All modifications must be allowed
+                metadata = concept.get_metadata()
+
+                for k, v in to_add["meta"].items():
+                    if k in self.forbidden_meta:
+                        return sheerka.ret(self.NAME, False, sheerka.err(ForbiddenAttribute(k)))
+
+                    try:
+                        if getattr(metadata, k) == v:
+                            same_values[k] = v
+                        else:
+                            setattr(metadata, k, v)
+                    except AttributeError:
+                        return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k)))
+
+                if same_values == to_add["meta"]:
+                    return sheerka.ret(self.NAME, False,
+                                       sheerka.err(NoModificationFound(concept, same_values)))
+
+            if "props" in to_add:
+                for k, v in to_add["props"].items():
+                    concept.add_prop(k, v)
+
+            if "variables" in to_add:
+                for k, v in to_add["variables"].items():
+                    # update existing or add new
+                    for i, var in enumerate(concept.get_metadata().variables):
+                        if var[0] == k:
+                            concept.get_metadata().variables[i] = (k, v)
+                            break
+                    else:
+                        concept.def_var(k, v)
+
+        if to_remove:
+            if "meta" in to_remove:
+                return sheerka.ret(self.NAME, False, sheerka.err(CannotRemoveMeta(to_remove["meta"])))
+
+            if "props" in to_remove:
+                for k, v in to_remove["props"].items():
+                    if k not in concept.get_metadata().props:
+                        return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k)))
+
+                    props = concept.get_metadata().props[k]
+                    try:
+                        if isinstance(v, (set, list, tuple)):
+                            for item in v:
+                                props.remove(item)
+                        else:
+                            props.remove(v)
+
+                        # remove empty sets
+                        if len(props) == 0:
+                            del concept.get_metadata().props[k]
+                    except KeyError:
+                        return sheerka.ret(self.NAME, False, sheerka.err(ValueNotFound(k, v)))
+
+            if "variables" in to_remove:
+                variables_to_remove = []
+                for k in to_remove["variables"]:
+                    for var_def in concept.get_metadata().variables:
+                        if var_def[0] == k:
+                            variables_to_remove.append(var_def)
+                            delattr(concept, var_def[0])
+                            break
+                    else:
+                        return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k)))
+                core.utils.remove_list_from_list(concept.get_metadata().variables, variables_to_remove)
+
+        concept.get_metadata().key = None
+        if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
+            concept.set_bnf(None)
+            self.bnp.ensure_bnf(context, concept)
+
+        concept.init_key()
+
+        return

src/core/sheerka/services/SheerkaConceptsAlgebra.py

@@ -5,6 +5,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import ensure_concept
 from core.concept import Concept
 from core.sheerka.Sheerka import Sheerka
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.sheerka_service import BaseService
 
 PROPERTIES_TO_COMPUTE = [BuiltinConcepts.ISA, BuiltinConcepts.HASA]
@@ -118,8 +119,10 @@ class SheerkaConceptsAlgebra(BaseService):
         if nb_props == 0:
             return res
 
-        all_concepts = self.sheerka.cache_manager.copy(Sheerka.CONCEPTS_BY_ID_ENTRY).values() \
+        concepts_service = self.sheerka.services[SheerkaConceptManager.NAME]
-            if self.sheerka.cache_manager.cache_only else self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY)
+
+        all_concepts = self.sheerka.cache_manager.copy(concepts_service.CONCEPTS_BY_ID_ENTRY).values() \
+            if self.sheerka.cache_manager.cache_only else self.sheerka.sdp.list(concepts_service.CONCEPTS_BY_ID_ENTRY)
 
         for c in all_concepts:
             score = self._compute_score(c, concept, step_b=round(1 / nb_props, 2))

src/core/sheerka/services/SheerkaCreateNewConcept.py

@@ -1,123 +0,0 @@
-import core.utils
-from core.builtin_concepts import BuiltinConcepts, ErrorConcept
-from core.builtin_helpers import ensure_concept
-from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs
-from core.global_symbols import EVENT_CONCEPT_CREATED
-from core.sheerka.services.sheerka_service import BaseService
-from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
-
-BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
-
-
-class SheerkaCreateNewConcept(BaseService):
-    """
-    Manages the creation of a new concept
-    """
-
-    NAME = "CreateNewConcept"
-
-    def __init__(self, sheerka):
-        super().__init__(sheerka)
-        self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS)  # BaseNodeParser
-
-    def initialize(self):
-        self.sheerka.bind_service_method(self.create_new_concept, True)
-        self.sheerka.bind_service_method(self.not_is_variable, False, visible=False)
-
-    def create_new_concept(self, context, concept: Concept):
-        """
-        Adds a new concept to the system
-        :param context:
-        :param concept:  DefConceptNode
-        :return: digest of the new concept
-        """
-        ensure_concept(concept)
-
-        sheerka = self.sheerka
-
-        concept.init_key()
-        init_bnf_ret_value = None
-
-        cache_manager = sheerka.cache_manager
-
-        if cache_manager.exists(sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
-            error = SheerkaDataProviderDuplicateKeyError(sheerka.CONCEPTS_BY_KEY_ENTRY + "." + concept.key,
-                                                         concept)
-            return sheerka.ret(
-                self.NAME,
-                False,
-                sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
-                error.args[0])
-
-        # set id before saving in db
-        sheerka.set_id_if_needed(concept, False)
-
-        # freeze attributes
-        freeze_concept_attrs(concept)
-
-        # compute new concepts_by_first_keyword
-        init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True)
-        if not init_ret_value.status:
-            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
-        concepts_by_first_keyword = init_ret_value.body
-
-        # computes resolved concepts_by_first_keyword
-        init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
-        if not init_ret_value.status:
-            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
-        resolved_concepts_by_first_keyword = init_ret_value.body
-
-        # if everything is fine
-        concept.freeze_definition_hash()
-        cache_manager.add_concept(concept)
-        cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
-        cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
-
-        if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
-            # allow search by definition when definition relevant
-            cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept)
-
-        # update references
-        for ref in self.compute_references(concept):
-            cache_manager.put(sheerka.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
-
-        # TODO : this line seems to be useless
-        # The grammar is never reset
-        if concept.get_bnf() and init_bnf_ret_value is not None and init_bnf_ret_value.status:
-            sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
-
-        # publish the new concept
-        sheerka.publish(context, EVENT_CONCEPT_CREATED, concept)
-
-        # process the return if needed
-        ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
-        return ret
-
-    def compute_references(self, concept):
-        """
-        We need to keep a track of all concepts used by the current concept
-        So that if one of these are modified, we can modify the current concept accordingly
-        :param concept:
-        :return:
-        """
-        refs = set()
-
-        if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
-            from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
-            other_concepts_visitor = BnfNodeConceptExpressionVisitor()
-            other_concepts_visitor.visit(concept.get_bnf())
-
-            for concept in other_concepts_visitor.references:
-                if isinstance(concept, str):
-                    concept = self.sheerka.get_by_key(concept)
-                refs.add(concept.id)
-
-        return refs
-
-    def not_is_variable(self, name):
-        """
-        Given a name tells if it refers to a variable name
-        :param name:
-        :return:
-        """
-        return not self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name)

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -4,6 +4,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate, ensure_concept
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, AllConceptParts, \
     concept_part_value
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
@@ -224,13 +225,12 @@ class SheerkaEvaluateConcept(BaseService):
         """
 
         def is_only_successful(r):
-            #            return False
             return context.sheerka.isinstance(r, BuiltinConcepts.RETURN_VALUE) and \
                    context.sheerka.isinstance(r.body, BuiltinConcepts.ONLY_SUCCESSFUL)
 
         def parse_token_concept(s):
             if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None):
-                return self.sheerka.resolve(identifier)
+                return self.sheerka.fast_resolve(identifier)
             return None
 
         for part_key in AllConceptParts:
@@ -247,11 +247,12 @@ class SheerkaEvaluateConcept(BaseService):
             if source.strip() == "":
                 concept.get_compiled()[part_key] = DoNotResolve(source)
             else:
-                # first case, when the metadata references another concept via c:xxx: keyword
                 if concept_found := parse_token_concept(source):
+                    # the  compiled can be a reference to another concept...
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
                     concept.get_compiled()[part_key] = concept_found
                 else:
+                    # ...or a list of ReturnValueConcept to resolve
                     res = parse_unrecognized(context,
                                              source,
                                              parsers="all",
@@ -272,11 +273,12 @@ class SheerkaEvaluateConcept(BaseService):
             if default_value.strip() == "":
                 concept.get_compiled()[var_name] = DoNotResolve(default_value)
             else:
-                # first case, when the metadata references another concept via c:xxx: keyword
                 if concept_found := parse_token_concept(default_value):
+                    # the  compiled can be a reference to another concept...
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
                     concept.get_compiled()[var_name] = concept_found
                 else:
+                    # ...or a list of ReturnValueConcept to resolve
                     res = parse_unrecognized(context,
                                              default_value,
                                              parsers="all",
@@ -285,7 +287,9 @@ class SheerkaEvaluateConcept(BaseService):
                     concept.get_compiled()[var_name] = res.body.body if is_only_successful(res) else res
 
         # Updates the cache of concepts when possible
-        if self.sheerka.has_id(concept.id):
+        # This piece of code is not used, a the compile part is removed by sheerka.new_from_template()
+        service = context.sheerka.services[SheerkaConceptManager.NAME]
+        if service.has_id(concept.id):
             self.sheerka.get_by_id(concept.id).set_compiled(concept.get_compiled())
 
     def resolve(self,

src/core/sheerka/services/SheerkaHasAManager.py

@@ -35,9 +35,8 @@ class SheerkaHasAManager(BaseService):
                                  name=BuiltinConcepts.HASA,
                                  concept=concept_a))
 
-        concept_a.add_prop(BuiltinConcepts.HASA, concept_b)
+        to_add = {"props": {BuiltinConcepts.HASA:  concept_b}}
-
+        return self.sheerka.modify_concept(context, concept_a, to_add, modify_source=True)
-        return self.sheerka.modify_concept(context, concept_a)
 
     def hasa(self, concept_a, concept_b):
         """

src/core/sheerka/services/SheerkaModifyConcept.py

@@ -1,110 +0,0 @@
-from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import ensure_concept
-from core.concept import NotInit, freeze_concept_attrs, Concept
-from core.sheerka.services.sheerka_service import BaseService
-
-
-class SheerkaModifyConcept(BaseService):
-    NAME = "ModifyConcept"
-
-    def __init__(self, sheerka):
-        super().__init__(sheerka)
-
-    def initialize(self):
-        self.sheerka.bind_service_method(self.modify_concept, True)
-        self.sheerka.bind_service_method(self.set_attr, True)
-        self.sheerka.bind_service_method(self.get_attr, False)
-
-    def modify_concept(self, context, concept):
-        """
-        Modify the definition of a concept
-        :param context:
-        :param concept:
-        :return:
-        """
-        old_version = self.sheerka.get_by_id(concept.id)
-
-        if old_version is None:
-            # nothing found in cache
-            return self.sheerka.ret(
-                self.NAME, False,
-                self.sheerka.new(
-                    BuiltinConcepts.UNKNOWN_CONCEPT,
-                    body=[("key", concept.key), ("id", concept.id)]))
-
-        if not self.sheerka.is_success(old_version) and concept.key != old_version.key:
-            # an error concept is returned
-            return self.sheerka.ret(
-                self.NAME, False,
-                old_version)
-
-        if old_version == concept:
-            # the concept is not modified
-            # This is an important sanity check. Do no remove because you don't understand it
-            return self.sheerka.ret(
-                self.NAME, False,
-                self.sheerka.new(
-                    BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
-                    body=concept))
-
-        # update attributes
-        freeze_concept_attrs(concept)
-
-        self.sheerka.cache_manager.update_concept(old_version, concept)
-
-        # TODO : update concept by first keyword : have a look at update_references() below
-        # TODO : update resolved by first keyword : have a look at update_references() below
-        # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
-        # TODO : Update concepts grammars : have a look at update_references() below
-
-        ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
-        return ret
-
-    def update_references(self, context, concept):
-        """
-        Updates all the concepts that reference concept
-        :param context:
-        :param concept:
-        :return:
-        """
-
-        refs = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_REFERENCES_ENTRY, concept.id)
-        if not refs:
-            return
-
-        for concept_id in refs:
-            # remove the grammar entry so that it can be recreated
-            self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
-
-    def set_attr(self, concept, attribute, value):
-        """
-        Modifies an attribute of a concept (concept.values)
-        :param context:
-        :param concept:
-        :param attribute:
-        :param value:
-        :return:
-        """
-        ensure_concept(concept)
-
-        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
-        concept.set_value(attr, value)
-        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-
-    def get_attr(self, concept, attribute):
-        """
-        Returns the attribute of a concept
-        :param context:
-        :param concept:
-        :param attribute:
-        :return:
-        """
-        ensure_concept()
-        if not self.sheerka.is_success(concept):
-            return concept
-
-        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
-
-        if (value := concept.get_value(attr)) == NotInit:
-            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute})
-        return value

src/core/sheerka/services/SheerkaRuleManager.py

@@ -623,7 +623,7 @@ class SheerkaRuleManager(BaseService):
         if rule.metadata.id is not None:
             return
 
-        rule.metadata.id = str(self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_KEYS_ENTRY, self.RULE_IDS))
+        rule.metadata.id = str(self.sheerka.cache_manager.get(self.sheerka.OBJECTS_IDS_ENTRY, self.RULE_IDS))
 
     def create_new_rule(self, context, rule):
         """

src/core/sheerka/services/SheerkaSetsManager.py

@@ -4,7 +4,7 @@ from cache.SetCache import SetCache
 from core.ast_helpers import UnreferencedVariablesVisitor
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF
-from core.sheerka.services.SheerkaModifyConcept import SheerkaModifyConcept
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.sheerka_service import BaseService
 
 GROUP_PREFIX = 'All_'
@@ -43,7 +43,6 @@ class SheerkaSetsManager(BaseService):
         context.log(f"Setting concept {concept} is a {concept_set}", who=self.NAME)
         core.builtin_helpers.ensure_concept(concept, concept_set)
 
-
         if BuiltinConcepts.ISA in concept.get_metadata().props and concept_set in concept.get_metadata().props[
             BuiltinConcepts.ISA]:
             return self.sheerka.ret(
@@ -53,11 +52,12 @@ class SheerkaSetsManager(BaseService):
 
         # KSI 20200709 add the concept, not its 'id' or 'key'
         # It will allow conditions handling if concept set has its WHERE or PRE set to something
-        concept.add_prop(BuiltinConcepts.ISA, concept_set)
+        to_add = {"props": {BuiltinConcepts.ISA: concept_set}}
-
+        res = self.sheerka.modify_concept(context, concept, to_add, modify_source=True)
-        res = self.sheerka.modify_concept(context, concept)
         if not res.status:
             return res
+        else:
+            concept = res.body.body
 
         res = self.add_concept_to_set(context, concept, concept_set)
 
@@ -88,7 +88,7 @@ class SheerkaSetsManager(BaseService):
         self.concepts_in_set.delete(concept_set.id)
 
         # update concept_set references
-        self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
+        self.sheerka.services[SheerkaConceptManager.NAME].update_references(context, concept_set)
 
         # remove the grammar entry so that it can be recreated
         self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_set.id)

src/evaluators/AddConceptInSetEvaluator.py

@@ -56,10 +56,6 @@ class AddConceptInSetEvaluator(OneReturnValueEvaluator):
                 parents=[return_value])
         concept = res.value
 
-        if sheerka.has_id(concept.id) and id(concept) == id(sheerka.get_by_id(concept.id)):
-            # hack because it is not possible to use sheerka.modify_concept() on a cache instance
-            concept = sheerka.new((concept.name, concept.id))
-
         res = _resolve(isa_node.set)
         if not res.status:
             return sheerka.ret(

src/evaluators/PythonEvaluator.py

@@ -158,7 +158,11 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         context.log(f"{evaluated=}", self.name)
         debugger.debug_var("ret", evaluated)
-        return sheerka.ret(self.name, True, evaluated, parents=[return_value])
+
+        if sheerka.isinstance(evaluated, BuiltinConcepts.RETURN_VALUE):
+            return sheerka.ret(self.name, evaluated.status, evaluated.body, parents=[return_value, evaluated])
+        else:
+            return sheerka.ret(self.name, True, evaluated, parents=[return_value])
 
     def get_globals(self, context, node, expression_only):
         """

src/parsers/BaseNodeParser.py

@@ -846,6 +846,7 @@ class BaseNodeParser(BaseParser):
     def get_concepts(self, token, to_keep, custom=None, to_map=None, strip_quotes=False):
         """
         Tries to find if there are concepts that match the value of the token
+        Caution: Returns the actual cache, not a copy
         :param token:
         :param to_keep: predicate to tell if the concept is eligible
         :param custom: lambda name -> List[Concepts] that gives extra concepts, according to the name
@@ -882,16 +883,17 @@ class BaseNodeParser(BaseParser):
         return custom_concepts if custom else None
 
     @staticmethod
-    def get_concepts_by_first_token(context, concepts, use_sheerka=False):
+    def get_concepts_by_first_token(context, concepts, use_sheerka=False, previous_entries=None):
         """
         Create the map describing the first token expected by a concept
         :param context:
         :param concepts: lists of concepts to parse
         :param use_sheerka: if True, update concepts_by_first_keyword from sheerka
+        :param previous_entries:
         :return:
         """
         sheerka = context.sheerka
-        res = sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka else {}
+        res = sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka else (previous_entries or {})
         for concept in concepts:
             keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
 
@@ -909,10 +911,15 @@ class BaseNodeParser(BaseParser):
         return sheerka.ret("BaseNodeParser", True, res)
 
     @staticmethod
-    def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword):
+    def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword, modified_concepts=None):
         sheerka = context.sheerka
         res = {}
 
+        def get_by_id(c_id):
+            if modified_concepts and c_id in modified_concepts:
+                return modified_concepts[c_id]
+            return sheerka.get_by_id(c_id)
+
         def resolve_concepts(concept_str):
             c_key, c_id = core.utils.unstr_concept(concept_str)
             if c_id in already_seen:
@@ -924,7 +931,7 @@ class BaseNodeParser(BaseParser):
             to_resolve = set()
             chicken_and_egg = set()
 
-            concept = sheerka.get_by_id(c_id)
+            concept = get_by_id(c_id)
 
             if sheerka.isaset(context, concept):
                 concepts = sheerka.get_set_elements(context, concept)

src/parsers/ExactConceptParser.py

@@ -76,15 +76,10 @@ class ExactConceptParser(BaseParser):
                         value = words[i]
                         concept.def_var_by_index(index, str_concept(value) if isinstance(value, tuple) else value)
                         concept.get_metadata().need_validation = True
-                        # if self.verbose_log.isEnabledFor(logging.DEBUG):
-                        #     prop_name = concept.get_metadata().variables[index][0]
-                        #     context.log(
-                        #         f"Added variable {index}: {prop_name}='{words[i]}'.",
-                        #         self.name)
 
                 already_recognized.append(concept)
 
-        by_name = sheerka.resolve(parser_input.as_text())
+        by_name = sheerka.fast_resolve(parser_input.as_text())
         core.builtin_helpers.set_is_evaluated(by_name)
         recognized = self.merge_concepts(already_recognized, by_name)
 

src/parsers/SequenceNodeParser.py

@@ -261,6 +261,13 @@ class SequenceNodeParser(BaseNodeParser):
         return make_unique(concepts_by_name + concepts_by_first_keyword, lambda c: c.id)
 
     def get_concepts_sequences(self):
+        """
+        Tries to find the concept.
+        TODO: KSI 20201206
+        I think that the code can be optimized as we create a new instance of each concept before validating
+        that we are going to keep it.
+        :return:
+        """
 
         forked = []
 

tests/BaseTest.py

@@ -79,7 +79,7 @@ class BaseTest:
             else:
                 c.init_key()
                 sheerka.set_id_if_needed(c, False)
-                sheerka.add_in_cache(c)
+                sheerka.test_only_add_in_cache(c)
 
             result.append(c)
 
@@ -93,7 +93,7 @@ class BaseTest:
 
         if create_new:
             sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache.clear()
-            sheerka.cache_manager.delete(sheerka.CONCEPTS_KEYS_ENTRY, SheerkaRuleManager.RULE_IDS)
+            sheerka.cache_manager.delete(sheerka.OBJECTS_IDS_ENTRY, SheerkaRuleManager.RULE_IDS)
             with sheerka.sdp.get_transaction(context.event.get_digest()) as transaction:
                 transaction.clear(SheerkaRuleManager.FORMAT_RULE_ENTRY)
 
@@ -146,8 +146,6 @@ class BaseTest:
         """True ret_val + add concept in cache"""
         if isinstance(obj, Concept):
             obj.init_key()
-            if sheerka.has_key(obj.key):
-                sheerka.add_in_cache(obj)
         return sheerka.ret(who, True, obj)
 
     @staticmethod
@@ -180,7 +178,7 @@ class BaseTest:
             concept.get_metadata().definition_type = DEFINITION_TYPE_BNF
         concept.init_key()
         sheerka.set_id_if_needed(concept, False)
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
         return concept
 
     @staticmethod

tests/cache/test_cache.py

@@ -539,9 +539,121 @@ class TestCache(TestUsingMemoryBasedSheerka):
 
         assert cache.get("key") == "value"
         cache.delete("key")
-        assert cache.get("value") is None
+        assert cache.get("key") is None
         assert cache.to_remove == {"key"}
 
+    def test_i_can_delete_values_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+        cache.reset_events()
+
+        cache.delete("key", "fake_value")
+        assert cache.get("key") == {"value1", "value2"}
+        assert len(cache) == 2
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value1")
+        assert cache.get("key") == {"value2"}
+        assert cache.to_add == {"key"}
+        assert len(cache) == 1
+
+        cache.delete("key", "value2")
+        assert cache.get("key") is None
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_key_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+
+        cache.delete("key")
+        assert cache.get("key") is None
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_a_key_that_does_not_exists(self):
+        cache = SetCache()
+        cache.delete("key")
+
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_from_a_key_from_list_id_needed(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+        cache.put("key", "value11")
+        cache.put("key2", "value2")
+        cache.put("key2", "value22")
+        cache.put("key2", "value222")
+        cache.put("key3", "value3")
+        cache.put("key3", "value33")
+        cache.put("key4", "value4")
+        cache.reset_events()
+
+        assert len(cache) == 8
+
+        # I can remove a whole key
+        cache.delete("key")
+        assert cache.get("key") is None
+        assert len(cache) == 6
+        assert cache.to_remove == {"key"}
+        assert cache.to_add == set()
+
+        # I can remove an element while a list is remaining
+        cache.reset_events()
+        cache.delete("key2", "value22")
+        assert cache.get("key2") == ["value2", "value222"]
+        assert len(cache) == 5
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while a single element is remaining
+        cache.reset_events()
+        cache.delete("key3", "value33")
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 4
+        assert cache.to_add == {"key3"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while nothing remains
+        cache.reset_events()
+        cache.delete("key4", "value4")
+        assert cache.get("key4") is None
+        assert len(cache) == 3
+        assert cache.to_remove == {"key4"}
+        assert cache.to_add == set()
+
+        # I do not remove when the value is not the same
+        cache.reset_events()
+        cache.delete("key3", "value33")  # value33 was already remove
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 3
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_deleting_a_list_if_need_entry_that_does_not_exist_is_not_an_error(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+
+        cache.reset_events()
+        cache.delete("key3")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key3", "value")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value2")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
     def test_initialized_key_is_removed_when_the_entry_is_found(self):
         caches = [Cache(), ListCache(), ListIfNeededCache(), SetCache()]
 

tests/cache/test_cache_manager.py

@@ -1,7 +1,10 @@
+import pytest
 from cache.Cache import Cache
-from cache.CacheManager import CacheManager
+from cache.CacheManager import CacheManager, ConceptNotFound
 from cache.DictionaryCache import DictionaryCache
 from cache.ListCache import ListCache
+from cache.ListIfNeededCache import ListIfNeededCache
+from core.concept import Concept
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -109,3 +112,46 @@ class TestCacheManager(TestUsingMemoryBasedSheerka):
         cache.put(False, {"key": "value", "key2": "value2", "key3": "value3"})
         cache_manager.commit(context)
         assert sheerka.sdp.get("test") == {"key": "value", "key2": "value2", "key3": "value3"}
+
+    def test_i_can_remove_a_concept_from_concepts_caches(self):
+        cache_manager = CacheManager(True)
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
+
+        sheerka, context, one, two, three, two_bis = self.init_concepts("one", "two", "three", Concept("two", body="2"))
+
+        for concept in [one, two, three, two_bis]:
+            cache_manager.add_concept(concept)
+
+        # sanity check
+        cache_def = cache_manager.caches["id"]
+        assert cache_def.cache.copy() == {one.id: one, two.id: two, three.id: three, two_bis.id: two_bis}
+        cache_def = cache_manager.caches["key"]
+        assert cache_def.cache.copy() == {one.key: one, two.key: [two, two_bis], three.key: three}
+
+        for cache_name in cache_manager.concept_caches:
+            cache_manager.caches[cache_name].cache.reset_events()
+
+        cache_manager.remove_concept(sheerka.new(("two", two_bis.id)))
+
+        cache_def = cache_manager.caches["id"]
+        assert cache_def.cache.copy() == {one.id: one, two.id: two, three.id: three}
+        assert cache_def.cache.to_remove == {two_bis.id}
+        assert cache_def.cache.to_add == set()
+        assert len(cache_def.cache) == 3
+
+        cache_def = cache_manager.caches["key"]
+        assert cache_def.cache.copy() == {one.key: one, two.key: two, three.key: three}
+        assert cache_def.cache.to_remove == set()
+        assert cache_def.cache.to_add == {"two"}
+        assert len(cache_def.cache) == 3
+
+    def test_i_cannot_remove_a_concept_that_does_not_exists(self):
+        cache_manager = CacheManager(True)
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
+
+        with pytest.raises(ConceptNotFound) as ex:
+            cache_manager.remove_concept(Concept("foo", id="1001"))
+
+        assert ex.value.concept == Concept("foo", id="1001")

tests/core/test_SheerkaConceptAlgebra.py

@@ -20,7 +20,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka):
 
         assert isinstance(res, Concept)
         assert res.get_metadata().props == {BuiltinConcepts.ISA: {male, human},
-                                      BuiltinConcepts.HASA: {car, licence}, }
+                                            BuiltinConcepts.HASA: {car, licence}, }
 
     def test_can_add_concepts_when_property_already_exist(self):
         sheerka, context, man, human, king, male = self.init_concepts(
@@ -58,7 +58,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka):
         res = sheerka.csub(context, new_foo, new_bar)
         assert isinstance(res, Concept)
         assert res.get_metadata().props == {BuiltinConcepts.ISA: {isa2},
-                                      BuiltinConcepts.HASA: {hasa2}, }
+                                            BuiltinConcepts.HASA: {hasa2}, }
 
     def test_i_can_recognize_myself_when_using_sdp_repository(self):
         sheerka, context, foo, isa1, hasa1, = self.init_concepts("foo", "isa1", "has1",

tests/core/test_SheerkaConceptManager.py

@@ -0,0 +1,739 @@
+import pytest
+from cache.CacheManager import ConceptNotFound
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs, NotInit, \
+    DEFINITION_TYPE_BNF
+from core.sheerka.Sheerka import Sheerka
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \
+    UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced
+from parsers.BaseNodeParser import BaseNodeParser
+from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
+    def test_i_can_create_a_concept(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = self.get_default_concept()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        res = sheerka.create_new_concept(context, concept)
+        sheerka.cache_manager.commit(context)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_found = res.value.body
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
+
+        assert concept_found.key == "__var__0 + __var__1"
+        assert concept_found.id == "1001"
+        assert get_concept_attrs(concept) == ['a', 'b']
+
+        # saved in cache
+        assert service.has_id(concept.id)
+        assert service.has_key(concept.key)
+        assert service.has_name(concept.name)
+        assert service.has_hash(concept.get_definition_hash())
+
+        # I can get the concept using various index
+        assert sheerka.get_by_id(concept.id) == concept
+        assert sheerka.get_by_key(concept.key) == concept
+        assert sheerka.get_by_name(concept.name) == concept
+        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
+
+        # I can get by the first entry
+        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
+        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
+
+        # saved in sdp
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
+        assert sheerka.sdp.exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
+
+    def test_i_cannot_create_a_bnf_concept_that_references_a_concept_that_cannot_be_resolved(self):
+        sheerka, context, one_1, one_1_0 = self.init_concepts(Concept("one", body="1"), Concept("one", body="1.0"))
+        twenty_one = Concept("twenty one", definition="'twenty' one", definition_type=DEFINITION_TYPE_BNF)
+
+        res = sheerka.create_new_concept(context, twenty_one)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
+        assert res.value.body == ("key", "one")
+
+    def test_i_can_add_a_concept_when_name_differs_from_the_key(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a")
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+        sheerka.cache_manager.commit(context)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_found = res.value.body
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
+
+        assert concept_found.key == "hello __var__0"
+        assert concept_found.id == "1001"
+
+        # saved in cache
+        assert service.has_id(concept.id)
+        assert service.has_key(concept.key)
+        assert service.has_name(concept.name)
+        assert service.has_hash(concept.get_definition_hash())
+
+        # I can get the concept using various index
+        assert sheerka.get_by_id(concept.id) == concept
+        assert sheerka.get_by_key(concept.key) == concept
+        assert sheerka.get_by_name(concept.name) == concept
+        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
+
+        # saved in sdp
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
+        assert sheerka.sdp.exists(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello")
+
+    def test_i_cannot_add_the_same_concept_twice(self):
+        """
+        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+        assert res.value.body == concept
+
+    def test_i_can_get_a_newly_created_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        from_cache = sheerka.get_by_key(concept.key)
+        assert from_cache is not None
+        assert from_cache == concept
+
+        from_cache = sheerka.get_by_id(concept.id)
+        assert from_cache is not None
+        assert from_cache == concept
+
+    def test_i_can_get_list_of_concept_when_same_key_using_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.get_metadata().body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get_by_key(concept1.key)
+        assert len(result) == 2
+        assert result[0] == concept1
+        assert result[1] == concept2
+
+    def test_concept_that_references_itself_is_correctly_created(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo", body="foo")
+
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert res.status
+
+    def test_i_can_get_by_name_when_created_with_def_definition(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = self.from_def_concept("plus", "a plus b", ["a", "b"])
+
+        res = sheerka.create_new_concept(context, concept)
+
+        assert res.status
+        assert sheerka.get_by_name(concept.name) == concept
+        assert sheerka.get_by_name(concept.get_metadata().definition) == concept
+
+        concept = Concept(name="foo", definition="foo", definition_type=DEFINITION_TYPE_DEF)
+        res = sheerka.create_new_concept(context, concept)
+
+        assert res.status
+        assert sheerka.get_by_name(concept.name) == concept  # it's not a list, ie the entry is not duplicated
+
+    def test_i_can_get_first_token_when_not_a_letter(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = Concept("--filter a").def_var("a")
+
+        res = sheerka.create_new_concept(context, concept)
+        assert res.status
+
+        # I can get by the first entry
+        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
+        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
+
+    @pytest.mark.parametrize("expression", [
+        "--'filter' ('one' | 'two') ",
+        "'--filter' ('one' | 'two') ",
+    ])
+    def test_i_can_get_first_token_when_bnf_concept_and_not_a_letter(self, expression):
+        sheerka, context, bnf_concept = self.init_concepts(
+            Concept("foo", definition=expression),
+            create_new=True)
+
+        # I can get by the first entry
+        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
+        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
+
+    def test_concept_references_are_updated_1(self):
+        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
+            "one",
+            "two",
+            "number",
+            "twenty",
+            Concept("twenties", definition="twenty one | two 'hundred'"),
+            create_new=True
+        )
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) is None
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
+
+    def test_concept_references_are_updated_2(self):
+        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
+            "one",
+            "two",
+            "number",
+            "twenty",
+            Concept("twenties", definition="twenty number"),
+            create_new=True
+        )
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) is None
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) is None
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
+        assert sheerka.cache_manager.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
+
+    @pytest.mark.parametrize("attr", [
+        "name",
+        "is_unique",
+        "body",
+        "where",
+        "pre",
+        "post",
+        "ret",
+        "definition",
+        "definition_type",
+        "desc",
+        "is_evaluated",
+        "need_validation",
+        "full_serialization",
+    ])
+    def test_i_can_modify_a_metadata_attribute(self, attr):
+        sheerka, context, foo = self.init_concepts("foo")
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {attr: "new value"}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert getattr(res.body.body.get_metadata(), attr) == "new value"
+
+    def test_i_can_modify_a_concept_when_at_least_one_attr_is_different(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo", body="a body"))
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"name": "foo", "body": "a body", "pre": "new pre"}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert getattr(res.body.body.get_metadata(), "name") == "foo"
+        assert getattr(res.body.body.get_metadata(), "body") == "a body"
+        assert getattr(res.body.body.get_metadata(), "pre") == "new pre"
+
+    def test_i_can_modify_add_a_property(self):
+        sheerka, context, one, foo = self.init_concepts("one", Concept("foo", props={BuiltinConcepts.ISA: {"value"}}))
+
+        res = sheerka.modify_concept(context, foo, to_add={"props": {BuiltinConcepts.ISA: "value2",
+                                                                     BuiltinConcepts.HASA: one}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.get_prop(BuiltinConcepts.ISA) == {"value", "value2"}
+        assert res.body.body.get_prop(BuiltinConcepts.HASA) == {sheerka.new("one")}
+
+    def test_i_can_modify_remove_a_property(self):
+        sheerka, context, foo = self.init_concepts(
+            Concept("foo", props={"a": {"value1", "value2", "value3"},
+                                  "b": {"value4"}}))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"props": {"a": {"value2", "value3"},
+                                                                        "b": "value4"}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.get_prop("a") == {"value1"}
+        assert res.body.body.get_prop("b") is None
+
+    def test_i_can_modify_add_variables(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a", "value"))
+
+        res = sheerka.modify_concept(context, foo, to_add={"variables": {"b": "some_value",
+                                                                         "a": "new_value",
+                                                                         "c": None}})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.get_metadata().variables == [("a", "new_value"), ("b", "some_value"), ("c", None)]
+        assert res.body.body.values() == {"a": NotInit, "b": NotInit, "c": NotInit}
+
+    def test_i_can_modify_remove_variables(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b", "value").def_var("c"))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"variables": ["a", "c"]})
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.get_metadata().variables == [("b", "value")]
+        assert res.body.body.values() == {"b": NotInit}
+
+    def test_i_can_modify_the_concept_source(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar")
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"body": "new value"}})
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert getattr(foo.get_metadata(), "body") is None
+
+        res = sheerka.modify_concept(context, bar, to_add={"meta": {"body": "new value"}}, modify_source=True)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert getattr(bar.get_metadata(), "body") == "new value"
+
+    def test_caches_are_updated_when_i_modify_the_properties_and_the_variables(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar", cache_only=False)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        to_add = {"meta": {"body": "metadata value"},
+                  "variables": {"var_name": "default value"},
+                  "props": {BuiltinConcepts.ISA: bar}}
+
+        res = sheerka.modify_concept(context, foo, to_add)
+        new_concept = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert new_concept.get_metadata().body == "metadata value"
+        assert new_concept.get_metadata().variables == [("var_name", "default value")]
+        assert new_concept.get_prop(BuiltinConcepts.ISA) == {bar}
+
+        # test that object
+        foo_from_sheerka = sheerka.get_by_key(new_concept.key)
+        assert foo_from_sheerka.get_metadata().body == "metadata value"
+        assert foo_from_sheerka.get_metadata().variables == [("var_name", "default value")]
+        assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == {bar}
+
+        # other caches are also updated
+        assert sheerka.get_by_id(new_concept.id).get_metadata().body == "metadata value"
+        assert sheerka.get_by_name(new_concept.name).get_metadata().body == "metadata value"
+        assert sheerka.get_by_hash(new_concept.get_definition_hash()).get_metadata().body == "metadata value"
+
+        # sdp is updated
+        sheerka.cache_manager.commit(context)
+        from_sdp = sheerka.sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
+        assert from_sdp.get_metadata().body == "metadata value"
+        assert from_sdp.get_metadata().variables == [("var_name", "default value")]
+        assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}
+
+        assert sheerka.sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).get_metadata().body == "metadata value"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).get_metadata().body == "metadata value"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_HASH_ENTRY,
+                               new_concept.get_definition_hash()).get_metadata().body == "metadata value"
+
+    def test_caches_are_update_when_i_modify_the_name(self):
+        sheerka, context, foo = self.init_concepts("foo", cache_only=False)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        sheerka.is_known(sheerka.get_by_name(foo.name))
+        sheerka.is_known(sheerka.get_by_key(foo.key))
+        sheerka.get_by_hash(foo.get_definition_hash())
+
+        to_add = {"meta": {"name": "bar"}}
+
+        res = sheerka.modify_concept(context, foo, to_add)
+        new_concept = res.body.body
+
+        assert new_concept.name == "bar"
+        assert sheerka.get_by_id(new_concept.id).name == "bar"
+        assert sheerka.get_by_key(new_concept.key).name == "bar"
+        assert sheerka.get_by_name(new_concept.name).name == "bar"
+        assert sheerka.get_by_hash(new_concept.get_definition_hash()).name == "bar"
+
+        assert not sheerka.is_known(sheerka.get_by_name(foo.name))
+        assert not sheerka.is_known(sheerka.get_by_key(foo.key))
+        assert not sheerka.is_known(sheerka.get_by_hash(foo.get_definition_hash()))
+
+        sheerka.cache_manager.commit(context)
+        assert sheerka.sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id).name == "bar"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).name == "bar"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).name == "bar"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_HASH_ENTRY, new_concept.get_definition_hash()).name == "bar"
+        assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, foo.key) is None
+        assert sheerka.sdp.get(service.CONCEPTS_BY_NAME_ENTRY, foo.name) is None
+        assert sheerka.sdp.get(service.CONCEPTS_BY_HASH_ENTRY, foo.get_definition_hash()) is None
+
+    def test_i_can_modify_a_concept_from_a_list_of_concepts(self):
+        sheerka, context, foo1, foo2 = self.init_concepts(
+            Concept("foo", body="1"),
+            Concept("foo", body="2"))
+
+        to_add = {"meta": {"body": "new_value"}}
+
+        res = sheerka.modify_concept(context, foo1, to_add)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+
+        new_concept = res.body.body
+
+        assert new_concept.id == foo1.id
+        assert res.body.body.get_metadata().body == "new_value"
+
+        assert sheerka.get_by_id(foo1.id).get_metadata().body == "new_value"
+        assert sheerka.get_by_id(foo2.id).get_metadata().body == "2"
+
+    def test_values_are_modified_when_variables_are_added_or_removed(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b"))
+
+        to_add = {"meta": {"body": "metadata value"},
+                  "variables": {"c": "default value"}}
+
+        to_remove = {"variables": ["a"]}
+
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+        res = sheerka.modify_concept(context, foo, to_add, to_remove)
+        new_concept = res.body.body
+
+        assert res.status
+        assert get_concept_attrs(foo) == ["b", "c"]
+        assert get_concept_attrs(new_concept) == ["b", "c"]
+
+    def test_key_is_modified_when_modifying_name_or_variables(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b"))
+
+        to_add = {"meta": {"name": "b bar c d"},
+                  "variables": {"c": None, "d": None}}
+
+        to_remove = {"variables": ["a"]}
+
+        res = sheerka.modify_concept(context, foo, to_add, to_remove)
+        new_concept = res.body.body
+
+        assert res.status
+        assert new_concept.key == "__var__0 bar __var__1 __var__2"
+
+    def test_key_is_modified_when_modifying_the_definition(self):
+        sheerka, context, foo = self.init_concepts(
+            Concept(name="foo", definition="foo a b", definition_type=DEFINITION_TYPE_DEF).def_var("a").def_var("b"))
+
+        to_add = {"meta": {"definition": "b bar c d"},
+                  "variables": {"c": None, "d": None}}
+
+        to_remove = {"variables": ["a"]}
+
+        res = sheerka.modify_concept(context, foo, to_add, to_remove)
+        new_concept = res.body.body
+
+        assert res.status
+        assert new_concept.key == "__var__0 bar __var__1 __var__2"
+
+    def test_bnf_is_modified_when_modifying_the_definition(self):
+        sheerka, context, one, two, foo = self.init_concepts(
+            "one",
+            "two",
+            Concept(name="foo", definition="'twenty' one"),
+            create_new=True
+        )
+
+        to_add = {"meta": {"definition": "'twenty' two"}}
+
+        res = sheerka.modify_concept(context, foo, to_add)
+        new_concept = res.body.body
+
+        assert res.status
+        assert new_concept.get_metadata().definition == "'twenty' two"
+        assert new_concept.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(two, rule_name='two'))
+
+    def test_concept_by_first_keyword_is_updated_after_concept_modification(self):
+        sheerka, context, foo, bar, baz = self.init_concepts(
+            Concept("foo"),
+            Concept("bar"),
+            Concept("baz", definition="foo"),
+            create_new=True)
+
+        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "foo": ["1001"],
+            "bar": ["1002"],
+            'c:|1001:': ['1003']}
+        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            'foo': ['1001', '1003'],
+            'bar': ['1002']}
+
+        to_add = {"meta": {"name": "bar"}}
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert res.status
+        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "bar": ["1002", "1001"],
+            'c:|1001:': ['1003']}
+        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            'bar': ['1002', '1001', '1003']}
+
+    def test_references_are_updated_after_concept_modification(self):
+        sheerka, context, one, twenty_one = self.init_concepts(
+            "onz",
+            Concept("twenty one", definition="'twenty' onz"),
+            create_new=True
+        )
+
+        assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(one, rule_name='onz'))
+
+        to_add = {"meta": {"name": "one"}}
+        res = sheerka.modify_concept(context, one, to_add)
+        modified = res.body.body
+
+        assert res.status
+
+        twenty_one = sheerka.get_by_name("twenty one")
+        assert twenty_one.get_metadata().definition == "'twenty' one"
+        assert twenty_one.get_bnf() is None
+
+        BaseNodeParser.ensure_bnf(context, twenty_one)
+        assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(modified, rule_name='one'))
+
+    def test_i_cannot_modify_without_any_modification(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        res = service.modify_concept(context, foo)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == NoModificationFound(foo)
+
+    def test_i_cannot_modify_forbidden_attributes(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        for attr in service.forbidden_meta:
+            res = service.modify_concept(context, foo, to_add={"meta": {attr: "new value"}})
+            assert not res.status
+            assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+            assert res.body.body == ForbiddenAttribute(attr)
+
+    def test_i_cannot_modify_unknown_attributes(self):
+        sheerka, context, foo = self.init_concepts("foo")
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"dummy": "new value"}})
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == UnknownAttribute("dummy")
+
+    def test_i_cannot_modify_if_all_new_values_are_the_same(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo", body="a body"))
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"name": "foo", "body": "a body"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == NoModificationFound(foo, {"name": "foo", "body": "a body"})
+
+    def test_i_cannot_remove_meta_attributes(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo"))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"meta": {"any_value": "foo"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == CannotRemoveMeta({"any_value": "foo"})
+
+    def test_i_cannot_remove_props_that_does_not_exists(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo"))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"props": {"any_value": "foo"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == UnknownAttribute("any_value")
+
+    def test_i_cannot_remove_props_value_that_does_not_exists(self):
+        # Need to returns an error, otherwise, we will save a concept that is not modified
+        sheerka, context, foo = self.init_concepts(Concept("foo", props={"a": {"value"}}))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"props": {"a": "dummy"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == ValueNotFound("a", "dummy")
+
+    def test_i_cannot_remove_variable_that_does_not_exists(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a"))
+
+        res = sheerka.modify_concept(context, foo, to_remove={"variables": ["b"]})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == UnknownAttribute("b")
+
+    def test_i_cannot_modify_a_concept_that_is_not_known(self):
+        sheerka, context = self.init_concepts()
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+
+        res = sheerka.modify_concept(context, foo, to_add={"meta": {"body": "new value"}})
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT)
+
+    def test_i_can_get_and_set_attribute(self):
+        sheerka, context = self.init_concepts()
+        foo = Concept("foo")
+        prop = Concept("property")
+        bar = Concept("bar")
+
+        res = sheerka.set_attr(foo, prop, bar)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        assert sheerka.get_attr(foo, prop) == bar
+
+    def test_i_cannot_remove_a_concept_which_has_reference(self):
+        sheerka, context, one, twenty_one = self.init_concepts(
+            Concept("one"),
+            Concept("twenty one", definition="'twenty' one"),
+            create_new=True)
+
+        res = sheerka.remove_concept(context, one)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == ConceptIsReferenced([twenty_one])
+
+    def test_i_can_remove_a_concept(self):
+        sheerka, context, one = self.init_concepts(
+            Concept("one"),
+            create_new=True)
+
+        # sanity check
+        assert sheerka.get_by_id(one.id) == one
+        assert sheerka.get_by_name(one.name) == one
+        assert sheerka.get_by_key(one.key) == one
+        assert sheerka.get_by_hash(one.get_definition_hash()) == one
+
+        res = sheerka.remove_concept(context, one)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        assert sheerka.isinstance(sheerka.get_by_id(one.id), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_name(one.name), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)
+
+    def test_i_cannot_remove_a_concept_that_does_not_exist(self):
+        sheerka, context = self.init_concepts()
+        one = Concept("one", id="1001")
+
+        res = sheerka.remove_concept(context, one)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == ConceptNotFound(one)
+
+
+class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
+    def test_i_can_add_several_concepts(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        hello = Concept("Hello world a").def_var("a")
+        res = sheerka.create_new_concept(context, hello)
+        sheerka.cache_manager.commit(context)
+        assert res.status
+
+        sheerka = self.get_sheerka()  # another instance
+        context = self.get_context(sheerka)
+        greeting = Concept("Greeting a").def_var("a")
+        res = sheerka.create_new_concept(context, greeting)
+        sheerka.cache_manager.commit(context)
+        assert res.status
+
+        sheerka = self.get_sheerka()  # another instance again
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, hello.key)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_KEY_ENTRY, greeting.key)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, hello.id)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_ID_ENTRY, greeting.id)
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, "Hello world a")
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_NAME_ENTRY, "Greeting a")
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, hello.get_definition_hash())
+        assert sheerka.sdp.exists(service.CONCEPTS_BY_HASH_ENTRY, greeting.get_definition_hash())
+
+        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello")
+        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting")
+
+    def test_i_cannot_add_the_same_concept_twice_using_sdp(self):
+        """
+        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
+        :return:
+        """
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(context, concept)
+        sheerka.cache_manager.commit(context)
+
+        sheerka.cache_manager.clear()
+        res = sheerka.create_new_concept(context, concept)
+
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+        assert res.value.body == concept
+
+    def test_new_entry_does_not_override_the_previous_ones(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        sheerka.create_new_concept(context, Concept("foo", body="1"))
+        sheerka.create_new_concept(context, Concept("foo", body="2"))
+        sheerka.cache_manager.commit(context)
+
+        assert len(sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 2
+
+        sheerka = self.get_sheerka()  # new instance
+        context = self.get_context(sheerka)
+        sheerka.create_new_concept(context, Concept("foo", body="3"))
+        sheerka.cache_manager.commit(context)
+
+        assert len(sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 3

tests/core/test_SheerkaCreateNewConcept.py

@@ -1,281 +0,0 @@
-import pytest
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs
-from core.sheerka.Sheerka import Sheerka
-
-from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
-from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-
-
-class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
-
-    def test_i_can_create_a_concept(self):
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = self.get_default_concept()
-
-        res = sheerka.create_new_concept(context, concept)
-        sheerka.cache_manager.commit(context)
-
-        assert res.status
-        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
-
-        concept_found = res.value.body
-        for prop in PROPERTIES_TO_SERIALIZE:
-            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
-
-        assert concept_found.key == "__var__0 + __var__1"
-        assert concept_found.id == "1001"
-        assert get_concept_attrs(concept) == ['a', 'b']
-
-        # saved in cache
-        assert sheerka.has_id(concept.id)
-        assert sheerka.has_key(concept.key)
-        assert sheerka.has_name(concept.name)
-        assert sheerka.has_hash(concept.get_definition_hash())
-
-        # I can get the concept using various index
-        assert sheerka.get_by_id(concept.id) == concept
-        assert sheerka.get_by_key(concept.key) == concept
-        assert sheerka.get_by_name(concept.name) == concept
-        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
-
-        # I can get by the first entry
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
-        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
-
-        # saved in sdp
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_KEY_ENTRY, concept.key)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
-
-    def test_i_can_add_a_concept_when_name_differs_from_the_key(self):
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a")
-
-        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
-        sheerka.cache_manager.commit(context)
-
-        assert res.status
-        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
-
-        concept_found = res.value.body
-        for prop in PROPERTIES_TO_SERIALIZE:
-            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
-
-        assert concept_found.key == "hello __var__0"
-        assert concept_found.id == "1001"
-
-        # saved in cache
-        assert sheerka.has_id(concept.id)
-        assert sheerka.has_key(concept.key)
-        assert sheerka.has_name(concept.name)
-        assert sheerka.has_hash(concept.get_definition_hash())
-
-        # I can get the concept using various index
-        assert sheerka.get_by_id(concept.id) == concept
-        assert sheerka.get_by_key(concept.key) == concept
-        assert sheerka.get_by_name(concept.name) == concept
-        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
-
-        # saved in sdp
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_KEY_ENTRY, concept.key)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello")
-
-    def test_i_cannot_add_the_same_concept_twice(self):
-        """
-        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
-        :return:
-        """
-        sheerka = self.get_sheerka()
-        concept = self.get_default_concept()
-
-        sheerka.create_new_concept(self.get_context(sheerka), concept)
-        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
-
-        assert not res.status
-        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
-        assert res.value.body == concept
-
-    def test_i_can_get_a_newly_created_concept(self):
-        sheerka = self.get_sheerka()
-        concept = self.get_default_concept()
-
-        sheerka.create_new_concept(self.get_context(sheerka), concept)
-
-        from_cache = sheerka.get_by_key(concept.key)
-        assert from_cache is not None
-        assert from_cache == concept
-
-        from_cache = sheerka.get_by_id(concept.id)
-        assert from_cache is not None
-        assert from_cache == concept
-
-    def test_i_can_get_list_of_concept_when_same_key_using_cache(self):
-        sheerka = self.get_sheerka()
-        concept1 = self.get_default_concept()
-        concept2 = self.get_default_concept()
-        concept2.get_metadata().body = "a+b"
-
-        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
-        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
-
-        assert res1.value.body.key == res2.value.body.key  # same key
-
-        result = sheerka.get_by_key(concept1.key)
-        assert len(result) == 2
-        assert result[0] == concept1
-        assert result[1] == concept2
-
-    def test_concept_that_references_itself_is_correctly_created(self):
-        sheerka = self.get_sheerka()
-        concept = Concept("foo", body="foo")
-
-        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
-
-        assert res.status
-
-    def test_i_can_get_by_name_when_created_with_def_definition(self):
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = self.from_def_concept("plus", "a plus b", ["a", "b"])
-
-        res = sheerka.create_new_concept(context, concept)
-
-        assert res.status
-        assert sheerka.get_by_name(concept.name) == concept
-        assert sheerka.get_by_name(concept.get_metadata().definition) == concept
-
-        concept = Concept(name="foo", definition="foo", definition_type=DEFINITION_TYPE_DEF)
-        res = sheerka.create_new_concept(context, concept)
-
-        assert res.status
-        assert sheerka.get_by_name(concept.name) == concept  # it's not a list, ie the entry is not duplicated
-
-    def test_i_can_get_first_token_when_not_a_letter(self):
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = Concept("--filter a").def_var("a")
-
-        res = sheerka.create_new_concept(context, concept)
-        assert res.status
-
-        # I can get by the first entry
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
-        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
-
-    @pytest.mark.parametrize("expression", [
-        "--'filter' ('one' | 'two') ",
-        "'--filter' ('one' | 'two') ",
-    ])
-    def test_i_can_get_first_token_when_bnf_concept_and_not_a_letter(self, expression):
-        sheerka, context, bnf_concept = self.init_concepts(
-            Concept("foo", definition=expression),
-            create_new=True)
-
-        # I can get by the first entry
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
-        assert sheerka.cache_manager.get(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
-
-    def test_concept_references_are_updated_1(self):
-        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
-            "one",
-            "two",
-            "number",
-            "twenty",
-            Concept("twenties", definition="twenty one | two 'hundred'"),
-            create_new=True
-        )
-
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, one.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, two.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, number.id) is None
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
-
-    def test_concept_references_are_updated_2(self):
-        sheerka, context, one, two, number, twenty, twenties = self.init_concepts(
-            "one",
-            "two",
-            "number",
-            "twenty",
-            Concept("twenties", definition="twenty number"),
-            create_new=True
-        )
-
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, one.id) is None
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, two.id) is None
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, number.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
-        assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
-
-
-class TestSheerkaCreateNewConceptFileBased(TestUsingFileBasedSheerka):
-    def test_i_can_add_several_concepts(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-        hello = Concept("Hello world a").def_var("a")
-        res = sheerka.create_new_concept(context, hello)
-        sheerka.cache_manager.commit(context)
-        assert res.status
-
-        sheerka = self.get_sheerka()  # another instance
-        context = self.get_context(sheerka)
-        greeting = Concept("Greeting a").def_var("a")
-        res = sheerka.create_new_concept(context, greeting)
-        sheerka.cache_manager.commit(context)
-        assert res.status
-
-        sheerka = self.get_sheerka()  # another instance again
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_KEY_ENTRY, hello.key)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_KEY_ENTRY, greeting.key)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, hello.id)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, greeting.id)
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, "Hello world a")
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_NAME_ENTRY, "Greeting a")
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, hello.get_definition_hash())
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, greeting.get_definition_hash())
-
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello")
-        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting")
-
-    def test_i_cannot_add_the_same_concept_twice_using_sdp(self):
-        """
-        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
-        :return:
-        """
-        sheerka = self.get_sheerka(cache_only=False)
-        context = self.get_context(sheerka)
-        concept = self.get_default_concept()
-
-        sheerka.create_new_concept(context, concept)
-        sheerka.cache_manager.commit(context)
-
-        sheerka.cache_manager.clear()
-        res = sheerka.create_new_concept(context, concept)
-
-        assert not res.status
-        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
-        assert res.value.body == concept
-
-    def test_new_entry_does_not_override_the_previous_ones(self):
-        sheerka = self.get_sheerka()
-        context = self.get_context(sheerka)
-        sheerka.create_new_concept(context, Concept("foo", body="1"))
-        sheerka.create_new_concept(context, Concept("foo", body="2"))
-        sheerka.cache_manager.commit(context)
-
-        assert len(sheerka.sdp.get(Sheerka.CONCEPTS_BY_KEY_ENTRY, "foo")) == 2
-
-        sheerka = self.get_sheerka()  # new instance
-        context = self.get_context(sheerka)
-        sheerka.create_new_concept(context, Concept("foo", body="3"))
-        sheerka.cache_manager.commit(context)
-
-        assert len(sheerka.sdp.get(Sheerka.CONCEPTS_BY_KEY_ENTRY, "foo")) == 3

tests/core/test_SheerkaEvaluateConcept.py

@@ -375,6 +375,26 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.get_metadata().is_evaluated
         assert sheerka.objvalue(evaluated) == 2
 
+    def test_i_can_evaluate_a_concept_that_references_another_concept_twice(self):
+        """
+        Test that a new instance of concept is return when the metadata refers to a concept
+        :return:
+        """
+        sheerka, context, predicate, foo = self.init_concepts(
+            Concept("Sometimes True", body="in_context('a')"),
+            Concept("foo", pre="c:Sometimes True:"))
+
+        foo1 = sheerka.new("foo")
+        foo1 = sheerka.evaluate_concept(context, foo1)  # 'a' is not in context, so it fails
+
+        context2 = self.get_context(sheerka)
+        context2.add_to_protected_hints('a')
+        foo2 = sheerka.new("foo")
+        foo2 = sheerka.evaluate_concept(context2, foo2)  # 'a' in context + new instance of 'Sometimes True'
+
+        assert sheerka.isinstance(foo1, BuiltinConcepts.CONDITION_FAILED)
+        assert sheerka.isinstance(foo2, "foo")
+
     def test_i_can_reference_sheerka(self):
         sheerka = self.get_sheerka()
 
@@ -624,8 +644,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
             eval_body=True
         )
 
-        sheerka.add_in_cache(one_str)
+        sheerka.test_only_add_in_cache(one_str)
-        sheerka.add_in_cache(one_digit)
+        sheerka.test_only_add_in_cache(one_digit)
 
         evaluated = sheerka.evaluate_concept(context, one_digit)
         assert evaluated.key == one_digit.key

tests/core/test_SheerkaModifyConcept.py

@@ -1,121 +0,0 @@
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts, get_concept_attrs
-
-from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
-from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-
-
-class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka):
-
-    def test_i_can_modify_a_concept(self):
-        sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True, cache_only=False)
-
-        assert get_concept_attrs(foo) == []
-
-        foo_instance = sheerka.new("foo")
-        foo_instance.get_metadata().body = "metadata value"  # modify metadata
-        foo_instance.def_var("var_name", "default value")  # modify definition of variables
-        foo_instance.add_prop(BuiltinConcepts.ISA, bar)  # modify property
-        foo_instance.set_value(ConceptParts.BODY, "body value")  # modify value
-        foo_instance.set_value("var_name", "var value")  # modify value
-        res = sheerka.modify_concept(context, foo_instance)
-
-        assert res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
-        assert res.body.body.get_metadata().body == "metadata value"
-        assert res.body.body.get_metadata().variables == [("var_name", "default value")]
-        assert res.body.body.get_prop(BuiltinConcepts.ISA) == {bar}
-        assert res.body.body.body == "body value"
-        assert res.body.body.get_value("var_name") == "var value"
-        assert get_concept_attrs(foo) == ["var_name"]
-
-        # test that object
-        foo_from_sheerka = sheerka.get_by_key("foo")
-        assert foo_from_sheerka.get_metadata().body == "metadata value"
-        assert foo_from_sheerka.get_metadata().variables == [("var_name", "default value")]
-        assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == {bar}
-        assert foo_from_sheerka.body == "body value"
-        assert foo_from_sheerka.get_value("var_name") == "var value"
-
-        # other caches are also updated
-        assert sheerka.get_by_id(foo.id).get_metadata().body == "metadata value"
-        assert sheerka.get_by_name(foo.name).get_metadata().body == "metadata value"
-        assert sheerka.get_by_hash(foo_instance.get_definition_hash()).get_metadata().body == "metadata value"
-
-        # sdp can be updated
-        sheerka.cache_manager.commit(context)
-        from_sdp = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, foo.id)
-        assert from_sdp.get_metadata().body == "metadata value"
-        assert from_sdp.get_metadata().variables == [("var_name", "default value")]
-        assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}
-        assert from_sdp.body == "body value"
-        assert from_sdp.get_value("var_name") == "var value"
-
-    def test_i_cannot_modify_a_concept_that_does_not_exists(self):
-        sheerka, context = self.init_concepts()
-
-        foo = Concept("foo").init_key()
-        sheerka.set_id_if_needed(foo, False)
-
-        res = sheerka.modify_concept(context, foo)
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert res.body.body == ("id", foo.id)
-
-    def test_i_cannot_modify_a_concept_that_returns_an_error(self):
-        sheerka, context = self.init_concepts()
-
-        foo = Concept("foo").init_key()
-        res = sheerka.modify_concept(context, foo)
-
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
-
-    def test_i_cannot_modify_if_the_concept_has_not_changed(self):
-        sheerka, context, foo = self.init_concepts("foo", create_new=True)
-        res = sheerka.modify_concept(context, foo)
-
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
-
-    def test_i_can_modify_a_concept_that_is_in_a_list(self):
-        sheerka, context, foo1, foo2 = self.init_concepts(
-            Concept("foo", body="1"),
-            Concept("foo", body="2"), create_new=True)
-
-        foo2_instance = sheerka.new("foo")[1]
-        foo2_instance.get_metadata().body = "value"
-
-        res = sheerka.modify_concept(context, foo2_instance)
-        assert res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
-        assert res.body.body.get_metadata().body == "value"
-
-        foo_from_sheerka = sheerka.new("foo")
-        assert foo_from_sheerka[0].get_metadata().body == "1"
-        assert foo_from_sheerka[1].get_metadata().body == "value"
-
-    def test_i_can_get_and_set_attribute(self):
-        sheerka, context = self.init_concepts()
-        foo = Concept("foo")
-        prop = Concept("property")
-        bar = Concept("bar")
-
-        res = sheerka.set_attr(foo, prop, bar)
-        assert res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
-
-        assert sheerka.get_attr(foo, prop) == bar
-
-
-class TestSheerkaModifyConceptUsingFile(TestUsingFileBasedSheerka):
-
-    def test_i_can_modify_a_concept_from_a_new_sheerka(self):
-        sheerka, context, foo = self.init_concepts("foo", create_new=True)
-        sheerka.cache_manager.commit(context)
-
-        sheerka = self.get_sheerka()
-        foo.add_prop("a", "b")
-        res = sheerka.modify_concept(context, foo)
-
-        assert res.status

tests/core/test_SheerkaRuleManager.py

@@ -33,7 +33,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_create_a_new_rule(self, action_type, cache_entry):
         sheerka, context = self.init_concepts(cache_only=False)
-        previous_rules_number = sheerka.cache_manager.caches[sheerka.CONCEPTS_KEYS_ENTRY].cache.copy()[
+        previous_rules_number = sheerka.cache_manager.caches[sheerka.OBJECTS_IDS_ENTRY].cache.copy()[
             SheerkaRuleManager.RULE_IDS]
 
         rule = Rule(action_type, "name", "True", "Hello world")

tests/core/test_sheerka.py

@@ -4,6 +4,7 @@ import pytest
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept, AllBuiltinConcepts
 from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit
 from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.tokenizer import Token, TokenKind
 
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
@@ -342,16 +343,17 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
     def test_builtin_concepts_are_initialized(self):
         sheerka = self.get_sheerka()
+        service = sheerka.services[SheerkaConceptManager.NAME]
         for concept_name in AllBuiltinConcepts:
-            assert sheerka.has_key(str(concept_name))
+            assert service.has_key(str(concept_name))
-            assert sheerka.sdp.get(sheerka.CONCEPTS_BY_KEY_ENTRY, str(concept_name)) is not None
+            assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, str(concept_name)) is not None
 
         # I can get back data from the sdp when the cache is empty
         sheerka.cache_manager.clear()
 
         # caches are empty
-        assert not sheerka.has_id("1")
+        assert not service.has_id("1")
-        assert not sheerka.has_key(str(BuiltinConcepts.SHEERKA))
+        assert not service.has_key(str(BuiltinConcepts.SHEERKA))
 
         assert sheerka.get_by_id("1") == sheerka  # use sdp
 
@@ -359,11 +361,12 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
     def test_builtin_concepts_can_be_updated(self):
         sheerka = self.get_sheerka()
+        service = sheerka.services[SheerkaConceptManager.NAME]
         before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
         before_parsing.get_metadata().desc = "I have a description"
         before_parsing.get_metadata().full_serialization = True
         with sheerka.sdp.get_transaction("Test") as transac:
-            transac.add(sheerka.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True)
+            transac.add(service.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True)
 
         sheerka = self.get_sheerka()  # another fresh new instance
         before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
@@ -390,11 +393,12 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
     def test_i_can_retrieve_from_sdp_when_cache_is_reset(self):
         sheerka, context, concept = self.init_concepts(Concept("foo", body="1"))
+        service = sheerka.services[SheerkaConceptManager.NAME]
         sheerka.cache_manager.commit(context)
 
         sheerka.cache_manager.clear()
         sheerka.get_by_key("foo")
-        assert sheerka.has_key("foo")
+        assert service.has_key("foo")
 
         # It's also updated when sdp returns more than one element
         concept2 = Concept("foo", body="2")
@@ -403,20 +407,20 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
         sheerka.cache_manager.clear()
         assert len(sheerka.get_by_key("foo")) == 2
-        assert sheerka.has_key("foo")
+        assert service.has_key("foo")
 
         # updated when by_id
         sheerka.cache_manager.clear()
         assert sheerka.get_by_id("1001") == concept
-        assert sheerka.has_id("1001")
+        assert service.has_id("1001")
 
         sheerka.cache_manager.clear()
         assert sheerka.get_by_name("foo") == [concept, concept2]
-        assert sheerka.has_name("foo")
+        assert service.has_name("foo")
 
         sheerka.cache_manager.clear()
         assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
-        assert sheerka.has_hash(concept.get_definition_hash())
+        assert service.has_hash(concept.get_definition_hash())
 
     def test_get_by_key_retrieve_all_elements(self):
         sheerka, context, *concepts = self.init_concepts(

tests/evaluators/test_AddConceptInSetEvaluator.py

@@ -41,7 +41,7 @@ class TestAddConceptInSetEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         foo = Concept("foo")
         context.sheerka.set_id_if_needed(foo, False)
-        context.sheerka.add_in_cache(foo)
+        context.sheerka.test_only_add_in_cache(foo)
 
         ret_val = get_isa_ret_val("foo", "bar")
         res = AddConceptInSetEvaluator().eval(context, ret_val)

tests/evaluators/test_ConceptEvaluator.py

@@ -93,10 +93,10 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
     def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(self):
         context = self.get_context()
         context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-        context.sheerka.add_in_cache(Concept(name="one").init_key())
+        context.sheerka.test_only_add_in_cache(Concept(name="one").init_key())
-        concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b")
+        concept_plus = context.sheerka.test_only_add_in_cache(Concept(name="a plus b")
-                                                    .def_var("a", "one")
+                                                              .def_var("a", "one")
-                                                    .def_var("b", "two").init_key())
+                                                              .def_var("b", "two").init_key())
 
         evaluator = ConceptEvaluator()
         item = self.pretval(concept_plus)

tests/evaluators/test_PythonEvaluator.py

@@ -3,6 +3,7 @@ import ast
 import pytest
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.concept import Concept, CB
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
 from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError, NamesWithAttributesVisitor
@@ -18,6 +19,10 @@ def get_obj_name(obj):
     return obj.name
 
 
+def return_return_value(status):
+    return ReturnValueConcept("who", status, f"the value is {status}")
+
+
 def get_source_code_node(source_code, concepts=None):
     if concepts:
         for concept_name, concept in sorted(concepts.items(), key=lambda kv: len(kv[0]), reverse=True):
@@ -132,8 +137,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         evaluated = PythonEvaluator().eval(context, parsed)
 
         assert evaluated.status
-        assert sheerka.has_key("foo")
+        assert sheerka.services[SheerkaConceptManager.NAME].has_key("foo")
-
 
     def test_i_can_eval_ast_expression_that_references_concepts(self):
         """
@@ -141,7 +145,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         :return:
         """
         context = self.get_context()
-        context.sheerka.add_in_cache(Concept("foo", body="1"))
+        context.sheerka.test_only_add_in_cache(Concept("foo", body="1"))
 
         parsed = PythonParser().parse(context, ParserInput("foo + 2"))
         evaluated = PythonEvaluator().eval(context, parsed)
@@ -155,7 +159,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         :return:
         """
         context = self.get_context()
-        context.sheerka.add_in_cache(Concept("foo"))
+        context.sheerka.test_only_add_in_cache(Concept("foo"))
 
         parsed = PythonParser().parse(context, ParserInput("def a(b):\n    return b\na(c:foo:)"))
         evaluated = PythonEvaluator().eval(context, parsed)
@@ -178,7 +182,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_eval_concept_token(self):
         context = self.get_context()
-        context.sheerka.add_in_cache(Concept("foo", body="2"))
+        context.sheerka.test_only_add_in_cache(Concept("foo", body="2"))
         context.add_to_short_term_memory("get_obj_name", get_obj_name)
 
         parsed = PythonParser().parse(context, ParserInput("get_obj_name(c:foo:)"))
@@ -190,7 +194,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_eval_when_expect_success(self):
         context = self.get_context()
-        context.sheerka.add_in_cache(Concept("foo", body="2"))
+        context.sheerka.test_only_add_in_cache(Concept("foo", body="2"))
 
         parsed = PythonParser().parse(context, ParserInput("foo==2"))
         python_evaluator = PythonEvaluator()
@@ -334,6 +338,23 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.value == "Print return values"
 
+    @pytest.mark.parametrize("method, expected_status", [
+        ("return_return_value(True)", True),
+        ("return_return_value(False)", False),
+    ])
+    def test_i_can_eval_a_function_that_returns_a_return_value(self, method, expected_status):
+        context = self.get_context()
+        context.add_to_short_term_memory("return_return_value", return_return_value)
+
+        parsed = FunctionParser().parse(context, ParserInput(method))
+        python_evaluator = PythonEvaluator()
+        evaluated = python_evaluator.eval(context, parsed)
+        ret_val = return_return_value(expected_status)
+
+        assert evaluated.status == expected_status
+        assert evaluated.value == ret_val.body
+        assert ret_val in evaluated.parents
+
     @pytest.mark.parametrize("text, expected", [
         ("foo.bar.baz", [["foo", "bar", "baz"]]),
         ("foo.bar.baz; one.two.three", [["foo", "bar", "baz"]]),

tests/non_reg/test_sheerka_non_reg.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit, CC
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
 from evaluators.PythonEvaluator import PythonEvalError
@@ -29,7 +30,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
     def test_i_can_recognize_concept_with_python_body(self):
         sheerka = self.get_sheerka()
         concept = Concept(name="one", body="1")
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
 
         text = "one"
         res = sheerka.evaluate_user_input(text)
@@ -45,8 +46,8 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
         sheerka = self.get_sheerka()
         concept_one = Concept(name="one")
         concept_un = Concept(name="un", body="one")
-        sheerka.add_in_cache(concept_one)
+        sheerka.test_only_add_in_cache(concept_one)
-        sheerka.add_in_cache(concept_un)
+        sheerka.test_only_add_in_cache(concept_un)
 
         res = sheerka.evaluate_user_input("un")
         return_value = res[0].value
@@ -61,7 +62,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
     def test_i_can_recognize_concept_with_no_body(self):
         sheerka = self.get_sheerka()
         concept = Concept(name="one")
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
 
         text = "one"
         res = sheerka.evaluate_user_input(text)
@@ -73,7 +74,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
     def test_is_unique_property_is_used_when_evaluating(self):
         sheerka = self.get_sheerka()
         concept = Concept(name="one", is_unique=True)
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
 
         text = "one"
         res = sheerka.evaluate_user_input(text)
@@ -113,14 +114,15 @@ as:
             assert getattr(concept_saved.get_metadata(), prop) == getattr(expected.get_metadata(), prop)
 
         # cache is up to date
-        assert sheerka.has_key(concept_saved.key)
+        service = sheerka.services[SheerkaConceptManager.NAME]
-        assert sheerka.has_id(concept_saved.id)
+        assert service.has_key(concept_saved.key)
-        assert sheerka.has_name(concept_saved.name)
+        assert service.has_id(concept_saved.id)
-        assert sheerka.has_hash(concept_saved.get_definition_hash())
+        assert service.has_name(concept_saved.name)
+        assert service.has_hash(concept_saved.get_definition_hash())
         assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {'+': ['1001']}
 
         # sdp is up to date
-        assert sheerka.sdp.exists(sheerka.CONCEPTS_BY_KEY_ENTRY, expected.key)
+        assert sheerka.sdp.exists(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, expected.key)
 
     def test_i_can_evaluate_def_concept_part_when_one_part_is_a_ref_of_another_concept(self):
         """
@@ -132,7 +134,7 @@ as:
 
         # concept 'a plus b' is known
         concept_a_plus_b = Concept(name="a plus b").def_var("a").def_var("b").init_key()
-        sheerka.add_in_cache(concept_a_plus_b)
+        sheerka.test_only_add_in_cache(concept_a_plus_b)
 
         res = sheerka.evaluate_user_input("def concept a xx b as a plus b")
         expected = Concept(name="a xx b", body="a plus b").def_var("a").def_var("b").init_key()
@@ -147,7 +149,7 @@ as:
         for prop in PROPERTIES_TO_SERIALIZE:
             assert getattr(concept_saved.get_metadata(), prop) == getattr(expected.get_metadata(), prop)
 
-        assert sheerka.has_key(concept_saved.key)
+        assert sheerka.services[SheerkaConceptManager.NAME].has_key(concept_saved.key)
 
     def test_i_cannot_evaluate_the_same_def_concept_twice(self):
         text = """
@@ -201,8 +203,8 @@ as:
 
     def test_i_can_recognize_concept_with_variable_and_python_as_body(self):
         sheerka = self.get_sheerka()
-        hello_a = sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_var("a"))
+        hello_a = sheerka.test_only_add_in_cache(Concept(name="hello a", body="'hello ' + a").def_var("a"))
-        sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
+        sheerka.test_only_add_in_cache(Concept(name="foo", body="'foo'"))
 
         res = sheerka.evaluate_user_input("hello foo")
         assert len(res) == 1
@@ -600,7 +602,7 @@ as:
     ])
     def test_i_can_manage_tokenizer_error(self, text):
         sheerka = self.get_sheerka()
-        sheerka.add_in_cache(Concept("foo"))
+        sheerka.test_only_add_in_cache(Concept("foo"))
 
         res = sheerka.evaluate_user_input(text)
 
@@ -610,7 +612,7 @@ as:
 
     def test_i_can_recognize_concept_from_string(self):
         sheerka = self.get_sheerka()
-        sheerka.add_in_cache(Concept("one", body="1"))
+        sheerka.test_only_add_in_cache(Concept("one", body="1"))
 
         res = sheerka.evaluate_user_input("'one'")
 
@@ -792,7 +794,7 @@ as:
 
         sheerka = self.init_scenario(definitions)
         context = self.get_context(sheerka)
-        sheerka.force_sya_def(context, [
+        sheerka.test_only_force_sya_def(context, [
             (sheerka.get_by_name("mult").id, 20, SyaAssociativity.Right),
             (sheerka.get_by_name("plus").id, 10, SyaAssociativity.Right),
         ])
@@ -954,7 +956,7 @@ as:
 
         # simulate that sheerka was stopped and restarted
         sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
-        sheerka.cache_manager.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "twenties").set_compiled({})
+        sheerka.cache_manager.get(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, "twenties").set_compiled({})
 
         res = sheerka.evaluate_user_input("eval twenty one")
         assert res[0].status
@@ -971,6 +973,7 @@ as:
         res = sheerka.evaluate_user_input("set_isa(last_created_concept(), number)")
 
         assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
         assert sheerka.isa(sheerka.new("one"), sheerka.new("number"))
 
     def test_i_can_evaluate_sya_and_ret_concepts(self):
@@ -1219,7 +1222,7 @@ class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
         assert res[0].status
         assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
 
-        saved_concept = sheerka.sdp.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "plus")
+        saved_concept = sheerka.sdp.get(SheerkaConceptManager.CONCEPTS_BY_KEY_ENTRY, "plus")
         assert saved_concept.key == "plus"
         assert saved_concept.get_metadata().definition == "a ('plus' plus)?"
         assert "a" in saved_concept.values()

tests/parsers/test_BaseNodeParser.py

@@ -48,7 +48,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
 
         bar = Concept("bar").init_key()
         sheerka.set_id_if_needed(bar, False)
-        sheerka.add_in_cache(bar)
+        sheerka.test_only_add_in_cache(bar)
 
         concept = Concept("foo").init_key()
         concept.set_bnf(bnf)
@@ -65,11 +65,11 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
 
         bar = Concept("bar").init_key()
         sheerka.set_id_if_needed(bar, False)
-        sheerka.add_in_cache(bar)
+        sheerka.test_only_add_in_cache(bar)
 
         baz = Concept("baz").init_key()
         sheerka.set_id_if_needed(baz, False)
-        sheerka.add_in_cache(baz)
+        sheerka.test_only_add_in_cache(baz)
 
         foo = Concept("foo").init_key()
         foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux")))
@@ -96,7 +96,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
 
         bar = Concept("bar").init_key()
         sheerka.set_id_if_needed(bar, False)
-        sheerka.add_in_cache(bar)
+        sheerka.test_only_add_in_cache(bar)
 
         foo = Concept("foo").init_key()
         foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux")))

tests/parsers/test_DefConceptParser.py

@@ -503,6 +503,15 @@ from give me the date !
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
         assert res.value.body == ("key", "unknown")
 
+    def test_i_cannot_parse_bnf_definition_referencing_multiple_concepts_sharing_the_same_name(self):
+        text = "def concept twenty one from bnf 'twenty' one"
+        sheerka, context, parser, *concepts = self.init_parser(Concept("one", body="1"), Concept("one", body="1.0"))
+        res = parser.parse(context, ParserInput(text))
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
+        assert res.value.body == ("key", "one")
+
     @pytest.mark.parametrize("text", [
         'def concept "def concept x"',
         'def concept "def concept x" as x',

tests/parsers/test_ExactConceptParser.py

@@ -210,7 +210,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
     # def test_i_can_detect_concept_from_tokens(self):
     #     context = self.get_context(self.get_sheerka(singleton=True))
     #     concept = get_concept("hello world", [])
-    #     context.sheerka.add_in_cache(concept)
+    #     context.sheerka.test_only_add_in_cache(concept)
     #
     #     source = "hello world"
     #     results = ExactConceptParser().parse(context, list(Tokenizer(source)))

tests/parsers/test_SyaNodeParser.py

@@ -66,7 +66,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
                                                       cmap["minus"],
                                                       CONCEPT_COMPARISON_CONTEXT)
 
-        # TestSyaNodeParser.sheerka.force_sya_def(context, [
+        # TestSyaNodeParser.sheerka.test_only_force_sya_def(context, [
         #     (cmap["plus"].id, 5, SyaAssociativity.Right),
         #     (cmap["mult"].id, 10, SyaAssociativity.Right),
         #     (cmap["minus"].id, 5, SyaAssociativity.Right)])

tests/parsers/test_UnrecognizedNodeParser.py

@@ -76,7 +76,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
     def setup_class(cls):
         t = TestUnrecognizedNodeParser()
         TestUnrecognizedNodeParser.sheerka, context, _ = t.init_parser(concepts_map, create_new=True)
-        TestUnrecognizedNodeParser.sheerka.force_sya_def(context, [
+        TestUnrecognizedNodeParser.sheerka.test_only_force_sya_def(context, [
             (concepts_map["mult"].id, 20, SyaAssociativity.Right),
             (concepts_map["plus"].id, 10, SyaAssociativity.Right),
         ])

tests/sheerkapickle/test_SheerkaPickler.py

@@ -138,7 +138,7 @@ class TestSheerkaPickler(TestUsingMemoryBasedSheerka):
 
         concept = Concept("foo").init_key()
         sheerka.set_id_if_needed(concept, False)
-        sheerka.add_in_cache(concept)
+        sheerka.test_only_add_in_cache(concept)
         obj = {concept: "a"}
         flatten = SheerkaPickler(sheerka).flatten(obj)
         assert flatten == {'c:foo|1001:': 'a'}