Implemented ConceptManager with concept creation, modification and deletion

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -172,7 +172,7 @@ class SheerkaComparisonManager(BaseService):
 
         cycles = self.detect_cycles(new)
         if cycles:
-            concepts_in_cycle = [self.sheerka.resolve(c) for c in cycles]
+            concepts_in_cycle = [self.sheerka.fast_resolve(c) for c in cycles]
             chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle)
             return self.sheerka.ret(self.NAME, False, chicken_an_egg)
 

src/core/sheerka/services/SheerkaConceptManager.py

@@ -0,0 +1,632 @@
+from dataclasses import dataclass
+
+import core.utils
+from cache.Cache import Cache
+from cache.CacheManager import ConceptNotFound
+from cache.ListIfNeededCache import ListIfNeededCache
+from cache.SetCache import SetCache
+from core.builtin_concepts import BuiltinConcepts, ErrorConcept, AllBuiltinConcepts, BuiltinUnique
+from core.builtin_helpers import ensure_concept
+from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, NotInit, \
+    ConceptMetadata
+from core.error import ErrorObj
+from core.global_symbols import EVENT_CONCEPT_CREATED
+from core.sheerka.services.sheerka_service import BaseService
+from core.tokenizer import Tokenizer, TokenKind
+from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
+
+BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
+
+
+@dataclass
+class NoModificationFound(ErrorObj):
+    """
+    Trying to modify a concept without modifying it
+    """
+    concept: Concept
+    attrs: dict = None
+
+
+@dataclass
+class ForbiddenAttribute(ErrorObj):
+    """
+    When trying to modify an attribute that must not be modified
+    """
+    attr: str
+
+
+@dataclass
+class UnknownAttribute(ErrorObj):
+    """
+    When trying to modify an attribute that does not exist
+    """
+    attr: str
+
+
+@dataclass
+class CannotRemoveMeta(ErrorObj):
+    """
+    When trying to remove a metadata attribute (ConceptMeta is a class, you cannot remove attr form it)
+    """
+    attrs: dict
+
+
+@dataclass
+class ValueNotFound(ErrorObj):
+    """
+    When trying to remove a value that does not exists (but the props/variable exists)
+    """
+    item: str
+    value: object
+
+
+@dataclass
+class ConceptIsReferenced(ErrorObj):
+    """
+    When trying to remove a concept that is referenced by other concept(s)
+    """
+    references: list
+
+
+class SheerkaConceptManager(BaseService):
+    NAME = "ConceptManager"
+
+    BUILTIN_CONCEPTS_IDS = "Builtins_Concepts_IDs"  # sequential key for builtin concepts
+    USER_CONCEPTS_IDS = "User_Concepts_IDs"  # sequential key for user defined concepts
+
+    CONCEPTS_IDS_ENTRY = "ConceptManager:Concepts_IDs"
+
+    CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID"  # to store all the concepts
+    CONCEPTS_BY_KEY_ENTRY = "ConceptManager:Concepts_By_Key"
+    CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name"
+    CONCEPTS_BY_HASH_ENTRY = "ConceptManager:Concepts_By_Hash"  # store hash of concepts definitions (not values)
+    CONCEPTS_REFERENCES_ENTRY = "ConceptManager:Concepts_References"  # tracks references between concepts
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS)  # BaseNodeParser
+        self.forbidden_meta = {"is_builtin", "key", "id", "props", "variables"}
+        self.allowed_meta = {attr for attr in vars(ConceptMetadata) if
+                             not attr.startswith("_") and attr not in self.forbidden_meta}
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.create_new_concept, True)
+        self.sheerka.bind_service_method(self.modify_concept, True)
+        self.sheerka.bind_service_method(self.remove_concept, True)
+        self.sheerka.bind_service_method(self.set_id_if_needed, True)
+        self.sheerka.bind_service_method(self.set_attr, True)
+        self.sheerka.bind_service_method(self.get_attr, False)
+        self.sheerka.bind_service_method(self.get_by_key, False, visible=False)
+        self.sheerka.bind_service_method(self.get_by_name, False, visible=False)
+        self.sheerka.bind_service_method(self.get_by_hash, False, visible=False)
+        self.sheerka.bind_service_method(self.get_by_id, False, visible=False)
+        self.sheerka.bind_service_method(self.not_is_variable, False, visible=False)
+
+        def params(cache_name):
+            return {
+                'default': lambda k: self.sheerka.sdp.get(cache_name, k),
+                'extend_exists': lambda k: self.sheerka.sdp.exists(cache_name, k)
+            }
+
+        register_concept_cache = self.sheerka.cache_manager.register_concept_cache
+
+        cache = Cache(**params(self.CONCEPTS_BY_ID_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
+
+        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_KEY_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
+
+        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_NAME_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
+
+        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
+
+        cache = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_REFERENCES_ENTRY, k))
+        self.sheerka.cache_manager.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache)
+
+    def initialize_deferred(self, context, is_first_time):
+        if is_first_time:
+            self.sheerka.cache_manager.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
+
+    def initialize_builtin_concepts(self):
+        """
+        Initializes the builtin concepts
+        :return: None
+        """
+        builtin_concepts_ids = {}
+
+        for key in AllBuiltinConcepts:
+            concept = self.sheerka if key == BuiltinConcepts.SHEERKA \
+                else self.sheerka.builtin_cache[str(key)]() if str(key) in self.sheerka.builtin_cache \
+                else Concept(key, True, False, key)
+
+            if key in BuiltinUnique:
+                concept.get_metadata().is_unique = True
+                concept.get_metadata().is_evaluated = True
+
+            from_db = self.sheerka.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key)
+            if from_db is None:
+                # self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
+                self.set_id_if_needed(concept, True)
+                self.sheerka.cache_manager.add_concept(concept)
+            else:
+                # self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
+                concept.update_from(from_db)
+
+            builtin_concepts_ids[key] = concept.id
+
+        return builtin_concepts_ids
+
+    def create_new_concept(self, context, concept: Concept):
+        """
+        Adds a new concept to the system
+        :param context:
+        :param concept:  DefConceptNode
+        :return: digest of the new concept
+        """
+        ensure_concept(concept)
+
+        sheerka = self.sheerka
+
+        concept.init_key()
+        init_bnf_ret_value = None
+
+        cache_manager = sheerka.cache_manager
+
+        if cache_manager.exists(self.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
+            error = SheerkaDataProviderDuplicateKeyError(self.CONCEPTS_BY_KEY_ENTRY + "." + concept.key, concept)
+            return sheerka.ret(
+                self.NAME,
+                False,
+                sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
+                error.args[0])
+
+        # set id before saving in db
+        sheerka.set_id_if_needed(concept, False)
+
+        # freeze attributes
+        freeze_concept_attrs(concept)
+
+        # check if the bnf definition is correctly computed
+        try:
+            self.bnp.ensure_bnf(context, concept)
+        except Exception as ex:
+            return sheerka.ret(self.NAME, False, ex.args[0])
+
+        # compute new concepts_by_first_keyword
+        init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True)
+        if not init_ret_value.status:
+            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
+        concepts_by_first_keyword = init_ret_value.body
+
+        # computes resolved concepts_by_first_keyword
+        init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
+        if not init_ret_value.status:
+            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
+        resolved_concepts_by_first_keyword = init_ret_value.body
+
+        # if everything is fine
+        concept.freeze_definition_hash()
+        cache_manager.add_concept(concept)
+        cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
+        cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
+
+        if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
+            # allow search by definition when definition relevant
+            cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept)
+
+        # update references
+        for ref in self.compute_references(concept):
+            cache_manager.put(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
+
+        # TODO : this line seems to be useless
+        # The grammar is never reset
+        if concept.get_bnf() and init_bnf_ret_value is not None and init_bnf_ret_value.status:
+            sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
+
+        # publish the new concept
+        sheerka.publish(context, EVENT_CONCEPT_CREATED, concept)
+
+        # process the return if needed
+        ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
+        return ret
+
+    def modify_concept(self, context, concept, to_add=None, to_remove=None, modify_source=False):
+        """
+        Modify the definition of a concept
+        :param context:
+        :param concept: concept to modify
+        :param to_add: meta, props or variables to add/update
+        :param to_remove: props or variables to remove
+        :param modify_source: update or not the concept given in parameter
+        :return:
+        """
+
+        # to_add is a dictionary
+        # to_add = {
+        #           'meta'     : {<key, value>} of metadata to add/update,
+        #           'props'    : {<key, value>} of properties to add/update,
+        #           'variables': {<key, value>} of variables to add/update,
+        #           }
+        # if the <key> already exists, the entry is updated, otherwise a new value is created
+        # for props, if the <key> already exists, a new entry is added to the set
+        #
+        # to_remove = {
+        #               'props'    : {<key, [value]>} entries to remove. 'value' can be a list or a single entry
+        #               'variables': [<key>] list of keys to remove
+        #              }
+        #
+        sheerka = self.sheerka
+        cache_manager = self.sheerka.cache_manager
+
+        if not to_add and not to_remove:
+            return sheerka.ret(self.NAME, False, sheerka.err(NoModificationFound(concept)))
+
+        if not sheerka.cache_manager.exists(self.CONCEPTS_BY_ID_ENTRY, concept.id):
+            return sheerka.ret(self.NAME, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept))
+
+        # modify the metadata. Almost all ConceptMetadata attributes except variables and props
+        new_concept = sheerka.new_from_template(concept, concept.key)  # reload from cache or database ?
+
+        res = self._update_concept(context, new_concept, to_add, to_remove)
+        if res is not None:
+            return res
+
+        freeze_concept_attrs(new_concept)
+
+        # To update concept by first keyword
+        # first remove the old references
+        keywords = self.bnp.get_first_tokens(sheerka, concept)  # keyword of the old concept
+        concepts_by_first_keyword = cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
+        for keyword in keywords:
+            try:
+                concepts_by_first_keyword[keyword].remove(concept.id)
+                if len(concepts_by_first_keyword[keyword]) == 0:
+                    del concepts_by_first_keyword[keyword]
+            except KeyError:  # only occurs in unit tests when concepts are created without create_new()
+                pass
+
+        # and then update
+        init_ret_value = self.bnp.get_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
+        if not init_ret_value.status:
+            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
+        concepts_by_first_keyword = init_ret_value.body
+
+        # computes resolved concepts_by_first_keyword
+        init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context,
+                                                                    concepts_by_first_keyword,
+                                                                    {new_concept.id: new_concept})
+        if not init_ret_value.status:
+            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
+        resolved_concepts_by_first_keyword = init_ret_value.body
+
+        # update concept that referenced the old concept and clear old references
+        self.update_references(context, concept, new_concept, to_add)
+        for ref in self.compute_references(concept):
+            cache_manager.delete(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
+
+        # compute new references
+        for ref in self.compute_references(new_concept):
+            cache_manager.put(self.CONCEPTS_REFERENCES_ENTRY, ref, new_concept.id)
+
+        cache_manager.update_concept(concept, new_concept)
+        cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
+        cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False,
+                          resolved_concepts_by_first_keyword)
+
+        # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
+        # TODO : Update concepts grammars : have a look at update_references() below
+        if modify_source:
+            self._update_concept(context, concept, to_add, to_remove)
+
+        ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=new_concept))
+        return ret
+
+    def remove_concept(self, context, concept):
+        """
+        Remove a concept
+        :param context:
+        :param concept:
+        :return:
+        """
+        sheerka = context.sheerka
+        refs = self.sheerka.cache_manager.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
+        if refs:
+            refs_instances = [sheerka.new_from_template(c, c.key) for c in [self.get_by_id(ref) for ref in refs]]
+            return sheerka.ret(self.NAME, False, sheerka.err(ConceptIsReferenced(refs_instances)))
+
+        try:
+            sheerka.cache_manager.remove_concept(concept)
+            return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS))
+        except ConceptNotFound as ex:
+            return sheerka.ret(self.NAME, False, sheerka.err(ex))
+
+    def set_attr(self, concept, attribute, value):
+        """
+        Modifies an attribute of a concept (concept.values)
+        :param context:
+        :param concept:
+        :param attribute:
+        :param value:
+        :return:
+        """
+        ensure_concept(concept)
+
+        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
+        concept.set_value(attr, value)
+        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def get_attr(self, concept, attribute):
+        """
+        Returns the attribute of a concept
+        :param context:
+        :param concept:
+        :param attribute:
+        :return:
+        """
+        ensure_concept()
+        if not self.sheerka.is_success(concept):
+            return concept
+
+        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
+
+        if (value := concept.get_value(attr)) == NotInit:
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute})
+        return value
+
+    def set_id_if_needed(self, obj: Concept, is_builtin: bool):
+        """
+        Set the key for the concept if needed
+        :param obj:
+        :param is_builtin:
+        :return:
+        """
+        if obj.get_metadata().id is not None:
+            return
+
+        entry_key = self.BUILTIN_CONCEPTS_IDS if is_builtin else self.USER_CONCEPTS_IDS
+        obj.get_metadata().id = str(self.sheerka.cache_manager.get(self.sheerka.OBJECTS_IDS_ENTRY, entry_key))
+        # self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
+
+    def get_by_key(self, concept_key, concept_id=None):
+        concept_key = str(concept_key) if isinstance(concept_key, BuiltinConcepts) else concept_key
+        return self.internal_get("key", concept_key, self.CONCEPTS_BY_KEY_ENTRY, concept_id)
+
+    def get_by_name(self, concept_name, concept_id=None):
+        return self.internal_get("name", concept_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
+
+    def get_by_hash(self, concept_hash, concept_id=None):
+        return self.internal_get("hash", concept_hash, self.CONCEPTS_BY_HASH_ENTRY, concept_id)
+
+    def get_by_id(self, concept_id):
+        return self.internal_get("id", concept_id, self.CONCEPTS_BY_ID_ENTRY, None)
+
+    def has_id(self, concept_id):
+        """
+        Returns True if a concept with this id exists in cache
+        It does not search in the remote repository
+        :param concept_id:
+        :return:
+        """
+        if concept_id is None:
+            return False
+        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
+
+    def has_key(self, concept_key):
+        """
+        Returns True if concept(s) with this key exist in cache
+        It does not search in the remote repository
+        :param concept_key:
+        :return:
+        """
+        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
+
+    def has_name(self, concept_name):
+        """
+        Returns True if concept(s) with this name exist in cache
+        It does not search in the remote repository
+        :param concept_name:
+        :return:
+        """
+        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
+
+    def has_hash(self, concept_hash):
+        """
+        Returns True if concept(s) with this hash exist in cache
+        It does not search in the remote repository
+        :param concept_hash:
+        :return:
+        """
+        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
+
+    def internal_get(self, index_name, key, cache_name, concept_id=None):
+        """
+        Tries to find an entry
+        :param index_name: name of the index (ex by_id, by_key...)
+        :param key: index value
+        :param cache_name: name of the cache (ex Concepts_By_ID...)
+        :param concept_id: id of the concept if none, in case where there are multiple results
+        :return:
+        """
+
+        if key is None:
+            return ErrorConcept(f"Concept '{key}' is undefined.")
+
+        concepts = self.sheerka.cache_manager.get(cache_name, key)
+        if concepts:
+            if concept_id is None:
+                return concepts
+
+            if not hasattr(concepts, "__iter__"):
+                return concepts
+
+            for c in concepts:
+                if c.id == concept_id:
+                    return c
+
+        metadata = [(index_name, key), ("id", concept_id)] if concept_id else (index_name, key)
+        return self.sheerka.get_unknown(metadata)
+
+    def update_references(self, context, concept, modified_concept=None, modifications=None):
+        """
+        Updates all the concepts that reference concept
+        :param context:
+        :param concept:
+        :param modified_concept:
+        :param modifications:
+        :return:
+        """
+
+        refs = self.sheerka.cache_manager.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
+        if not refs:
+            return
+
+        for concept_id in refs:
+            # remove the grammar entry so that it can be recreated
+            self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
+
+            # reset the bnf definition if needed
+            if modified_concept:
+                if self.has_id(concept_id):
+                    to_update = self.get_by_id(concept_id)
+                    metadata = to_update.get_metadata()
+                    if metadata.definition_type == DEFINITION_TYPE_BNF and self._name_has_changed(modifications):
+                        tokens = list(Tokenizer(metadata.definition))
+                        modified = False
+                        for i, token in enumerate(tokens):
+                            if token.type == TokenKind.IDENTIFIER and token.value == concept.name:
+                                clone = token.clone()
+                                clone.value = modified_concept.name
+                                tokens[i] = clone
+                                modified = True
+
+                        if modified:
+                            to_update.get_metadata().definition = core.utils.get_text_from_tokens(tokens)
+                            to_update.set_bnf(None)
+
+    def compute_references(self, concept):
+        """
+        We need to keep a track of all concepts used by the current concept
+        So that if one of these are modified, we can modify the current concept accordingly
+        :param concept:
+        :return:
+        """
+        refs = set()
+
+        if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
+            from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
+            other_concepts_visitor = BnfNodeConceptExpressionVisitor()
+            other_concepts_visitor.visit(concept.get_bnf())
+
+            for concept in other_concepts_visitor.references:
+                if isinstance(concept, str):
+                    concept = self.sheerka.get_by_key(concept)
+                refs.add(concept.id)
+
+        return refs
+
+    def not_is_variable(self, name):
+        """
+        Given a name tells if it refers to a variable name
+        :param name:
+        :return:
+        """
+        return not self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name)
+
+    @staticmethod
+    def _name_has_changed(to_add):
+        if to_add is None or "meta" not in to_add:
+            return False
+
+        return "name" in to_add["meta"]
+
+    @staticmethod
+    def _definition_has_changed(to_add):
+        if to_add is None or "meta" not in to_add:
+            return False
+
+        return "definition" in to_add["meta"]
+
+    def _update_concept(self, context, concept, to_add, to_remove):
+        sheerka = context.sheerka
+        same_values = {}
+
+        if to_add:
+            if "meta" in to_add:
+                # All modifications must be allowed
+                metadata = concept.get_metadata()
+
+                for k, v in to_add["meta"].items():
+                    if k in self.forbidden_meta:
+                        return sheerka.ret(self.NAME, False, sheerka.err(ForbiddenAttribute(k)))
+
+                    try:
+                        if getattr(metadata, k) == v:
+                            same_values[k] = v
+                        else:
+                            setattr(metadata, k, v)
+                    except AttributeError:
+                        return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k)))
+
+                if same_values == to_add["meta"]:
+                    return sheerka.ret(self.NAME, False,
+                                       sheerka.err(NoModificationFound(concept, same_values)))
+
+            if "props" in to_add:
+                for k, v in to_add["props"].items():
+                    concept.add_prop(k, v)
+
+            if "variables" in to_add:
+                for k, v in to_add["variables"].items():
+                    # update existing or add new
+                    for i, var in enumerate(concept.get_metadata().variables):
+                        if var[0] == k:
+                            concept.get_metadata().variables[i] = (k, v)
+                            break
+                    else:
+                        concept.def_var(k, v)
+
+        if to_remove:
+            if "meta" in to_remove:
+                return sheerka.ret(self.NAME, False, sheerka.err(CannotRemoveMeta(to_remove["meta"])))
+
+            if "props" in to_remove:
+                for k, v in to_remove["props"].items():
+                    if k not in concept.get_metadata().props:
+                        return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k)))
+
+                    props = concept.get_metadata().props[k]
+                    try:
+                        if isinstance(v, (set, list, tuple)):
+                            for item in v:
+                                props.remove(item)
+                        else:
+                            props.remove(v)
+
+                        # remove empty sets
+                        if len(props) == 0:
+                            del concept.get_metadata().props[k]
+                    except KeyError:
+                        return sheerka.ret(self.NAME, False, sheerka.err(ValueNotFound(k, v)))
+
+            if "variables" in to_remove:
+                variables_to_remove = []
+                for k in to_remove["variables"]:
+                    for var_def in concept.get_metadata().variables:
+                        if var_def[0] == k:
+                            variables_to_remove.append(var_def)
+                            delattr(concept, var_def[0])
+                            break
+                    else:
+                        return sheerka.ret(self.NAME, False, sheerka.err(UnknownAttribute(k)))
+                core.utils.remove_list_from_list(concept.get_metadata().variables, variables_to_remove)
+
+        concept.get_metadata().key = None
+        if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
+            concept.set_bnf(None)
+            self.bnp.ensure_bnf(context, concept)
+
+        concept.init_key()
+
+        return

src/core/sheerka/services/SheerkaConceptsAlgebra.py

@@ -5,6 +5,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import ensure_concept
 from core.concept import Concept
 from core.sheerka.Sheerka import Sheerka
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.sheerka_service import BaseService
 
 PROPERTIES_TO_COMPUTE = [BuiltinConcepts.ISA, BuiltinConcepts.HASA]
@@ -118,8 +119,10 @@ class SheerkaConceptsAlgebra(BaseService):
         if nb_props == 0:
             return res
 
-        all_concepts = self.sheerka.cache_manager.copy(Sheerka.CONCEPTS_BY_ID_ENTRY).values() \
-            if self.sheerka.cache_manager.cache_only else self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY)
+        concepts_service = self.sheerka.services[SheerkaConceptManager.NAME]
+
+        all_concepts = self.sheerka.cache_manager.copy(concepts_service.CONCEPTS_BY_ID_ENTRY).values() \
+            if self.sheerka.cache_manager.cache_only else self.sheerka.sdp.list(concepts_service.CONCEPTS_BY_ID_ENTRY)
 
         for c in all_concepts:
             score = self._compute_score(c, concept, step_b=round(1 / nb_props, 2))

src/core/sheerka/services/SheerkaCreateNewConcept.py

@@ -1,123 +0,0 @@
-import core.utils
-from core.builtin_concepts import BuiltinConcepts, ErrorConcept
-from core.builtin_helpers import ensure_concept
-from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs
-from core.global_symbols import EVENT_CONCEPT_CREATED
-from core.sheerka.services.sheerka_service import BaseService
-from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
-
-BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
-
-
-class SheerkaCreateNewConcept(BaseService):
-    """
-    Manages the creation of a new concept
-    """
-
-    NAME = "CreateNewConcept"
-
-    def __init__(self, sheerka):
-        super().__init__(sheerka)
-        self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS)  # BaseNodeParser
-
-    def initialize(self):
-        self.sheerka.bind_service_method(self.create_new_concept, True)
-        self.sheerka.bind_service_method(self.not_is_variable, False, visible=False)
-
-    def create_new_concept(self, context, concept: Concept):
-        """
-        Adds a new concept to the system
-        :param context:
-        :param concept:  DefConceptNode
-        :return: digest of the new concept
-        """
-        ensure_concept(concept)
-
-        sheerka = self.sheerka
-
-        concept.init_key()
-        init_bnf_ret_value = None
-
-        cache_manager = sheerka.cache_manager
-
-        if cache_manager.exists(sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
-            error = SheerkaDataProviderDuplicateKeyError(sheerka.CONCEPTS_BY_KEY_ENTRY + "." + concept.key,
-                                                         concept)
-            return sheerka.ret(
-                self.NAME,
-                False,
-                sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
-                error.args[0])
-
-        # set id before saving in db
-        sheerka.set_id_if_needed(concept, False)
-
-        # freeze attributes
-        freeze_concept_attrs(concept)
-
-        # compute new concepts_by_first_keyword
-        init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True)
-        if not init_ret_value.status:
-            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
-        concepts_by_first_keyword = init_ret_value.body
-
-        # computes resolved concepts_by_first_keyword
-        init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
-        if not init_ret_value.status:
-            return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
-        resolved_concepts_by_first_keyword = init_ret_value.body
-
-        # if everything is fine
-        concept.freeze_definition_hash()
-        cache_manager.add_concept(concept)
-        cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
-        cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
-
-        if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
-            # allow search by definition when definition relevant
-            cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept)
-
-        # update references
-        for ref in self.compute_references(concept):
-            cache_manager.put(sheerka.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
-
-        # TODO : this line seems to be useless
-        # The grammar is never reset
-        if concept.get_bnf() and init_bnf_ret_value is not None and init_bnf_ret_value.status:
-            sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
-
-        # publish the new concept
-        sheerka.publish(context, EVENT_CONCEPT_CREATED, concept)
-
-        # process the return if needed
-        ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
-        return ret
-
-    def compute_references(self, concept):
-        """
-        We need to keep a track of all concepts used by the current concept
-        So that if one of these are modified, we can modify the current concept accordingly
-        :param concept:
-        :return:
-        """
-        refs = set()
-
-        if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
-            from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
-            other_concepts_visitor = BnfNodeConceptExpressionVisitor()
-            other_concepts_visitor.visit(concept.get_bnf())
-
-            for concept in other_concepts_visitor.references:
-                if isinstance(concept, str):
-                    concept = self.sheerka.get_by_key(concept)
-                refs.add(concept.id)
-
-        return refs
-
-    def not_is_variable(self, name):
-        """
-        Given a name tells if it refers to a variable name
-        :param name:
-        :return:
-        """
-        return not self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name)

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -4,6 +4,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate, ensure_concept
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, AllConceptParts, \
     concept_part_value
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
@@ -224,13 +225,12 @@ class SheerkaEvaluateConcept(BaseService):
         """
 
         def is_only_successful(r):
-            #            return False
             return context.sheerka.isinstance(r, BuiltinConcepts.RETURN_VALUE) and \
                    context.sheerka.isinstance(r.body, BuiltinConcepts.ONLY_SUCCESSFUL)
 
         def parse_token_concept(s):
             if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None):
-                return self.sheerka.resolve(identifier)
+                return self.sheerka.fast_resolve(identifier)
             return None
 
         for part_key in AllConceptParts:
@@ -247,11 +247,12 @@ class SheerkaEvaluateConcept(BaseService):
             if source.strip() == "":
                 concept.get_compiled()[part_key] = DoNotResolve(source)
             else:
-                # first case, when the metadata references another concept via c:xxx: keyword
                 if concept_found := parse_token_concept(source):
+                    # the  compiled can be a reference to another concept...
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
                     concept.get_compiled()[part_key] = concept_found
                 else:
+                    # ...or a list of ReturnValueConcept to resolve
                     res = parse_unrecognized(context,
                                              source,
                                              parsers="all",
@@ -272,11 +273,12 @@ class SheerkaEvaluateConcept(BaseService):
             if default_value.strip() == "":
                 concept.get_compiled()[var_name] = DoNotResolve(default_value)
             else:
-                # first case, when the metadata references another concept via c:xxx: keyword
                 if concept_found := parse_token_concept(default_value):
+                    # the  compiled can be a reference to another concept...
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
                     concept.get_compiled()[var_name] = concept_found
                 else:
+                    # ...or a list of ReturnValueConcept to resolve
                     res = parse_unrecognized(context,
                                              default_value,
                                              parsers="all",
@@ -285,7 +287,9 @@ class SheerkaEvaluateConcept(BaseService):
                     concept.get_compiled()[var_name] = res.body.body if is_only_successful(res) else res
 
         # Updates the cache of concepts when possible
-        if self.sheerka.has_id(concept.id):
+        # This piece of code is not used, a the compile part is removed by sheerka.new_from_template()
+        service = context.sheerka.services[SheerkaConceptManager.NAME]
+        if service.has_id(concept.id):
             self.sheerka.get_by_id(concept.id).set_compiled(concept.get_compiled())
 
     def resolve(self,

src/core/sheerka/services/SheerkaHasAManager.py

@@ -35,9 +35,8 @@ class SheerkaHasAManager(BaseService):
                                  name=BuiltinConcepts.HASA,
                                  concept=concept_a))
 
-        concept_a.add_prop(BuiltinConcepts.HASA, concept_b)
-
-        return self.sheerka.modify_concept(context, concept_a)
+        to_add = {"props": {BuiltinConcepts.HASA:  concept_b}}
+        return self.sheerka.modify_concept(context, concept_a, to_add, modify_source=True)
 
     def hasa(self, concept_a, concept_b):
         """

src/core/sheerka/services/SheerkaModifyConcept.py

@@ -1,110 +0,0 @@
-from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import ensure_concept
-from core.concept import NotInit, freeze_concept_attrs, Concept
-from core.sheerka.services.sheerka_service import BaseService
-
-
-class SheerkaModifyConcept(BaseService):
-    NAME = "ModifyConcept"
-
-    def __init__(self, sheerka):
-        super().__init__(sheerka)
-
-    def initialize(self):
-        self.sheerka.bind_service_method(self.modify_concept, True)
-        self.sheerka.bind_service_method(self.set_attr, True)
-        self.sheerka.bind_service_method(self.get_attr, False)
-
-    def modify_concept(self, context, concept):
-        """
-        Modify the definition of a concept
-        :param context:
-        :param concept:
-        :return:
-        """
-        old_version = self.sheerka.get_by_id(concept.id)
-
-        if old_version is None:
-            # nothing found in cache
-            return self.sheerka.ret(
-                self.NAME, False,
-                self.sheerka.new(
-                    BuiltinConcepts.UNKNOWN_CONCEPT,
-                    body=[("key", concept.key), ("id", concept.id)]))
-
-        if not self.sheerka.is_success(old_version) and concept.key != old_version.key:
-            # an error concept is returned
-            return self.sheerka.ret(
-                self.NAME, False,
-                old_version)
-
-        if old_version == concept:
-            # the concept is not modified
-            # This is an important sanity check. Do no remove because you don't understand it
-            return self.sheerka.ret(
-                self.NAME, False,
-                self.sheerka.new(
-                    BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
-                    body=concept))
-
-        # update attributes
-        freeze_concept_attrs(concept)
-
-        self.sheerka.cache_manager.update_concept(old_version, concept)
-
-        # TODO : update concept by first keyword : have a look at update_references() below
-        # TODO : update resolved by first keyword : have a look at update_references() below
-        # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
-        # TODO : Update concepts grammars : have a look at update_references() below
-
-        ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
-        return ret
-
-    def update_references(self, context, concept):
-        """
-        Updates all the concepts that reference concept
-        :param context:
-        :param concept:
-        :return:
-        """
-
-        refs = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_REFERENCES_ENTRY, concept.id)
-        if not refs:
-            return
-
-        for concept_id in refs:
-            # remove the grammar entry so that it can be recreated
-            self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
-
-    def set_attr(self, concept, attribute, value):
-        """
-        Modifies an attribute of a concept (concept.values)
-        :param context:
-        :param concept:
-        :param attribute:
-        :param value:
-        :return:
-        """
-        ensure_concept(concept)
-
-        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
-        concept.set_value(attr, value)
-        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-
-    def get_attr(self, concept, attribute):
-        """
-        Returns the attribute of a concept
-        :param context:
-        :param concept:
-        :param attribute:
-        :return:
-        """
-        ensure_concept()
-        if not self.sheerka.is_success(concept):
-            return concept
-
-        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
-
-        if (value := concept.get_value(attr)) == NotInit:
-            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute})
-        return value

src/core/sheerka/services/SheerkaRuleManager.py

@@ -623,7 +623,7 @@ class SheerkaRuleManager(BaseService):
         if rule.metadata.id is not None:
             return
 
-        rule.metadata.id = str(self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_KEYS_ENTRY, self.RULE_IDS))
+        rule.metadata.id = str(self.sheerka.cache_manager.get(self.sheerka.OBJECTS_IDS_ENTRY, self.RULE_IDS))
 
     def create_new_rule(self, context, rule):
         """

src/core/sheerka/services/SheerkaSetsManager.py

@@ -4,7 +4,7 @@ from cache.SetCache import SetCache
 from core.ast_helpers import UnreferencedVariablesVisitor
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF
-from core.sheerka.services.SheerkaModifyConcept import SheerkaModifyConcept
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.sheerka_service import BaseService
 
 GROUP_PREFIX = 'All_'
@@ -43,7 +43,6 @@ class SheerkaSetsManager(BaseService):
         context.log(f"Setting concept {concept} is a {concept_set}", who=self.NAME)
         core.builtin_helpers.ensure_concept(concept, concept_set)
 
-
         if BuiltinConcepts.ISA in concept.get_metadata().props and concept_set in concept.get_metadata().props[
             BuiltinConcepts.ISA]:
             return self.sheerka.ret(
@@ -53,11 +52,12 @@ class SheerkaSetsManager(BaseService):
 
         # KSI 20200709 add the concept, not its 'id' or 'key'
         # It will allow conditions handling if concept set has its WHERE or PRE set to something
-        concept.add_prop(BuiltinConcepts.ISA, concept_set)
-
-        res = self.sheerka.modify_concept(context, concept)
+        to_add = {"props": {BuiltinConcepts.ISA: concept_set}}
+        res = self.sheerka.modify_concept(context, concept, to_add, modify_source=True)
         if not res.status:
             return res
+        else:
+            concept = res.body.body
 
         res = self.add_concept_to_set(context, concept, concept_set)
 
@@ -88,7 +88,7 @@ class SheerkaSetsManager(BaseService):
         self.concepts_in_set.delete(concept_set.id)
 
         # update concept_set references
-        self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
+        self.sheerka.services[SheerkaConceptManager.NAME].update_references(context, concept_set)
 
         # remove the grammar entry so that it can be recreated
         self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_set.id)