Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

src/core/sheerka/ExecutionContext.py

@@ -144,9 +144,9 @@ class ExecutionContext:
 
     def add_preprocess(self, name, **kwargs):
         preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
-        preprocess.set_prop("name", name)
+        preprocess.set_value("name", name)
         for k, v in kwargs.items():
-            preprocess.set_prop(k, v)
+            preprocess.set_value(k, v)
 
         if not self.preprocess:
             self.preprocess = []
@@ -168,9 +168,9 @@ class ExecutionContext:
         if isinstance(self.obj, Concept):
             if self.obj.key == key:
                 return self.obj
-            for prop in self.obj.props:
-                if prop == key:
-                    value = self.obj.props[prop].value
+            for var_name in self.obj.values:
+                if var_name == key:
+                    value = self.obj.get_value(var_name)
                     if isinstance(value, Concept):
                         return value
 
@@ -180,16 +180,16 @@ class ExecutionContext:
                 if k == key:
                     return c
 
-        return self.sheerka.get(key)
+        return self.sheerka.get_by_key(key)
 
     def new_concept(self, key, **kwargs):
         # search in obj
         if self.obj:
             if self.obj.key == key:
                 return self.sheerka.new_from_template(self.obj, key, **kwargs)
-            for prop in self.obj.props:
-                if prop == key:
-                    value = self.obj.props[prop].value
+            for var_name in self.obj.values:
+                if var_name == key:
+                    value = self.obj.get_value(var_name)
                     if isinstance(value, Concept):
                         return self.sheerka.new_from_template(value, key, **kwargs)
                     else:
@@ -327,6 +327,7 @@ class ExecutionContext:
             bag[prop] = getattr(self, prop)
         bag["status"] = self.get_status()
         bag["elapsed"] = self.elapsed
+        bag["elapsed_str"] = self.elapsed_str
         bag["digest"] = self.event.get_digest() if self.event else None
         return bag
 

src/core/sheerka/Services/SheerkaCreateNewConcept.py

@@ -1,9 +1,9 @@
+import core.utils
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept
 from core.concept import Concept
-from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError, SheerkaDataProviderRef
-import core.utils
+from sdp.sheerkaDataProvider_Old import SheerkaDataProviderDuplicateKeyError
 
-BNF_NODE_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
+BNF_NODE_PARSER_CLASS = "parsers.BnfNodeParser_Old.BnfNodeParser"
 BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
 
 
@@ -15,122 +15,80 @@ class SheerkaCreateNewConcept:
     def __init__(self, sheerka):
         self.sheerka = sheerka
         self.logger_name = self.create_new_concept.__name__
-        self.base_lexer_parser = core.utils.get_class(BASE_NODE_PARSER_CLASS)("BaseNodeParser", 0)
+        self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS)  # BaseNodeParser
 
     def create_new_concept(self, context, concept: Concept):
         """
         Adds a new concept to the system
         :param context:
         :param concept:  DefConceptNode
-        :param logger
         :return: digest of the new concept
         """
 
+        sheerka = self.sheerka
+
         concept.init_key()
-        concepts_definitions = None
         init_bnf_ret_value = None
 
-        sdp = self.sheerka.sdp
+        cache_manager = sheerka.cache_manager
 
-        # checks for duplicate concepts
-        # TODO checks if it exists in cache first
-
-        if sdp.exists(self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
-            error = SheerkaDataProviderDuplicateKeyError(self.sheerka.CONCEPTS_ENTRY + "." + concept.key, concept)
-            return self.sheerka.ret(
+        if cache_manager.exists(sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
+            error = SheerkaDataProviderDuplicateKeyError(sheerka.CONCEPTS_BY_KEY_ENTRY + "." + concept.key,
+                                                         concept)
+            return sheerka.ret(
                 self.logger_name,
                 False,
-                self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
+                sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
                 error.args[0])
 
         # set id before saving in db
-        self.sheerka.set_id_if_needed(concept, False)
+        sheerka.set_id_if_needed(concept, False)
 
-        # add the BNF if known
-        if concept.bnf:
-            concepts_definitions = self.sheerka.get_concepts_definitions(context)
-            concepts_definitions[concept] = concept.bnf
+        # update the dictionary of concepts by first key
+        init_ret_value = self.bnp.get_concepts_by_first_keyword(context, [concept], True)
+        if not init_ret_value.status:
+            return sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
+        concepts_by_first_keyword = init_ret_value.body
 
-            # check if it's a valid BNF or whether it breaks the known rules
-            bnf_lexer_parser = self.sheerka.parsers[BNF_NODE_PARSER_CLASS]()
-            with context.push(self.sheerka.name, desc=f"Initializing concept definition for {concept}") as sub_context:
-                sub_context.concepts[concept.key] = concept  # the concept is not in the real cache yet
-                init_bnf_ret_value = bnf_lexer_parser.initialize(sub_context, concepts_definitions)
-                sub_context.add_values(return_values=init_bnf_ret_value)
-                if not init_bnf_ret_value.status:
-                    return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_bnf_ret_value.value))
+        # update resolved dictionary
+        init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
+        if not init_ret_value.status:
+            return sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
+        resolved_concepts_by_first_keyword = init_ret_value.body
 
         # update concept definition by key
-        init_sya_ret_value = self.base_lexer_parser.initialize(context, [concept], use_sheerka=True)
-        if not init_sya_ret_value.status:
-            return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_sya_ret_value.value))
-        concepts_by_first_keyword = init_sya_ret_value.body
+        # init_sya_ret_value = self.bnp.initialize(context, [concept], use_sheerka=True)
+        # if not init_sya_ret_value.status:
+        #     return sheerka.ret(self.logger_name, False, ErrorConcept(init_sya_ret_value.value))
+        # concepts_by_first_keyword = init_sya_ret_value.body
 
         concept.freeze_definition_hash()
 
-        # save the new concept in sdp
-        try:
-            # TODO : needs to make these calls atomic (or at least one single call)
-            # save the new concept
-            concept.metadata.full_serialization = True
-            result = sdp.add(
-                context.event.get_digest(),
-                self.sheerka.CONCEPTS_ENTRY,
-                concept,
-                use_ref=True)
-            concept.metadata.full_serialization = False
+        cache_manager.add_concept(concept)
+        cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
+        cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
 
-            # update the concept (I hope that it's enough)
-            concept.set_origin(result.digest)
-
-            # save it by id
-            sdp.add(
-                context.event.get_digest(),
-                self.sheerka.CONCEPTS_BY_ID_ENTRY,
-                SheerkaDataProviderRef(concept.id, result.digest))
-
-            # save it by name
-            sdp.add(
-                context.event.get_digest(),
-                self.sheerka.CONCEPTS_BY_NAME_ENTRY,
-                SheerkaDataProviderRef(concept.name, result.digest))
-
-            # records the hash
-            sdp.add(
-                context.event.get_digest(),
-                self.sheerka.CONCEPTS_BY_HASH_ENTRY,
-                SheerkaDataProviderRef(concept.get_definition_hash(), result.digest))
-
-            # update the definition table
-            if concepts_definitions is not None:
-                sdp.set(
-                    context.event.get_digest(),
-                    self.sheerka.CONCEPTS_DEFINITIONS_ENTRY,
-                    bnf_lexer_parser.encode_grammar(init_bnf_ret_value.body),
-                    use_ref=True)
-                self.sheerka.concepts_definitions_cache = None  # invalidate cache
-
-            # update the concepts by first keyword
-            sdp.set(context.event.get_digest(),
-                    self.sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
-                    concepts_by_first_keyword)
-
-        except SheerkaDataProviderDuplicateKeyError as error:
-            context.log_error("Failed to create a new concept.", who=self.logger_name)
-            return self.sheerka.ret(
-                self.logger_name,
-                False,
-                self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
-                error.args[0])
-
-        # Updates the caches
-        self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
-        self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
-        self.sheerka.cache_by_id[concept.id] = concept
-        if init_bnf_ret_value is not None and init_bnf_ret_value.status:
-            self.sheerka.concepts_grammars = init_bnf_ret_value.body
-        self.sheerka.concepts_by_first_keyword = concepts_by_first_keyword
+        if concept.bnf and init_bnf_ret_value is not None and init_bnf_ret_value.status:
+            sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
 
         # process the return if needed
-        ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
+        ret = sheerka.ret(self.logger_name, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
         return ret
+
+    # def load_concepts_nodes_definitions(self, context):
+    #     """
+    #     Gets from sdp what is need to parse nodes
+    #     :return:
+    #     """
+    #     sdp = self.sheerka.sdp
+    #
+    #     concepts_by_first_keyword = sdp.get(
+    #         self.sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
+    #         load_origin=False) or {}
+    #
+    #     init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
+    #     if not init_ret_value.status:
+    #         return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
+    #     resolved_concepts_by_first_keyword = init_ret_value.body
+    #
+    #     return concepts_by_first_keyword, resolved_concepts_by_first_keyword

src/core/sheerka/Services/SheerkaDump.py

@@ -1,9 +1,10 @@
+import os
+import pprint
+
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka.ExecutionContext import ExecutionContext
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
-import pprint
-import os
 
 
 def get_pp():
@@ -17,7 +18,7 @@ class SheerkaDump:
         self.sheerka = sheerka
 
     def dump_concepts(self):
-        lst = self.sheerka.sdp.list(self.sheerka.CONCEPTS_ENTRY)
+        lst = self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_KEY_ENTRY)
         for item in lst:
             if hasattr(item, "__iter__"):
                 for i in item:
@@ -25,10 +26,6 @@ class SheerkaDump:
             else:
                 self.sheerka.log.info(item)
 
-    def dump_definitions(self):
-        defs = self.sheerka.sdp.get(self.sheerka.CONCEPTS_DEFINITIONS_ENTRY)
-        self.sheerka.log.info(defs)
-
     def dump_desc(self, *concept_names, eval=False):
         first = True
         event = Event(f"Dumping description", "")
@@ -37,7 +34,7 @@ class SheerkaDump:
             if isinstance(concept_name, Concept):
                 concepts = concept_name
             else:
-                concepts = self.sheerka.get(concept_name)
+                concepts = self.sheerka.get_by_key(concept_name)
                 if self.sheerka.isinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT):
                     self.sheerka.log.error(f"Concept '{concept_name}' is unknown")
                     return False
@@ -59,8 +56,8 @@ class SheerkaDump:
                 self.sheerka.log.info(f"where  : {c.metadata.where}")
                 if eval:
                     self.sheerka.log.info(f"value  : {value}")
-                for p in c.props:
-                    self.sheerka.log.info(f"{p}: {c.get_prop(p)}")
+                for v in c.values:
+                    self.sheerka.log.info(f"{v}: {c.get_value(v)}")
                 else:
                     self.sheerka.log.info("No property")
 

src/core/sheerka/Services/SheerkaEvaluateConcept.py

@@ -1,6 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
 from core.builtin_helpers import expect_one
+from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,
@@ -91,32 +91,27 @@ class SheerkaEvaluateConcept:
                     concept.compiled[part_key] = res
                     sub_context.add_values(return_values=res)
 
-        for prop, default_value in concept.metadata.props:
-            if prop in concept.compiled:
+        for var_name, default_value in concept.metadata.variables:
+            if var_name in concept.compiled:
                 continue
 
             if default_value is None or not isinstance(default_value, str):
                 continue
 
             if default_value.strip() == "":
-                concept.compiled[prop] = DoNotResolve(default_value)
+                concept.compiled[var_name] = DoNotResolve(default_value)
             else:
-                with context.push(desc=f"Initializing AST for property {prop}") as sub_context:
+                with context.push(desc=f"Initializing AST for property {var_name}") as sub_context:
                     sub_context.add_inputs(source=default_value)
                     to_parse = self.sheerka.ret(context.who, True,
                                                 self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value))
                     res = self.sheerka.execute(context, to_parse, steps)
-                    concept.compiled[prop] = res
+                    concept.compiled[var_name] = res
                     sub_context.add_values(return_values=res)
 
         # Updates the cache of concepts when possible
-        if concept.key in self.sheerka.cache_by_key:
-            entry = self.sheerka.cache_by_key[concept.key]
-            if isinstance(entry, list):
-                # TODO : manage when there are multiple entries
-                pass
-            else:
-                self.sheerka.cache_by_key[concept.key].compiled = concept.compiled
+        if self.sheerka.has_id(concept.id):
+            self.sheerka.get_by_id(concept.id).compiled = concept.compiled
 
     def resolve(self, context, to_resolve, current_prop, current_concept, force_evaluation):
         if isinstance(to_resolve, DoNotResolve):
@@ -198,7 +193,6 @@ class SheerkaEvaluateConcept:
         It means that if the where clause is True, will evaluate the body
         :param context:
         :param concept:
-        :param evaluate_body: If false, only evaluate body when necessary
         :return: value of the evaluation or error
         """
 
@@ -208,26 +202,26 @@ class SheerkaEvaluateConcept:
         self.initialize_concept_asts(context, concept)
 
         # to make sure of the order, it don't use ConceptParts.get_parts()
-        # props must be evaluated first, body must be evaluated before where
+        # variables must be evaluated first, body must be evaluated before where
         all_metadata_to_eval = self.choose_metadata_to_eval(context, concept)
 
         for metadata_to_eval in all_metadata_to_eval:
-            if metadata_to_eval == "props":
-                for prop_name in (p for p in concept.props if p in concept.compiled):
-                    prop_ast = concept.compiled[prop_name]
+            if metadata_to_eval == "variables":
+                for var_name in (v for v in concept.variables() if v in concept.compiled):
+                    prop_ast = concept.compiled[var_name]
 
                     if isinstance(prop_ast, list):
                         # Do not send the current concept for the properties
-                        resolved = self.resolve_list(context, prop_ast, prop_name, None, True)
+                        resolved = self.resolve_list(context, prop_ast, var_name, None, True)
                     else:
                         # Do not send the current concept for the properties
-                        resolved = self.resolve(context, prop_ast, prop_name, None, True)
+                        resolved = self.resolve(context, prop_ast, var_name, None, True)
 
                     if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
-                        resolved.set_prop("concept", concept)  # since current concept was not sent
+                        resolved.set_value("concept", concept)  # since current concept was not sent
                         return resolved
                     else:
-                        concept.set_prop(prop_name, resolved)
+                        concept.set_value(var_name, resolved)
             else:
                 part_key = ConceptParts(metadata_to_eval)
 
@@ -245,7 +239,7 @@ class SheerkaEvaluateConcept:
                     if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
                         return resolved
                     else:
-                        concept.values[part_key] = self.get_infinite_recursion_resolution(resolved) or resolved
+                        concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved)
 
         #
         # TODO : Validate the PRE condition
@@ -253,8 +247,8 @@ class SheerkaEvaluateConcept:
 
         # validate where clause
         if ConceptParts.WHERE in concept.values:
-            where_value = concept.values[ConceptParts.WHERE]
-            if not (where_value is None or self.sheerka.value(where_value)):
+            where_value = concept.get_value(ConceptParts.WHERE)
+            if not (where_value is None or self.sheerka.objvalue(where_value)):
                 return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
 
         #
@@ -267,7 +261,7 @@ class SheerkaEvaluateConcept:
 
     def choose_metadata_to_eval(self, context, concept):
         if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
-            return ["pre", "post", "props", "body", "where"]
+            return ["pre", "post", "variables", "body", "where"]
 
         metadata = ["pre", "post"]
         if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.metadata.need_validation:
@@ -310,9 +304,9 @@ class SheerkaEvaluateConcept:
             if not isinstance(return_value.body.source, str):
                 continue
 
-            for prop_name in (p[0] for p in concept.metadata.props):
-                if prop_name in return_value.body.source:
-                    needed.append("props")
+            for var_name in (p[0] for p in concept.metadata.variables):
+                if var_name in return_value.body.source:
+                    needed.append("variables")
                     break
 
             if "self" in return_value.body.source:

src/core/sheerka/Services/SheerkaExecute.py

@@ -230,12 +230,12 @@ class SheerkaExecute:
 
         for preprocess in context.preprocess:
             for e in parsers_or_evaluators:
-                if self.matches(e.name, preprocess.get_prop("name")):
-                    for prop, value in preprocess.props.items():
-                        if prop == "name":
+                if self.matches(e.name, preprocess.get_value("name")):
+                    for var_name in preprocess.values:
+                        if var_name == "name":
                             continue
-                        if hasattr(e, prop):
-                            setattr(e, prop, value.value)
+                        if hasattr(e, var_name):
+                            setattr(e, var_name, preprocess.get_value(var_name))
         return parsers_or_evaluators[0] if single_one else parsers_or_evaluators
 
     @staticmethod

src/core/sheerka/Services/SheerkaModifyConcept.py

@@ -1,5 +1,4 @@
 from core.builtin_concepts import BuiltinConcepts
-from sdp.sheerkaDataProvider import SheerkaDataProviderRef
 
 
 class SheerkaModifyConcept:
@@ -8,52 +7,35 @@ class SheerkaModifyConcept:
         self.logger_name = self.modify_concept.__name__
 
     def modify_concept(self, context, concept):
+        old_version = self.sheerka.get_by_id(concept.id)
 
-        sdp = self.sheerka.sdp
-
-        try:
-            # modify the entry
-            concept.metadata.full_serialization = True
-            result = sdp.modify(
-                context.event.get_digest(),
-                self.sheerka.CONCEPTS_ENTRY,
-                concept.key,
-                concept)
-            concept.metadata.full_serialization = False
-
-            # update reference entry
-            sdp.modify(
-                context.event.get_digest(),
-                self.sheerka.CONCEPTS_BY_ID_ENTRY,
-                concept.id,
-                SheerkaDataProviderRef(concept.id, result.digest, concept.get_origin()))
-
-            # update name entry
-            sdp.modify(
-                context.event.get_digest(),
-                self.sheerka.CONCEPTS_BY_NAME_ENTRY,
-                concept.name,
-                SheerkaDataProviderRef(concept.name, result.digest, concept.get_origin()))
-
-            # update the hash entry
-            sdp.modify(
-                context.event.get_digest(),
-                self.sheerka.CONCEPTS_BY_HASH_ENTRY,
-                concept.get_original_definition_hash(),
-                SheerkaDataProviderRef(concept.get_definition_hash(), result.digest, concept.get_origin()))
-
-        except IndexError as error:
-            context.log_error(f"Failed to update concept '{concept}'.", who=self.logger_name)
+        if old_version is None:
+            # nothing found in cache
             return self.sheerka.ret(
-                self.logger_name,
-                False,
-                self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept),
-                error.args[0])
+                self.logger_name, False,
+                self.sheerka.new(
+                    BuiltinConcepts.UNKNOWN_CONCEPT,
+                    body=[("key", concept.key), ("id", concept.id)]))
 
-        # update cache
-        self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
-        self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
-        self.sheerka.cache_by_id[concept.id] = concept
+        if not self.sheerka.is_success(old_version) and concept.key != old_version.key:
+            # an error concept is returned
+            return self.sheerka.ret(
+                self.logger_name, False,
+                old_version)
+
+        if old_version == concept:
+            # the concept is not modified
+            return self.sheerka.ret(
+                self.logger_name, False,
+                self.sheerka.new(
+                    BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
+                    body=concept))
+
+        self.sheerka.cache_manager.update_concept(old_version, concept)
+
+        # TODO : update concept by first keyword
+        # TODO : update resolved by first keyword
+        # TODO : update concets grammars
 
         ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
         return ret

src/core/sheerka/Services/SheerkaSetsManager.py

@@ -1,7 +1,7 @@
-from core.ast.nodes import python_to_concept
-from core.builtin_concepts import BuiltinConcepts, ErrorConcept
-from core.concept import Concept, ConceptParts
 import core.builtin_helpers
+from core.ast.nodes import python_to_concept
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, ConceptParts
 
 GROUP_PREFIX = 'All_'
 
@@ -20,13 +20,16 @@ class SheerkaSetsManager:
         :return:
         """
 
-        context.log(f"Setting that concept {concept} is a {concept_set}", who=self.logger_name)
+        context.log(f"Setting concept {concept} is a {concept_set}", who=self.logger_name)
 
-        isa = [] if BuiltinConcepts.ISA not in concept.props else concept.get_prop(BuiltinConcepts.ISA)
-        if concept_set not in isa:
-            isa.append(concept_set)
+        if BuiltinConcepts.ISA in concept.metadata.props and concept_set in concept.metadata.props[BuiltinConcepts.ISA]:
+            return self.sheerka.ret(
+                self.logger_name,
+                False,
+                self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
+
+        concept.add_prop(BuiltinConcepts.ISA, concept_set)
 
-        concept.set_prop(BuiltinConcepts.ISA, isa)
         res = self.sheerka.modify_concept(context, concept)
         if not res.status:
             return res
@@ -47,32 +50,34 @@ class SheerkaSetsManager:
         assert concept.id
         assert concept_set.id
 
-        try:
-            result = self.sheerka.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id)
-            if result.already_exists:  # concept already in set
-                return self.sheerka.ret(
-                    self.logger_name,
-                    False,
-                    self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
-            else:
-                return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-        except Exception as error:
-            context.log_error("Failed to add to set.", who=self.logger_name)
-            return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
+        set_elements = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, concept_set.id)
+        if set_elements and concept.id in set_elements:
+            return self.sheerka.ret(
+                self.logger_name,
+                False,
+                self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
+
+        self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_GROUPS_ENTRY, concept_set.id, concept.id)
+        return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
     def add_concepts_to_set(self, context, concepts, concept_set):
         """Adding multiple concepts at the same time"""
 
         context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.logger_name)
-        previous = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept_set.id)
+        already_in_set = []
+        for concept in concepts:
+            res = self.add_concept_to_set(context, concept, concept_set)
+            if self.sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_IN_SET):
+                already_in_set.append(res.body.body)
 
-        new_ids = [c.id for c in concepts] if previous is None else previous + [c.id for c in concepts]
-        try:
-            self.sheerka.sdp.set(context.event.get_digest(), GROUP_PREFIX + concept_set.id, new_ids)
-            return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-        except Exception as error:
-            context.log_error("Failed to add to set.", who=self.logger_name)
-            return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
+        if already_in_set:
+            body = self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
+                                    body=already_in_set,
+                                    concept_set=concept_set)
+        else:
+            body = self.sheerka.new(BuiltinConcepts.SUCCESS)
+
+        return self.sheerka.ret(self.logger_name, len(already_in_set) != len(concepts), body)
 
     def get_set_elements(self, context, concept):
         """
@@ -83,38 +88,41 @@ class SheerkaSetsManager:
         :return:
         """
 
-        # noinspection PyShadowingNames
-        def _get_set_elements(context, concept, sub_concept):
-            if not (isinstance(sub_concept, Concept) and sub_concept.id):
+        def _get_set_elements(sub_concept):
+            if not self.isaset(context, sub_concept):
                 return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
 
-            ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + sub_concept.id)
-            if ids:
-                if concept.metadata.where:
-                    new_condition = self._validate_where_clause(concept)
-                    if not new_condition:
-                        return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
-                    else:
-                        # This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
-                        # to do it properly now. It will be enhanced later
-                        concepts = self._get_concepts(context, ids, True)
-                        globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
-                        locals_ = {}
-                        exec(new_condition, globals_, locals_)
-                        return locals_["result"]
-                else:
-                    return self._get_concepts(context, ids, False)
+            # first, try to see if sub_context has it's own group entry
+            ids = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, sub_concept.id)
+            concepts = self._get_concepts(context, ids, True)
 
-            # it may be a concept that references a set
-            if not sub_concept.metadata.is_evaluated:
-                with context.push(desc=f"Evaluating concept {sub_concept}") as sub_context:
-                    sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-                    evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept)
-                    if evaluated.key != concept.key:
-                        return False
-            return _get_set_elements(context, concept, sub_concept.body)
+            # aggregate with en entries from its body
+            sub_concept = core.builtin_helpers.ensure_evaluated(context, sub_concept)
+            if not self.sheerka.is_success(sub_concept):
+                return sub_concept
 
-        return _get_set_elements(context, concept, concept)
+            if self.isaset(context, sub_concept.body):
+                other_concepts = _get_set_elements(sub_concept.body)
+                if not self.sheerka.is_success(other_concepts):
+                    return other_concepts
+                concepts.extend(other_concepts)
+
+            # apply the where clause if any
+            if sub_concept.metadata.where:
+                new_condition = self._validate_where_clause(sub_concept)
+                if not new_condition:
+                    return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=sub_concept)
+
+                # This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
+                # to do it properly now. It will be enhanced later
+                globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
+                locals_ = {}
+                exec(new_condition, globals_, locals_)
+                concepts = locals_["result"]
+
+            return concepts
+
+        return _get_set_elements(concept)
 
     def isinset(self, a, b):
         """
@@ -135,17 +143,15 @@ class SheerkaSetsManager:
         if not (a.id and b.id):
             return False
 
-        if self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id):
-            return True
-
-        return False
+        group_elements = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, b.id)
+        return group_elements and a.id in group_elements
 
     def isa(self, a, b):
 
-        if BuiltinConcepts.ISA not in a.props:
+        if BuiltinConcepts.ISA not in a.metadata.props:
             return False
 
-        for c in a.get_prop(BuiltinConcepts.ISA):
+        for c in a.metadata.props[BuiltinConcepts.ISA]:
             if c == b:
                 return True
             if self.isa(c, b):
@@ -163,21 +169,19 @@ class SheerkaSetsManager:
         """"""
 
         if not (isinstance(concept, Concept) and concept.id):
-            return None
+            return False
+
+        # check if it has a group
+        # TODO: use cache instead of directly requesting sdp
+        if self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_GROUPS_ENTRY, concept.id):
+            return True
 
         # it may be a concept that references a set
-        if not concept.metadata.is_evaluated:
-            with context.push(desc=f"Evaluating concept {concept}") as sub_context:
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-                evaluated = self.sheerka.evaluate_concept(sub_context, concept)
-                if evaluated.key != concept.key:
-                    return False
+        concept = core.builtin_helpers.ensure_evaluated(context, concept)
+        if not context.sheerka.is_success(concept):
+            return False
 
-        if concept.body:
-            return self.isaset(context, concept.body)
-
-        res = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
-        return res is not None
+        return self.isaset(context, concept.body)
 
     def _validate_where_clause(self, concept):
         python_parser_result = [r for r in concept.compiled[ConceptParts.WHERE] if r.who == "parsers.Python"]
@@ -190,7 +194,7 @@ class SheerkaSetsManager:
         if len(names) != 1 or names[0] != concept.metadata.body:
             return None
 
-        condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.value(x)")
+        condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.objvalue(x)")
         expression = f"""
 result=[]
 for x in xx__concepts__xx:
@@ -218,10 +222,11 @@ for x in xx__concepts__xx:
 
         result = []
         with context.push(desc=f"Evaluating concepts of a set") as sub_context:
+            sub_context.add_inputs(ids=ids)
             sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
             for element_id in ids:
                 concept = self.sheerka.get_by_id(element_id)
                 evaluated = self.sheerka.evaluate_concept(sub_context, concept)
                 result.append(evaluated)
-
+            sub_context.add_inputs(return_value=result)
         return result

src/core/sheerka/Services/SheerkaVariableManager.py

@@ -20,35 +20,29 @@ class Variable:
 
 
 class SheerkaVariableManager:
-    VARIABLES_ENTRY = "All_Variables"  # to store all the concepts
 
     def __init__(self, sheerka):
         self.sheerka = sheerka
 
     def record(self, context, who, key, value):
-        """Persist a variable"""
-        # first check if there is a previous version of the variable
-        try:
-            old = self.sheerka.sdp.get(self.VARIABLES_ENTRY, who + "." + key)
-            if old.value == value:
-                return
+        """
 
-            parent = getattr(old, Serializer.ORIGIN)
-        except IndexError:
-            parent = None
+        :param context:
+        :param who: entity that owns the key (acts as a namespace)
+        :param key:
+        :param value:
+        :return:
+        """
 
-        variable = Variable(context.event.get_digest(), who, key, value, [parent] if parent else None)
-        self.sheerka.sdp.set(context.event.get_digest(), self.VARIABLES_ENTRY, variable, use_ref=True)
+        variable = Variable(context.event.get_digest(), who, key, value, None)
+        self.sheerka.cache_manager.put(self.sheerka.VARIABLES_ENTRY, variable.get_key(), variable)
 
     def load(self, who, key):
-        variable = self.sheerka.sdp.get_safe(self.VARIABLES_ENTRY, who + "." + key)
+        variable = self.sheerka.cache_manager.get(self.sheerka.VARIABLES_ENTRY, who + "." + key)
         if variable is None:
             return None
 
         return variable.value
 
     def delete(self, context, who, key):
-        self.sheerka.sdp.remove(
-            context.event.get_digest(),
-            self.VARIABLES_ENTRY,
-            lambda _key, _var: _key == who + "." + key)
+        self.sheerka.cache_manager.delete(self.sheerka.VARIABLES_ENTRY, who + "." + key)

src/core/sheerka/Sheerka.py

@@ -2,6 +2,12 @@ import logging
 
 import core.builtin_helpers
 import core.utils
+from cache.Cache import Cache
+from cache.CacheManager import CacheManager
+from cache.DictionaryCache import DictionaryCache
+from cache.IncCache import IncCache
+from cache.ListIfNeededCache import ListIfNeededCache
+from cache.SetCache import SetCache
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
     UnknownConcept
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
@@ -18,8 +24,7 @@ from core.sheerka_logger import console_handler
 from printer.SheerkaPrinter import SheerkaPrinter
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 
-CONCEPT_LEXER_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
-BNF_PARSER_CLASS = "parsers.BnfParser.BnfParser"
+BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
 CONCEPTS_FILE = "_concepts.txt"
 
 
@@ -28,45 +33,36 @@ class Sheerka(Concept):
     Main controller for the project
     """
 
-    CONCEPTS_ENTRY = "All_Concepts"  # to store all the concepts
-    CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"
+    CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"  # to store all the concepts
+    CONCEPTS_BY_KEY_ENTRY = "Concepts_By_Key"
     CONCEPTS_BY_NAME_ENTRY = "Concepts_By_Name"
     CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash"  # store hash of concepts definitions (not values)
-    CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions"  # to store definitions (bnf) of concepts
-    CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
-    CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
 
+    CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
+    RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Resolved_Concepts_By_First_Keyword"
+    CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
+    RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY = "Resolved_Concepts_Sya_Definitions"
+    CONCEPTS_GRAMMARS_ENTRY = "Concepts_Grammars"
+
+    CONCEPTS_GROUPS_ENTRY = "Concepts_Groups"
+    VARIABLES_ENTRY = "Variables"  # entry for admin or internal variables
+
+    CONCEPTS_KEYS_ENTRY = "Concepts_Keys"
     BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts"  # sequential key for builtin concepts
     USER_CONCEPTS_KEYS = "User_Concepts"  # sequential key for user defined concepts
 
-    def __init__(self, skip_builtins_in_db=False, debug=False, loggers=None):
+    def __init__(self, cache_only=False, debug=False, loggers=None):
         self.init_logging(debug, loggers)
         self.loggers = loggers
 
         super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
         self.log.debug("Starting Sheerka.")
 
-        # cache of the most used concepts
-        # Note that these are only templates
-        # They are used as a footprint for instantiation
-        # Except of source when the concept is supposed to be unique
-        # key is the key of the concept (not the name or the id)
-        self.cache_by_key = {}
-        self.cache_by_id = {}
-        self.cache_by_name = {}
+        self.bnp = None  # reference to the BaseNodeParser class (to compute first keyword token)
 
-        # cache for concept definitions,
-        # Primarily used for unit test that does not have access to sdp
-        self.concepts_definitions_cache = {}
-
-        #
-        # cache for concepts grammars
-        # a grammar is a resolved BNF
-        self.concepts_grammars = {}
-
-        # cache for SYA concepts
-        self.concepts_by_first_keyword = {}
-        self.sya_definitions = {}
+        # # Cache for concepts grammars
+        # # To be shared between BNFNode parsers instances
+        # self.concepts_grammars = {}
 
         # a concept can be instantiated
         # ex: File is a concept, but File('foo.txt') is an instance
@@ -78,6 +74,8 @@ class Sheerka(Concept):
         self.rules = []
 
         self.sdp: SheerkaDataProvider = None  # SheerkaDataProvider
+        self.cache_manager = CacheManager(cache_only)
+
         self.builtin_cache = {}  # cache for builtin concepts
         self.parsers = {}  # cache for builtin parsers
         self.evaluators = []  # cache for builtin evaluators
@@ -85,8 +83,6 @@ class Sheerka(Concept):
         self.evaluators_prefix: str = None
         self.parsers_prefix: str = None
 
-        self.skip_builtins_in_db = skip_builtins_in_db
-
         self.execute_handler = SheerkaExecute(self)
         self.create_new_concept_handler = SheerkaCreateNewConcept(self)
         self.modify_concept_handler = SheerkaModifyConcept(self)
@@ -100,36 +96,65 @@ class Sheerka(Concept):
         self.during_restore = False
         self._builtins_classes_cache = None
 
-    def initialize(self, root_folder: str = None):
+        self.save_execution_context = True
+
+    @property
+    def resolved_concepts_by_first_keyword(self):
+        """
+        We return the cache as we will be interested by statistics
+        :return:
+        """
+        return self.cache_manager.caches[self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY].cache
+
+    @property
+    def resolved_sya_def(self):
+        """
+
+        :return:
+        """
+        return self.cache_manager.caches[self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY].cache
+
+    @property
+    def concepts_grammars(self):
+        return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
+
+    def initialize(self, root_folder: str = None, save_execution_context=True):
         """
         Starting Sheerka
         Loads the current configuration
         Notes that when it's the first time, it also create the needed working folders
         :param root_folder: root configuration folder
+        :param save_execution_context:
         :return: ReturnValue(Success or Error)
         """
 
+        self.save_execution_context = save_execution_context
+
         try:
             from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
             initialize_pickle_handlers()
 
             self.sdp = SheerkaDataProvider(root_folder, self)
-            if self.sdp.first_time:
-                self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
+            self.initialize_caching()
 
-            event = Event("Initializing Sheerka.", user=self.name)
+            event = Event("Initializing Sheerka.", user_id=self.name)
             self.sdp.save_event(event)
             with ExecutionContext(self.key, event, self, "Initializing Sheerka.", self.init_log) as exec_context:
+                if self.sdp.first_time:
+                    self.first_time_initialisation(exec_context)
 
-                self.initialize_builtin_concepts()
                 self.initialize_builtin_parsers()
                 self.initialize_builtin_evaluators()
-                self.initialize_bnf_parsing(exec_context)
-                self.initialize_sya_parsing()
+                self.initialize_builtin_concepts()
+                self.initialize_concept_node_parsing(exec_context)
                 res = ReturnValueConcept(self, True, self)
 
                 exec_context.add_values(return_values=res)
-                if not self.skip_builtins_in_db:
+
+                if self.cache_manager.is_dirty:
+                    self.cache_manager.commit(exec_context)
+
+                if save_execution_context:
                     self.sdp.save_result(exec_context, is_admin=True)
                 self.init_log.debug(f"Sheerka successfully initialized")
 
@@ -138,6 +163,59 @@ class Sheerka(Concept):
 
         return res
 
+    def initialize_caching(self):
+
+        def params(cache_name):
+            return {
+                'default': lambda k: self.sdp.get(cache_name, k),
+                'extend_exists': lambda k: self.sdp.exists(cache_name, k)
+            }
+
+        cache = IncCache(default=lambda k: self.sdp.get(self.CONCEPTS_KEYS_ENTRY, k))
+        self.cache_manager.register_cache(self.CONCEPTS_KEYS_ENTRY, cache)
+
+        register_concept_cache = self.cache_manager.register_concept_cache
+
+        cache = Cache(**params(self.CONCEPTS_BY_ID_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
+
+        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_KEY_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
+
+        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_NAME_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
+
+        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
+        register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
+
+        cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, k))
+        self.cache_manager.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
+        self.cache_manager.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None)  # to init from sdp
+
+        cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, k))
+        self.cache_manager.register_cache(self.CONCEPTS_SYA_DEFINITION_ENTRY, cache)
+        self.cache_manager.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, None)  # to init from sdp
+
+        cache = SetCache(default=lambda k: self.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k))
+        self.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, cache)
+
+        cache = Cache(default=lambda k: self.sdp.get(self.VARIABLES_ENTRY, k))
+        self.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True)
+
+        cache = DictionaryCache()
+        self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
+
+        cache = DictionaryCache()
+        self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, cache, persist=False)
+
+        cache = Cache()
+        self.cache_manager.register_cache(self.CONCEPTS_GRAMMARS_ENTRY, cache, persist=False)
+
+    def first_time_initialisation(self, context):
+
+        self.cache_manager.put(self.CONCEPTS_KEYS_ENTRY, self.USER_CONCEPTS_KEYS, 1000)
+        self.variable_handler.record(context, self.name, "save_execution_context", True)
+
     def initialize_builtin_concepts(self):
         """
         Initializes the builtin concepts
@@ -160,18 +238,16 @@ class Sheerka(Concept):
             if not concept.metadata.is_unique and str(key) in builtins_classes:
                 self.builtin_cache[key] = builtins_classes[str(key)]
 
-            if not self.skip_builtins_in_db:
-                from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.metadata.key)
-                if from_db is None:
-                    self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
-                    self.set_id_if_needed(concept, True)
-                    concept.metadata.full_serialization = True
-                    self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
-                else:
-                    self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
-                    concept.update_from(from_db)
+            from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.metadata.key)
+            if from_db is None:
+                self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
+                self.set_id_if_needed(concept, True)
+                self.cache_manager.add_concept(concept)
+            else:
+                self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
+                concept.update_from(from_db)
 
-            self.add_in_cache(concept)
+        return
 
     def initialize_builtin_parsers(self):
         """
@@ -187,17 +263,23 @@ class Sheerka(Concept):
             if parser.__module__ == base_class.__module__:
                 continue
 
-            if parser.__module__ in modules_to_skip:
-                continue
-
             qualified_name = core.utils.get_full_qualified_name(parser)
             self.init_log.debug(f"Adding builtin parser '{qualified_name}'")
             temp_result[qualified_name] = parser
 
+        # keep a reference to base_node_parser
+        self.bnp = temp_result[BASE_NODE_PARSER_CLASS]
+
         # Now we sort the parser by name.
         #  It's not important for the logic of their usage as they have their priority anyway,
         #  We do that for the unit tests. They are to complicated to write otherwise
         for name in sorted(temp_result.keys()):
+            parser = temp_result[name]
+
+            if parser.__module__ in modules_to_skip:
+                # base node parser module does not contains any valid parser
+                continue
+
             self.parsers[name] = temp_result[name]
 
     def initialize_builtin_evaluators(self):
@@ -214,55 +296,39 @@ class Sheerka(Concept):
             self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
             self.evaluators.append(evaluator)
 
-    def initialize_bnf_parsing(self, execution_context):
-        self.init_log.debug("Initializing concepts grammars.")
-        definitions = self.get_concepts_definitions(execution_context)
+    def initialize_concept_node_parsing(self, context):
+        self.init_log.debug("Initializing concept node parsing.")
 
-        if definitions is None:
-            self.init_log.debug("No BNF defined")
-            return
+        concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
+        res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
+        self.cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
 
-        lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]()
-        ret_val = lexer_parser.initialize(execution_context, definitions)
-        if not ret_val.status:
-            self.init_log.error("Failed to initialize concepts definitions " + str(ret_val.body))
-            return
+        # sya = self.bnf.resolve_sya_associativity_and_precedence()
+        # self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, sya)
+        #
+        #
+        # self.concepts_by_first_keyword, \
+        # self.resolved_concepts_by_first_keyword = \
+        #     self.create_new_concept_handler.load_concepts_nodes_definitions(context)
 
-        self.concepts_grammars = lexer_parser.concepts_grammars
+        # self.concepts_by_first_keyword = self.sdp.get_safe(
+        #     self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
+        #     load_origin=False) or {}
+        #
+        # self.sya_definitions = self.sdp.get_safe(
+        #     self.CONCEPTS_SYA_DEFINITION_ENTRY,
+        #     load_origin=False) or {}
+        #
+        # init_ret_value = self.bnp.resolve_concepts_by_first_keyword(self, self.concepts_by_first_keyword)
+        # if not init_ret_value.status:
+        #     return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
+        # self.resolved_concepts_by_first_keyword = init_ret_value.body
 
-    def initialize_sya_parsing(self):
-        self.init_log.debug("Initializing sya definitions.")
-
-        self.concepts_by_first_keyword = self.sdp.get_safe(
-            self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
-            load_origin=False) or {}
-
-        self.sya_definitions = self.sdp.get_safe(
-            self.CONCEPTS_SYA_DEFINITION_ENTRY,
-            load_origin=False) or {}
-
-    def reset(self):
-        self.reset_cache()
-        self.concepts_by_first_keyword = {}
-        self.concepts_grammars = {}
-        self.sya_definitions = {}
+    def reset(self, cache_only=False):
+        self.cache_manager.clear()
+        self.cache_manager.cache_only = cache_only
+        self.printer_handler.reset()
         self.sdp.reset()
-        self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
-
-    def reset_cache(self, filter_to_use=None):
-        """
-        reset the different cache that exists
-        :param filter_to_use:
-        :return:
-        """
-        if filter_to_use is None:
-            self.cache_by_key = {}
-            self.cache_by_id = {}
-            self.cache_by_name = {}
-        else:
-            raise NotImplementedError()
-
-        return self
 
     def evaluate_user_input(self, text: str, user_name="kodjo"):
         """
@@ -294,7 +360,10 @@ class Sheerka(Concept):
             ret = self.execute(execution_context, [user_input, reduce_requested], steps)
             execution_context.add_values(return_values=ret)
 
-            if not self.skip_builtins_in_db:
+            if self.cache_manager.is_dirty:
+                self.cache_manager.commit(execution_context)
+
+            if self.save_execution_context and self.variable_handler.load(self.name, "save_execution_context"):
                 self.sdp.save_result(execution_context)
 
         # # hack to save valid concept definition
@@ -302,6 +371,8 @@ class Sheerka(Concept):
         #     if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
         #         with open(CONCEPTS_FILE, "a") as f:
         #             f.write(text + "\n")
+
+        self._last_execution = execution_context
         return ret
 
     def print(self, result, instructions=None):
@@ -343,8 +414,8 @@ class Sheerka(Concept):
         if obj.metadata.id is not None:
             return
 
-        entry = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
-        obj.metadata.id = self.sdp.get_next_key(entry)
+        key = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
+        obj.metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key))
         self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
 
     def create_new_concept(self, context, concept: Concept):
@@ -380,21 +451,25 @@ class Sheerka(Concept):
         """
         return self.sets_handler.set_isa(context, concept, concept_set)
 
-    def set_sya_def(self, context, list_of_def):
+    def force_sya_def(self, context, list_of_def):
         """
         Set the precedence and/or the associativity of a concept
+        FOR TESTS PURPOSE. TO REMOVE EVENTUALLY
         :param context:
         :param list_of_def list of tuple(concept_id, precedence (int), SyaAssociativity)
         :return:
         """
 
         # validate the entries
+        # If one entry is an invalid concept, rollback everything
         for concept_id, precedence, associativity in list_of_def:
             if concept_id == BuiltinConcepts.UNKNOWN_CONCEPT:
                 return self.ret(self.name,
                                 False,
                                 self.new(BuiltinConcepts.ERROR, body=f"Concept {concept_id} is not known"))
 
+        sya_def = self.cache_manager.copy(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY) or {}
+
         # update the definitions
         for concept_id, precedence, associativity in list_of_def:
             if precedence is None and associativity is None:
@@ -403,12 +478,10 @@ class Sheerka(Concept):
                 except KeyError:
                     pass
             else:
-                self.sya_definitions[concept_id] = (precedence, associativity.value)
+                sya_def[concept_id] = (precedence, associativity)
 
-        # then save
-        self.sdp.set(context.event.get_digest(),
-                     self.CONCEPTS_SYA_DEFINITION_ENTRY,
-                     self.sya_definitions)
+        # put in cache
+        self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, False, sya_def)
 
         return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
 
@@ -448,122 +521,108 @@ class Sheerka(Concept):
         if concept.key is None:
             raise KeyError()
 
-        self.cache_by_key[concept.key] = concept
-
-        if concept.id:
-            self.cache_by_id[concept.id] = concept
+        self.cache_manager.add_concept(concept)
 
         return concept
 
-    def get(self, concept_key, concept_id=None):
+    #
+    # def get(self, concept_key, concept_id=None):
+    #     """
+    #     Tries to find a concept
+    #     What is return must be used a template for another concept.
+    #     You must not modify the returned concept
+    #     :param concept_key: key of the concept
+    #     :param concept_id: when multiple concepts with the same key, use the id
+    #     :return:
+    #     """
+    #
+    #     by_key = self.get_by_key(concept_key)
+    #     if self.is_known(by_key):
+    #         return by_key
+    #
+    #     # else return by name
+    #     by_name = self.get_by_name(concept_key)
+    #     if self.is_known(by_name):
+    #         return by_name
+    #
+    #     return by_key  # return not found for key
+
+    def get_by_key(self, concept_key, concept_id=None):
+        concept_key = str(concept_key) if isinstance(concept_key, BuiltinConcepts) else concept_key
+        return self.internal_get("key", concept_key, self.CONCEPTS_BY_KEY_ENTRY, concept_id)
+
+    def get_by_name(self, concept_name, concept_id=None):
+        return self.internal_get("name", concept_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
+
+    def get_by_hash(self, concept_hash, concept_id=None):
+        return self.internal_get("hash", concept_hash, self.CONCEPTS_BY_HASH_ENTRY, concept_id)
+
+    def get_by_id(self, concept_id):
+        return self.internal_get("id", concept_id, self.CONCEPTS_BY_ID_ENTRY, None)
+
+    def internal_get(self, index_name, key, cache_name, concept_id=None):
         """
-        Tries to find a concept
-        What is return must be used a template for another concept.
-        You must not modify the returned concept
-        :param concept_key: key of the concept
-        :param concept_id: when multiple concepts with the same key, use the id
+        Tries to find an entry
+        :param index_name: name of the index (ex by_id, by_key...)
+        :param key: index value
+        :param cache_name: name of the cache (ex Concepts_By_ID...)
+        :param concept_id: id of the concept if none, in case where there are multiple results
         :return:
         """
 
-        by_key = self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
-        if self.is_known(by_key):
-            return by_key
+        if key is None:
+            return ErrorConcept(f"Concept '{key}' is undefined.")
 
-        # else return by name
-        by_name = self.internal_get("name", concept_key, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
-        if self.is_known(by_name):
-            return by_name
+        concepts = self.cache_manager.get(cache_name, key)
+        if concepts:
+            if concept_id is None:
+                return concepts
 
-        return by_key  # return not found for key
+            if not hasattr(concepts, "__iter__"):
+                return concepts
 
-    def get_by_key(self, concept_key, concept_id=None):
-        return self.internal_get("key", concept_key, self.cache_by_key, self.CONCEPTS_ENTRY, concept_id)
+            for c in concepts:
+                if c.id == concept_id:
+                    return c
 
-    def get_by_name(self, concept_name, concept_id=None):
-        return self.internal_get("name", concept_name, self.cache_by_name, self.CONCEPTS_BY_NAME_ENTRY, concept_id)
+        metadata = [(index_name, key), ("id", concept_id)] if concept_id else (index_name, key)
+        return self._get_unknown(metadata)
 
-    def get_by_id(self, concept_id):
-        if concept_id is None:
-            return ErrorConcept("Concept id is undefined.")
-
-        # first search in cache
-        if concept_id in self.cache_by_id:
-            result = self.cache_by_id[concept_id]
-        else:
-            result = self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id)
-            if result is None:
-                result = self._get_unknown(('id', concept_id))
-            else:
-                self.cache_by_id[concept_id] = result
-
-        return result
-
-    def internal_get(self, index_name, index_value, cache_to_use, sdp_entry, concept_id=None):
+    def has_id(self, concept_id):
         """
-        Tries to find an entry
-        :param index_name:
-        :param index_value:
-        :param cache_to_use:
-        :param sdp_entry:
+        Returns True if a concept with this id exists in cache
+        It does not search in the remote repository
         :param concept_id:
         :return:
         """
+        return self.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
 
-        if index_value is None:
-            return ErrorConcept(f"Concept {index_name} is undefined.")
+    def has_key(self, concept_key):
+        """
+        Returns True if concept(s) with this key exist in cache
+        It does not search in the remote repository
+        :param concept_key:
+        :return:
+        """
+        return self.cache_manager.has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
 
-        if isinstance(index_value, BuiltinConcepts):
-            index_value = str(index_value)
+    def has_name(self, concept_name):
+        """
+        Returns True if concept(s) with this name exist in cache
+        It does not search in the remote repository
+        :param concept_name:
+        :return:
+        """
+        return self.cache_manager.has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
 
-        # first search in cache
-        if index_value in cache_to_use:
-            result = cache_to_use[index_value]
-        else:
-            result = self.sdp.get_safe(sdp_entry, index_value)
-            if result is None:
-                metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
-                result = self._get_unknown(metadata)
-                # Do not put in cache_by_key or cache_by_id unknown concept
-                # TODO: implement an MRU cache for them
-            else:
-                cache_to_use[index_value] = result
-                for r in (result if isinstance(result, list) else [result]):
-                    if r.id:
-                        self.cache_by_id[r.id] = r
-
-        if not (isinstance(result, list) and concept_id):
-            return result
-
-        # result is a list, but we have the concept_id to discriminate
-        for c in result:
-            if c.id == concept_id:
-                return c
-
-        metadata = [(index_name, index_value), ("id", concept_id)] if concept_id else (index_name, index_value)
-        return self._get_unknown(metadata)
-
-    def get_concepts_definitions(self, context):
-
-        if self.concepts_definitions_cache:
-            return self.concepts_definitions_cache
-
-        encoded_bnf = self.sdp.get_safe(
-            self.CONCEPTS_DEFINITIONS_ENTRY,
-            load_origin=False) or {}
-
-        self.concepts_definitions_cache = {}
-        bnf_parser = self.parsers[BNF_PARSER_CLASS]()
-        for k, v in encoded_bnf.items():
-            key, id_ = core.utils.unstr_concept(k)
-            concept = self.new((key, id_))
-            context.log(f"Parsing BNF definition for {concept}", context.who)
-            rule_result = bnf_parser.parse(context, v)
-            if rule_result.status:
-                self.concepts_definitions_cache[concept] = rule_result.value.value
-            else:
-                self.log.error(f"Failed to load bnf rule for concept {key}")
-
-        return self.concepts_definitions_cache
+    def has_hash(self, concept_hash):
+        """
+        Returns True if concept(s) with this hash exist in cache
+        It does not search in the remote repository
+        :param concept_hash:
+        :return:
+        """
+        return self.cache_manager.has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
 
     def new(self, concept_key, **kwargs):
         """
@@ -578,7 +637,7 @@ class Sheerka(Concept):
         else:
             concept_id = None
 
-        template = self.get_by_id(concept_id) if not concept_key else self.get(concept_key, concept_id)
+        template = self.get_by_id(concept_id) if not concept_key else self.get_by_key(concept_key, concept_id)
 
         # manage concept not found
         if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \
@@ -599,7 +658,7 @@ class Sheerka(Concept):
 
         # otherwise, create another instance
         concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
-        concept.update_from(template)
+        concept.update_from(template, update_value=False)
         concept.freeze_definition_hash()
 
         if len(kwargs) == 0:
@@ -608,10 +667,10 @@ class Sheerka(Concept):
         # update the properties, values, attributes
         # Not quite sure that this is the correct process order
         for k, v in kwargs.items():
-            if k in concept.props:
-                concept.set_prop(k, v)
+            if k in concept.values:
+                concept.set_value(k, v)
             elif k in PROPERTIES_FOR_NEW:
-                concept.values[ConceptParts(k)] = v
+                concept.set_value(ConceptParts(k), v)
             elif hasattr(concept, k):
                 setattr(concept, k, v)
             else:
@@ -639,12 +698,12 @@ class Sheerka(Concept):
             message=message,
             parents=parents)
 
-    def value(self, obj, reduce_simple_list=False):
+    def objvalue(self, obj, reduce_simple_list=False):
         if obj is None:
             return None
 
-        if hasattr(obj, "get_value"):
-            return obj.get_value()
+        if hasattr(obj, "get_obj_value"):
+            return obj.get_obj_value()
 
         if not isinstance(obj, Concept):
             return obj
@@ -657,7 +716,18 @@ class Sheerka(Concept):
         else:
             body_to_use = obj.body
 
-        return self.value(body_to_use)
+        return self.objvalue(body_to_use)
+
+    def objvalues(self, objs):
+        if not (isinstance(objs, list) or
+                self.isinstance(objs, BuiltinConcepts.LIST) or
+                self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
+            objs = [objs]
+
+        if isinstance(objs, list):
+            return (self.objvalue(obj) for obj in objs)
+
+        return (self.objvalue(obj) for obj in objs.body)
 
     def value_by_concept(self, obj, concept):
         if obj is None:
@@ -678,8 +748,8 @@ class Sheerka(Concept):
         if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
             return obj
 
-        if isinstance(obj, list):
-            return obj
+        if isinstance(obj, (list, set, tuple)):
+            return [self.get_error(o) for o in obj]
 
         if self.isinstance(obj, BuiltinConcepts.RETURN_VALUE):
             if obj.status:
@@ -687,19 +757,10 @@ class Sheerka(Concept):
 
             if self.isinstance(obj.body, BuiltinConcepts.PARSER_RESULT):
                 return self.get_error(obj.body.body)
+            else:
+                return obj.body
 
-        return NotImplementedError()
-
-    def get_values(self, objs):
-        if not (isinstance(objs, list) or
-                self.isinstance(objs, BuiltinConcepts.LIST) or
-                self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
-            objs = [objs]
-
-        if isinstance(objs, list):
-            return (self.value(obj) for obj in objs)
-
-        return (self.value(obj) for obj in objs.body)
+        raise NotImplementedError()
 
     def is_success(self, obj):
         if isinstance(obj, bool):  # quick win
@@ -761,8 +822,12 @@ class Sheerka(Concept):
         return self.parsers_prefix + name
 
     def concepts(self):
+        """
+        List of all known concepts (look up in sdp)
+        :return:
+        """
         res = []
-        lst = self.sdp.list(self.CONCEPTS_ENTRY)
+        lst = self.sdp.list(self.CONCEPTS_BY_ID_ENTRY)
         for item in lst:
             if isinstance(item, list):
                 res.extend(item)
@@ -818,10 +883,10 @@ class Sheerka(Concept):
         # the metadata can be a list, if several attributes where given
         # (key, 'not_found), (id, invalid_id)
 
-        unknown_concept = UnknownConcept()
-        unknown_concept.set_metadata_value(ConceptParts.BODY, metadata)
+        unknown_concept = UnknownConcept()  # don't use new() for prevent circular reference
+        unknown_concept.set_value(ConceptParts.BODY, metadata)
         for meta in (metadata if isinstance(metadata, list) else [metadata]):
-            unknown_concept.set_prop(meta[0], meta[1])
+            unknown_concept.set_value(meta[0], meta[1])
         unknown_concept.metadata.is_evaluated = True
         return unknown_concept