Implemented SheerkaOntology

src/core/ast_helpers.py

@@ -2,6 +2,7 @@ import ast
 from dataclasses import dataclass
 
 from cache.FastCache import FastCache
+from core.global_symbols import NotFound
 
 
 @dataclass
@@ -56,12 +57,12 @@ class UnreferencedNamesVisitor(ast.NodeVisitor):
 
     def get_names(self, node):
         names = UnreferencedNamesVisitor.cache.get(node)
-        if names is not None:
-            return names
+        if names is NotFound:
+            self.visit(node)
+            UnreferencedNamesVisitor.cache.put(node, self.names)
+            return self.names
 
-        self.visit(node)
-        UnreferencedNamesVisitor.cache.put(node, self.names)
-        return self.names
+        return names
 
     def visit_Name(self, node):
         self.names.add(node.id)

src/core/builtin_concepts.py

@@ -351,7 +351,7 @@ class PythonSecurityError(Concept, ErrorObj):
         self._metadata.is_evaluated = True
 
 
-class NotFound(Concept, ErrorObj):
+class NotFoundConcept(Concept, ErrorObj):
     ALL_ATTRIBUTES = []
 
     def __init__(self, body=None):

src/core/builtin_concepts_ids.py

@@ -95,6 +95,8 @@ class BuiltinConcepts:
     INVALID_GREATEST_OPERATION = "__INVALID_GREATEST_OPERATION"
     NEW_RULE = "__NEW_RULE"
     UNKNOWN_RULE = "__UNKNOWN_RULE"
+    ONTOLOGY_ALREADY_DEFINED = "__ONTOLOGY_ALREADY_DEFINED"
+    ONTOLOGY_REMOVED = "__ONTOLOGY_REMOVED"
 
     NODE = "__NODE"
     GENERIC_NODE = "__GENERIC_NODE"
@@ -167,6 +169,7 @@ BuiltinErrors = [
     BuiltinConcepts.NOT_FOUND,
     BuiltinConcepts.INVALID_LESSER_OPERATION,
     BuiltinConcepts.INVALID_GREATEST_OPERATION,
+    BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED
 ]
 
 BuiltinContainers = [

src/core/builtin_helpers.py

@@ -1,10 +1,15 @@
+import ast
 import logging
 
+from cache.Cache import Cache
+from core.ast_helpers import ast_to_props
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, NotInit, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value
+from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value
+from core.global_symbols import NotInit, NotFound
 from core.rule import Rule
 from core.sheerka.services.SheerkaExecute import SheerkaExecute
 from core.tokenizer import Keywords
+from core.utils import as_bag
 from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
     RuleNode
 from parsers.BaseParser import BaseParser, ParsingError
@@ -510,11 +515,11 @@ def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers
 
 def update_compiled(context, concept, errors, parsers=None):
     """
-    recursively iterate thru concept.get_compiled() to replace LexerNode into concepts or list of ReturnValueConcept
+    recursively iterate over concept.get_compiled() to replace LexerNode into concepts or list of ReturnValueConcept
     When parsing using a LexerNodeParser (SyaNodeParser, BnfNodeParser...)
     the result will be a LexerNode.
     In the specific case of a ConceptNode, the compiled variables will also be LexerNode (UnrecognizedTokensNode...)
-    This function iterate thru the compile to transform these nodes into concept of compiled AST
+    This function iterate over the compile to transform these nodes into concept of compiled AST
     :param context:
     :param concept:
     :param errors: a list the must be initialized by the caller
@@ -648,3 +653,59 @@ def ensure_concept_or_rule(*items):
     else:
         if not isinstance(items, (Concept, Rule)):
             raise TypeError(f"'{items}' must be a concept or rule")
+
+
+expressions_cache = Cache()
+
+
+def evaluate_expression(expr, bag):
+    """
+    Try to evaluate expr in context of bag
+    :param expr:
+    :param bag:
+    :return:
+    """
+
+    if expr is None or expr.strip() == "":
+        return None
+
+    if expr in bag:
+        return bag[expr]
+
+    props_definitions = expressions_cache.get(expr)
+    if props_definitions is NotFound:
+        _ast = ast.parse(expr, mode="eval")
+        props_definitions = []
+        ast_to_props(props_definitions, _ast.body, None)
+        props_definitions.reverse()
+        expressions_cache.put(expr, props_definitions)
+
+    return evaluate_object(bag, props_definitions)
+
+
+def evaluate_object(bag, properties):
+    """
+    Evaluate the properties of an object
+    Works with evaluate_expression
+    :param bag:
+    :param properties: List of ast_helpers.PropDef
+    :return:
+    """
+    for prop in properties:
+        try:
+            obj = bag[prop.prop]
+        except KeyError:
+            try:
+                obj = bag["self"][prop.prop]
+            except Exception:
+                raise NameError(prop.prop)
+
+        if obj is None:
+            return None
+
+        if prop.index is not None:
+            obj = obj[prop.index]
+
+        bag = as_bag(obj)
+
+    return obj

src/core/concept.py

@@ -2,10 +2,12 @@ import hashlib
 from collections import namedtuple
 from copy import deepcopy
 from dataclasses import dataclass
+from threading import RLock
 from typing import Union
 
 import core.utils
 from core.builtin_concepts_ids import BuiltinDynamicAttrs
+from core.global_symbols import NotInit
 from core.tokenizer import Tokenizer, TokenKind
 
 PROPERTIES_FOR_DIGEST = ("name", "key",
@@ -21,19 +23,6 @@ DEFINITION_TYPE_BNF = "bnf"
 DEFINITION_TYPE_DEF = "def"
 
 
-class NotInitialized:
-    value = "**NotInit**"
-
-    def __repr__(self):
-        return self.value
-
-    def __eq__(self, other):
-        return isinstance(other, NotInitialized)
-
-
-NotInit = NotInitialized()
-
-
 class ConceptParts:
     """
     Lists metadata that can contains some code
@@ -75,6 +64,7 @@ class ConceptMetadata:
 
 
 ALL_ATTRIBUTES = {}
+all_attributes_lock = RLock()
 
 
 def get_concept_attrs(concept):
@@ -86,15 +76,28 @@ def get_concept_attrs(concept):
     except KeyError:
         pass
 
-    all_attributes = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
-    if concept.id and concept.key not in BuiltinDynamicAttrs:
-        ALL_ATTRIBUTES[concept.id] = all_attributes
-    return all_attributes
+    with all_attributes_lock:
+        all_attributes = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
+        if concept.id and concept.key not in BuiltinDynamicAttrs:
+            ALL_ATTRIBUTES[concept.id] = all_attributes
+        return all_attributes
 
 
 def freeze_concept_attrs(concept):
-    if concept.key not in BuiltinDynamicAttrs:
-        ALL_ATTRIBUTES[concept.id] = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
+    with all_attributes_lock:
+        if concept.key not in BuiltinDynamicAttrs:
+            ALL_ATTRIBUTES[concept.id] = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"]
+
+
+def copy_concepts_attrs():
+    with all_attributes_lock:
+        return ALL_ATTRIBUTES.copy()
+
+
+def load_concepts_attrs(attrs):
+    global ALL_ATTRIBUTES
+    with all_attributes_lock:
+        ALL_ATTRIBUTES = attrs
 
 
 class Concept:

src/core/global_symbols.py

@@ -8,3 +8,35 @@ EVENT_CONCEPT_CREATED = "evt_cc"
 # comparison context
 RULE_COMPARISON_CONTEXT = "Rule"
 CONCEPT_COMPARISON_CONTEXT = "Sya"
+
+
+class CustomType:
+
+    def __init__(self, value):
+        self.value = value
+
+    def __repr__(self):
+        return self.value
+
+    def __eq__(self, other):
+        return isinstance(other, CustomType) and self.value == other.value
+
+
+class NotInitType(CustomType):
+    def __init__(self):
+        super(NotInitType, self).__init__("**NotInit**")
+
+
+class NotFoundType(CustomType):
+    def __init__(self):
+        super(NotFoundType, self).__init__("**NotFound**")
+
+
+class RemovedType(CustomType):
+    def __init__(self):
+        super(RemovedType, self).__init__("**Removed**")
+
+
+NotInit = NotInitType()
+NotFound = NotFoundType()
+Removed = RemovedType()

src/core/rule.py

@@ -5,7 +5,7 @@ import core.utils
 
 ACTION_TYPE_PRINT = "print"
 ACTION_TYPE_EXEC = "exec"
-ACTION_TYPE_DEFERRED = "deferred"
+ACTION_TYPE_DEFERRED = "deferred"  # KSI 2021-04-01 What is it for ? I definitely need some proper documentation
 
 
 @dataclass
@@ -27,8 +27,9 @@ class Rule:
                  predicate=None,
                  action=None,
                  priority=None,
+                 rule_id=None,
                  is_enabled=None):
-        self.metadata = RuleMetadata(action_type, name, predicate, action, is_enabled=is_enabled)
+        self.metadata = RuleMetadata(action_type, name, predicate, action, id=rule_id, is_enabled=is_enabled)
         self.compiled_predicate = None
         self.compiled_action = None
         from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
@@ -36,7 +37,10 @@ class Rule:
         self.error_sink = None
 
     def __repr__(self):
-        return f"Rule(#{self.metadata.id}, when '{self.metadata.predicate}' {self.metadata.action_type} '{self.metadata.action}', priority={self.priority})"
+        rule_id = f"#{self.metadata.id}"
+        if self.name:
+            rule_id += f" ({self.metadata.name})"
+        return f"Rule({rule_id}, when '{self.metadata.predicate}' {self.metadata.action_type} '{self.metadata.action}', priority={self.priority})"
 
     def __eq__(self, other):
         if id(other) == id(self):
@@ -57,6 +61,22 @@ class Rule:
                      self.metadata.action_type,
                      self.metadata.action))
 
+    def __deepcopy__(self, memodict={}):
+        copy = Rule(self.metadata.action_type,
+                    self.name,
+                    self.metadata.predicate,
+                    self.metadata.action,
+                    self.priority,
+                    self.id,
+                    self.metadata.is_enabled)
+        copy.compiled_predicate = self.compiled_predicate
+        copy.compiled_action = self.compiled_action
+
+        return copy
+
+    def __copy__(self):
+        return self.__deepcopy__()
+
     def set_id(self, rule_id):
         self.metadata.id = rule_id
         return self

src/core/sheerka/Sheerka.py

@@ -5,21 +5,20 @@ from dataclasses import dataclass
 import core.builtin_helpers
 import core.utils
 from cache.Cache import Cache
-from cache.CacheManager import CacheManager
 from cache.DictionaryCache import DictionaryCache
 from cache.IncCache import IncCache
 from core.builtin_concepts import ErrorConcept, ReturnValueConcept, UnknownConcept
 from core.builtin_concepts_ids import BuiltinErrors, BuiltinConcepts
-from core.concept import Concept, ConceptParts, NotInit, get_concept_attrs
+from core.concept import Concept, ConceptParts, get_concept_attrs
 from core.error import ErrorObj
-from core.global_symbols import EVENT_USER_INPUT_EVALUATED
+from core.global_symbols import EVENT_USER_INPUT_EVALUATED, NotInit, NotFound
 from core.profiling import profile
 from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager, OntologyAlreadyExists
 from core.sheerka_logger import console_handler
-from core.simple_debug import my_debug
 from core.tokenizer import Token, TokenKind
 from printer.SheerkaPrinter import SheerkaPrinter
-from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
+from sdp.sheerkaDataProvider import Event
 
 BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
 EXIT_COMMANDS = ("quit", "exit", "bye")
@@ -64,18 +63,19 @@ class Sheerka(Concept):
     ALL_ATTRIBUTES = []
 
     def __init__(self, cache_only=False, debug=False, loggers=None):
+        super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
+
         self.init_logging(debug, loggers)
         self.loggers = loggers
+        self.cache_only = cache_only
 
-        super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
         # self.log.debug("Starting Sheerka.")
 
         self.bnp = None  # reference to the BaseNodeParser class (to compute first keyword token)
         self.return_value_concept_id = None
         self.error_concept_id = None
 
-        self.sdp: SheerkaDataProvider = None
-        self.cache_manager = CacheManager(cache_only)
+        self.om: SheerkaOntologyManager = None
 
         self.services = {}  # sheerka plugins
 
@@ -105,29 +105,20 @@ class Sheerka(Concept):
         self.locals = {}
         self.concepts_ids = None
 
-    @property
-    def resolved_concepts_by_first_keyword(self):
-        """
-        We return the cache as we will be interested by statistics
-        :return:
-        """
-        return self.cache_manager.caches[self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY].cache
-
-    @property
-    def resolved_sya_def(self):
-        """
-
-        :return:
-        """
-        return self.cache_manager.caches[self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY].cache
-
     @property
     def concepts_grammars(self):
-        return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
+        """
+        Quick access to BNF grammars
+        """
+        return self.om.current_cache_manager().caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
 
     @property
     def chicken_and_eggs(self):
-        return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
+        return self.om.current_cache_manager().caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
+
+    @property
+    def root_folder(self):
+        return self.om.root_folder
 
     def bind_service_method(self, bound_method, has_side_effect, as_name=None, visible=True):
         """
@@ -147,7 +138,7 @@ class Sheerka(Concept):
                 self.methods_with_context.add(as_name)
             self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect)
 
-        setattr(self, as_name, bound_method)
+        setattr(self, bound_method.__name__, bound_method)
 
     def initialize(self, root_folder: str = None, save_execution_context=None, enable_process_return_values=None):
         """
@@ -171,7 +162,7 @@ class Sheerka(Concept):
             from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
             initialize_pickle_handlers()
 
-            self.sdp = SheerkaDataProvider(root_folder, self)
+            self.om = SheerkaOntologyManager(self, root_folder, self.cache_only)
             self.builtin_cache = self.get_builtins_classes_as_dict()
 
             self.initialize_caching()
@@ -181,33 +172,38 @@ class Sheerka(Concept):
             self.initialize_builtin_evaluators()
 
             event = Event("Initializing Sheerka.", user_id=self.name)
-            self.sdp.save_event(event)
+            self.om.save_event(event)
             with ExecutionContext(self.key,
                                   event,
                                   self,
                                   BuiltinConcepts.INIT_SHEERKA,
                                   None,
                                   desc="Initializing Sheerka.") as exec_context:
-                if self.sdp.first_time:
+                if self.om.current_sdp().first_time:
                     self.first_time_initialisation(exec_context)
 
                 self.initialize_builtin_concepts()
                 self.initialize_concept_node_parsing(exec_context)
 
-                self.initialize_services_deferred(exec_context, self.sdp.first_time)
+                self.initialize_services_deferred(exec_context, self.om.current_sdp().first_time)
 
                 res = ReturnValueConcept(self, True, self)
                 exec_context.add_values(return_values=res)
 
-                if self.cache_manager.is_dirty:
-                    self.cache_manager.commit(exec_context)
+                if self.om.is_dirty():
+                    self.om.commit(exec_context)
 
                 if self.save_execution_context:
-                    self.sdp.save_result(exec_context, is_admin=True)
+                    self.om.save_result(exec_context, is_admin=True)
+
+                # append the other ontologies if needed
+                self.om.freeze()
+                self.initialize_ontologies(exec_context)
+
                 # self.init_log.debug(f"Sheerka successfully initialized")
 
         except IOError as e:
-            res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
+            res = ReturnValueConcept(self.name, False, self.new(BuiltinConcepts.ERROR, body=e))
 
         finally:
             self.during_initialisation = False
@@ -216,28 +212,28 @@ class Sheerka(Concept):
 
     def initialize_caching(self):
 
-        cache = IncCache(default=lambda k: self.sdp.get(self.OBJECTS_IDS_ENTRY, k))
-        self.cache_manager.register_cache(self.OBJECTS_IDS_ENTRY, cache)
+        cache = IncCache().auto_configure(self.OBJECTS_IDS_ENTRY)
+        self.om.register_cache(self.OBJECTS_IDS_ENTRY, cache)
 
-        cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, k))
-        self.cache_manager.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
-        self.cache_manager.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None)  # to init from sdp
+        cache = DictionaryCache().auto_configure(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
+        self.om.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
+        self.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None)  # to init from sdp
 
-        cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, k))
-        self.cache_manager.register_cache(self.CONCEPTS_SYA_DEFINITION_ENTRY, cache)
-        self.cache_manager.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, None)  # to init from sdp
+        cache = DictionaryCache().auto_configure(self.CONCEPTS_SYA_DEFINITION_ENTRY)
+        self.om.register_cache(self.CONCEPTS_SYA_DEFINITION_ENTRY, cache)
+        self.om.get(self.CONCEPTS_SYA_DEFINITION_ENTRY, None)  # to init from sdp
 
-        cache = DictionaryCache()
-        self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
+        cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
+        self.om.register_cache(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache, persist=False)
 
-        cache = DictionaryCache()
-        self.cache_manager.register_cache(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, cache, persist=False)
+        cache = DictionaryCache().auto_configure(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY)
+        self.om.register_cache(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, cache, persist=False)
 
-        cache = Cache()
-        self.cache_manager.register_cache(self.CONCEPTS_GRAMMARS_ENTRY, cache, persist=False)
+        cache = Cache().auto_configure(self.CONCEPTS_GRAMMARS_ENTRY)
+        self.om.register_cache(self.CONCEPTS_GRAMMARS_ENTRY, cache, persist=False)
 
-        cache = Cache()
-        self.cache_manager.register_cache(self.CHICKEN_AND_EGG_CONCEPTS_ENTRY, cache, persist=False)
+        cache = Cache().auto_configure(self.CHICKEN_AND_EGG_CONCEPTS_ENTRY)
+        self.om.register_cache(self.CHICKEN_AND_EGG_CONCEPTS_ENTRY, cache, persist=False)
 
     def initialize_services(self):
         """
@@ -261,13 +257,12 @@ class Sheerka(Concept):
         :return:
         """
         # self.init_log.debug("Initializing services (deferred)")
-
         for service in self.services.values():
             if hasattr(service, "initialize_deferred"):
                 service.initialize_deferred(context, is_first_time)
 
     def first_time_initialisation(self, context):
-        self.record_var(context, self.name, "save_execution_context", True)
+        self.record_var(context, self.name, "save_execution_context", self.save_execution_context)
 
     def initialize_builtin_concepts(self):
         """
@@ -339,27 +334,18 @@ class Sheerka(Concept):
     def initialize_concept_node_parsing(self, context):
         # self.init_log.debug("Initializing concepts by first keyword.")
 
-        concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
+        concepts_by_first_keyword = self.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
         res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
-        self.cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
+        self.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
 
-    def reset(self, cache_only=False):
-        if self.cache_manager.cache_only != cache_only:
-            self.cache_manager.reset(cache_only)
-            self.initialize_caching()
-            for service in self.services.values():
-                if hasattr(service, "initialize"):
-                    service.initialize()
-        else:
-            self.cache_manager.clear()
+    def initialize_ontologies(self, context):
+        ontologies = self.om.current_sdp().load_ontologies()
+        if not ontologies:
+            return
 
-        for service in self.services.values():
-            if hasattr(service, "reset"):
-                service.reset()
-
-        self.printer_handler.reset()
-        self.sdp.reset()
-        self.locals = {}
+        for ontology_name in list(reversed(ontologies))[1:]:
+            self.om.push_ontology(ontology_name, False)
+            self.initialize_services_deferred(context, False)
 
     # @profile(filename="profile_80")
     def evaluate_user_input(self, text: str, user_name="kodjo"):
@@ -371,9 +357,9 @@ class Sheerka(Concept):
         :return:
         """
         # self.log.debug(f"Processing user input '{text}', {user_name=}.")
-        my_debug(f"****************** Processing user input '{text}', {user_name=}.***********************************")
+        # my_debug(f"****************** Processing user input '{text}', {user_name=}.***********************************")
         event = Event(text, user_name)
-        self.sdp.save_event(event)
+        self.om.save_event(event)
 
         with ExecutionContext(self.key,
                               event,
@@ -391,8 +377,8 @@ class Sheerka(Concept):
             ret = self.execute(execution_context, [user_input, reduce_requested], EXECUTE_STEPS)
             execution_context.add_values(return_values=ret)
 
-            if self.cache_manager.is_dirty:
-                self.cache_manager.commit(execution_context)
+            if self.om.is_dirty:
+                self.om.commit(execution_context)
 
             self.publish(execution_context, EVENT_USER_INPUT_EVALUATED)
 
@@ -489,13 +475,13 @@ class Sheerka(Concept):
                 return None
 
             if key[1]:
-                concept = self.cache_manager.get(self.CONCEPTS_BY_ID_ENTRY, key[1])
+                concept = self.om.get(self.CONCEPTS_BY_ID_ENTRY, key[1])
             else:
-                concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key[0])
+                concept = self.om.get(self.CONCEPTS_BY_NAME_ENTRY, key[0])
         else:
-            concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key)
+            concept = self.om.get(self.CONCEPTS_BY_NAME_ENTRY, key)
 
-        if concept is None:
+        if concept is NotFound:
             return None
         return new_instances(concept) if return_new else concept
 
@@ -556,6 +542,76 @@ class Sheerka(Concept):
         concept._metadata.is_evaluated = True  # because we have manually set the variables
         return concept
 
+    def push_ontology(self, context, name, cache_only=False):
+
+        try:
+            if self.om.already_on_top(name):
+                return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
+        except OntologyAlreadyExists:
+            return self.ret(self.name, False, self.new(BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED, body=name))
+
+        # record sheerka and services states
+        self.om.record_sheerka_state()
+        for service in self.services.values():
+            if hasattr(service, "save_state"):
+                service.save_state(context)
+            if hasattr(service, "reset_state"):
+                service.reset_state()
+
+        self.om.push_ontology(name, cache_only)
+
+        # Not the first time for this ontology. Update the services
+        if name in self.om.current_sdp().load_ontologies():
+            self.initialize_services_deferred(context, False)
+
+        self.om.save_ontologies()
+
+        return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
+
+    def pop_ontology(self):
+        ontology = self.om.pop_ontology()
+
+        self.om.reset_sheerka_state()
+        for service in self.services.values():
+            if hasattr(service, "restore_state"):
+                service.restore_state()
+            if hasattr(service, "reset_state"):
+                service.reset_state()
+
+        self.om.save_ontologies()
+        return self.ret(self.name, True, self.new(BuiltinConcepts.ONTOLOGY_REMOVED, body=ontology))
+
+    def get_ontology(self, context):
+        self.om.record_sheerka_state()
+        for service in self.services.values():
+            if hasattr(service, "save_state"):
+                service.save_state(context)
+
+        return self.om.get_ontology()
+
+    def add_ontology(self, context, ontology):
+        """
+        Add the previously recorded ontology on the top
+        """
+
+        # save the state of the current ontology
+        self.om.record_sheerka_state()
+        for service in self.services.values():
+            if hasattr(service, "save_state"):
+                service.save_state(context)
+            # if hasattr(service, "reset_state"): # no need to do it twice
+            #     service.reset_state()
+
+        self.om.add_ontology(ontology)
+
+        # update sheerka with this new ontology
+        self.om.reset_sheerka_state()
+        for service in self.services.values():
+            if hasattr(service, "restore_state"):
+                service.restore_state()
+            if hasattr(service, "reset_state"):
+                service.reset_state()
+
     def ret(self, who: str, status: bool, value, parents=None):
         """
         Creates and returns a ReturnValue concept
@@ -665,7 +721,7 @@ class Sheerka(Concept):
         if not isinstance(obj, Concept):
             return True
 
-        return obj.key != str(BuiltinConcepts.UNKNOWN_CONCEPT)
+        return obj.key not in (BuiltinConcepts.UNKNOWN_CONCEPT, BuiltinConcepts.UNKNOWN_RULE)
 
     @staticmethod
     def isinstance(a, b):
@@ -791,40 +847,6 @@ class Sheerka(Concept):
     def test_error(self):
         raise Exception("I can raise an error")
 
-    def test_only_force_sya_def(self, context, list_of_def):
-        """
-        Set the precedence and/or the associativity of a concept
-        FOR TESTS PURPOSE. TO REMOVE EVENTUALLY
-        :param context:
-        :param list_of_def list of tuple(concept_id, precedence (int), SyaAssociativity)
-        :return:
-        """
-
-        # validate the entries
-        # If one entry is an invalid concept, rollback everything
-        for concept_id, precedence, associativity in list_of_def:
-            if concept_id == BuiltinConcepts.UNKNOWN_CONCEPT:
-                return self.ret(self.name,
-                                False,
-                                self.new(BuiltinConcepts.ERROR, body=f"Concept {concept_id} is not known"))
-
-        sya_def = self.cache_manager.copy(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY) or {}
-
-        # update the definitions
-        for concept_id, precedence, associativity in list_of_def:
-            if precedence is None and associativity is None:
-                try:
-                    del self.sya_definitions[concept_id]
-                except KeyError:
-                    pass
-            else:
-                sya_def[concept_id] = (precedence, associativity)
-
-        # put in cache
-        self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, False, sya_def)
-
-        return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
-
     def test_only_add_in_cache(self, concept: Concept):
         """
         Adds a concept template in cache.
@@ -840,7 +862,7 @@ class Sheerka(Concept):
         if concept.key is None:
             raise KeyError()
 
-        self.cache_manager.add_concept(concept)
+        self.om.add_concept(concept)
 
         return concept
 
@@ -848,7 +870,7 @@ class Sheerka(Concept):
 def to_profile():
     sheerka = Sheerka()
     sheerka.initialize(save_execution_context=False, enable_process_return_values=False)
-    event = Event("test", "kodjoko")
+    event = Event("test", "kodjo")
     execution_context = ExecutionContext(sheerka.name,
                                          event,
                                          sheerka,

src/core/sheerka/SheerkaOntologyManager.py

@@ -0,0 +1,470 @@
+from cache.CacheManager import CacheManager
+from cache.DictionaryCache import DictionaryCache
+from core.concept import copy_concepts_attrs, load_concepts_attrs
+from core.global_symbols import NotFound, Removed
+from core.utils import sheerka_deepcopy
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+
+
+class OntologyManagerFrozen(Exception):
+    """
+    Raised when you try to add a cache manager while the ontology manager is frozen
+    """
+    pass
+
+
+class OntologyManagerNotFrozen(Exception):
+    """
+    Raised when you try to push or pop a cache manager while the ontology manager is not frozen
+    """
+    pass
+
+
+class OntologyManagerCannotPopLatest(Exception):
+    """
+    Raised when you try pop the latest cache manager
+    """
+    pass
+
+
+class OntologyAlreadyExists(Exception):
+    """
+    When the ontology exists AND is not the top layer
+    """
+
+    def __init__(self, name):
+        self.name = name
+
+
+class AlternateSdp:
+    def __init__(self, ontologies):
+        self.names = [o.name for o in ontologies]
+        self.cache_managers = [o.cache_manager for o in ontologies]
+
+    def get(self, cache_name, key):
+        last = len(self.cache_managers) - 1
+        for i, cache_manager in enumerate(self.cache_managers):
+            value = cache_manager.get(cache_name, key)
+            if value is not NotFound:
+                return value
+
+            if i != last:
+                # forget than the key was requested
+                cache_manager.remove_initialized_key(cache_name, key)
+
+        return NotFound
+
+    def alt_get(self, cache_name, key):
+        last = len(self.cache_managers) - 1
+        for i, cache_manager in enumerate(self.cache_managers):
+            value = cache_manager.alt_get(cache_name, key)
+            if value is not NotFound:
+                return value
+
+            if i != last:
+                # forget than the key was requested
+                cache_manager.remove_initialized_key(cache_name, key)
+
+        return NotFound
+
+    def exists(self, cache_name, key):
+        for cache_manager in self.cache_managers:
+            if cache_manager.exists(cache_name, key):
+                return True
+        return False
+
+
+class Ontology:
+    def __init__(self, name, cache_manager: CacheManager, alt_sdp: AlternateSdp):
+        self.name = name
+        self.cache_manager = cache_manager
+        self.alt_sdp = alt_sdp
+        self.concepts_attributes = None
+        self.local_variables = None
+
+    def __repr__(self):
+        return f"Ontology('{self.name}')"
+
+
+class SheerkaOntologyManager:
+    ROOT_ONTOLOGY_NAME = "__default__"
+
+    def __init__(self, sheerka, root_folder, cache_only):
+        self.sheerka = sheerka
+        self.root_folder = root_folder
+        self.cache_only = cache_only
+        self.frozen = False
+
+        ref_cache_manager = CacheManager(self.cache_only, sdp=SheerkaDataProvider(root_folder, self.sheerka))
+        self.ontologies = [Ontology(self.ROOT_ONTOLOGY_NAME, ref_cache_manager, None)]
+
+    @property
+    def ontologies_names(self):
+        return [o.name for o in self.ontologies]
+
+    def freeze(self):
+        self.frozen = True
+        return self
+
+    def test_only_unfreeze(self):
+        # To remove ASAP
+        self.frozen = False
+        return self
+
+    def push_ontology(self, name, cache_only=None):
+        """
+        Add an ontology layer
+        :param name: name of the layer
+        :param cache_only:
+        """
+        if not self.frozen:
+            raise OntologyManagerNotFrozen()
+
+        # pseudo clone cache manager
+        cache_manager = CacheManager(cache_only or self.cache_only, sdp=self.get_sdp(name))
+        for cache_name, cache_def in self.current_cache_manager().caches.items():
+            clone = cache_def.cache.clone()
+            if cache_name in self.current_cache_manager().concept_caches:
+                cache_manager.register_concept_cache(cache_name, clone, cache_def.get_key, cache_def.use_ref)
+            else:
+                cache_manager.register_cache(cache_name, clone, cache_def.persist, cache_def.use_ref)
+
+            # Dictionary cache special treatment
+            if isinstance(clone, DictionaryCache):
+                clone.put(False, cache_def.cache.copy())  # only a shadow copy for now
+                clone.reset_events()
+
+        alt_sdp = AlternateSdp(self.ontologies)
+        self.ontologies.insert(0, Ontology(name, cache_manager, alt_sdp))
+        return self
+
+    def pop_ontology(self):
+        """
+        Remove the top ontology layer
+        """
+        if not self.frozen:
+            raise OntologyManagerNotFrozen()
+
+        if len(self.ontologies) == 1:
+            raise OntologyManagerCannotPopLatest()
+
+        return self.ontologies.pop(0)
+
+    def add_ontology(self, ontology: Ontology):
+        """
+        Put back a previously created ontology
+        :param ontology: how to get the items
+        """
+        if not self.frozen:
+            raise OntologyManagerNotFrozen()
+
+        ontology.alt_sdp = AlternateSdp(self.ontologies)
+        self.ontologies.insert(0, ontology)
+        for cache_def in ontology.cache_manager.caches.values():
+            cache_def.cache.reset_initialized_keys()
+
+        return self
+
+    def get_ontology(self, name=None):
+        """
+        Return the first ontology with the corresponding name
+        When no is given, return the top ontology
+        """
+        if name is None:
+            return self.ontologies[0]
+
+        for ontology in self.ontologies:
+            if ontology.name == name:
+                return ontology
+
+        raise KeyError(name)
+
+    def save_ontologies(self):
+        self.current_sdp().save_ontologies(self.ontologies_names)
+
+    # def load_ontologies(self):
+    #     ontologies = self.current_sdp().load_ontologies()
+    #     if not ontologies:
+    #         return
+    #
+    #     for ontology_name in list(reversed(ontologies))[1:]:
+    #         self.push_ontology(ontology_name)
+
+    def already_on_top(self, name):
+        """
+        Returns True if the ontology 'name' is already on the top
+        Raises a OntologyAlreadyExists exception if the ontology exists, but not at the top
+        """
+        if self.ontologies[0].name == name:
+            return True
+
+        if name in self.ontologies_names:
+            raise OntologyAlreadyExists(name)
+
+        return False
+
+    def record_sheerka_state(self):
+        """
+        The current ontology can keep extra information
+        """
+        # TODO persist these information ?
+        self.current_ontology().concepts_attributes = copy_concepts_attrs()
+        self.current_ontology().local_variables = sheerka_deepcopy(self.sheerka.locals)
+
+    def reset_sheerka_state(self):
+        if self.current_ontology().concepts_attributes is not None:
+            load_concepts_attrs(self.current_ontology().concepts_attributes)
+        if self.current_ontology().local_variables is not None:
+            self.sheerka.locals = self.current_ontology().local_variables
+
+    def current_cache_manager(self) -> CacheManager:
+        return self.ontologies[0].cache_manager
+
+    def current_sdp(self) -> SheerkaDataProvider:
+        return self.ontologies[0].cache_manager.sdp
+
+    def current_ontology(self) -> Ontology:
+        return self.ontologies[0]
+
+    def register_concept_cache(self, name, cache, get_key, use_ref):
+        """
+        Define which type of cache along with how to compute the key
+        :param name:
+        :param cache:
+        :param get_key:
+        :param use_ref:
+        :return:
+        """
+        if self.frozen:
+            raise OntologyManagerFrozen
+
+        return self.current_cache_manager().register_concept_cache(name, cache, get_key, use_ref)
+
+    def register_cache(self, name, cache, persist=True, use_ref=False):
+        """
+        Define which type of cache along with how to compute the key
+        :param name:
+        :param cache:
+        :param persist:
+        :param use_ref:
+        :return:
+        """
+        if self.frozen:
+            raise OntologyManagerFrozen
+
+        return self.current_cache_manager().register_cache(name, cache, persist, use_ref)
+
+    def add_concept(self, concept):
+        """
+        We need multiple indexes to retrieve a concept
+        So the new concept is dispatched into multiple caches
+        :param concept:
+        :return:
+        """
+        return self.current_cache_manager().add_concept(concept)
+
+    def update_concept(self, old, new):
+        """
+        Update a concept.
+        :param old:  old version of the concept
+        :param new: new version of the concept
+        :return:
+        """
+
+        return self.current_cache_manager().update_concept(old, new, self.ontologies[0].alt_sdp)
+
+    def remove_concept(self, concept):
+        """
+        Remove a concept from all caches
+        :param concept:
+        :return:
+        """
+        return self.current_cache_manager().remove_concept(concept, self.ontologies[0].alt_sdp)
+
+    def get(self, cache_name, key):
+        """
+        Browses the ontologies, looking for the data 'key' in entry 'cache_name'
+        If a value is found in a low level cache, updates the top level one
+        :param cache_name:
+        :param key:
+        :return:
+        """
+        value = self.current_cache_manager().get(cache_name, key, self.ontologies[0].alt_sdp)
+        return NotFound if value is Removed else value
+
+    def exists(self, cache_name, key):
+        """
+        Browses the ontologies to check if the data 'key' is defined in entry 'cache_name'
+        :param cache_name:
+        :param key:
+        :return:
+        """
+        for ontology in self.ontologies:
+            if ontology.cache_manager.exists(cache_name, key):
+                return True
+
+        return False
+
+    def list(self, entry, cache_only=False):
+        """
+        list all entries
+        """
+        return list(self.get_all(entry, cache_only).values())
+
+    def list_by_key(self, entry, key):
+        """
+        List all entries of a given key
+        If the values are lists, sets of dictionaries, they will be concatenated
+        Otherwise it will raise an error
+        """
+        res = None
+
+        def update_values(_res, values_):
+            if values_ is NotFound:
+                return _res
+            elif values_ is Removed:
+                _res.clear()
+            elif isinstance(values_, dict):
+                if _res is None:
+                    _res = values_.copy()
+                elif isinstance(_res, dict):
+                    _res.update(values_)
+                else:
+                    raise ValueError(f"Expecting dict while found '{values_}'")
+            elif isinstance(values_, list):
+                if _res is None:
+                    _res = values_.copy()
+                elif isinstance(_res, list):
+                    _res.extend(values_)
+                else:
+                    raise ValueError(f"Expecting list while found '{values_}'")
+            else:
+                raise NotImplementedError()
+
+            return _res
+
+        for ontology in reversed(self.ontologies):
+
+            from_cache_values = ontology.cache_manager.get(entry, key)
+            if from_cache_values is not NotFound:
+                res = update_values(res, from_cache_values)
+            else:
+                from_sdp_values = ontology.cache_manager.sdp.get(entry, key)
+                res = update_values(res, from_sdp_values)
+
+        return res
+
+    def get_all(self, entry, cache_only=False):
+        """
+        Return all key, value from all ontologies
+        First look in sdp, then override with the cache, for all ontologies
+        :param entry: cache name / sdp entry
+        :param cache_only: Do no fetch data from remote sdp
+        """
+        res = {}
+        for ontology in reversed(self.ontologies):
+
+            if not cache_only:
+                # get values from sdp
+                values = ontology.cache_manager.sdp.get(entry)
+                if values is Removed:
+                    res.clear()
+
+                elif values is not NotFound:
+                    for k, v in values.items():
+                        if v is Removed:
+                            del res[k]
+                        else:
+                            res[k] = v
+
+            # override with the values from cache
+            try:
+                cache = ontology.cache_manager.get_cache(entry)
+
+                if cache.is_cleared():
+                    res.clear()
+
+                for k in cache:
+                    v = cache.alt_get(k)  # Do not use get(), because of IncCache()
+                    if v is Removed:
+                        del res[k]
+                    else:
+                        res[k] = v
+
+            except KeyError:
+                pass
+
+        return res
+
+    def put(self, cache_name, key, value):
+        """
+        Add to a cache
+        :param cache_name:
+        :param key:
+        :param value:
+        :return:
+        """
+        return self.current_cache_manager().put(cache_name, key, value, self.ontologies[0].alt_sdp)
+
+    def delete(self, cache_name, key, value=None):
+        """
+        Delete an entry
+        :param cache_name:
+        :param key:
+        :param value:
+        :return:
+        """
+        return self.current_cache_manager().delete(cache_name, key, value, self.ontologies[0].alt_sdp)
+
+    def populate(self, cache_name, populate_function, get_key_function, reset_events=False, all_ontologies=False):
+        """
+        Populate a specific cache with a bunch of items
+        :param cache_name:
+        :param populate_function: how to get the items
+        :param get_key_function: how to get the key, out of an item
+        :param reset_events: reset the to_add and to_remove events after the populate
+        :param all_ontologies: populate all ontology layers
+        :return:
+        """
+        self.current_cache_manager().populate(cache_name, populate_function, get_key_function, reset_events)
+        if all_ontologies:
+            for ontology in self.ontologies[1:]:
+                ontology.cache_manager.populate(cache_name, populate_function, get_key_function, reset_events)
+
+    def copy(self, cache_name):
+        """
+        get a copy the content of the top ontology layer
+        :param self:
+        :param cache_name:
+        :return:
+        """
+        return self.current_cache_manager().caches[cache_name].cache.copy()
+
+    def commit(self, context):
+        """
+        Persist all the caches into a physical persistence storage
+        :param context:
+        :return:
+        """
+        return self.current_cache_manager().commit(context)
+
+    def clear(self, cache_name=None):
+        return self.current_cache_manager().clear(cache_name)
+
+    def get_sdp(self, name=None):
+        """
+        Return new instance of SheerkaDataProvider
+        """
+        if name:
+            return SheerkaDataProvider(self.root_folder, self.sheerka, name)
+        else:
+            return self.current_sdp()
+
+    def save_event(self, event):
+        return self.current_sdp().save_event(event)
+
+    def save_result(self, execution_context, is_admin):
+        return self.current_sdp().save_result(execution_context, is_admin)
+
+    def is_dirty(self):
+        return self.current_cache_manager().is_dirty

src/core/sheerka/services/SheerkaAdmin.py

@@ -1,5 +1,6 @@
 import sys
 import time
+from operator import attrgetter
 from os import path
 
 from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
@@ -27,13 +28,16 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.extended_isinstance, False)
         self.sheerka.bind_service_method(self.is_container, False)
         self.sheerka.bind_service_method(self.format_rules, False)
+        self.sheerka.bind_service_method(self.admin_push_ontology, True, as_name="push_ontology")
+        self.sheerka.bind_service_method(self.admin_pop_ontology, True, as_name="pop_ontology")
+        self.sheerka.bind_service_method(self.ontologies, False)
 
     def caches_names(self):
         """
         Returns the name of all the caches
         :return:
         """
-        return list(self.sheerka.cache_manager.caches.keys())
+        return list(self.sheerka.om.current_cache_manager().caches.keys())
 
     def cache(self, name, *keys):
         """
@@ -42,13 +46,13 @@ class SheerkaAdmin(BaseService):
         :param keys: look for a specific key. May ask to sdp if the key is not in cache
         :return:
         """
-        if name not in self.sheerka.cache_manager.caches:
+        if name not in self.sheerka.om.current_cache_manager().caches:
             return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"cache": name})
 
         if not keys:
-            return self.sheerka.cache_manager.caches[name].cache.copy()
+            return self.sheerka.om.current_cache_manager().caches[name].cache.copy()
 
-        return {key: self.sheerka.cache_manager.get(name, key) for key in keys}
+        return {key: self.sheerka.om.get(name, key) for key in keys}
 
     def restore(self, concept_file=CONCEPTS_FILE_TO_USE):
         """
@@ -119,7 +123,8 @@ class SheerkaAdmin(BaseService):
             raise e
 
     def concepts(self):
-        return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY))
+        concepts = sorted(self.sheerka.om.list(self.sheerka.CONCEPTS_BY_ID_ENTRY), key=lambda item: int(item.id))
+        return self.sheerka.new(BuiltinConcepts.TO_LIST, body=concepts)
 
     def desc(self, *concepts):
         ensure_concept(*concepts)
@@ -146,7 +151,6 @@ class SheerkaAdmin(BaseService):
     def format_rules(self):
         return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_format_rules())
 
-
     def extended_isinstance(self, a, b):
         """
         switch between sheerka.isinstance and builtin.isinstance
@@ -171,3 +175,13 @@ class SheerkaAdmin(BaseService):
             return False
 
         return obj.key in BuiltinContainers
+
+    def admin_push_ontology(self, context, name):
+        return self.sheerka.push_ontology(context, name, False)
+
+    def admin_pop_ontology(self):
+        return self.sheerka.pop_ontology()
+
+    def ontologies(self):
+        ontologies = self.sheerka.om.ontologies_names
+        return self.sheerka.new(BuiltinConcepts.TO_LIST, body=ontologies)

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -3,8 +3,9 @@ from dataclasses import dataclass
 from cache.Cache import Cache
 from cache.ListCache import ListCache
 from core.builtin_concepts import BuiltinConcepts
-from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, \
-    CONCEPT_COMPARISON_CONTEXT
+from core.global_symbols import EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_PRECEDENCE_MODIFIED, \
+    RULE_COMPARISON_CONTEXT, \
+    CONCEPT_COMPARISON_CONTEXT, NotFound
 from core.builtin_helpers import ensure_concept_or_rule
 from core.concept import Concept
 from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
@@ -136,8 +137,8 @@ class SheerkaComparisonManager(BaseService):
 
     def _add_comparison(self, context, comparison_obj):
         key = self._compute_key(comparison_obj.property, comparison_obj.context)
-        previous = self.sheerka.cache_manager.get(self.COMPARISON_ENTRY, key)
-        new = previous.copy() if previous else []
+        previous = self.sheerka.om.get(self.COMPARISON_ENTRY, key)
+        new = previous.copy() if isinstance(previous, list) else []
 
         for co in new:
             if co.property == comparison_obj.property and \
@@ -176,10 +177,10 @@ class SheerkaComparisonManager(BaseService):
             chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle)
             return self.sheerka.ret(self.NAME, False, chicken_an_egg)
 
-        self.sheerka.cache_manager.put(self.COMPARISON_ENTRY, key, comparison_obj)
-        self.sheerka.cache_manager.put(self.RESOLVED_COMPARISON_ENTRY, key, self._compute_weights(new,
-                                                                                                  lesser_objs_ids,
-                                                                                                  greatest_objs_ids))
+        self.sheerka.om.put(self.COMPARISON_ENTRY, key, comparison_obj)
+        self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key, self._compute_weights(new,
+                                                                                       lesser_objs_ids,
+                                                                                       greatest_objs_ids))
 
         if comparison_obj.property == BuiltinConcepts.PRECEDENCE:
             if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT:
@@ -190,11 +191,11 @@ class SheerkaComparisonManager(BaseService):
         return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
     def initialize(self):
-        cache = ListCache(default=lambda k: self.sheerka.sdp.get(self.COMPARISON_ENTRY, k))
-        self.sheerka.cache_manager.register_cache(self.COMPARISON_ENTRY, cache, True, True)
+        cache = ListCache().auto_configure(self.COMPARISON_ENTRY)
+        self.sheerka.om.register_cache(self.COMPARISON_ENTRY, cache, True, True)
 
-        cache = Cache()
-        self.sheerka.cache_manager.register_cache(self.RESOLVED_COMPARISON_ENTRY, cache, persist=False)
+        cache = Cache().auto_configure(self.RESOLVED_COMPARISON_ENTRY)
+        self.sheerka.om.register_cache(self.RESOLVED_COMPARISON_ENTRY, cache, persist=False)
 
         self.sheerka.bind_service_method(self.set_is_greater_than, True)
         self.sheerka.bind_service_method(self.set_is_less_than, True)
@@ -325,19 +326,26 @@ class SheerkaComparisonManager(BaseService):
         return self._get_partition(weighted_concept)
 
     def get_concepts_weights(self, prop_name, comparison_context="#"):
-        weighted_concepts = self.sheerka.cache_manager.get(
-            self.RESOLVED_COMPARISON_ENTRY,
-            self._compute_key(prop_name, comparison_context))
+        # KSI 2021-01-10 This implementation seems to be too complicated
+        # Chances are that there is a better way to implement this.
+        # Note that I don't want to use a DictionaryCache for the RESOLVED_COMPARISON_ENTRY
+        # as I don't need to have all the keys in memory at the same time
+        # Anyway...
 
-        if weighted_concepts is None:
-            key = self._compute_key(prop_name, comparison_context)
-            entries = self.sheerka.cache_manager.get(self.COMPARISON_ENTRY, key)
+        # If the weighted_concepts is in the TOP LAYER cache, we can use it
+        key_to_use = self._compute_key(prop_name, comparison_context)
+        if self.sheerka.om.current_cache_manager().has(self.RESOLVED_COMPARISON_ENTRY, key_to_use):
+            weighted_concepts = self.sheerka.om.get(self.RESOLVED_COMPARISON_ENTRY, key_to_use)
 
+        else:
+            # otherwise, either it's not computed yet or it does not include the info of the current layer
+            # In both case, it is safer to recompute the weights
+            entries = self.sheerka.om.list_by_key(self.COMPARISON_ENTRY, key_to_use)
             if entries is None:
-                return {}
+                weighted_concepts = {}  # Why not put it in cache ???
             else:
                 weighted_concepts = self._compute_weights(entries)
-                self.sheerka.cache_manager.put(self.RESOLVED_COMPARISON_ENTRY, key, weighted_concepts)
+                self.sheerka.om.put(self.RESOLVED_COMPARISON_ENTRY, key_to_use, weighted_concepts)
 
         return weighted_concepts
 

src/core/sheerka/services/SheerkaConceptManager.py

@@ -8,10 +8,9 @@ from cache.SetCache import SetCache
 from core.builtin_concepts import ErrorConcept
 from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, BuiltinUnique
 from core.builtin_helpers import ensure_concept
-from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, NotInit, \
-    ConceptMetadata
+from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata
 from core.error import ErrorObj
-from core.global_symbols import EVENT_CONCEPT_CREATED
+from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer, TokenKind
 from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
@@ -101,34 +100,28 @@ class SheerkaConceptManager(BaseService):
         self.sheerka.bind_service_method(self.get_by_name, False, visible=False)
         self.sheerka.bind_service_method(self.get_by_hash, False, visible=False)
         self.sheerka.bind_service_method(self.get_by_id, False, visible=False)
-        self.sheerka.bind_service_method(self.not_is_variable, False, visible=False)
+        self.sheerka.bind_service_method(self.is_not_a_variable, False, visible=False)
 
-        def params(cache_name):
-            return {
-                'default': lambda k: self.sheerka.sdp.get(cache_name, k),
-                'extend_exists': lambda k: self.sheerka.sdp.exists(cache_name, k)
-            }
+        register_concept_cache = self.sheerka.om.register_concept_cache
 
-        register_concept_cache = self.sheerka.cache_manager.register_concept_cache
-
-        cache = Cache(**params(self.CONCEPTS_BY_ID_ENTRY))
+        cache = Cache().auto_configure(self.CONCEPTS_BY_ID_ENTRY)
         register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
 
-        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_KEY_ENTRY))
+        cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_KEY_ENTRY)
         register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
 
-        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_NAME_ENTRY))
+        cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_NAME_ENTRY)
         register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
 
-        cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
+        cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_HASH_ENTRY)
         register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
 
-        cache = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_REFERENCES_ENTRY, k))
-        self.sheerka.cache_manager.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache)
+        cache = SetCache().auto_configure(self.CONCEPTS_REFERENCES_ENTRY)
+        self.sheerka.om.register_cache(self.CONCEPTS_REFERENCES_ENTRY, cache)
 
     def initialize_deferred(self, context, is_first_time):
         if is_first_time:
-            self.sheerka.cache_manager.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
+            self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
 
     def initialize_builtin_concepts(self):
         """
@@ -146,11 +139,11 @@ class SheerkaConceptManager(BaseService):
                 concept.get_metadata().is_unique = True
                 concept.get_metadata().is_evaluated = True
 
-            from_db = self.sheerka.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key)
-            if from_db is None:
+            from_db = self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key)
+            if from_db is NotFound:
                 # self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
                 self.set_id_if_needed(concept, True)
-                self.sheerka.cache_manager.add_concept(concept)
+                self.sheerka.om.add_concept(concept)
             else:
                 # self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
                 concept.update_from(from_db)
@@ -173,9 +166,9 @@ class SheerkaConceptManager(BaseService):
         concept.init_key()
         init_bnf_ret_value = None
 
-        cache_manager = sheerka.cache_manager
+        ontology = sheerka.om
 
-        if cache_manager.exists(self.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
+        if ontology.exists(self.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
             error = SheerkaDataProviderDuplicateKeyError(self.CONCEPTS_BY_KEY_ENTRY + "." + concept.key, concept)
             return sheerka.ret(
                 self.NAME,
@@ -186,9 +179,6 @@ class SheerkaConceptManager(BaseService):
         # set id before saving in db
         sheerka.set_id_if_needed(concept, False)
 
-        # freeze attributes
-        freeze_concept_attrs(concept)
-
         # check if the bnf definition is correctly computed
         try:
             self.bnp.ensure_bnf(context, concept)
@@ -196,7 +186,7 @@ class SheerkaConceptManager(BaseService):
             return sheerka.ret(self.NAME, False, ex.args[0])
 
         # compute new concepts_by_first_keyword
-        init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True)
+        init_ret_value = self.bnp.compute_concepts_by_first_token(context, [concept], True)
         if not init_ret_value.status:
             return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
         concepts_by_first_keyword = init_ret_value.body
@@ -208,18 +198,20 @@ class SheerkaConceptManager(BaseService):
         resolved_concepts_by_first_keyword = init_ret_value.body
 
         # if everything is fine
+        freeze_concept_attrs(concept)
         concept.freeze_definition_hash()
-        cache_manager.add_concept(concept)
-        cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
-        cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
+
+        ontology.add_concept(concept)
+        ontology.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
+        ontology.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
 
         if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
             # allow search by definition when definition relevant
-            cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept)
+            ontology.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept)
 
         # update references
         for ref in self.compute_references(concept):
-            cache_manager.put(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
+            ontology.put(self.CONCEPTS_REFERENCES_ENTRY, ref, concept.id)
 
         # TODO : this line seems to be useless
         # The grammar is never reset
@@ -246,7 +238,7 @@ class SheerkaConceptManager(BaseService):
 
         # to_add is a dictionary
         # to_add = {
-        #           'meta'     : {<key, value>} of metadata to add/update,
+        #           'meta'     : {<key, value>} of metadata to update,
         #           'props'    : {<key, value>} of properties to add/update,
         #           'variables': {<key, value>} of variables to add/update,
         #           }
@@ -259,12 +251,12 @@ class SheerkaConceptManager(BaseService):
         #              }
         #
         sheerka = self.sheerka
-        cache_manager = self.sheerka.cache_manager
+        cache_manager = self.sheerka.om
 
         if not to_add and not to_remove:
             return sheerka.ret(self.NAME, False, sheerka.err(NoModificationFound(concept)))
 
-        if not sheerka.cache_manager.exists(self.CONCEPTS_BY_ID_ENTRY, concept.id):
+        if not sheerka.om.exists(self.CONCEPTS_BY_ID_ENTRY, concept.id):
             return sheerka.ret(self.NAME, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept))
 
         # modify the metadata. Almost all ConceptMetadata attributes except variables and props
@@ -274,8 +266,6 @@ class SheerkaConceptManager(BaseService):
         if res is not None:
             return res
 
-        freeze_concept_attrs(new_concept)
-
         # To update concept by first keyword
         # first remove the old references
         keywords = self.bnp.get_first_tokens(sheerka, concept)  # keyword of the old concept
@@ -289,7 +279,7 @@ class SheerkaConceptManager(BaseService):
                 pass
 
         # and then update
-        init_ret_value = self.bnp.get_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
+        init_ret_value = self.bnp.compute_concepts_by_first_token(context, [new_concept], False, concepts_by_first_keyword)
         if not init_ret_value.status:
             return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
         concepts_by_first_keyword = init_ret_value.body
@@ -316,6 +306,10 @@ class SheerkaConceptManager(BaseService):
         cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False,
                           resolved_concepts_by_first_keyword)
 
+        # everything seems to be fine. Update the list of attributes
+        # Caution. Must be done AFTER update_concept()
+        freeze_concept_attrs(new_concept)
+
         # TODO : update when definition_type = DEFINITION_TYPE_DEF : have a look at update_references() below
         # TODO : Update concepts grammars : have a look at update_references() below
         if modify_source:
@@ -332,13 +326,13 @@ class SheerkaConceptManager(BaseService):
         :return:
         """
         sheerka = context.sheerka
-        refs = self.sheerka.cache_manager.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
-        if refs:
+        refs = self.sheerka.om.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
+        if refs is not NotFound:
             refs_instances = [sheerka.new_from_template(c, c.key) for c in [self.get_by_id(ref) for ref in refs]]
             return sheerka.ret(self.NAME, False, sheerka.err(ConceptIsReferenced(refs_instances)))
 
         try:
-            sheerka.cache_manager.remove_concept(concept)
+            sheerka.om.remove_concept(concept)
             return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS))
         except ConceptNotFound as ex:
             return sheerka.ret(self.NAME, False, sheerka.err(ex))
@@ -387,7 +381,7 @@ class SheerkaConceptManager(BaseService):
             return
 
         entry_key = self.BUILTIN_CONCEPTS_IDS if is_builtin else self.USER_CONCEPTS_IDS
-        obj.get_metadata().id = str(self.sheerka.cache_manager.get(self.sheerka.OBJECTS_IDS_ENTRY, entry_key))
+        obj.get_metadata().id = str(self.sheerka.om.get(self.sheerka.OBJECTS_IDS_ENTRY, entry_key))
         # self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
 
     def get_by_key(self, concept_key, concept_id=None):
@@ -412,7 +406,7 @@ class SheerkaConceptManager(BaseService):
         """
         if concept_id is None:
             return False
-        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
+        return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
 
     def has_key(self, concept_key):
         """
@@ -421,7 +415,7 @@ class SheerkaConceptManager(BaseService):
         :param concept_key:
         :return:
         """
-        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
+        return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_KEY_ENTRY, concept_key)
 
     def has_name(self, concept_name):
         """
@@ -430,7 +424,7 @@ class SheerkaConceptManager(BaseService):
         :param concept_name:
         :return:
         """
-        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
+        return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_NAME_ENTRY, concept_name)
 
     def has_hash(self, concept_hash):
         """
@@ -439,7 +433,7 @@ class SheerkaConceptManager(BaseService):
         :param concept_hash:
         :return:
         """
-        return self.sheerka.cache_manager.has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
+        return self.sheerka.om.current_cache_manager().has(self.CONCEPTS_BY_HASH_ENTRY, concept_hash)
 
     def internal_get(self, index_name, key, cache_name, concept_id=None):
         """
@@ -454,8 +448,8 @@ class SheerkaConceptManager(BaseService):
         if key is None:
             return ErrorConcept(f"Concept '{key}' is undefined.")
 
-        concepts = self.sheerka.cache_manager.get(cache_name, key)
-        if concepts:
+        concepts = self.sheerka.om.get(cache_name, key)
+        if concepts is not NotFound:
             if concept_id is None:
                 return concepts
 
@@ -479,13 +473,13 @@ class SheerkaConceptManager(BaseService):
         :return:
         """
 
-        refs = self.sheerka.cache_manager.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
-        if not refs:
+        refs = self.sheerka.om.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
+        if refs is NotFound:
             return
 
         for concept_id in refs:
             # remove the grammar entry so that it can be recreated
-            self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
+            self.sheerka.om.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
 
             # reset the bnf definition if needed
             if modified_concept:
@@ -527,13 +521,13 @@ class SheerkaConceptManager(BaseService):
 
         return refs
 
-    def not_is_variable(self, name):
+    def is_not_a_variable(self, name):
         """
         Given a name tells if it refers to a variable name
         :param name:
         :return:
         """
-        return not self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name)
+        return self.sheerka.om.get(self.sheerka.CONCEPTS_BY_NAME_ENTRY, name) is NotFound
 
     @staticmethod
     def _name_has_changed(to_add):

src/core/sheerka/services/SheerkaConceptsAlgebra.py

@@ -119,10 +119,9 @@ class SheerkaConceptsAlgebra(BaseService):
         if nb_props == 0:
             return res
 
-        concepts_service = self.sheerka.services[SheerkaConceptManager.NAME]
+        concepts_manager = self.sheerka.services[SheerkaConceptManager.NAME]
 
-        all_concepts = self.sheerka.cache_manager.copy(concepts_service.CONCEPTS_BY_ID_ENTRY).values() \
-            if self.sheerka.cache_manager.cache_only else self.sheerka.sdp.list(concepts_service.CONCEPTS_BY_ID_ENTRY)
+        all_concepts = self.sheerka.om.list(concepts_manager.CONCEPTS_BY_ID_ENTRY)
 
         for c in all_concepts:
             score = self._compute_score(c, concept, step_b=round(1 / nb_props, 2))

src/core/sheerka/services/SheerkaDebugManager.py

@@ -3,17 +3,17 @@ import re
 from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, NotInit
+from core.builtin_helpers import evaluate_expression
+from core.concept import Concept
+from core.global_symbols import NotInit, NotFound
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.services.sheerka_service import BaseService
 from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS, PRIMITIVES_TYPES
-from core.utils import evaluate_expression, as_bag
+from core.utils import as_bag
 from parsers.BaseNodeParser import SourceCodeWithConceptNode, UnrecognizedTokensNode
 
 pp = pprint.PrettyPrinter(indent=2, width=CONSOLE_COLUMNS)
 
-NotFound = "** Not Found **"
-
 
 class ConceptDebugObj:
     def __init__(self, concept, **kwargs):
@@ -278,12 +278,21 @@ class SheerkaDebugManager(BaseService):
     def __init__(self, sheerka):
         super().__init__(sheerka)
         self.activated = False  # is debug activated
-        self.explicit = False  # No need to activate context debug when debug mode is on
-        self.context_cache = set()  # debug for specific context
-        self.variable_cache = set()  # debug for specific variable
+        self.explicit = False  # No need to activate context debug when debug mode is on # to remove ?
+        self.context_cache = set()  # debug for specific context # to remove ?
+        self.variable_cache = set()  # debug for specific variable # to remove ?
         self.debug_vars_settings = []
         self.debug_rules_settings = []
         self.debug_concepts_settings = []
+        self.state_vars = [
+            "activated",
+            "explicit",  # to remove ?
+            "context_cache",  # to remove ?
+            "variable_cache",  # to remove ?
+            "debug_vars_settings",
+            "debug_rules_settings",
+            "debug_concepts_settings"
+        ]
 
     def initialize(self):
         # TO REMOVE ???
@@ -307,13 +316,7 @@ class SheerkaDebugManager(BaseService):
         # self.sheerka.bind_service_method(self.get_debug_settings, False, as_name="debug_settings")
 
     def initialize_deferred(self, context, is_first_time):
-        self.restore_values("activated",
-                            "explicit",
-                            "context_cache",
-                            "variable_cache",
-                            "debug_vars_settings",
-                            "debug_rules_settings",
-                            "debug_concepts_settings")
+        self.restore_state()
 
     def reset(self):
         """
@@ -327,6 +330,12 @@ class SheerkaDebugManager(BaseService):
         self.debug_rules_settings.clear()
         self.debug_concepts_settings.clear()
 
+    def save_state(self, context):
+        self.store_values(context, *self.state_vars)
+
+    def restore_state(self):
+        self.restore_values(*self.state_vars)
+
     def set_debug(self, context, value=True):
         self.activated = value
         self.sheerka.record_var(context, self.NAME, "activated", self.activated)
@@ -728,6 +737,16 @@ class SheerkaDebugManager(BaseService):
 
     @staticmethod
     def parse_debug_args(item_name, *args, **kwargs):
+        """
+        Returns
+        i : item to debug. It can be a Concept, Rule or a variable
+        s : Service to debug (so far, it is SheerkaService)
+        m : Method within the serice
+        c_id : Context id
+        c_children : True / False to allow debugging of context children
+        d : Debug id
+        e : enable / disabled
+        """
         service, method_name, context_id, context_children, item, debug_id, enabled = None, None, None, False, None, None, True
         if len(args) > 0:
             if args[0] is None or args[0] == "":

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -2,8 +2,9 @@ from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate, ensure_concept
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, AllConceptParts, \
+from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \
     concept_part_value
+from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService
@@ -605,7 +606,7 @@ class SheerkaEvaluateConcept(BaseService):
             # # update the cache for concepts with no variables
             # Cannot use cache. See the comment at the beginning of this method
             # if len(concept.get_metadata().variables) == 0:
-            #     self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
+            #     self.sheerka.om.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
 
             if not concept.get_metadata().is_builtin:
                 self.sheerka.register_object(sub_context, concept.name, concept)

src/core/sheerka/services/SheerkaEventManager.py

@@ -1,5 +1,5 @@
 from threading import RLock
-
+from core.global_symbols import NotFound
 from core.sheerka.services.sheerka_service import BaseService
 
 
@@ -19,6 +19,16 @@ class SheerkaEventManager(BaseService):
         self.sheerka.bind_service_method(self.subscribe, True, visible=False)
         self.sheerka.bind_service_method(self.publish, True, visible=False)
 
+    def save_state(self, context):
+        with self._lock:
+            copy = self.subscribers.copy()
+            self.sheerka.record_internal_var(context, self.NAME, "subscribers", copy)
+
+    def restore_state(self):
+        with self._lock:
+            if (from_cache := self.sheerka.load_internal_var(self.NAME, "subscribers")) is not NotFound:
+                self.subscribers = from_cache
+
     def subscribe(self, topic, callback):
         """
         To subscribe to a topic, just give the callback to call
@@ -51,9 +61,10 @@ class SheerkaEventManager(BaseService):
             except KeyError:
                 pass
 
-    def reset_topic(self, topic):
+    def test_only_reset_topic(self, topic):
         """
-        Remove all subsccribers from a given topic
+        Remove all subscribers from a given topic
+        TO REMOVE once sheerka ontology is fully implemented
         :param topic:
         :return:
         """

src/core/sheerka/services/SheerkaExecute.py

@@ -1,6 +1,8 @@
 import core.utils
 from cache.Cache import Cache
+from cache.FastCache import FastCache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
+from core.global_symbols import NotFound
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer, TokenKind, Token
 
@@ -166,7 +168,7 @@ class SheerkaExecute(BaseService):
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
-        self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
+        self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=20)
         self.instantiated_evaluators = None
         self.evaluators_by_name = None
 
@@ -192,10 +194,12 @@ class SheerkaExecute(BaseService):
     def initialize(self):
         self.sheerka.bind_service_method(self.execute, True)
 
-        self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False)
         self.reset_registered_evaluators()
         self.reset_registered_parsers()
 
+    def reset_state(self):
+        self.pi_cache.clear()
+
     def reset_registered_evaluators(self):
         # instantiate evaluators, once for all, only keep when it's enabled
         self.instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators]
@@ -340,7 +344,7 @@ class SheerkaExecute(BaseService):
 
         if tokens is None or self.pi_cache.has(text):
             pi = self.pi_cache.get(text)
-            if pi is None:  # when CacheManager.cache_only is True
+            if pi is NotFound:  # when CacheManager.cache_only is True
                 pi = ParserInput(text)
                 self.pi_cache.put(text, pi)
             return pi

src/core/sheerka/services/SheerkaFunctionsParametersHistory.py

@@ -3,6 +3,7 @@ from operator import itemgetter
 from typing import Tuple, Dict, List
 
 from cache.Cache import Cache
+from core.global_symbols import NotFound
 from core.sheerka.services.sheerka_service import BaseService, ServiceObj
 
 
@@ -29,10 +30,10 @@ class SheerkaFunctionsParametersHistory(BaseService):
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
-        self.cache = Cache(max_size=1024, default=lambda k: self.sheerka.sdp.get(self.FUNCTIONS_PARAMETERS_ENTRY, k))
 
     def initialize(self):
-        self.sheerka.cache_manager.register_cache(self.FUNCTIONS_PARAMETERS_ENTRY, self.cache, True, True)
+        cache = Cache(max_size=1024).auto_configure(self.FUNCTIONS_PARAMETERS_ENTRY)
+        self.sheerka.om.register_cache(self.FUNCTIONS_PARAMETERS_ENTRY, cache, True, True)
         return self
 
     def record_function_parameter(self, context, func_name: str, param_number: int, param_value: str):
@@ -44,8 +45,11 @@ class SheerkaFunctionsParametersHistory(BaseService):
         :param param_value:
         :return:
         """
-        old = self.cache.get(func_name)
-        if old is not None:
+        old = self.sheerka.om.get(self.FUNCTIONS_PARAMETERS_ENTRY, func_name)
+        if old is NotFound:
+            obj = FunctionParametersObj(context.event.get_digest(), func_name, {param_number: [(param_value, 1)]})
+            self.sheerka.om.put(self.FUNCTIONS_PARAMETERS_ENTRY, func_name, obj)
+        else:
             if param_number in old.params:
                 lst = old.params[param_number]
                 for i, value in enumerate(lst):  # value is a tuple (param_value, counter)
@@ -56,10 +60,7 @@ class SheerkaFunctionsParametersHistory(BaseService):
                     lst.append((param_value, 1))
             else:
                 old.params[param_number] = [(param_value, 1)]
-            self.cache.put(func_name, old)
-        else:
-            obj = FunctionParametersObj(context.event.get_digest(), func_name, {param_number: [(param_value, 1)]})
-            self.cache.put(func_name, obj)
+            self.sheerka.om.put(self.FUNCTIONS_PARAMETERS_ENTRY, func_name, old)
 
     def get_function_parameters(self, func_name: str, param_number: int):
         """
@@ -68,8 +69,8 @@ class SheerkaFunctionsParametersHistory(BaseService):
         :param param_number:
         :return:
         """
-        values = self.cache.get(func_name)
-        if values is None:
+        values = self.sheerka.om.get(self.FUNCTIONS_PARAMETERS_ENTRY, func_name)
+        if values is NotFound:
             return []
 
         if param_number not in values.params:

src/core/sheerka/services/SheerkaHistoryManager.py

@@ -67,10 +67,10 @@ class SheerkaHistoryManager(BaseService):
         :return:
         """
 
-        events = list(self.sheerka.sdp.load_events(depth, start))
+        events = list(self.sheerka.om.current_sdp().load_events(depth, start))
         for event in events:
             try:
-                result = self.sheerka.sdp.load_result(event.get_digest())
+                result = self.sheerka.om.current_sdp().load_result(event.get_digest())
             except (IOError, KeyError):
                 result = None
             yield History(event, result)

src/core/sheerka/services/SheerkaIsAManager.py

@@ -4,21 +4,18 @@ from cache.SetCache import SetCache
 from core.ast_helpers import UnreferencedVariablesVisitor
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF
+from core.global_symbols import NotFound
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.sheerka_service import BaseService
 
-GROUP_PREFIX = 'All_'
 
-
-class SheerkaSetsManager(BaseService):
-    NAME = "SetsManager"
-    CONCEPTS_GROUPS_ENTRY = "SetsManager:Concepts_Groups"
-    CONCEPTS_IN_GROUPS_ENTRY = "SetsManager:Concepts_In_Groups"  # cache for get_set_elements()
+class SheerkaIsAManager(BaseService):
+    NAME = "IsAManager"
+    CONCEPTS_GROUPS_ENTRY = "IsAManager:Concepts_Groups"
+    CONCEPTS_IN_GROUPS_ENTRY = "IsAManager:Concepts_In_Groups"  # cache for get_set_elements()
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
-        self.sets = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k))
-        self.concepts_in_set = Cache()
 
     def initialize(self):
         self.sheerka.bind_service_method(self.set_isa, True)
@@ -28,8 +25,10 @@ class SheerkaSetsManager(BaseService):
         self.sheerka.bind_service_method(self.isa, False)
         self.sheerka.bind_service_method(self.isaset, True)  # concept is evaluated, need to change the code
 
-        self.sheerka.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, self.sets)
-        self.sheerka.cache_manager.register_cache(self.CONCEPTS_IN_GROUPS_ENTRY, self.concepts_in_set, persist=False)
+        cache = SetCache().auto_configure(self.CONCEPTS_GROUPS_ENTRY)
+        self.sheerka.om.register_cache(self.CONCEPTS_GROUPS_ENTRY, cache)
+        cache = Cache().auto_configure(self.CONCEPTS_IN_GROUPS_ENTRY)
+        self.sheerka.om.register_cache(self.CONCEPTS_IN_GROUPS_ENTRY, cache, persist=False)
 
     def set_isa(self, context, concept, concept_set):
         """
@@ -43,8 +42,8 @@ class SheerkaSetsManager(BaseService):
         context.log(f"Setting concept {concept} is a {concept_set}", who=self.NAME)
         core.builtin_helpers.ensure_concept(concept, concept_set)
 
-        if BuiltinConcepts.ISA in concept.get_metadata().props and concept_set in concept.get_metadata().props[
-            BuiltinConcepts.ISA]:
+        if BuiltinConcepts.ISA in concept.get_metadata().props and \
+            concept_set in concept.get_metadata().props[BuiltinConcepts.ISA]:
             return self.sheerka.ret(
                 self.NAME,
                 False,
@@ -75,23 +74,23 @@ class SheerkaSetsManager(BaseService):
         context.log(f"Adding concept {concept} to set {concept_set}", who=self.NAME)
         core.builtin_helpers.ensure_concept(concept, concept_set)
 
-        set_elements = self.sets.get(concept_set.id)
-        if set_elements and concept.id in set_elements:
+        set_elements = self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, concept_set.id)
+        if set_elements is not NotFound and concept.id in set_elements:
             return self.sheerka.ret(
                 self.NAME,
                 False,
                 self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
 
-        self.sets.put(concept_set.id, concept.id)
+        self.sheerka.om.put(self.CONCEPTS_GROUPS_ENTRY, concept_set.id, concept.id)
 
         # invalidate the cache of what contains concept_set
-        self.concepts_in_set.delete(concept_set.id)
+        self.sheerka.om.delete(self.CONCEPTS_IN_GROUPS_ENTRY, concept_set.id)
 
         # update concept_set references
         self.sheerka.services[SheerkaConceptManager.NAME].update_references(context, concept_set)
 
         # remove the grammar entry so that it can be recreated
-        self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_set.id)
+        self.sheerka.om.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_set.id)
 
         return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
@@ -116,7 +115,7 @@ class SheerkaSetsManager(BaseService):
                                     concept_set=concept_set)
         else:
             body = self.sheerka.new(BuiltinConcepts.SUCCESS)
-            self.concepts_in_set.delete(concept_set.id)
+            self.sheerka.om.delete(self.CONCEPTS_IN_GROUPS_ENTRY, concept_set.id)
 
         return self.sheerka.ret(self.NAME, len(already_in_set) != len(concepts), body)
 
@@ -136,7 +135,7 @@ class SheerkaSetsManager(BaseService):
                 return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
 
             # first, try to see if sub_concept has it's own group entry
-            ids = self.sets.get(sub_concept.id)
+            ids = self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, sub_concept.id)
             concepts = self._get_concepts(context, ids, True)
 
             # aggregate with en entries from its body
@@ -166,13 +165,13 @@ class SheerkaSetsManager(BaseService):
             return concepts
 
         # already in cache ?
-        if res := self.concepts_in_set.get(concept.id):
+        if (res := self.sheerka.om.get(self.CONCEPTS_IN_GROUPS_ENTRY, concept.id)) is not NotFound:
             return res
 
         res = _get_set_elements(concept)
 
         # put in cache
-        self.concepts_in_set.put(concept.id, res)
+        self.sheerka.om.put(self.CONCEPTS_IN_GROUPS_ENTRY, concept.id, res)
         return res
 
     def isinset(self, a, b):
@@ -190,8 +189,8 @@ class SheerkaSetsManager(BaseService):
         if not (a.id and b.id):
             return False
 
-        group_elements = self.sets.get(b.id)
-        return group_elements and a.id in group_elements
+        group_elements = self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, b.id)
+        return group_elements is not NotFound and a.id in group_elements
 
     def isa(self, a, b):
 
@@ -226,7 +225,7 @@ class SheerkaSetsManager(BaseService):
 
         # check if it has a group
         # TODO: use cache instead of directly requesting sdp
-        if self.sets.get(concept.id):
+        if self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, concept.id) is not NotFound:
             return True
 
         # it may be a concept that references a set
@@ -267,7 +266,7 @@ for x in xx__concepts__xx:
         :return:
         """
 
-        if not ids:
+        if ids in (None, NotFound):
             return []
 
         if not evaluate:

src/core/sheerka/services/SheerkaMemory.py

@@ -4,7 +4,7 @@ from cache.FastCache import FastCache
 from cache.ListIfNeededCache import ListIfNeededCache
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.global_symbols import EVENT_CONTEXT_DISPOSED
+from core.global_symbols import EVENT_CONTEXT_DISPOSED, NotFound
 from core.sheerka.services.sheerka_service import BaseService, ServiceObj
 
 
@@ -17,13 +17,11 @@ class SheerkaMemory(BaseService):
     NAME = "Memory"
     GLOBAL = "global"
 
-    SHORT_TERM_OBJECTS_ENTRY = "Memory:ShortTermMemoryObjects"
     OBJECTS_ENTRY = "Memory:Objects"
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
         self.short_term_objects = FastCache()
-        self.memory_objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k))
         self.registration = {}
 
     def initialize(self):
@@ -35,15 +33,20 @@ class SheerkaMemory(BaseService):
         self.sheerka.bind_service_method(self.get_from_memory, False)
         self.sheerka.bind_service_method(self.register_object, True, visible=False)
         self.sheerka.bind_service_method(self.unregister_object, True, visible=False)
-        self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False)
+        self.sheerka.bind_service_method(self.commit_registered_objects, True, visible=False)
         self.sheerka.bind_service_method(self.memory, False)
         self.sheerka.bind_service_method(self.mem, False)
 
-        self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.memory_objects, persist=True, use_ref=True)
+        cache = ListIfNeededCache().auto_configure(self.OBJECTS_ENTRY)
+        self.sheerka.om.register_cache(self.OBJECTS_ENTRY, cache, persist=True, use_ref=True)
 
     def reset(self):
         self.short_term_objects.clear()
-        self.memory_objects.clear()
+        self.sheerka.om.clear(self.OBJECTS_ENTRY)
+
+    def reset_state(self):
+        self.short_term_objects.clear()
+        self.registration.clear()
 
     def initialize_deferred(self, context, is_first_time):
         self.sheerka.subscribe(EVENT_CONTEXT_DISPOSED, self.remove_context)
@@ -55,7 +58,7 @@ class SheerkaMemory(BaseService):
                 return self.short_term_objects.cache[id_to_use][key]
             except KeyError:
                 if context is None:
-                    return None
+                    return NotFound
 
                 context = context.get_parent()
 
@@ -92,12 +95,12 @@ class SheerkaMemory(BaseService):
         :param concept:
         :return:
         """
-        self.memory_objects.put(key, MemoryObject(context.event.get_digest(), concept))
+        self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), concept))
 
     def get_from_memory(self, context, key):
         """"
         """
-        return self.memory_objects.get(key)
+        return self.sheerka.om.get(SheerkaMemory.OBJECTS_ENTRY, key)
 
     def register_object(self, context, key, concept):
         """
@@ -126,7 +129,7 @@ class SheerkaMemory(BaseService):
         except KeyError:
             pass
 
-    def add_registered_objects(self, context):
+    def commit_registered_objects(self, context):
         """
         Adds all registered memory_objects
         :param context:
@@ -147,7 +150,7 @@ class SheerkaMemory(BaseService):
             name_to_use = name.name if isinstance(name, Concept) else name
             self.unregister_object(context, name_to_use)
             obj = self.get_from_memory(context, name_to_use)
-            if obj is None:
+            if obj is NotFound:
                 return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
 
             if isinstance(obj, list):
@@ -156,8 +159,7 @@ class SheerkaMemory(BaseService):
             return obj.obj
 
         res = {}
-        for k in self.memory_objects:
-            obj = self.memory_objects.get(k)
+        for k, obj in self.sheerka.om.get_all(SheerkaMemory.OBJECTS_ENTRY).items():
             if isinstance(obj, list):
                 obj = obj[-1]
             res[k] = obj.obj
@@ -165,5 +167,5 @@ class SheerkaMemory(BaseService):
         return res
 
     def mem(self):
-        keys = sorted([k for k in self.memory_objects])
+        keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)])
         return {"keys": keys, "len": len(keys)}

src/core/sheerka/services/SheerkaQuestion.py

@@ -10,27 +10,10 @@ class SheerkaQuestion(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-       # self.sheerka.bind_service_method(self.question, False)
         self.sheerka.bind_service_method(self.is_question, False)
 
-    # def question(self, context, q):
-    #     """
-    #     Evaluate q in the context in a question
-    #     :param context:
-    #     :param q:
-    #     :return:
-    #     """
-    #
-    #     if isinstance(q, Concept):
-    #         with context.push(BuiltinConcepts.EVALUATE_CONCEPT, q, desc=f"Evaluating question '{q}'") as sub_context:
-    #             sub_context.global_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
-    #             sub_context.global_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
-    #
-    #             evaluated = self.sheerka.evaluate_concept(sub_context, q)
-    #
-    #         return evaluated
-
-    def is_question(self, context):
+    @staticmethod
+    def is_question(context):
         """
         Returns True if a question is asked
         :return:

src/core/sheerka/services/SheerkaResultManager.py

@@ -2,7 +2,7 @@ import ast
 
 from cache.Cache import Cache
 from core.builtin_concepts import BuiltinConcepts
-from core.global_symbols import EVENT_USER_INPUT_EVALUATED, EVENT_CONCEPT_CREATED
+from core.global_symbols import EVENT_USER_INPUT_EVALUATED, EVENT_CONCEPT_CREATED, NotFound
 from core.sheerka.services.sheerka_service import BaseService
 from core.utils import CONSOLE_COLORS_MAP as CCM
 from core.utils import as_bag
@@ -13,6 +13,9 @@ MAX_EXECUTION_HISTORY = 100
 class SheerkaResultConcept(BaseService):
     NAME = "Result"
 
+    # SheerkaResultConcept seems to be a concept that must not support multiple ontology layers
+    # We must have always access to everything that was done, whatever the ontology
+
     def __init__(self, sheerka, page_size=30):
         super().__init__(sheerka)
         self.page_size = page_size
@@ -20,6 +23,7 @@ class SheerkaResultConcept(BaseService):
         self.last_execution = None
         self.last_created_concept = None
         self.last_created_concept_id = None
+        self.state_vars = ["last_created_concept_id"]
 
     def initialize(self):
         self.sheerka.bind_service_method(self.get_results_by_digest, True)  # digest is recorded
@@ -27,20 +31,26 @@ class SheerkaResultConcept(BaseService):
         self.sheerka.bind_service_method(self.get_last_results, True)  # digest is recorded
         self.sheerka.bind_service_method(self.get_results, False)
         self.sheerka.bind_service_method(self.get_execution_item, False)
-        self.sheerka.bind_service_method(self.get_last_ret, False, as_name="last_ret")
+        self.sheerka.bind_service_method(self.get_last_return_value, False, as_name="last_ret")
         self.sheerka.bind_service_method(self.get_last_created_concept, False, as_name="last_created_concept")
 
     def initialize_deferred(self, context, is_first_time):
-        self.restore_values("last_created_concept_id")
+        self.restore_values(*self.state_vars)
         self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.user_input_evaluated)
         self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.new_concept_created)
 
-    def reset(self):
+    def test_only_reset(self):
         self.executions_contexts_cache.clear()
         self.last_execution = None
         self.last_created_concept = None
         self.last_created_concept_id = None
 
+    def save_state(self, context):
+        self.store_values(context, *self.state_vars)
+
+    def restore_state(self):
+        self.restore_values(*self.state_vars)
+
     @staticmethod
     def get_predicate(**kwargs):
         if len(kwargs) == 0:
@@ -81,7 +91,7 @@ class SheerkaResultConcept(BaseService):
         :param record_digest:
         :return:
         """
-        if digest is None:
+        if digest is NotFound:
             return None
 
         if filter is not None:
@@ -92,8 +102,8 @@ class SheerkaResultConcept(BaseService):
                 result = self.executions_contexts_cache.get(digest)
                 event = result.event
             else:
-                result = self.sheerka.sdp.load_result(digest)
-                event = self.sheerka.sdp.load_event(digest)  # there is no real need for a cache of the events
+                result = self.sheerka.om.current_sdp().load_result(digest)
+                event = self.sheerka.om.current_sdp().load_event(digest)  # really needed ?
 
             if record_digest:
                 context.log(f"Recording digest '{digest}'")
@@ -141,7 +151,7 @@ class SheerkaResultConcept(BaseService):
         start = len(self.executions_contexts_cache)
         consumed = 0
         while True:
-            for event in self.sheerka.sdp.load_events(self.page_size, start):
+            for event in self.sheerka.om.current_sdp().load_events(self.page_size, start):
                 consumed += 1
                 if event.message.startswith(command):
                     return self.get_results_by_digest(context, event.get_digest(), filter, record_digest, **kwargs)
@@ -200,14 +210,14 @@ class SheerkaResultConcept(BaseService):
         :return:
         """
         digest = self.sheerka.load_var(self.NAME, "digest")
-        if digest is None:
+        if digest is NotFound:
             return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body="no digest")
 
         try:
             if digest in self.executions_contexts_cache:
                 result = self.executions_contexts_cache.get(digest)
             else:
-                result = self.sheerka.sdp.load_result(digest)
+                result = self.sheerka.om.current_sdp().load_result(digest)
             items = list(self.as_list(result, self.get_predicate(id=item_id)))
 
             if len(items) == 0:
@@ -227,7 +237,7 @@ class SheerkaResultConcept(BaseService):
         """
         if self.sheerka.save_execution_context:
             try:
-                self.sheerka.sdp.save_result(execution_context)
+                self.sheerka.om.current_sdp().save_result(execution_context)
             except Exception as ex:
                 print(f"{CCM['red']}Failed to save execution context. Reason: {ex}{CCM['reset']}")
                 pass
@@ -236,7 +246,7 @@ class SheerkaResultConcept(BaseService):
         self.executions_contexts_cache.put(execution_context.event.get_digest(), execution_context)
         self.last_execution = execution_context
 
-    def get_last_ret(self, context):
+    def get_last_return_value(self, context):
         """
         Return the last return value(s)
         :return:
@@ -248,7 +258,7 @@ class SheerkaResultConcept(BaseService):
         if event_id is not None:
             try:
 
-                execution_result = self.sheerka.sdp.load_result(event_id)
+                execution_result = self.sheerka.om.current_sdp().load_result(event_id)
                 return execution_result.values["return_values"]
 
             except FileNotFoundError as ex:
@@ -278,9 +288,9 @@ class SheerkaResultConcept(BaseService):
         page_size = 2
         consumed = 0
         while True:
-            for event in self.sheerka.sdp.load_events(page_size, start):
+            for event in self.sheerka.om.current_sdp().load_events(page_size, start):
                 consumed += 1
-                if self.sheerka.sdp.has_result(event.get_digest()):
+                if self.sheerka.om.current_sdp().has_result(event.get_digest()):
                     return event.get_digest()
 
             if consumed < page_size:

src/core/sheerka/services/SheerkaRuleManager.py

@@ -4,10 +4,11 @@ from dataclasses import dataclass
 from typing import Union
 
 from cache.Cache import Cache
+from cache.ListIfNeededCache import ListIfNeededCache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.builtin_helpers import parse_unrecognized, only_successful, ensure_rule
 from core.concept import Concept
-from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT
+from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound
 from core.rule import Rule
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Keywords, TokenKind, Token, IterParser
@@ -509,22 +510,25 @@ class SheerkaRuleManager(BaseService):
     RULE_IDS = "Rules_Ids"
     FORMAT_RULE_ENTRY = "RuleManager:FormatRules"
     EXEC_RULE_ENTRY = "RuleManager:ExecRules"
+    RULES_BY_NAME_ENTRY = "RuleManager:Rules_By_Name"
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
-        self.format_rule_cache = Cache(default=lambda k: self.sheerka.sdp.get(self.FORMAT_RULE_ENTRY, k))
-        self.exec_rule_cache = Cache(default=lambda k: self.sheerka.sdp.get(self.EXEC_RULE_ENTRY, k))
-
         self._format_rules = None  # sorted by priority
 
     def initialize(self):
         self.sheerka.bind_service_method(self.create_new_rule, True, visible=False)
         self.sheerka.bind_service_method(self.get_rule_by_id, False)
+        self.sheerka.bind_service_method(self.get_rule_by_name, False)
         self.sheerka.bind_service_method(self.dump_desc_rule, False, as_name="desc_rule")
         self.sheerka.bind_service_method(self.get_format_rules, False, visible=False)
 
-        self.sheerka.cache_manager.register_cache(self.FORMAT_RULE_ENTRY, self.format_rule_cache, True, True)
-        self.sheerka.cache_manager.register_cache(self.EXEC_RULE_ENTRY, self.exec_rule_cache, True, True)
+        cache = Cache().auto_configure(self.FORMAT_RULE_ENTRY)
+        self.sheerka.om.register_cache(self.FORMAT_RULE_ENTRY, cache, True, True)
+        cache = Cache().auto_configure(self.EXEC_RULE_ENTRY)
+        self.sheerka.om.register_cache(self.EXEC_RULE_ENTRY, cache, True, True)
+        cache = ListIfNeededCache().auto_configure(self.RULES_BY_NAME_ENTRY)
+        self.sheerka.om.register_cache(self.RULES_BY_NAME_ENTRY, cache, True, True)
 
     def initialize_deferred(self, context, is_first_time):
 
@@ -533,20 +537,29 @@ class SheerkaRuleManager(BaseService):
             self.init_builtin_rules(context)
         else:
             # adds the other rules (when it's not the first time)
-            self.format_rule_cache.populate(lambda: self.sheerka.sdp.list(self.FORMAT_RULE_ENTRY), lambda rule: rule.id)
-            self.exec_rule_cache.populate(lambda: self.sheerka.sdp.list(self.EXEC_RULE_ENTRY), lambda rule: rule.id)
-            self.format_rule_cache.reset_events()
-            self.exec_rule_cache.reset_events()
+            self.sheerka.om.populate(self.FORMAT_RULE_ENTRY,
+                                     lambda sdp: sdp.list(self.FORMAT_RULE_ENTRY),
+                                     lambda rule: rule.id,
+                                     reset_events=True,
+                                     all_ontologies=True)
+            self.sheerka.om.populate(self.EXEC_RULE_ENTRY,
+                                     lambda sdp: sdp.list(self.EXEC_RULE_ENTRY),
+                                     lambda rule: rule.id,
+                                     reset_events=True,
+                                     all_ontologies=True)
 
-        # compile all the rules
-        for rule_id in self.format_rule_cache:
-            rule = self.init_rule(context, self.format_rule_cache.get(rule_id))
+        # compile all format the rules
+        for rule_id, rule_def in self.sheerka.om.get_all(self.FORMAT_RULE_ENTRY, cache_only=True).items():
+            rule = self.init_rule(context, rule_def)
 
         # update rules priorities
         self.update_rules_priorities(context)
 
         self.sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities)
 
+    def reset_state(self):
+        self._format_rules = None
+
     def update_rules_priorities(self, context):
         """
         Ask the SheerkaComparisonManager for the priorities
@@ -555,9 +568,8 @@ class SheerkaRuleManager(BaseService):
         # get the priorities
         rules_weights = self.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT)
 
-        # compile all the rules
-        for rule_id in self.format_rule_cache:
-            rule = self.format_rule_cache.get(rule_id)
+        # update the priorities
+        for rule in self.sheerka.om.list(self.FORMAT_RULE_ENTRY, cache_only=True):
             if rule.str_id in rules_weights:
                 rule.priority = rules_weights[rule.str_id]
 
@@ -623,7 +635,7 @@ class SheerkaRuleManager(BaseService):
         if rule.metadata.id is not None:
             return
 
-        rule.metadata.id = str(self.sheerka.cache_manager.get(self.sheerka.OBJECTS_IDS_ENTRY, self.RULE_IDS))
+        rule.metadata.id = str(self.sheerka.om.get(self.sheerka.OBJECTS_IDS_ENTRY, self.RULE_IDS))
 
     def create_new_rule(self, context, rule):
         """
@@ -642,10 +654,14 @@ class SheerkaRuleManager(BaseService):
 
         # save it
         if rule.metadata.action_type == "print":
-            self.sheerka.cache_manager.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule)
+            self.sheerka.om.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule)
             self._format_rules = None
         else:
-            self.sheerka.cache_manager.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule)
+            self.sheerka.om.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule)
+
+        # save by name if needed
+        if rule.metadata.name:
+            self.sheerka.om.put(self.RULES_BY_NAME_ENTRY, rule.metadata.name, rule)
 
         # process the return if needed
         ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_RULE, body=rule))
@@ -721,17 +737,28 @@ class SheerkaRuleManager(BaseService):
         if rule_id is None:
             return None
 
-        rule = self.format_rule_cache.get(rule_id)
-        if rule:
+        rule = self.sheerka.om.get(self.FORMAT_RULE_ENTRY, rule_id)
+        if rule is not NotFound:
             return rule
 
-        rule = self.exec_rule_cache.get(rule_id)
-        if rule:
+        rule = self.sheerka.om.get(self.EXEC_RULE_ENTRY, rule_id)
+        if rule is not NotFound:
             return rule
 
         metadata = [("id", rule_id)]
         return self.sheerka.new(BuiltinConcepts.UNKNOWN_RULE, body=metadata)
 
+    def get_rule_by_name(self, rule_name):
+        if rule_name is None:
+            return None
+
+        rule = self.sheerka.om.get(self.RULES_BY_NAME_ENTRY, rule_name)
+        if rule is NotFound:
+            metadata = [("name", rule_name)]
+            return self.sheerka.new(BuiltinConcepts.UNKNOWN_RULE, body=metadata)
+
+        return rule
+
     def dump_desc_rule(self, rules):
         """
         dumps the definition of a rule
@@ -759,7 +786,9 @@ class SheerkaRuleManager(BaseService):
         if self._format_rules:
             return self._format_rules
 
-        self._format_rules = sorted(self.format_rule_cache.get_all(), key=operator.attrgetter('priority'), reverse=True)
+        self._format_rules = sorted(self.sheerka.om.list(self.FORMAT_RULE_ENTRY, cache_only=True),
+                                    key=operator.attrgetter('priority'),
+                                    reverse=True)
         return self._format_rules
 
     def add_evaluators(self, source, ret_vals):

src/core/sheerka/services/SheerkaVariableManager.py

@@ -3,6 +3,7 @@ from typing import List
 
 from cache.Cache import Cache
 from core.builtin_concepts import BuiltinConcepts
+from core.global_symbols import NotFound
 from core.sheerka.services.sheerka_service import ServiceObj, BaseService
 
 
@@ -23,32 +24,52 @@ class Variable(ServiceObj):
         return f"({self.who}){self.key}={self.value}"
 
 
+@dataclass
+class InternalObj:
+    obj: object
+
+    def __deepcopy__(self, memodict={}):
+        return self
+
+    def __copy__(self):
+        return self
+
+
 class SheerkaVariableManager(BaseService):
     NAME = "VariableManager"
-    VARIABLES_ENTRY = "VariableManager:Variables"  # entry for admin or internal variables
+    VARIABLES_ENTRY = "VariableManager:Variables"  # entry for variables which will be copied in sdp
+    INTERNAL_VARIABLES_ENTRY = "VariableManager:InternalVariables"  # internal to current process (can store lambda)
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
-        self.bound = {
-            "sheerka.enable_process_return_values": "enable_process_return_values",
-            "sheerka.save_execution_context": "save_execution_context"
+        self.bound_variables = {
+            self.sheerka.name: {"enable_process_return_values", "save_execution_context"}
         }
 
     def initialize(self):
         self.sheerka.bind_service_method(self.record_var, True, visible=False)
         self.sheerka.bind_service_method(self.load_var, False, visible=False)
+        self.sheerka.bind_service_method(self.record_internal_var, True, visible=False)
+        self.sheerka.bind_service_method(self.load_internal_var, False, visible=False)
         self.sheerka.bind_service_method(self.delete_var, True, visible=False)
         self.sheerka.bind_service_method(self.set_var, True)
         self.sheerka.bind_service_method(self.get_var, False)
         self.sheerka.bind_service_method(self.list_vars, False)
 
-        cache = Cache()
-        cache.populate(lambda: self.sheerka.sdp.list(self.VARIABLES_ENTRY), lambda var: var.get_key())
-        self.sheerka.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True)
+        cache = Cache().auto_configure(self.VARIABLES_ENTRY)
+        self.sheerka.om.register_cache(self.VARIABLES_ENTRY, cache, True, True)
+        cache.populate(lambda sdp: sdp.list(self.VARIABLES_ENTRY), lambda var: var.get_key())
 
-        for variable in cache.get_all():
-            if variable.key in self.bound:
-                setattr(self.sheerka, self.bound[variable.key], variable.value)
+        internal_vars = Cache().auto_configure(self.INTERNAL_VARIABLES_ENTRY)
+        self.sheerka.om.register_cache(self.INTERNAL_VARIABLES_ENTRY, internal_vars, False, False)
+
+    def initialize_deferred(self, context, first_time):
+        # update bound variables
+        for who, keys in self.bound_variables.items():
+            for key in keys:
+                if (variable := self.sheerka.om.get(self.VARIABLES_ENTRY, f"{who}|{key}")) is not NotFound:
+                    service = self.sheerka if who == self.sheerka.name else self.sheerka.services[who]
+                    setattr(service, key, variable.value)
 
     def record_var(self, context, who, key, value):
         """
@@ -61,34 +82,49 @@ class SheerkaVariableManager(BaseService):
         """
 
         variable = Variable(context.event.get_digest(), who, key, value, None)
-        self.sheerka.cache_manager.put(self.VARIABLES_ENTRY, variable.get_key(), variable)
+        self.sheerka.om.put(self.VARIABLES_ENTRY, variable.get_key(), variable)
 
         # TODO: manage credentials
-        if key in self.bound:
-            setattr(self.sheerka, self.bound[key], value)
-
-        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+        if who in self.bound_variables and key in self.bound_variables[who]:
+            service = self.sheerka if who == self.sheerka.name else self.sheerka.services[who]
+            setattr(service, key, value)
 
     def load_var(self, who, key):
-        variable = self.sheerka.cache_manager.get(self.VARIABLES_ENTRY, who + "|" + key)
-        if variable is None:
-            return None
+        variable = self.sheerka.om.get(self.VARIABLES_ENTRY, who + "|" + key)
+        if variable is NotFound:
+            return NotFound
 
         return variable.value
 
+    def record_internal_var(self, context, who, key, value):
+        """
+        Stores the value in the internal cache
+        This cache is not pushed to the remote repository
+        :param context:
+        :param who: entity that owns the key (acts as a namespace)
+        :param key:
+        :param value:
+        """
+        self.sheerka.om.put(self.INTERNAL_VARIABLES_ENTRY, f"{who}|{key}", InternalObj(value))
+
+    def load_internal_var(self, who, key):
+        value = self.sheerka.om.get(self.INTERNAL_VARIABLES_ENTRY, f"{who}|{key}")
+        return NotFound if value is NotFound else value.obj
+
     def delete_var(self, context, who, key):
-        self.sheerka.cache_manager.delete(self.VARIABLES_ENTRY, who + "|" + key)
+        self.sheerka.om.delete(self.VARIABLES_ENTRY, who + "|" + key)
 
     def set_var(self, context, key, value):
-        return self.record_var(context, context.event.user_id, key, value)
+        self.record_var(context, context.event.user_id, key, value)
+        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
     def get_var(self, context, key):
         return self.load_var(context.event.user_id, key)
 
     def list_vars(self, context, all_vars=False):
         if all_vars:
-            res = [str(v) for v in self.sheerka.cache_manager.copy(self.VARIABLES_ENTRY).values()]
+            res = [str(v) for v in self.sheerka.om.copy(self.VARIABLES_ENTRY).values()]
         else:
-            res = [str(v) for v in self.sheerka.cache_manager.copy(self.VARIABLES_ENTRY).values() if
+            res = [str(v) for v in self.sheerka.om.copy(self.VARIABLES_ENTRY).values() if
                    v.who == context.event.user_id]
         return res

src/core/sheerka/services/sheerka_service.py

@@ -1,5 +1,8 @@
 from dataclasses import dataclass
 
+from core.global_symbols import NotFound
+from core.utils import sheerka_deepcopy
+
 
 @dataclass
 class ServiceObj:
@@ -21,6 +24,13 @@ class BaseService:
         """
         pass
 
+    def store_values(self, context, *args):
+        """
+        Use variable Manager to store the state of the service
+        """
+        for prop_name in args:
+            self.sheerka.record_var(context, self.NAME, prop_name, sheerka_deepcopy(getattr(self, prop_name)))
+
     def restore_values(self, *args):
         """
         Use Variable Manager to restore the state of a service
@@ -28,5 +38,5 @@ class BaseService:
         :return:
         """
         for prop_name in args:
-            if (value := self.sheerka.load_var(self.NAME, prop_name)) is not None:
+            if (value := self.sheerka.load_var(self.NAME, prop_name)) is not NotFound:
                 setattr(self, prop_name, value)

src/core/simple_debug.py

@@ -23,13 +23,13 @@ def my_debug(*args, check_started=None):
             if debug_name not in debug_activated:
                 return
 
-    # with open("debug.txt", "a") as f:
-    #     for arg in args:
-    #         if isinstance(arg, list):
-    #             for item in arg:
-    #                 f.write(f"{item}\n")
-    #         else:
-    #             f.write(f"{arg}\n")
+    with open("debug.txt", "a") as f:
+        for arg in args:
+            if isinstance(arg, list):
+                for item in arg:
+                    f.write(f"{item}\n")
+            else:
+                f.write(f"{arg}\n")
 
 
 def start_debug(debug_name=default_debug_name, msg=None):

src/core/utils.py

@@ -5,8 +5,7 @@ import os
 import pkgutil
 from copy import deepcopy
 
-from cache.Cache import Cache
-from core.ast_helpers import ast_to_props
+from core.global_symbols import CustomType
 from core.tokenizer import TokenKind, Tokenizer
 from pyparsing import *
 
@@ -35,8 +34,6 @@ CONSOLE_COLORS_MAP = {
 
 PRIMITIVES_TYPES = (str, bool, type(None), int, float, list, dict, set, bytes, tuple, type)
 
-expressions_cache = Cache()
-
 ESC = Literal('\x1b')
 integer = Word(nums)
 escapeSeq = Combine(ESC + '[' + Optional(delimitedList(integer, ';')) +
@@ -603,59 +600,6 @@ def flatten_all_children(item, get_children):
     return inner_get_all_children(item)
 
 
-def evaluate_expression(expr, bag):
-    """
-    Try to evaluate expr in context of bag
-    :param expr:
-    :param bag:
-    :return:
-    """
-
-    if expr is None or expr.strip() == "":
-        return None
-
-    if expr in bag:
-        return bag[expr]
-
-    props_definitions = expressions_cache.get(expr)
-    if props_definitions is None:
-        _ast = ast.parse(expr, mode="eval")
-        props_definitions = []
-        ast_to_props(props_definitions, _ast.body, None)
-        props_definitions.reverse()
-        expressions_cache.put(expr, props_definitions)
-
-    return evaluate_object(bag, props_definitions)
-
-
-def evaluate_object(bag, properties):
-    """
-    Evaluate the properties of an object
-    Works with evaluate_expression
-    :param bag:
-    :param properties: List of ast_helpers.PropDef
-    :return:
-    """
-    for prop in properties:
-        try:
-            obj = bag[prop.prop]
-        except KeyError:
-            try:
-                obj = bag["self"][prop.prop]
-            except Exception:
-                raise NameError(prop.prop)
-
-        if obj is None:
-            return None
-
-        if prop.index is not None:
-            obj = obj[prop.index]
-
-        bag = as_bag(obj)
-
-    return obj
-
-
 def get_text_from_tokens(tokens, custom_switcher=None, tracker=None):
     """
     Create the source code, from the list of token
@@ -729,7 +673,9 @@ def sheerka_deepcopy(obj):
         return instance
 
     from core.concept import Concept
-    if isinstance(obj, dict):
+    if isinstance(obj, CustomType):
+        return obj
+    elif isinstance(obj, dict):
         res = {sheerka_deepcopy(k): sheerka_deepcopy(v) for k, v in obj.items()}
         return res
     elif isinstance(obj, list):
@@ -759,6 +705,7 @@ class NextIdManager:
     """
     solely return the next integer
     """
+
     def __init__(self):
         self.id = -1