Added first version of DebugManager. Implemented draft of the rule engine

src/core/sheerka/ExecutionContext.py

@@ -1,17 +1,26 @@
 import logging
+import os
+import pprint
 import time
 
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
-from core.concept import Concept
+from core.concept import Concept, get_concept_attrs
+from core.global_symbols import CONTEXT_DISPOSED
 from core.sheerka.services.SheerkaExecute import NO_MATCH
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
-from core.sheerka_logger import get_logger
+from core.utils import CONSOLE_COLORS_MAP as CCM
 from sdp.sheerkaDataProvider import Event
 
+try:
+    rows, columns = os.popen('stty size', 'r').read().split()
+except ValueError:
+    rows, columns = 50, 80
+
+pp = pprint.PrettyPrinter(indent=2, width=columns)
+
 DEBUG_TAB_SIZE = 4
 
 PROPERTIES_TO_SERIALIZE = ("_id",
-                           "_bag",
                            "_children",
                            "_start",
                            "_stop",
@@ -50,19 +59,17 @@ class ExecutionContext:
                  logger=None,
                  global_hints=None,
                  errors=None,
-                 **kwargs):
+                 obj=None,
+                 concepts=None):
 
         self._id = ExecutionContext.get_id(event.get_digest()) if event else None
         self._parent = None
         self._children = []
-        self._tab = ""
-        self._bag = {}  # context variables
         self._start = 0  # when the execution starts (to measure elapsed time)
         self._stop = 0  # when the execution stops (to measure elapses time)
         self._logger = logger
         self._format_instructions = None  # how to print the execution context
-        self._stat_log = get_logger("stats")
-        self._show_stats = False
+        self._push = None
 
         self.who = who  # who is asking
         self.event = event  # what was the (original) trigger
@@ -70,6 +77,8 @@ class ExecutionContext:
         self.action = action
         self.action_context = action_context
         self.desc = desc  # human description of what is going on
+        self.preprocess_parsers = None
+        self.preprocess_evaluators = None
         self.preprocess = None
         self.stm = False  # True if the context has short term memory entries
 
@@ -80,13 +89,11 @@ class ExecutionContext:
 
         self.inputs = {}  # what were the parameters of the execution context
         self.values = {}  # what was produced by the execution context
-        self.obj = kwargs.pop("obj", None)  # current obj we are working on
+        self.obj = obj
+        self.concepts = concepts
 
-        self.concepts = kwargs.pop("concepts", {})  # known concepts specific to this context
-
-        # update the other elements
-        for k, v in kwargs.items():
-            self._bag[k] = v
+        self_debug, self.debug_mode = sheerka.get_context_debug_mode(self.id)
+        self.debug_enabled = self_debug is not None
 
     @property
     def elapsed(self):
@@ -117,24 +124,19 @@ class ExecutionContext:
         """
         return self._children
 
-    def __getattr__(self, item):
-        if item in self._bag:
-            return self._bag[item]
-
-        raise AttributeError(f"'ExecutionContext' object has no attribute '{item}'")
-
     def __enter__(self):
         self._start = time.time_ns()
-        self.log_new()
+        # self.log_new()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._push:
+            return
+
         if self.stm:
-            self.sheerka.services[SheerkaMemory.NAME].remove_context(self)
+            self.sheerka.publish(self, CONTEXT_DISPOSED)
 
         self._stop = time.time_ns()
-        if self._show_stats:
-            self._stat_log.debug(f"[{self._id:2}]" + self._tab + "Execution time: " + self.elapsed_str)
 
     def __repr__(self):
         msg = f"ExecutionContext(who={self.who}, id={self._id}, action={self.action}, context={self.action_context}"
@@ -143,11 +145,6 @@ class ExecutionContext:
         msg += ")"
         return msg
 
-    # def __str__(self):
-    #     msg = self.desc or "New Context"
-    #     msg += f", who={self.who}, id={self.id}"
-    #     return msg
-
     def __eq__(self, other):
         if id(self) == id(other):
             return True
@@ -168,12 +165,12 @@ class ExecutionContext:
 
         return True
 
-    def push(self, action: BuiltinConcepts, action_context, who=None, desc=None, logger=None, **kwargs):
+    def push(self, action: BuiltinConcepts, action_context, who=None, desc=None, logger=None, obj=None, concepts=None):
+        if self._push:
+            return self._push
+
         who = who or self.who
         logger = logger or self._logger
-        _kwargs = {"obj": self.obj, "concepts": self.concepts}
-        _kwargs.update(self._bag)
-        _kwargs.update(kwargs)
         new = ExecutionContext(
             who,
             self.event,
@@ -184,19 +181,40 @@ class ExecutionContext:
             logger,
             self.global_hints,
             self.errors,
-            **_kwargs)
+            obj or self.obj,
+            concepts or self.concepts)
         new._parent = self
-        new._tab = self._tab + " " * DEBUG_TAB_SIZE
         new.preprocess = self.preprocess
+        new.preprocess_parsers = self.preprocess_parsers
+        new.preprocess_evaluators = self.preprocess_evaluators
         new.protected_hints.update(self.protected_hints)
 
+        if new.debug_mode is None and self.debug_mode == "protected":
+            new.debug_mode = "protected"
+            new.debug_enabled = True
+
         self._children.append(new)
 
         return new
 
+    def deactivate_push(self):
+        self._push = self.push(BuiltinConcepts.NOP, None)
+        self._push._push = self._push
+        if self.stm:
+            bag = self.sheerka.services[SheerkaMemory.NAME].get_all_short_term_memory(self)
+            self.sheerka.add_many_to_short_term_memory(self._push, bag)
+
+    def activate_push(self):
+        if self._push:
+            if self._push.stm:
+                self.sheerka.publish(self._push, CONTEXT_DISPOSED)
+            self._push._stop = time.time_ns()
+
+            self._push = None
+
     def add_preprocess(self, name, **kwargs):
         preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
-        preprocess.set_value("name", name)
+        preprocess.set_value("preprocess_name", name)
         for k, v in kwargs.items():
             preprocess.set_value(k, v)
 
@@ -206,13 +224,17 @@ class ExecutionContext:
         return self
 
     def add_inputs(self, **kwargs):
-        for k, v in kwargs.items():
-            self.inputs[k] = v
+        if self._push:
+            return
+
+        self.inputs.update(kwargs)
         return self
 
     def add_values(self, **kwargs):
-        for k, v in kwargs.items():
-            self.values[k] = v
+        if self._push:
+            return
+
+        self.values.update(kwargs)
         return self
 
     def add_to_short_term_memory(self, key, concept):
@@ -224,6 +246,9 @@ class ExecutionContext:
         """
         self.sheerka.add_to_short_term_memory(self, key, concept)
 
+    def clear_short_term_memory(self):
+        self.sheerka.clear_short_term_memory(self)
+
     def get_from_short_term_memory(self, key):
         """
 
@@ -237,11 +262,10 @@ class ExecutionContext:
         if isinstance(self.obj, Concept):
             if self.obj.key == key:
                 return self.obj
-            for var_name in self.obj.values:
-                if var_name == key:
-                    value = self.obj.get_value(var_name)
-                    if isinstance(value, Concept):
-                        return value
+            if key in get_concept_attrs(self.obj):
+                value = self.obj.get_value(key)
+                if isinstance(value, Concept):
+                    return value
 
         # search in concepts
         if self.concepts:
@@ -296,8 +320,34 @@ class ExecutionContext:
             to_str = self.return_value_to_str(r)
             self._logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
-    def debug(self, text):
-        print(text)
+    def get_debugger(self, who, method_name):
+        return self.sheerka.get_debugger(self, who, method_name)
+
+    def debug(self, who, method_name, variable_name, text, is_error=False):
+        activated = self.sheerka.debug_activated_for(who)
+        if activated:
+            str_text = pp.pformat(text)
+            color = 'red' if is_error else 'green'
+            if "\n" not in str(str_text):
+                self.sheerka.debug(
+                    f"[{self._id:3}] {CCM[color]}{who}.{method_name}.{variable_name}: {CCM['reset']}{str_text}")
+            else:
+                self.sheerka.debug(f"[{self._id:3}] {CCM[color]}{who}.{method_name}.{variable_name}: {CCM['reset']}")
+                self.sheerka.debug(str_text)
+
+    def debug_entering(self, who, method_name, **kwargs):
+        if self.sheerka.debug_activated_for(who):
+            str_text = pp.pformat(kwargs)
+            if "\n" not in str(str_text):
+                self.sheerka.debug(
+                    f"[{self._id:3}] {CCM['blue']}Entering {who}.{method_name} with {CCM['reset']}{str_text}")
+            else:
+                self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}Entering {who}.{method_name}:{CCM['reset']}")
+                self.sheerka.debug(f"[{self._id:3}] {str_text}")
+
+    def debug_log(self, who, text):
+        if self.sheerka.debug_activated_for(who):
+            self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}{text}{CCM['reset']}")
 
     def get_parent(self):
         return self._parent
@@ -384,9 +434,6 @@ class ExecutionContext:
         And it removes the visibility from the other attributes/methods
         """
         bag = {}
-        for k, v in self._bag.items():
-            bag[k] = v
-            bag["bag." + k] = v
         for prop in ("id", "who", "action", "desc", "obj", "inputs", "values", "concepts"):
             bag[prop] = getattr(self, prop)
         bag["context"] = self.action_context
@@ -396,6 +443,7 @@ class ExecutionContext:
         bag["elapsed"] = self.elapsed
         bag["elapsed_str"] = self.elapsed_str
         bag["digest"] = self.event.get_digest() if self.event else None
+        bag["_children"] = self._children
         return bag
 
     @staticmethod
@@ -438,3 +486,17 @@ class ExecutionContext:
                 break
 
             current = current._parent
+
+    def has_parent(self, context_id):
+        current = self
+
+        while current._parent:
+            current = current._parent
+            if current.id == context_id:
+                return True
+            if current.id < context_id:
+                return False
+
+        return False
+
+

src/core/sheerka/Sheerka.py

@@ -11,8 +11,10 @@ from cache.IncCache import IncCache
 from cache.ListIfNeededCache import ListIfNeededCache
 from cache.SetCache import SetCache
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
-    UnknownConcept
-from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
+    UnknownConcept, AllBuiltinConcepts
+from core.concept import Concept, ConceptParts, NotInit, get_concept_attrs
+from core.error import ErrorObj
+from core.profiling import profile
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka_logger import console_handler
 from core.tokenizer import Token, TokenKind
@@ -66,15 +68,18 @@ class Sheerka(Concept):
     MAX_EXECUTION_HISTORY = 100
     MAX_RETURN_VALUES_HISTORY = 100
 
+    ALL_ATTRIBUTES = []
+
     def __init__(self, cache_only=False, debug=False, loggers=None):
         self.init_logging(debug, loggers)
         self.loggers = loggers
 
         super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
-        self.log.debug("Starting Sheerka.")
+        # self.log.debug("Starting Sheerka.")
 
         self.bnp = None  # reference to the BaseNodeParser class (to compute first keyword token)
         self.return_value_concept_id = None
+        self.error_concept_id = None
 
         # a concept can be instantiated
         # ex: File is a concept, but File('foo.txt') is an instance
@@ -85,7 +90,7 @@ class Sheerka(Concept):
         # ex: hello => say('hello')
         self.rules = []
 
-        self.sdp: SheerkaDataProvider = None  # SheerkaDataProvider
+        self.sdp: SheerkaDataProvider = None
         self.cache_manager = CacheManager(cache_only)
 
         self.services = {}  # sheerka plugins
@@ -103,6 +108,7 @@ class Sheerka(Concept):
         self._builtins_classes_cache = None
 
         self.save_execution_context = True
+        self.enable_process_return_values = False
 
         self.methods_with_context = {"test_using_context"}  # only the names, the method is defined in sheerka_methods
         self.sheerka_methods = {
@@ -171,17 +177,22 @@ class Sheerka(Concept):
         """
         self.sheerka_pipeables[func_name] = SheerkaMethod(function, has_side_effect)
 
-    def initialize(self, root_folder: str = None, save_execution_context=True):
+    def initialize(self, root_folder: str = None, save_execution_context=None, enable_process_return_values=None):
         """
         Starting Sheerka
         Loads the current configuration
         Notes that when it's the first time, it also create the needed working folders
         :param root_folder: root configuration folder
         :param save_execution_context:
+        :param enable_process_return_values:
         :return: ReturnValue(Success or Error)
         """
 
-        self.save_execution_context = save_execution_context
+        if save_execution_context is not None:
+            self.save_execution_context = save_execution_context
+
+        if enable_process_return_values is not None:
+            self.enable_process_return_values = enable_process_return_values
 
         try:
             from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
@@ -189,7 +200,10 @@ class Sheerka(Concept):
 
             self.sdp = SheerkaDataProvider(root_folder, self)
             self.initialize_caching()
+            self.get_builtin_parsers()
+            self.get_builtin_evaluators()
             self.initialize_services()
+            self.initialize_builtin_evaluators()
 
             event = Event("Initializing Sheerka.", user_id=self.name)
             self.sdp.save_event(event)
@@ -198,25 +212,24 @@ class Sheerka(Concept):
                                   self,
                                   BuiltinConcepts.INIT_SHEERKA,
                                   None,
-                                  desc="Initializing Sheerka.",
-                                  logger=self.init_log) as exec_context:
+                                  desc="Initializing Sheerka.") as exec_context:
                 if self.sdp.first_time:
                     self.first_time_initialisation(exec_context)
 
-                self.initialize_builtin_parsers()
-                self.initialize_builtin_evaluators()
                 self.initialize_builtin_concepts()
                 self.initialize_concept_node_parsing(exec_context)
-                res = ReturnValueConcept(self, True, self)
 
+                self.initialize_services_deferred(exec_context, self.sdp.first_time)
+
+                res = ReturnValueConcept(self, True, self)
                 exec_context.add_values(return_values=res)
 
                 if self.cache_manager.is_dirty:
                     self.cache_manager.commit(exec_context)
 
-                if save_execution_context:
+                if self.save_execution_context:
                     self.sdp.save_result(exec_context, is_admin=True)
-                self.init_log.debug(f"Sheerka successfully initialized")
+                # self.init_log.debug(f"Sheerka successfully initialized")
 
         except IOError as e:
             res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
@@ -276,7 +289,7 @@ class Sheerka(Concept):
         Introspect to find services and bind them
         :return:
         """
-        self.init_log.debug("Initializing services")
+        # self.init_log.debug("Initializing services")
 
         core.utils.import_module_and_sub_module('core.sheerka.services')
         base_class = "core.sheerka.services.sheerka_service.BaseService"
@@ -286,49 +299,63 @@ class Sheerka(Concept):
                 instance.initialize()
             self.services[service.NAME] = instance
 
+    def initialize_services_deferred(self, context, is_first_time):
+        """
+        Initialize part of services that may takes some time or that need the execution context
+        TODO: Create a separate thread for these initialisations as they may take time
+        :return:
+        """
+        # self.init_log.debug("Initializing services (deferred)")
+
+        for service in self.services.values():
+            if hasattr(service, "initialize_deferred"):
+                service.initialize_deferred(context, is_first_time)
+
     def first_time_initialisation(self, context):
 
         self.cache_manager.put(self.CONCEPTS_KEYS_ENTRY, self.USER_CONCEPTS_KEYS, 1000)
-        self.record(context, self.name, "save_execution_context", True)
+        self.record_var(context, self.name, "save_execution_context", True)
 
     def initialize_builtin_concepts(self):
         """
         Initializes the builtin concepts
         :return: None
         """
-        self.init_log.debug("Initializing builtin concepts")
+        # self.init_log.debug("Initializing builtin concepts")
         builtins_classes = self.get_builtins_classes_as_dict()
 
         # this all initialization of the builtins seems to be little bit complicated
         # why do we need to update it from DB ?
-        for key in BuiltinConcepts:
+        for key in AllBuiltinConcepts:
             concept = self if key == BuiltinConcepts.SHEERKA \
                 else builtins_classes[str(key)]() if str(key) in builtins_classes \
                 else Concept(key, True, False, key)
 
             if key in BuiltinUnique:
-                concept.metadata.is_unique = True
-                concept.metadata.is_evaluated = True
+                concept._metadata.is_unique = True
+                concept._metadata.is_evaluated = True
 
-            if not concept.metadata.is_unique and str(key) in builtins_classes:
+            if not concept._metadata.is_unique and str(key) in builtins_classes:
                 self.builtin_cache[key] = builtins_classes[str(key)]
 
-            from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept.metadata.key)
+            from_db = self.cache_manager.get(self.CONCEPTS_BY_KEY_ENTRY, concept._metadata.key)
             if from_db is None:
-                self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
+                # self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
                 self.set_id_if_needed(concept, True)
                 self.cache_manager.add_concept(concept)
 
                 if key == BuiltinConcepts.RETURN_VALUE:
                     self.return_value_concept_id = concept.id
+                elif key == BuiltinConcepts.ERROR:
+                    self.error_concept_id = concept.id
 
             else:
-                self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
+                # self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
                 concept.update_from(from_db)
 
         return
 
-    def initialize_builtin_parsers(self):
+    def get_builtin_parsers(self):
         """
         Init the parsers
         :return:
@@ -343,7 +370,7 @@ class Sheerka(Concept):
                 continue
 
             qualified_name = core.utils.get_full_qualified_name(parser)
-            self.init_log.debug(f"Adding builtin parser '{qualified_name}'")
+            # self.init_log.debug(f"Adding builtin parser '{qualified_name}'")
             temp_result[qualified_name] = parser
 
         # keep a reference to base_node_parser
@@ -361,22 +388,29 @@ class Sheerka(Concept):
 
             self.parsers[name] = temp_result[name]
 
+    def get_builtin_evaluators(self):
+        """
+        get all evaluators
+        :return:
+        """
+        core.utils.import_module_and_sub_module("evaluators")
+        evaluators = core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator")
+        evaluators.extend(core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator"))
+
+        for evaluator in evaluators:
+            self.evaluators.append(evaluator)
+
     def initialize_builtin_evaluators(self):
         """
         Init the evaluators
         :return:
         """
-        core.utils.import_module_and_sub_module("evaluators")
-        for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator"):
-            self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
-            self.evaluators.append(evaluator)
-
-        for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator"):
-            self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
-            self.evaluators.append(evaluator)
+        for evaluator in self.evaluators:
+            if hasattr(evaluator, "initialize"):
+                evaluator.initialize(self)
 
     def initialize_concept_node_parsing(self, context):
-        self.init_log.debug("siInitializing concepts by first keyword.")
+        # self.init_log.debug("Initializing concepts by first keyword.")
 
         concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
         res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
@@ -391,11 +425,16 @@ class Sheerka(Concept):
                     service.initialize()
         else:
             self.cache_manager.clear()
+
+        for service in self.services.values():
+            if hasattr(service, "reset"):
+                service.reset()
+
         self.printer_handler.reset()
         self.sdp.reset()
         self.locals = {}
 
-    # @profile()
+    # @profile(filename="profile_80")
     def evaluate_user_input(self, text: str, user_name="kodjo"):
         """
         Note to KSI: If you try to add execution context to this function,
@@ -404,19 +443,20 @@ class Sheerka(Concept):
         :param user_name:
         :return:
         """
-        self.log.debug(f"Processing user input '{text}', {user_name=}.")
+        # self.log.debug(f"Processing user input '{text}', {user_name=}.")
         event = Event(text, user_name)
-        evt_digest = self.sdp.save_event(event)
-        self.log.debug(f"{evt_digest=}")
+        self.sdp.save_event(event)
 
         with ExecutionContext(self.key,
                               event,
                               self,
                               BuiltinConcepts.PROCESS_INPUT,
                               text,
-                              desc=f"Evaluating '{text}'",
-                              logger=self.log) as execution_context:
+                              desc=f"Evaluating '{text}'") as execution_context:
+
             user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
+
+            # TODO. Must be a context hint, not a return value
             reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
 
             ret = self.execute(execution_context, [user_input, reduce_requested], EXECUTE_STEPS)
@@ -425,17 +465,21 @@ class Sheerka(Concept):
             if self.cache_manager.is_dirty:
                 self.cache_manager.commit(execution_context)
 
-            try:
-                if self.save_execution_context and self.load(self.name, "save_execution_context"):
+            # exec_count = ExecutionContext.ids[execution_context.event.get_digest()]
+            # print("Execution Context Count:", exec_count)
+            if self.save_execution_context:
+                try:
+                    # if exec_count > 3400:
+                    #     print("Saving result. digest=", execution_context.event.get_digest())
                     self.sdp.save_result(execution_context)
-            except Exception as ex:
-                self.log.error(f"Failed to save execution context. Reason: {ex}")
+                except Exception as ex:
+                    print(f"Failed to save execution context. Reason: {ex}")
+                    pass
+                    # self.log.error(f"Failed to save execution context. Reason: {ex}")
 
-        # # hack to save valid concept definition
-        # if not self.during_restore:
-        #     if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
-        #         with open(CONCEPTS_FILE, "a") as f:
-        #             f.write(text + "\n")
+            # Do not save execution contexts from process_return_values
+            if self.enable_process_return_values:
+                self.process_return_values(execution_context, ret)
 
         self.execution_count += 1
         self._last_execution = execution_context
@@ -461,17 +505,16 @@ class Sheerka(Concept):
     def set_id_if_needed(self, obj: Concept, is_builtin: bool):
         """
         Set the key for the concept if needed
-        For test purpose only !!!!!
         :param obj:
         :param is_builtin:
         :return:
         """
-        if obj.metadata.id is not None:
+        if obj._metadata.id is not None:
             return
 
         key = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
-        obj.metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key))
-        self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
+        obj._metadata.id = str(self.cache_manager.get(self.CONCEPTS_KEYS_ENTRY, key))
+        # self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
 
     def force_sya_def(self, context, list_of_def):
         """
@@ -588,9 +631,9 @@ class Sheerka(Concept):
         # ##############
         # if the entry is a concept token, use its values.
         if isinstance(concept, Token):
-            if concept.type != TokenKind.CONCEPT:
+            if concept.type == TokenKind.RULE:  # do not recognize rules !!!
                 return None
-            concept = concept.value
+            concept = concept.value  # concept is now a tuple
 
         if isinstance(concept, str) and \
             concept.startswith("c:") and \
@@ -607,7 +650,7 @@ class Sheerka(Concept):
             if concept[1]:
                 if self.is_known(found := self.get_by_id(concept[1])):
                     instance = self.new_from_template(found, found.key)
-                    instance.metadata.is_evaluated = True
+                    instance._metadata.is_evaluated = True
                     return instance
             elif concept[0]:
                 if self.is_known(found := self.get_by_name(concept[0])):
@@ -626,6 +669,28 @@ class Sheerka(Concept):
 
         return None
 
+    def fast_resolve(self, key, return_new=True):
+        def new_instances(concepts):
+            if hasattr(concepts, "__iter__"):
+                return [self.new_from_template(c, c.key) for c in concepts]
+            return self.new_from_template(concepts, concepts.key)
+
+        if isinstance(key, Token):
+            if key.type == TokenKind.RULE:  # do not recognize rules !!!
+                return None
+
+            if key.value[1]:
+                concept = self.cache_manager.get(self.CONCEPTS_BY_ID_ENTRY, key.value[1])
+            else:
+                concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key.value[0])
+
+        else:
+            concept = self.cache_manager.get(self.CONCEPTS_BY_NAME_ENTRY, key)
+
+        if concept is None:
+            return None
+        return new_instances(concept) if return_new else concept
+
     def has_id(self, concept_id):
         """
         Returns True if a concept with this id exists in cache
@@ -694,13 +759,13 @@ class Sheerka(Concept):
     def new_from_template(self, template, key, **kwargs):
         # core.utils.my_debug(f"Created {template}, {key=}, {kwargs=}")
         # manage singleton
-        if template.metadata.is_unique:
+        if template.get_metadata().is_unique:
             return template
 
         # otherwise, create another instance
         concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
         concept.update_from(template, update_value=False)
-        concept.freeze_definition_hash()
+        # concept.freeze_definition_hash()
 
         if len(kwargs) == 0:
             return concept
@@ -708,17 +773,17 @@ class Sheerka(Concept):
         # update the properties, values, attributes
         # Not quite sure that this is the correct process order
         for k, v in kwargs.items():
-            if k in concept.values:
+            if k in get_concept_attrs(concept):
                 concept.set_value(k, v)
-            elif k in PROPERTIES_FOR_NEW:
-                concept.set_value(ConceptParts(k), v)
+            elif k == "body":
+                concept.set_value(ConceptParts.BODY, v)
             elif hasattr(concept, k):
                 setattr(concept, k, v)
             else:
                 return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept)
 
         # TODO : add the concept to the list of known concepts (self.instances)
-        concept.metadata.is_evaluated = True  # because we have manually set the variables
+        concept._metadata.is_evaluated = True  # because we have manually set the variables
         return concept
 
     def ret(self, who: str, status: bool, value, message=None, parents=None):
@@ -732,22 +797,16 @@ class Sheerka(Concept):
         :return:
         """
 
-        # 1 second saved every twenty seconds in unit tests
         return ReturnValueConcept(
             who=who,
             status=status,
             value=value,
-            message=message,
             parents=parents,
             concept_id=self.return_value_concept_id
         )
-        # return self.new(
-        #     BuiltinConcepts.RETURN_VALUE,
-        #     who=who,
-        #     status=status,
-        #     value=value,
-        #     message=message,
-        #     parents=parents)
+
+    def err(self, body):
+        return ErrorConcept(body, self.error_concept_id)
 
     def objvalue(self, obj, reduce_simple_list=False):
         if obj is None:
@@ -759,7 +818,7 @@ class Sheerka(Concept):
         if not isinstance(obj, Concept):
             return obj
 
-        if obj.body is BuiltinConcepts.NOT_INITIALIZED:
+        if obj.body is NotInit:
             return obj
 
         if reduce_simple_list and (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1:
@@ -796,7 +855,7 @@ class Sheerka(Concept):
         return self.value_by_concept(obj.body, concept)
 
     def get_error(self, obj):
-        if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
+        if isinstance(obj, Concept) and obj._metadata.is_builtin and obj.key in BuiltinErrors:
             return obj
 
         if isinstance(obj, (list, set, tuple)):
@@ -848,9 +907,9 @@ class Sheerka(Concept):
     def test(self):
         return f"I have access to Sheerka !"
 
-    def test_using_context(self, context, param1, param2):
+    def test_using_context(self, context, param):
         event = context.event.get_digest()
-        return f"I have access to Sheerka ! {param1=}, {param2=}, {event=}."
+        return f"I have access to Sheerka ! {param=}, {event=}."
 
     def test_error(self):
         raise Exception("I can raise an error")
@@ -863,14 +922,17 @@ class Sheerka(Concept):
         if isinstance(obj, ReturnValueConcept):
             return obj.status
 
+        if isinstance(obj, ErrorObj):
+            return False
+
         # other cases ?
         # ...
 
         # manage internal errors
-        if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
+        if isinstance(obj, Concept) and obj._metadata.is_builtin and obj.key in BuiltinErrors:
             return False
 
-        return obj
+        return bool(obj)
 
     @staticmethod
     def is_known(obj):
@@ -914,9 +976,7 @@ class Sheerka(Concept):
 
         unknown_concept = UnknownConcept()  # don't use new() for prevent circular reference
         unknown_concept.set_value(ConceptParts.BODY, metadata)
-        for meta in (metadata if isinstance(metadata, list) else [metadata]):
-            unknown_concept.set_value(meta[0], meta[1])
-        unknown_concept.metadata.is_evaluated = True
+        unknown_concept._metadata.is_evaluated = True
         return unknown_concept
 
     @staticmethod
@@ -924,7 +984,7 @@ class Sheerka(Concept):
         res = {}
         for c in core.utils.get_classes("core.builtin_concepts"):
             if issubclass(c, Concept) and c != Concept:
-                res[c().metadata.key] = c
+                res[c()._metadata.key] = c
 
         return res
 
@@ -967,3 +1027,29 @@ class Sheerka(Concept):
         logging.addLevelName(logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR))
         # uncomment the following line to enable colors
         # logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit)
+
+
+def to_profile():
+    sheerka = Sheerka()
+    sheerka.initialize(save_execution_context=False, enable_process_return_values=False)
+    event = Event("test", "kodjoko")
+    execution_context = ExecutionContext(sheerka.name,
+                                         event,
+                                         sheerka,
+                                         BuiltinConcepts.PROCESS_INPUT,
+                                         None)
+
+    profile_push(execution_context)
+
+
+@profile(filename="profile_push")
+def profile_push(execution_context):
+    for i in range(177942):
+        execution_context.push(BuiltinConcepts.NOP,
+                               {"action": "fake"},
+                               execution_context.sheerka.name,
+                               desc="a proper description")
+
+
+if __name__ == '__main__':
+    to_profile()

src/core/sheerka/services/SheerkaAdmin.py

@@ -1,12 +1,14 @@
+import sys
 import time
 from os import path
 
-from core.builtin_concepts import BuiltinConcepts
+from core.builtin_concepts import BuiltinConcepts, BuiltinContainers
+from core.concept import Concept
 from core.sheerka.services.sheerka_service import BaseService
 
 CONCEPTS_FILE_LITE = "_concepts_lite.txt"
-CONCEPTS_FILE_ALL_CONCEPTS = "_concepts.txt"
-CONCEPTS_FILE_TO_USE = CONCEPTS_FILE_ALL_CONCEPTS
+CONCEPTS_FILE_FULL = "_concepts_full.txt"
+CONCEPTS_FILE_TO_USE = CONCEPTS_FILE_FULL
 
 
 class SheerkaAdmin(BaseService):
@@ -22,6 +24,10 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.concepts, False)
         self.sheerka.bind_service_method(self.last_created_concept, False)
         self.sheerka.bind_service_method(self.last_ret, False)
+        self.sheerka.bind_service_method(self.last_error_ret, False)
+        self.sheerka.bind_service_method(self.extended_isinstance, False)
+        self.sheerka.bind_service_method(self.is_container, False)
+        self.sheerka.bind_service_method(self.format_rules, False)
 
     def caches_names(self):
         """
@@ -53,18 +59,19 @@ class SheerkaAdmin(BaseService):
 
         def restore_from_file(file_name):
             _nb_lines, _nb_instructions, _nb_lines_in_error = 0, 0, 0
-            if not path.exists(file_name):
-                self.sheerka.log.error(f"\u001b[31mFile '{file_name}' is not found !\u001b[0m")
+            file_path = path.join(path.dirname(sys.argv[0]), file_name)
+            if not path.exists(file_path):
+                print(f"\u001b[31mFile '{file_path}' is not found !\u001b[0m")
                 return 0, 0, 1
 
-            with open(file_name, "r") as f:
+            with open(file_path, "r") as f:
                 for line in f.readlines():
                     _nb_lines += 1
                     line = line.strip()
 
                     if line.startswith("#import "):
                         to_import = "_concepts_" + line[8:] + ".txt"
-                        self.sheerka.log.info(f"Importing {to_import}")
+                        print(f"Importing {to_import}")
                         res = restore_from_file(to_import)
                         _nb_lines += res[0]
                         _nb_instructions += res[1]
@@ -74,42 +81,49 @@ class SheerkaAdmin(BaseService):
                     if line == "" or line.startswith("#"):
                         continue
 
-                    self.sheerka.log.info(line)
+                    print(line)
                     _nb_instructions += 1
                     res = self.sheerka.evaluate_user_input(line)
                     if len(res) > 1 or not res[0].status:
                         _nb_lines_in_error += 1
-                        self.sheerka.log.error("\u001b[31mError detected !\u001b[0m")
+                        print("\u001b[31mError detected !\u001b[0m")
 
             return _nb_lines, _nb_instructions, _nb_lines_in_error
 
-        if concept_file == "full":
-            concept_file = CONCEPTS_FILE_ALL_CONCEPTS
-
-        elif not concept_file.startswith("_concepts"):
+        if not concept_file.startswith("_concepts"):
             concept_file = f"_concepts_{concept_file}.txt"
 
         try:
             start = time.time_ns()
             self.sheerka.during_restore = True
+            self.sheerka.save_execution_context = False
+            enable_process_return_values_previous_value = self.sheerka.enable_process_return_values
+            self.sheerka.enable_process_return_values = False
+
             nb_lines, nb_instructions, nb_lines_in_error = restore_from_file(concept_file)
+
+            self.sheerka.enable_process_return_values = enable_process_return_values_previous_value
+            self.sheerka.save_execution_context = True
             self.sheerka.during_restore = False
             stop = time.time_ns()
 
             nano_sec = stop - start
             dt = nano_sec / 1e6
             elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
-            self.sheerka.log.info(f"Imported {nb_lines} line(s) in {elapsed}.")
-            self.sheerka.log.info(f"{nb_instructions} instruction(s).")
+            print(f"Imported {nb_lines} line(s) in {elapsed}.")
+            print(f"{nb_instructions} instruction(s).")
             if nb_lines_in_error > 0:
-                self.sheerka.log.info(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m")
+                print(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m")
             else:
-                self.sheerka.log.info(f"No error.")
+                print(f"No error.")
         except IOError as e:
             raise e
 
     def concepts(self):
-        return self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY)
+        return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY))
+
+    def format_rules(self):
+        return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_format_rules())
 
     def last_created_concept(self, use_history=False):
         for exec_result in reversed(self.sheerka.last_executions):
@@ -124,4 +138,53 @@ class SheerkaAdmin(BaseService):
         return self.sheerka.new(BuiltinConcepts.NOT_FOUND)
 
     def last_ret(self, context, index=-1):
-        return self.sheerka.last_return_values[index]
+        try:
+            last = self.sheerka.last_return_values[index]
+            return last[0] if isinstance(last, list) and len(last) == 1 else last
+        except IndexError:
+            return None
+
+    def last_error_ret(self, context, index=-1):
+        while index >= -len(self.sheerka.last_return_values):
+            last = self.sheerka.last_return_values[index]
+            last = [last] if not hasattr(last, "__iter__") else last
+            last = [ret_val for ret_val in last if not ret_val.status]
+            if len(last) == 0:
+                index -= 1
+                continue
+
+            if len(last) > 1:
+                return context.sheerka.ret(SheerkaAdmin.NAME,
+                                           False,
+                                           context.sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=last))
+
+            return last[0]
+
+        return context.sheerka.ret(SheerkaAdmin.NAME,
+                                   False,
+                                   context.sheerka.new(BuiltinConcepts.NOT_FOUND))
+
+    def extended_isinstance(self, a, b):
+        """
+        switch between sheerka.isinstance and builtin.isinstance
+        :param a:
+        :param b:
+        :return:
+        """
+
+        if isinstance(b, (type, tuple)):
+            return isinstance(a, b)
+
+        return self.sheerka.isinstance(a, b)
+
+    @staticmethod
+    def is_container(obj):
+        """
+        A container concept is a builtin concept that embed a result
+        :param obj:
+        :return:
+        """
+        if not isinstance(obj, Concept):
+            return False
+
+        return obj.key in BuiltinContainers

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -3,7 +3,11 @@ from dataclasses import dataclass
 from cache.Cache import Cache
 from cache.ListCache import ListCache
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import ensure_concept, Concept
+from core.global_symbols import CONCEPT_PRECEDENCE_MODIFIED, RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, \
+    CONCEPT_COMPARISON_CONTEXT
+from core.builtin_helpers import ensure_concept_or_rule
+from core.concept import Concept
+from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
 from core.sheerka.services.sheerka_service import ServiceObj, BaseService
 
 
@@ -43,7 +47,7 @@ class SheerkaComparisonManager(BaseService):
         :return:
         """
         if isinstance(prop_name, Concept):
-            prefix = prop_name.key if prop_name.metadata.is_builtin else prop_name.id
+            prefix = prop_name.key if prop_name.get_metadata().is_builtin else prop_name.id
         else:
             prefix = prop_name
 
@@ -66,9 +70,11 @@ class SheerkaComparisonManager(BaseService):
         for _ in range(len(comparison_objs)):
             for comparison_obj in comparison_objs:
                 if comparison_obj.op == ">":
-                    values[comparison_obj.a] = values[comparison_obj.b] + 1
+                    if values[comparison_obj.a] <= values[comparison_obj.b]:
+                        values[comparison_obj.a] = values[comparison_obj.b] + 1
                 else:
-                    values[comparison_obj.b] = values[comparison_obj.a] + 1
+                    if values[comparison_obj.b] <= values[comparison_obj.a]:
+                        values[comparison_obj.b] = values[comparison_obj.a] + 1
 
         return values
 
@@ -128,17 +134,17 @@ class SheerkaComparisonManager(BaseService):
             res.setdefault(v, []).append(k)
         return res
 
-    def _add_comparison(self, comparison_obj):
+    def _add_comparison(self, context, comparison_obj):
         key = self._compute_key(comparison_obj.property, comparison_obj.context)
         previous = self.sheerka.cache_manager.get(self.COMPARISON_ENTRY, key)
         new = previous.copy() if previous else []
 
         for co in new:
             if co.property == comparison_obj.property and \
-                    co.a == comparison_obj.a and \
-                    co.b == comparison_obj.b and \
-                    co.op == comparison_obj.op and \
-                    co.context == comparison_obj.context:
+                co.a == comparison_obj.a and \
+                co.b == comparison_obj.b and \
+                co.op == comparison_obj.op and \
+                co.context == comparison_obj.context:
                 return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED))
 
         new.append(comparison_obj)
@@ -166,7 +172,7 @@ class SheerkaComparisonManager(BaseService):
 
         cycles = self.detect_cycles(new)
         if cycles:
-            concepts_in_cycle = [self.sheerka.get_by_id(c) for c in cycles]
+            concepts_in_cycle = [self.sheerka.resolve(c) for c in cycles]
             chicken_an_egg = self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_cycle)
             return self.sheerka.ret(self.NAME, False, chicken_an_egg)
 
@@ -175,6 +181,12 @@ class SheerkaComparisonManager(BaseService):
                                                                                                   lesser_objs_ids,
                                                                                                   greatest_objs_ids))
 
+        if comparison_obj.property == BuiltinConcepts.PRECEDENCE:
+            if comparison_obj.context == CONCEPT_COMPARISON_CONTEXT:
+                self.sheerka.publish(context, CONCEPT_PRECEDENCE_MODIFIED)
+            elif comparison_obj.context == RULE_COMPARISON_CONTEXT:
+                self.sheerka.publish(context, RULE_PRECEDENCE_MODIFIED)
+
         return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
     def initialize(self):
@@ -191,41 +203,51 @@ class SheerkaComparisonManager(BaseService):
         self.sheerka.bind_service_method(self.get_partition, False)
         self.sheerka.bind_service_method(self.get_concepts_weights, False)
 
-    def set_is_greater_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
+    def set_is_greater_than(self, context, prop_name, item_a, item_b, comparison_context="#"):
         """
         Records that the property of concept a is greater than concept b's one
         :param context:
         :param prop_name:
-        :param concept_a:
-        :param concept_b:
+        :param item_a:
+        :param item_b:
         :param comparison_context:
         :return:
         """
-        context.log(f"Setting concept {concept_a} is greater than {concept_b}", who=self.NAME)
-        ensure_concept(concept_a, concept_b)
+        context.log(f"Setting item {item_a} is greater than {item_b}", who=self.NAME)
+        ensure_concept_or_rule(item_a, item_b)
 
         event_digest = context.event.get_digest()
-        comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, ">", comparison_context)
-        return self._add_comparison(comparison_obj)
+        comparison_obj = ComparisonObj(event_digest,
+                                       prop_name,
+                                       item_a.str_id,
+                                       item_b.str_id,
+                                       ">",
+                                       comparison_context)
+        return self._add_comparison(context, comparison_obj)
 
-    def set_is_less_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
+    def set_is_less_than(self, context, prop_name, item_a, item_b, comparison_context="#"):
         """
         Records that the property of concept a is lesser than concept b's one
         :param context:
         :param prop_name:
-        :param concept_a:
-        :param concept_b:
+        :param item_a:
+        :param item_b:
         :param comparison_context:
         :return:
         """
-        context.log(f"Setting concept {concept_a} is less than {concept_b}", who=self.NAME)
-        ensure_concept(concept_a, concept_b)
+        context.log(f"Setting item {item_a} is less than {item_b}", who=self.NAME)
+        ensure_concept_or_rule(item_a, item_b)
 
         event_digest = context.event.get_digest()
-        comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, "<", comparison_context)
-        return self._add_comparison(comparison_obj)
+        comparison_obj = ComparisonObj(event_digest,
+                                       prop_name,
+                                       item_a.str_id,
+                                       item_b.str_id,
+                                       "<",
+                                       comparison_context)
+        return self._add_comparison(context, comparison_obj)
 
-    def set_is_lesser(self, context, prop_name, concept, comparison_context="#"):
+    def set_is_lesser(self, context, prop_name, item, comparison_context="#"):
         """
         Records that the concept is less than any other concept if no direct comparison is given
 
@@ -235,18 +257,23 @@ class SheerkaComparisonManager(BaseService):
          * All lesser concepts that have no comparison directive are greater than the others (and share the same weight)
         :param context:
         :param prop_name:
-        :param concept:
+        :param item:
         :param comparison_context:
         :return:
         """
-        context.log(f"Setting concept {concept} is lesser", who=self.NAME)
-        ensure_concept(concept)
+        context.log(f"Setting item {item} is lesser", who=self.NAME)
+        ensure_concept_or_rule(item)
 
         event_digest = context.event.get_digest()
-        comparison_obj = ComparisonObj(event_digest, prop_name, concept.id, None, "<<", comparison_context)
-        return self._add_comparison(comparison_obj)
+        comparison_obj = ComparisonObj(event_digest,
+                                       prop_name,
+                                       item.str_id,
+                                       None,
+                                       "<<",
+                                       comparison_context)
+        return self._add_comparison(context, comparison_obj)
 
-    def set_is_greatest(self, context, prop_name, concept, comparison_context="#"):
+    def set_is_greatest(self, context, prop_name, item, comparison_context="#"):
         """
         Records that the concept is greater than any other concept if no direct comparison is given
 
@@ -256,16 +283,21 @@ class SheerkaComparisonManager(BaseService):
         * All greatest concepts that have no comparison directive are less than the others (and share the same weight)
         :param context:
         :param prop_name:
-        :param concept:
+        :param item:
         :param comparison_context:
         :return:
         """
-        context.log(f"Setting concept {concept} is greatest", who=self.NAME)
-        ensure_concept(concept)
+        context.log(f"Setting item {item} is greatest", who=self.NAME)
+        ensure_concept_or_rule(item)
 
         event_digest = context.event.get_digest()
-        comparison_obj = ComparisonObj(event_digest, prop_name, concept.id, None, ">>", comparison_context)
-        return self._add_comparison(comparison_obj)
+        comparison_obj = ComparisonObj(event_digest,
+                                       prop_name,
+                                       item.str_id,
+                                       None,
+                                       ">>",
+                                       comparison_context)
+        return self._add_comparison(context, comparison_obj)
 
     def set_are_equivalent(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
         """
@@ -281,9 +313,6 @@ class SheerkaComparisonManager(BaseService):
         """
         pass
 
-    def set_are_equiv(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
-        pass
-
     def get_partition(self, prop_name, comparison_context="#"):
         """
         Returns the equivalent classes for the property, using the comparison_context

src/core/sheerka/services/SheerkaConceptsAlgebra.py

@@ -2,7 +2,8 @@ from dataclasses import dataclass
 from operator import attrgetter
 
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ensure_concept
+from core.builtin_helpers import ensure_concept
+from core.concept import Concept
 from core.sheerka.Sheerka import Sheerka
 from core.sheerka.services.sheerka_service import BaseService
 
@@ -81,13 +82,13 @@ class SheerkaConceptsAlgebra(BaseService):
         :param key:
         :return:
         """
-        if key not in source.metadata.props:
+        if key not in source.get_metadata().props:
             return
 
-        if key in destination.metadata.props:
-            destination.metadata.props[key].update(source.metadata.props[key])
+        if key in destination.get_metadata().props:
+            destination.get_metadata().props[key].update(source.get_metadata().props[key])
         else:
-            destination.metadata.props[key] = source.metadata.props[key].copy()
+            destination.get_metadata().props[key] = source.get_metadata().props[key].copy()
 
     def sub_props(self, destination, source, key):
         """
@@ -97,11 +98,11 @@ class SheerkaConceptsAlgebra(BaseService):
         :param key:
         :return:
         """
-        if key not in source.metadata.props or key not in destination.metadata.props:
+        if key not in source.get_metadata().props or key not in destination.get_metadata().props:
             return
 
-        for item in source.metadata.props[key]:
-            destination.metadata.props[key].discard(item)
+        for item in source.get_metadata().props[key]:
+            destination.get_metadata().props[key].discard(item)
 
     def recognize(self, concept, all_scores=False):
         """
@@ -118,7 +119,7 @@ class SheerkaConceptsAlgebra(BaseService):
             return res
 
         all_concepts = self.sheerka.cache_manager.copy(Sheerka.CONCEPTS_BY_ID_ENTRY).values() \
-            if self.sheerka.cache_manager.cache_only else self.sheerka.concepts()
+            if self.sheerka.cache_manager.cache_only else self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY)
 
         for c in all_concepts:
             score = self._compute_score(c, concept, step_b=round(1 / nb_props, 2))
@@ -127,8 +128,8 @@ class SheerkaConceptsAlgebra(BaseService):
 
         if len(res) == 0:
             props = []
-            for p in [p for p in PROPERTIES_TO_COMPUTE if p in concept.metadata.props]:
-                props.append((p, concept.metadata.props[p]))
+            for p in [p for p in PROPERTIES_TO_COMPUTE if p in concept.get_metadata().props]:
+                props.append((p, concept.get_metadata().props[p]))
             return self.sheerka.get_unknown(props)
 
         res.sort(key=attrgetter('score'), reverse=True)
@@ -158,9 +159,9 @@ class SheerkaConceptsAlgebra(BaseService):
 
         # adds step_b for every property that are in both a and b
         for prop in PROPERTIES_TO_COMPUTE:
-            if prop in b.metadata.props and prop in a.metadata.props:
-                for prop_value in b.metadata.props[prop]:
-                    if prop_value in a.metadata.props[prop]:
+            if prop in b.get_metadata().props and prop in a.get_metadata().props:
+                for prop_value in b.get_metadata().props[prop]:
+                    if prop_value in a.get_metadata().props[prop]:
                         score += step_b
 
         if not step_a:
@@ -171,11 +172,11 @@ class SheerkaConceptsAlgebra(BaseService):
 
         # remove step_a for every property that is in a, but not in b
         for prop in PROPERTIES_TO_COMPUTE:
-            if prop in a.metadata.props and prop not in a.metadata.props:
-                score += step_a * len(a.metadata.props)
-            elif prop in a.metadata.props and prop in a.metadata.props:
-                for prop_value in a.metadata.props[prop]:
-                    if prop_value not in b.metadata.props[prop]:
+            if prop in a.get_metadata().props and prop not in a.get_metadata().props:
+                score += step_a * len(a.get_metadata().props)
+            elif prop in a.get_metadata().props and prop in a.get_metadata().props:
+                for prop_value in a.get_metadata().props[prop]:
+                    if prop_value not in b.get_metadata().props[prop]:
                         score -= step_b
 
         return score
@@ -189,6 +190,6 @@ class SheerkaConceptsAlgebra(BaseService):
         """
         nb_props = 0
         for prop in PROPERTIES_TO_COMPUTE:
-            if prop in concept.metadata.props:
-                nb_props += len(concept.metadata.props[prop])
+            if prop in concept.get_metadata().props:
+                nb_props += len(concept.get_metadata().props[prop])
         return nb_props

src/core/sheerka/services/SheerkaCreateNewConcept.py

@@ -1,6 +1,7 @@
 import core.utils
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept
-from core.concept import Concept, DEFINITION_TYPE_DEF, ensure_concept, DEFINITION_TYPE_BNF
+from core.builtin_helpers import ensure_concept
+from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs
 from core.sheerka.services.sheerka_service import BaseService
 from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
 
@@ -28,7 +29,6 @@ class SheerkaCreateNewConcept(BaseService):
         :param concept:  DefConceptNode
         :return: digest of the new concept
         """
-
         ensure_concept(concept)
 
         sheerka = self.sheerka
@@ -50,6 +50,9 @@ class SheerkaCreateNewConcept(BaseService):
         # set id before saving in db
         sheerka.set_id_if_needed(concept, False)
 
+        # freeze attributes
+        freeze_concept_attrs(concept)
+
         # compute new concepts_by_first_keyword
         init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True)
         if not init_ret_value.status:
@@ -68,9 +71,9 @@ class SheerkaCreateNewConcept(BaseService):
         cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
         cache_manager.put(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, resolved_concepts_by_first_keyword)
 
-        if concept.metadata.definition_type == DEFINITION_TYPE_DEF and concept.metadata.definition != concept.name:
+        if concept.get_metadata().definition_type == DEFINITION_TYPE_DEF and concept.get_metadata().definition != concept.name:
             # allow search by definition when definition relevant
-            cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.metadata.definition, concept)
+            cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.get_metadata().definition, concept)
 
         # update references
         for ref in self.compute_references(concept):
@@ -78,7 +81,7 @@ class SheerkaCreateNewConcept(BaseService):
 
         # TODO : this line seems to be useless
         # The grammar is never reset
-        if concept.bnf and init_bnf_ret_value is not None and init_bnf_ret_value.status:
+        if concept.get_bnf() and init_bnf_ret_value is not None and init_bnf_ret_value.status:
             sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
 
         # process the return if needed
@@ -94,10 +97,10 @@ class SheerkaCreateNewConcept(BaseService):
         """
         refs = set()
 
-        if concept.metadata.definition_type == DEFINITION_TYPE_BNF:
+        if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
             from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
             other_concepts_visitor = BnfNodeConceptExpressionVisitor()
-            other_concepts_visitor.visit(concept.bnf)
+            other_concepts_visitor.visit(concept.get_bnf())
 
             for concept in other_concepts_visitor.references:
                 if isinstance(concept, str):

src/core/sheerka/services/SheerkaDebugManager.py

@@ -0,0 +1,440 @@
+import os
+import pprint
+import re
+from dataclasses import dataclass
+
+from core.builtin_concepts import BuiltinConcepts
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.services.sheerka_service import BaseService
+from core.utils import CONSOLE_COLORS_MAP as CCM
+from core.utils import evaluate_expression, as_bag
+
+try:
+    rows, columns = os.popen('stty size', 'r').read().split()
+except ValueError:
+    rows, columns = 50, 80
+
+pp = pprint.PrettyPrinter(indent=2, width=columns)
+
+
+class BaseDebugLogger:
+    ids = {}
+
+    @staticmethod
+    def next_id(hint):
+        if hint in BaseDebugLogger.ids:
+            BaseDebugLogger.ids[hint] += 1
+        else:
+            BaseDebugLogger.ids[hint] = 0
+        return BaseDebugLogger.ids[hint]
+
+    def __init__(self, debug_manager, who, method_name, context_id, debug_id):
+        pass
+
+    def debug_entering(self, **kwargs):
+        pass
+
+    def debug_var(self, name, value, is_error=False):
+        pass
+
+    def debug_rule(self, rule, results):
+        pass
+
+    def debug_log(self, text):
+        pass
+
+
+class NullDebugLogger(BaseDebugLogger):
+    def __init__(self):
+        pass
+
+
+class ConsoleDebugLogger(BaseDebugLogger):
+
+    def __init__(self, debug_manager, service_name, method_name, context_id, debug_id):
+        BaseDebugLogger.__init__(self, debug_manager, service_name, method_name, context_id, debug_id)
+        self.debug_manager = debug_manager
+        self.service_name = service_name
+        self.method_name = method_name
+        self.context_id = context_id
+        self.debug_id = debug_id
+        self.is_highlighted = ""
+
+    def debug_entering(self, **kwargs):
+        super().debug_entering(**kwargs)
+
+        str_text = f"{CCM['blue']}Entering {self.service_name}.{self.method_name} with {CCM['reset']}"
+        str_vars = pp.pformat(kwargs)
+        if "\n" not in str(str_vars):
+            self.debug_manager.debug(self.prefix() + str_text + str_vars)
+        else:
+            self.debug_manager.debug(self.prefix() + str_text)
+            self.debug_manager.debug(self.prefix() + str_vars)
+
+    def debug_var(self, name, value, is_error=False):
+        enabled = self.debug_manager.compute_var_debug(self.service_name,
+                                                       self.method_name,
+                                                       self.context_id,
+                                                       name,
+                                                       self.context_id)
+        if enabled == False:
+            return
+
+        color = 'red' if is_error else 'green'
+        str_text = f"{CCM[color]}..{name}={CCM['reset']}"
+        str_vars = "" if isinstance(enabled, str) else pp.pformat(value)
+        if "\n" not in str(str_vars):
+            self.debug_manager.debug(self.prefix() + str_text + str_vars)
+        else:
+            self.debug_manager.debug(self.prefix() + str_text)
+            self.debug_manager.debug(self.prefix() + str_vars)
+
+    def debug_rule(self, rule, results):
+        if not self.debug_manager.compute_debug_rule(rule.id, self.context_id, self.debug_id):
+            return
+
+        str_text = f"{CCM['green']}..results({rule.id})={CCM['reset']}"
+        str_vars = pp.pformat(results)
+        if "\n" not in str(str_vars):
+            self.debug_manager.debug(self.prefix() + str_text + str_vars)
+        else:
+            self.debug_manager.debug(self.prefix() + str_text)
+            self.debug_manager.debug(self.prefix() + str_vars)
+
+    def debug_log(self, text):
+        self.debug_manager.debug(self.prefix() + f"{CCM['blue']}..{text}{CCM['reset']}")
+
+    def prefix(self):
+        return f"[{self.context_id:2}][{self.debug_id:2}] {self.is_highlighted}"
+
+
+@dataclass
+class DebugVarSetting:
+    service_name: str
+    method_name: str
+    variable_name: str
+    context_id: int
+    context_children: bool
+    debug_id: int
+    debug_children: bool
+
+    enabled: bool
+
+
+@dataclass
+class DebugRuleSetting:
+    rule_id: str
+    context_id: int
+    debug_id: int
+
+    enabled: bool
+
+
+class SheerkaDebugManager(BaseService):
+    NAME = "Debug"
+    PREFIX = "debug."
+
+    children_activation_regex = re.compile(r"(\d+)\+")
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.activated = False  # is debug activated
+        self.explicit = False  # No need to activate context debug when debug mode is on
+        self.context_cache = set()  # debug for specific context
+        self.variable_cache = set()  # debug for specific variable
+        self.debug_vars_settings = []
+        self.debug_rules_settings = []
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.set_debug, True)
+        self.sheerka.bind_service_method(self.set_explicit, True)
+        self.sheerka.bind_service_method(self.activate_debug_for, True)
+        self.sheerka.bind_service_method(self.deactivate_debug_for, True)
+        self.sheerka.bind_service_method(self.debug_activated, False)
+        self.sheerka.bind_service_method(self.debug_activated_for, False)
+        self.sheerka.bind_service_method(self.get_context_debug_mode, False)
+        self.sheerka.bind_service_method(self.debug_rule, True)
+        self.sheerka.bind_service_method(self.debug_rule_activated, False)
+        self.sheerka.bind_service_method(self.inspect, False)
+        self.sheerka.bind_service_method(self.debug, False, visible=False)
+        self.sheerka.bind_service_method(self.get_debugger, False)
+        self.sheerka.bind_service_method(self.debug_var, False)
+        self.sheerka.bind_service_method(self.reset_debug, False)
+        self.sheerka.bind_service_method(self.get_debug_settings, False, as_name="debug_settings")
+
+    def initialize_deferred(self, context, is_first_time):
+        self.restore_values("activated",
+                            "explicit",
+                            "context_cache",
+                            "variable_cache",
+                            "debug_vars_settings",
+                            "debug_rules_settings")
+
+    def reset(self):
+        """
+        For test purpose
+        :return:
+        """
+        self.activated = False
+        self.context_cache.clear()
+        self.variable_cache.clear()
+        self.debug_vars_settings.clear()
+        self.debug_rules_settings.clear()
+
+    def set_debug(self, context, value=True):
+        self.activated = value
+        self.sheerka.record_var(context, self.NAME, "activated", self.activated)
+        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def set_explicit(self, context, value=True):
+        self.explicit = value
+        self.sheerka.record_var(context, self.NAME, "explicit", self.explicit)
+        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def activate_debug_for(self, context, debug_id, children=False):
+        """
+
+        :param context:
+        :param debug_id: if debug_id is str, activate variable cache, context_cache otherwise
+        :param children:
+        :return:
+        """
+        # preprocess
+        if isinstance(debug_id, str) and (m := self.children_activation_regex.match(debug_id)):
+            debug_id = int(m.group(1))
+            children = True
+
+        if isinstance(debug_id, str):
+            self.variable_cache.add(debug_id)
+            self.sheerka.record_var(context, self.NAME, "variable_cache", self.variable_cache)
+        else:
+            self.context_cache.add(debug_id)
+            if children:
+                self.context_cache.add(str(debug_id) + "+")
+            self.sheerka.record_var(context, self.NAME, "context_cache", self.context_cache)
+        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def deactivate_debug_for(self, context, debug_id, children=False):
+        if isinstance(debug_id, str):
+            self.variable_cache.discard(debug_id)
+            self.sheerka.record_var(context, self.NAME, "variable_cache", self.variable_cache)
+        else:
+            self.context_cache.discard(debug_id)
+            if children:
+                self.context_cache.discard(str(debug_id) + "+")
+            self.sheerka.record_var(context, self.NAME, "context_cache", self.context_cache)
+        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def debug_activated(self):
+        return self.activated
+
+    def debug_activated_for(self, debug_id):
+        if not self.activated:
+            return None
+
+        return debug_id in self.variable_cache
+
+    def debug_rule_activated(self, rule_id, context_id):
+        """
+
+        :param rule_id:
+        :param context_id:
+        :return:
+        """
+        key = f"{rule_id}|{context_id}"
+        return key in self.rules_cache
+
+    def get_context_debug_mode(self, context_id):
+        if not self.activated:
+            return None, None
+
+        debug_for_children = "protected" if str(context_id) + "+" in self.context_cache else None
+        debug_for_self = "private" if not self.explicit or context_id in self.context_cache else None
+
+        return debug_for_self, debug_for_children
+
+    def inspect(self, context, context_id, *props):
+        """
+        Print
+        :param context:
+        :param context_id:
+        :return:
+        """
+        to_inspect = self.sheerka.get_execution_item(context, context_id)
+        if not isinstance(to_inspect, ExecutionContext):
+            return to_inspect
+
+        if not props:
+            props = ["inputs", "values.return_values"]
+
+        bag = as_bag(to_inspect)
+        res = {}
+        for prop in props:
+            res[prop] = evaluate_expression(prop, bag)
+        # res = {
+        #     "return_values": to_inspect.values.get("return_values", None)
+        # }
+
+        pp.pprint(res)
+        return None
+
+    def debug(self, *args, **kwargs):
+        print(*args, **kwargs)
+
+    def get_debugger(self, context, who, method_name):
+        if self.compute_debug(who, method_name, context):
+            debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id))
+            return ConsoleDebugLogger(self, who, method_name, context.id, debug_id)
+
+        return NullDebugLogger()
+
+    def debug_var(self, context,
+                  service=None,
+                  method=None,
+                  variable=None,
+                  context_id=None,
+                  context_children=False,
+                  debug_id=None,
+                  debug_children=False,
+                  enabled=True):
+
+        for setting in self.debug_vars_settings:
+            if setting.service_name == service and \
+                setting.method_name == method and \
+                setting.variable_name == variable and \
+                setting.context_id == context_id and \
+                setting.context_children == context_children and \
+                setting.debug_id == debug_id and \
+                setting.debug_children == debug_children:
+                setting.enabled = enabled
+                break
+        else:
+            self.debug_vars_settings.append(DebugVarSetting(service,
+                                                            method,
+                                                            variable,
+                                                            context_id,
+                                                            context_children,
+                                                            debug_id,
+                                                            debug_children,
+                                                            enabled))
+
+        self.sheerka.record_var(context, self.NAME, "debug_vars_settings", self.debug_vars_settings)
+        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def reset_debug(self, context):
+        self.debug_vars_settings.clear()
+        self.debug_rules_settings.clear()
+        self.sheerka.record_var(context, self.NAME, "debug_vars_settings", self.debug_vars_settings)
+        self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_vars_settings)
+        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def compute_debug(self, service_name, method_name, context):
+        if not self.activated:
+            return False
+
+        selected = []
+        for setting in self.debug_vars_settings:
+            if setting.service_name is None and setting.method_name is None and setting.context_id is None:
+                continue
+
+            if (setting.service_name is None or setting.service_name == service_name) and \
+                (setting.method_name is None or setting.method_name == method_name) and \
+                (setting.context_id is None or setting.context_id == context.id or (
+                    setting.context_children and context.has_parent(setting.context_id))):
+                selected.append(setting.enabled)
+
+        if len(selected) == 0:
+            return False
+
+        res = selected[0]
+        for enabled in selected[1:]:
+            res &= enabled
+
+        return res
+
+    def compute_var_debug(self, service_name, method_name, context_id, variable_name, debug_id):
+        if not self.activated:
+            return False
+
+        selected = []
+        for setting in self.debug_vars_settings:
+            if setting.variable_name is None and setting.debug_id is None:
+                continue
+
+            if (setting.service_name is None or setting.service_name == service_name) and \
+                (setting.method_name is None or setting.method_name == method_name) and \
+                (setting.context_id is None or setting.context_id == context_id) and \
+                (setting.variable_name is None or
+                 setting.variable_name == "*" or
+                 setting.variable_name == variable_name) and \
+                (setting.debug_id is None or setting.debug_id == debug_id):
+                selected.append(setting.enabled)
+
+        if len(selected) == 0:
+            return False
+
+        res = selected[0]
+        for enabled in selected[1:]:
+            if res == False or enabled == False:
+                return False
+
+            if isinstance(res, str):
+                continue
+
+            res = enabled
+
+        return res
+
+    def debug_rule(self, context, rule=None, context_id=None, debug_id=None, enabled=True):
+        """
+        Add a debug rule request
+        :param context:
+        :param rule:
+        :param context_id:
+        :param debug_id:
+        :param enabled:
+        :return:
+        """
+        rule = str(rule) if rule is not None else None
+        for setting in self.debug_rules_settings:
+            if setting.rule_id == rule and \
+                setting.context_id == context_id and \
+                setting.debug_id == debug_id:
+                setting.enabled = enabled
+                break
+        else:
+            self.debug_rules_settings.append(DebugRuleSetting(rule,
+                                                              context_id,
+                                                              debug_id,
+                                                              enabled))
+
+        self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings)
+        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def compute_debug_rule(self, rule_id, context_id, debug_id):
+        if not self.activated:
+            return False
+
+        selected = []
+        for setting in self.debug_rules_settings:
+            if (setting.rule_id is None or setting.rule_id == rule_id) and \
+                (setting.context_id is None or setting.context_id == context_id) and \
+                (setting.debug_id is None or setting.debug_id == debug_id):
+                selected.append(setting.enabled)
+
+        if len(selected) == 0:
+            return False
+
+        res = selected[0]
+        for enabled in selected[1:]:
+            res &= enabled
+
+        return res
+
+    def reset_debug_rules(self, context):
+        self.debug_rules_settings.clear()
+        self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings)
+        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
+    def get_debug_settings(self):
+        return self.debug_vars_settings

src/core/sheerka/services/SheerkaDump.py

@@ -21,7 +21,7 @@ class SheerkaDump(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.dump_desc, True, "desc") # because concept is evaluated
+        self.sheerka.bind_service_method(self.dump_desc, True, "desc")  # has_side_effect 'cause concept is evaluated
         self.sheerka.bind_service_method(self.dump_sdp, False, "dump_sdp")
 
     def dump_desc(self, *concept_names, eval=False):
@@ -34,7 +34,7 @@ class SheerkaDump(BaseService):
             else:
                 concepts = self.sheerka.get_by_key(concept_name)
                 if self.sheerka.isinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT):
-                    self.sheerka.log.error(f"Concept '{concept_name}' is unknown")
+                    print(f"Concept '{concept_name}' is unknown")
                     return False
 
             if not hasattr(concepts, "__iter__"):
@@ -46,36 +46,35 @@ class SheerkaDump(BaseService):
                     value = evaluated.body if evaluated.key == c.key else evaluated
 
                 if not first:
-                    self.sheerka.log.info("")
-                self.sheerka.log.info(f"id          : {c.id}")
-                self.sheerka.log.info(f"name        : {c.name}")
-                self.sheerka.log.info(f"key         : {c.key}")
-                self.sheerka.log.info(f"definition  : {c.metadata.definition}")
-                self.sheerka.log.info(f"type        : {c.metadata.definition_type}")
-                self.sheerka.log.info(f"body        : {c.metadata.body}")
-                self.sheerka.log.info(f"where       : {c.metadata.where}")
-                self.sheerka.log.info(f"pre         : {c.metadata.pre}")
-                self.sheerka.log.info(f"post        : {c.metadata.post}")
-                self.sheerka.log.info(f"ret         : {c.metadata.ret}")
-                self.sheerka.log.info(f"vars        : {c.metadata.variables}")
-                self.sheerka.log.info(f"props       : {c.metadata.props}")
+                    print("")
+                print(f"id          : {c.id}")
+                print(f"name        : {c.name}")
+                print(f"key         : {c.key}")
+                print(f"definition  : {c.get_metadata().definition}")
+                print(f"type        : {c.get_metadata().definition_type}")
+                print(f"body        : {c.get_metadata().body}")
+                print(f"where       : {c.get_metadata().where}")
+                print(f"pre         : {c.get_metadata().pre}")
+                print(f"post        : {c.get_metadata().post}")
+                print(f"ret         : {c.get_metadata().ret}")
+                print(f"vars        : {c.get_metadata().variables}")
+                print(f"props       : {c.get_metadata().props}")
                 if eval:
-                    self.sheerka.log.info(f"value  : {value}")
-                if c.values:
-                    for v in c.values:
-                        self.sheerka.log.info(f"{v}: {c.get_value(v)}")
+                    print(f"value  : {value}")
+                for v in c.values():
+                    print(f"{v}: {c.get_value(v)}")
                 else:
-                    self.sheerka.log.info("No variable")
+                    print("No variable")
 
-                self.sheerka.log.info(f"digest : {c.get_origin()}")
+                print(f"digest : {c.get_origin()}")
 
                 if self.sheerka.isaset(context, c):
                     items = self.sheerka.get_set_elements(context, c)
-                    self.sheerka.log.info(f"elements  : {items}")
+                    print(f"elements  : {items}")
 
                 first = False
 
     def dump_sdp(self):
         snapshot = self.sheerka.sdp.get_snapshot(SheerkaDataProvider.HeadFile)
         state = self.sheerka.sdp.load_state(snapshot)
-        self.sheerka.log.info(get_pp().pformat(state.data))
+        print(get_pp().pformat(state.data))

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -1,8 +1,9 @@
 from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, ensure_concept
+from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate, ensure_concept
+from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit, AllConceptParts, \
+    concept_part_value
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
@@ -47,8 +48,10 @@ class SheerkaEvaluateConcept(BaseService):
 
         parent = context.get_parent()
         while parent is not None:
-            if parent.who == context.who and parent.action == BuiltinConcepts.EVALUATING_CONCEPT and \
-                parent.obj == concept and parent.obj.compiled == concept.compiled:
+            if (parent.who == context.who and
+                parent.action == BuiltinConcepts.EVALUATING_CONCEPT and
+                parent.obj == concept and
+                parent.obj.get_compiled() == concept.get_compiled()):
                 return True
 
             parent = parent.get_parent()
@@ -90,15 +93,15 @@ class SheerkaEvaluateConcept(BaseService):
         vars_needed = False
         body_needed = False
 
-        if concept_part in concept.compiled and concept.compiled[concept_part] is not None:
-            concept_part_source = getattr(concept.metadata, concept_part.value)
+        if concept_part in concept.get_compiled() and concept.get_compiled()[concept_part] is not None:
+            concept_part_source = getattr(concept.get_metadata(), concept_part_value(concept_part))
 
             assert concept_part_source is not None
 
             tokens = [t.str_value for t in Tokenizer(concept_part_source)]
 
             if check_vars:
-                for var_name in (v[0] for v in concept.metadata.variables):
+                for var_name in (v[0] for v in concept.get_metadata().variables):
                     if var_name in tokens:
                         vars_needed = True
                         ret.append("variables")
@@ -106,9 +109,9 @@ class SheerkaEvaluateConcept(BaseService):
 
             if check_body and "self" in tokens:
                 body_needed = True
-                ret.append("body")
+                ret.append(ConceptParts.BODY)
 
-            ret.append(concept_part.value)
+            ret.append(concept_part)
 
         return ret, vars_needed, body_needed
 
@@ -121,16 +124,16 @@ class SheerkaEvaluateConcept(BaseService):
         :param var_name:
         :return:
         """
-        if concept.metadata.where is None or concept.metadata.where.strip() == "":
+        if concept.get_metadata().where is None or concept.get_metadata().where.strip() == "":
             return None
 
-        ret = ExpressionParser().parse(context, ParserInput(concept.metadata.where))
+        ret = ExpressionParser().parse(context, ParserInput(concept.get_metadata().where))
         if not ret.status:
             # TODO: manage invalid where clause
             return None
         expr = ret.body.body
 
-        to_trueify = [v[0] for v in concept.metadata.variables if v[0] != var_name]
+        to_trueify = [v[0] for v in concept.get_metadata().variables if v[0] != var_name]
         trueified_where = str(TrueifyVisitor(to_trueify, [var_name]).visit(expr))
 
         tokens = [t.str_value for t in Tokenizer(trueified_where)]
@@ -140,7 +143,7 @@ class SheerkaEvaluateConcept(BaseService):
                 compiled = compile(trueified_where, "<where clause>", "eval")
             except Exception:
                 pass
-            return WhereClauseDef(concept, concept.metadata.where, trueified_where, var_name, compiled)
+            return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, compiled)
         else:
             return None
 
@@ -153,7 +156,8 @@ class SheerkaEvaluateConcept(BaseService):
         :return:
         """
         ret = []
-        for r in [r for r in return_values if r.status]:
+        valid_return_values = [r for r in return_values if r.status]
+        for r in valid_return_values:
             if where_clause_def.compiled:
                 try:
                     if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}):
@@ -177,10 +181,13 @@ class SheerkaEvaluateConcept(BaseService):
         if len(ret) > 0:
             return ret
 
+        reason = [r.body for r in return_values] if len(valid_return_values) == 0 else None
+
         return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED,
                                 body=where_clause_def.clause,
                                 concept=where_clause_def.concept,
-                                prop=where_clause_def.prop)
+                                prop=where_clause_def.prop,
+                                reason=reason)
 
     def manage_infinite_recursion(self, context):
         """
@@ -194,7 +201,7 @@ class SheerkaEvaluateConcept(BaseService):
         concepts_found = set()
         while parent and parent.obj:
             if parent.who == context.who and parent.action == BuiltinConcepts.EVALUATING_CONCEPT:
-                body = parent.obj.metadata.body
+                body = parent.obj.get_metadata().body
                 try:
                     return self.sheerka.ret(self.NAME, True, InfiniteRecursionResolved(eval(body)))
                 except Exception:
@@ -226,11 +233,11 @@ class SheerkaEvaluateConcept(BaseService):
                 return self.sheerka.resolve(identifier)
             return None
 
-        for part_key in ConceptParts:
-            if part_key in concept.compiled:
+        for part_key in AllConceptParts:
+            if part_key in concept.get_compiled():
                 continue
 
-            source = getattr(concept.metadata, part_key.value)
+            source = getattr(concept.get_metadata(), concept_part_value(part_key))
             if source is None:  # or not isinstance(source, str):
                 continue
 
@@ -238,22 +245,22 @@ class SheerkaEvaluateConcept(BaseService):
                 raise Exception("Invalid concept init. metadata must be a string")
 
             if source.strip() == "":
-                concept.compiled[part_key] = DoNotResolve(source)
+                concept.get_compiled()[part_key] = DoNotResolve(source)
             else:
                 # first case, when the metadata references another concept via c:xxx: keyword
                 if concept_found := parse_token_concept(source):
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
-                    concept.compiled[part_key] = concept_found
+                    concept.get_compiled()[part_key] = concept_found
                 else:
                     res = parse_unrecognized(context,
                                              source,
                                              parsers="all",
                                              prop=part_key,
                                              filter_func=only_successful)
-                    concept.compiled[part_key] = res.body.body if is_only_successful(res) else res
+                    concept.get_compiled()[part_key] = res.body.body if is_only_successful(res) else res
 
-        for var_name, default_value in concept.metadata.variables:
-            if var_name in concept.compiled:
+        for var_name, default_value in concept.get_metadata().variables:
+            if var_name in concept.get_compiled():
                 continue
 
             if default_value is None:
@@ -263,23 +270,23 @@ class SheerkaEvaluateConcept(BaseService):
                 raise Exception("Invalid concept init. variable metadata must be a string")
 
             if default_value.strip() == "":
-                concept.compiled[var_name] = DoNotResolve(default_value)
+                concept.get_compiled()[var_name] = DoNotResolve(default_value)
             else:
                 # first case, when the metadata references another concept via c:xxx: keyword
                 if concept_found := parse_token_concept(default_value):
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
-                    concept.compiled[var_name] = concept_found
+                    concept.get_compiled()[var_name] = concept_found
                 else:
                     res = parse_unrecognized(context,
                                              default_value,
                                              parsers="all",
                                              prop=var_name,
                                              filter_func=only_successful)
-                    concept.compiled[var_name] = res.body.body if is_only_successful(res) else res
+                    concept.get_compiled()[var_name] = res.body.body if is_only_successful(res) else res
 
         # Updates the cache of concepts when possible
         if self.sheerka.has_id(concept.id):
-            self.sheerka.get_by_id(concept.id).compiled = concept.compiled
+            self.sheerka.get_by_id(concept.id).set_compiled(concept.get_compiled())
 
     def resolve(self,
                 context,
@@ -325,8 +332,8 @@ class SheerkaEvaluateConcept(BaseService):
         with context.push(BuiltinConcepts.EVALUATING_ATTRIBUTE,
                           current_prop,
                           desc=desc,
-                          obj=current_concept,
-                          path=path) as sub_context:
+                          obj=current_concept) as sub_context:
+            sub_context.add_inputs(path=path)
 
             if force_evaluation:
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
@@ -352,7 +359,7 @@ class SheerkaEvaluateConcept(BaseService):
             else:
                 # update short term memory with current concept variables
                 if current_concept:
-                    for var in current_concept.metadata.variables:
+                    for var in current_concept.get_metadata().variables:
                         value = current_concept.get_value(var[0])
                         if value != NotInit:
                             sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0]))
@@ -433,23 +440,23 @@ class SheerkaEvaluateConcept(BaseService):
         :return: value of the evaluation or error
         """
 
-        if concept.metadata.is_evaluated:
+        if concept.get_metadata().is_evaluated:
             return concept
 
         # I cannot use cache because of concept like 'number'.
         # They don't have variables, but their values change every time they are instantiated
         # TODO: Need to find a way to cache despite of them
         # need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)
-        # if need_body and len(concept.metadata.variables) == 0 and context.sheerka.has_id(concept.id):
+        # if need_body and len(concept.get_metadata().variables) == 0 and context.sheerka.has_id(concept.id):
         #     from_cache = context.sheerka.get_by_id(concept.id)
-        #     if from_cache.metadata.is_evaluated:
+        #     if from_cache.get_metadata().is_evaluated:
         #         concept.set_value(ConceptParts.BODY, from_cache.body)
-        #         concept.metadata.is_evaluated = True
+        #         concept.get_metadata().is_evaluated = True
         #         return concept
 
         desc = f"Evaluating concept {concept}"
-        with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc, eval_body=eval_body) as sub_context:
-
+        with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc) as sub_context:
+            sub_context.add_inputs(eval_body=eval_body)
             if eval_body:
                 # ask for body evaluation
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
@@ -466,8 +473,8 @@ class SheerkaEvaluateConcept(BaseService):
 
             for metadata_to_eval in all_metadata_to_eval:
                 if metadata_to_eval == "variables":
-                    for var_name in (v for v in concept.variables() if v in concept.compiled):
-                        prop_ast = concept.compiled[var_name]
+                    for var_name in (v for v in concept.variables() if v in concept.get_compiled()):
+                        prop_ast = concept.get_compiled()[var_name]
 
                         w_clause = self.get_where_clause_def(context, concept, var_name)
                         # TODO, manage when the where clause cannot be parsed
@@ -485,17 +492,17 @@ class SheerkaEvaluateConcept(BaseService):
                         else:
                             concept.set_value(var_name, resolved)
                 else:
-                    part_key = ConceptParts(metadata_to_eval)
+                    part_key = metadata_to_eval
 
                     # do not evaluate where when the body is a set
                     # Indeed, the way that the where clause is expressed is not a valid python or concept code
                     if part_key == ConceptParts.WHERE and self.sheerka.isaset(sub_context, concept.body):
                         continue
 
-                    if part_key not in concept.compiled or concept.compiled[part_key] is None:
+                    if part_key not in concept.get_compiled() or concept.get_compiled()[part_key] is None:
                         continue
 
-                    metadata_ast = concept.compiled[part_key]
+                    metadata_ast = concept.get_compiled()[part_key]
 
                     # if part_key is PRE, POST or WHERE, the concept need to be evaluated
                     # if we want the predicates to be resolved => so force_eval = True
@@ -514,7 +521,7 @@ class SheerkaEvaluateConcept(BaseService):
                     # validate PRE and WHERE condition
                     if part_key in (ConceptParts.PRE, ConceptParts.WHERE) and not self.sheerka.objvalue(resolved):
                         return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED,
-                                                body=getattr(concept.metadata, metadata_to_eval),
+                                                body=getattr(concept.get_metadata(), concept_part_value(metadata_to_eval)),
                                                 concept=concept,
                                                 prop=part_key)
 
@@ -524,19 +531,19 @@ class SheerkaEvaluateConcept(BaseService):
 
             concept.init_key()  # Necessary for old unit tests. To remove someday
 
-            if "body" in all_metadata_to_eval:
-                concept.metadata.is_evaluated = True
+            if ConceptParts.BODY in all_metadata_to_eval:
+                concept.get_metadata().is_evaluated = True
 
             # # update the cache for concepts with no variables
             # Cannot use cache. See the comment at the beginning of this method
-            # if len(concept.metadata.variables) == 0:
+            # if len(concept.get_metadata().variables) == 0:
             #     self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
 
-            if not concept.metadata.is_builtin:
+            if not concept.get_metadata().is_builtin:
                 self.sheerka.register_object(sub_context, concept.name, concept)
 
             # manage RET metadata
-            if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values:
+            if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values():
                 return concept.get_value(ConceptParts.RET)
             else:
                 return concept
@@ -547,7 +554,7 @@ class SheerkaEvaluateConcept(BaseService):
         needed, variables, body = self.get_needed_metadata(concept, ConceptParts.PRE, True, True)
         to_eval.extend(needed)
 
-        if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.metadata.need_validation:
+        if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.get_metadata().need_validation:
             # What are the cases where we do not need a validation ?
             # see test_sheerka_non_reg::test_i_can_evaluate_bnf_concept_with_where_clause()
             # res = sheerka.evaluate_user_input("foobar")
@@ -572,7 +579,7 @@ class SheerkaEvaluateConcept(BaseService):
                 to_eval.append('variables')
 
             if not body:
-                to_eval.append("body")
+                to_eval.append(ConceptParts.BODY)
 
         return to_eval
 

src/core/sheerka/services/SheerkaEvaluateRules.py

@@ -0,0 +1,114 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.builtin_helpers import expect_one
+from core.sheerka.services.sheerka_service import BaseService
+from evaluators.ConceptEvaluator import ConceptEvaluator
+
+DISABLED_RULES = "#disabled#"
+LOW_PRIORITY_RULES = "#low_priority#"
+
+
+class SheerkaEvaluateRules(BaseService):
+    NAME = "EvaluateRules"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.evaluators_by_name = None
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.evaluate_format_rules, False)
+        self.reset_evaluators()
+
+    def reset_evaluators(self):
+        # instantiate evaluators, once for all, only keep when it's enabled
+        evaluators = [e_class() for e_class in self.sheerka.evaluators]
+        evaluators = [e for e in evaluators if e.enabled]
+        self.evaluators_by_name = {e.short_name: e for e in evaluators}
+
+    def evaluate_format_rules(self, context, bag, disabled):
+        return self.evaluate_rules(context, self.sheerka.get_format_rules(), bag, disabled)
+
+    def evaluate_rules(self, context, rules, bag, disabled):
+        """
+        evaluate the format rules, in the context of 'bag'
+        CAUTION : the rules MUST be sorted by priority
+        :param context:
+        :param rules:
+        :param bag:
+        :param disabled: disabled rules (because they have already been fired or whatever)
+        :return: { True : list of success, False :list of failed, '#disabled"': list of disabled...}
+        """
+        with context.push(BuiltinConcepts.EVALUATING_RULES, bag, desc="Evaluating rules...") as sub_context:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+            sub_context.add_inputs(bag=bag)
+
+            debugger = sub_context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rules")
+            debugger.debug_entering(bag=bag)
+
+            results = {}
+
+            sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
+            success_priority = None
+            for rule in rules:
+                if not rule.metadata.is_enabled or rule.id in disabled:
+                    results.setdefault(DISABLED_RULES, []).append(rule)
+                    continue
+
+                if success_priority and rule.priority != success_priority:
+                    results.setdefault(LOW_PRIORITY_RULES, []).append(rule)
+                    continue
+
+                res = self.evaluate_rule(sub_context, rule, bag)
+                ok = res.status and self.sheerka.is_success(self.sheerka.objvalue(res))
+                results.setdefault(ok, []).append(rule)
+                if ok and success_priority is None:
+                    success_priority = rule.priority
+
+            debugger.debug_var("results", self.get_debug_format(results))
+
+            sub_context.add_values(rules_result=results)
+            return results
+
+    def evaluate_rule(self, context, rule, bag):
+        """
+        Evaluate all the predicate
+        :param context:
+        :param rule:
+        :param bag:
+        :return:
+        """
+
+        results = []
+        for rule_predicate in rule.compiled_predicate:
+
+            if rule_predicate.source in bag:
+                # simple case where the rule is an item of the bag. No need of complicate evaluation
+                results.append(context.sheerka.ret(self.NAME, True, bag[rule_predicate.source]))
+
+            else:
+
+                # do not forget to reset the 'is_evaluated' in the case of a concept
+                if rule_predicate.evaluator == ConceptEvaluator.NAME:
+                    rule_predicate.concept.get_metadata().is_evaluated = False
+
+                evaluator = self.evaluators_by_name[rule_predicate.evaluator]
+                results.append(evaluator.eval(context, rule_predicate.predicate))
+
+        debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule")
+        debugger.debug_rule(rule, results)
+        # if context.sheerka.debug_rule_activated(rule_id, context.id):
+        #     context.debug(SheerkaEvaluateRules.NAME, "evaluate_rules", f"result(#{rule_id})", results)
+
+        return expect_one(context, results)
+
+    @staticmethod
+    def get_debug_format(result):
+        """
+        Return the same dictionary, the with the short formatting of the rules
+        eg without the action clause
+        :param result:
+        :return:
+        """
+        return {key: [str(r) if key == True else r.short_str() for r in rules] for key, rules in result.items()}

src/core/sheerka/services/SheerkaEventManager.py

@@ -0,0 +1,60 @@
+from threading import RLock
+
+from core.sheerka.services.sheerka_service import BaseService
+
+
+class SheerkaEventManager(BaseService):
+    """
+    This class implement a very basic publish and subscribe mechanism
+    It supposes that the subscriber has a little knowledge of how the publisher works
+    """
+    NAME = "EventManager"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self._lock = RLock()
+        self.subscribers = {}
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.subscribe, True, visible=False)
+        self.sheerka.bind_service_method(self.publish, True, visible=False)
+
+    def subscribe(self, topic, callback):
+        """
+        To subscribe to a topic, just give the callback to call
+        Note that the callback must be a function whose first argument is a context
+        :param topic:
+        :param callback:
+        :return:
+        """
+        with self._lock:
+            self.subscribers.setdefault(topic, []).append(callback)
+
+    def publish(self, context, topic, data=None):
+        """
+        Publish on a topic
+        The data is not mandatory
+        :param context:
+        :param topic:
+        :param data:
+        :return:
+        """
+        with self._lock:
+            try:
+                subscribers = self.subscribers[topic]
+                if data:
+                    for callback in subscribers:
+                        callback(context, data)
+                else:
+                    for callback in subscribers:
+                        callback(context)
+            except KeyError:
+                pass
+
+    def reset_topic(self, topic):
+        """
+        Remove all subsccribers from a given topic
+        :param topic:
+        :return:
+        """
+        self.subscribers[topic].clear()

src/core/sheerka/services/SheerkaExecute.py

@@ -5,6 +5,16 @@ from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer, TokenKind, Token
 
 NO_MATCH = "** No Match **"
+EVALUATOR_STEPS = [
+    BuiltinConcepts.BEFORE_PARSING,
+    BuiltinConcepts.AFTER_PARSING,
+    BuiltinConcepts.BEFORE_EVALUATION,
+    BuiltinConcepts.EVALUATION,
+    BuiltinConcepts.AFTER_EVALUATION,
+    BuiltinConcepts.BEFORE_RENDERING,
+    BuiltinConcepts.RENDERING,
+    BuiltinConcepts.AFTER_RENDERING,
+]
 
 
 class ParserInput:
@@ -74,10 +84,10 @@ class ParserInput:
             if self.start == 0 and self.end == self.length:
                 self.sub_text = self.text
                 return self.sub_text
-            self.sub_text = self.get_text_from_tokens(self.tokens[self.start:self.end])
+            self.sub_text = core.utils.get_text_from_tokens(self.tokens[self.start:self.end])
             return self.sub_text
         else:
-            return self.get_text_from_tokens(self.as_tokens(), custom_switcher, tracker)
+            return core.utils.get_text_from_tokens(self.as_tokens(), custom_switcher, tracker)
 
     def as_tokens(self):
         if self.sub_tokens:
@@ -145,36 +155,6 @@ class ParserInput:
             return True
         return False
 
-    @staticmethod
-    def get_text_from_tokens(tokens, custom_switcher=None, tracker=None):
-        """
-        Create the source code, from the list of token
-        :param tokens: list of tokens
-        :param custom_switcher: to override the behaviour (the return value) of some token
-        :param tracker: keep track of the original token value when custom switched
-        :return:
-        """
-        if tokens is None:
-            return ""
-        res = ""
-
-        if not hasattr(tokens, "__iter__"):
-            tokens = [tokens]
-
-        switcher = {
-            TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value),
-        }
-
-        if custom_switcher:
-            switcher.update(custom_switcher)
-
-        for token in tokens:
-            value = switcher.get(token.type, lambda t: t.value)(token)
-            res += value
-            if tracker is not None and token.type in custom_switcher:
-                tracker[value] = token.value
-        return res
-
 
 class SheerkaExecute(BaseService):
     """
@@ -187,18 +167,156 @@ class SheerkaExecute(BaseService):
     def __init__(self, sheerka):
         super().__init__(sheerka)
         self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
+        self.instantiated_evaluators = None
+        self.evaluators_by_name = None
+        self.grouped_evaluators_cache = {}  # key=step, value=tuple(evaluators for this step, sorted priorities)
+        self.old_values = []
 
     def initialize(self):
         self.sheerka.bind_service_method(self.execute, True)
 
         self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False)
+        self.reset_evaluators()
+
+    def reset_evaluators(self):
+        # instantiate evaluators, once for all, only keep when it's enabled
+        self.instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators]
+        self.instantiated_evaluators = [e for e in self.instantiated_evaluators if e.enabled]
+        self.evaluators_by_name = {e.short_name: e for e in self.instantiated_evaluators}
+
+        # get default evaluators by process step
+        for process_step in EVALUATOR_STEPS:
+            self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped_evaluators(
+                [e for e in self.instantiated_evaluators if process_step in e.steps])
+
+    # @staticmethod
+    # def get_grouped_evaluators(instantiated_evaluators, process_step):
+    #     """
+    #     For a given list of evaluators and a given process step
+    #     Computes
+    #         * the evaluators eligible for this step
+    #         * the list of sorted priorities for theses evaluators
+    #     :param instantiated_evaluators:
+    #     :param process_step:
+    #     :return:
+    #     """
+    #     grouped = {}
+    #     for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]:
+    #         grouped.setdefault(evaluator.priority, []).append(evaluator)
+    #
+    #     sorted_groups = sorted(grouped.keys(), reverse=True)
+    #     return grouped, sorted_groups
+
+    @staticmethod
+    def get_grouped_evaluators(evaluators):
+        """
+        For a given list of evaluators,
+        group them by priorities
+        sort the priorities
+        :param evaluators:
+        :return: tuple({priority: List of evaluators with this priority}, list of sorted priorities)
+        """
+        grouped = {}
+        for evaluator in evaluators:
+            grouped.setdefault(evaluator.priority, []).append(evaluator)
+
+        sorted_groups = sorted(grouped.keys(), reverse=True)
+        return grouped, sorted_groups
+
+    def preprocess(self, items, preprocess_definitions):
+        for preprocess in preprocess_definitions:
+            for item in items:
+                if self.matches(item.name, preprocess.get_value("preprocess_name")):
+                    for var_name, value in preprocess.values().items():
+                        if var_name == "preprocess_name":
+                            continue
+                        if hasattr(item, var_name):
+                            self.old_values.append((item, var_name, getattr(item, var_name)))
+                            setattr(item, var_name, value)
+
+    def preprocess_old(self, context, parsers_or_evaluators, mode):
+        if mode == "parsers":
+            if not context.preprocess and not context.preprocess_parsers:
+                return parsers_or_evaluators
+            items = context.preprocess_parsers
+        elif mode == "evaluators":
+            if not context.preprocess and not context.preprocess_evaluators:
+                return parsers_or_evaluators
+            items = context.preprocess_evaluators
+        else:
+            raise ValueError(mode)
+
+        if not hasattr(parsers_or_evaluators, "__iter__"):
+            single_one = True
+            parsers_or_evaluators = [parsers_or_evaluators]
+        else:
+            single_one = False
+
+        if items:
+            res = []
+            for item in items:
+                for e in parsers_or_evaluators:
+                    if item == e.name:
+                        res.append(e)
+                        break
+                else:
+                    raise ValueError(f"{item} not found.")
+            parsers_or_evaluators = res
+
+        if context.preprocess:
+            for preprocess in context.preprocess:
+                for e in parsers_or_evaluators:
+                    if self.matches(e.name, preprocess.get_value("name")):
+                        for var_name in preprocess.values:
+                            if var_name == "name":
+                                continue
+                            if hasattr(e, var_name):
+                                self.old_values.append((e, var_name, getattr(e, var_name)))
+                                setattr(e, var_name, preprocess.get_value(var_name))
+
+        return parsers_or_evaluators[0] if single_one else parsers_or_evaluators
+
+    def get_evaluators(self, context, process_step):
+        """
+        Returns the list of evaluators to use for a specific test
+        :param context:
+        :param process_step:
+        :return:
+        """
+        # Normal case, the evaluators are the default one
+        if not context.preprocess_evaluators and not context.preprocess:
+            return self.grouped_evaluators_cache[f"{process_step}|__default"]
+
+        # First case, only use a subset of evaluators
+        if context.preprocess_evaluators and not context.preprocess:
+            key = str(process_step) + "|" + "|".join(context.preprocess_evaluators)
+            try:
+                return self.grouped_evaluators_cache[key]
+            except KeyError:
+                evaluators = [self.evaluators_by_name[e] for e in context.preprocess_evaluators]
+                grouped = self.get_grouped_evaluators(evaluators)
+                self.grouped_evaluators_cache[key] = grouped
+                return grouped
+
+        # final case, evaluators attributes are modified by the context
+        # So first, get the modified evaluators
+        evaluators = [self.evaluators_by_name[e] for e in
+                      context.preprocess_evaluators] if context.preprocess_evaluators else self.instantiated_evaluators
+        self.preprocess(evaluators, context.preprocess)
+        evaluators = [e for e in evaluators if e.enabled]  # make sure they are still enabled
+        key = str(process_step) + "|" + "|".join([e.name for e in evaluators if e.enabled])
+        try:
+            return self.grouped_evaluators_cache[key]
+        except KeyError:
+            grouped = self.get_grouped_evaluators(evaluators)
+            self.grouped_evaluators_cache[key] = grouped
+            return grouped
 
     def get_parser_input(self, text, tokens=None):
         """
         Returns new or existing parser input
         :param text:
         :param tokens:
-        :param length:
         :return:
         """
 
@@ -212,7 +330,7 @@ class SheerkaExecute(BaseService):
                 self.pi_cache.put(text, pi)
             return pi
 
-        key = text or ParserInput.get_text_from_tokens(tokens)
+        key = text or core.utils.get_text_from_tokens(tokens)
         pi = ParserInput(key, tokens)
         self.pi_cache.put(key, pi)
         return pi
@@ -251,7 +369,7 @@ class SheerkaExecute(BaseService):
 
         # group the parsers by priorities
         instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()]
-        instantiated_parsers = self.preprocess(context, instantiated_parsers)
+        instantiated_parsers = self.preprocess_old(context, instantiated_parsers, "parsers")
 
         grouped_parsers = {}
         for parser in [p for p in instantiated_parsers if p.enabled]:
@@ -272,13 +390,12 @@ class SheerkaExecute(BaseService):
 
                     # if self.sheerka.log.isEnabledFor(logging.DEBUG):
                     #     debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \
-                    #         else "'" + BaseParser.get_text_from_tokens(to_parse) + "' as tokens"
+                    #         else "'" + core.utils.get_text_from_tokens(to_parse) + "' as tokens"
                     #     context.log(f"Parsing {debug_text}")
 
                     with context.push(BuiltinConcepts.PARSING,
                                       {"parser": parser.name},
-                                      desc=f"Parsing using {parser.name}",
-                                      logger=parser.verbose_log) as sub_context:
+                                      desc=f"Parsing using {parser.name}") as sub_context:
                         sub_context.add_inputs(to_parse=to_parse)
                         res = parser.parse(sub_context, to_parse)
                         if res is not None:
@@ -318,27 +435,13 @@ class SheerkaExecute(BaseService):
         if not isinstance(return_values, list):
             return_values = [return_values]
 
-        # group the evaluators by priority and sort them
-        # The first one to be applied will be the one with the highest priority
-        grouped_evaluators = {}
-        instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators]
+        grouped_evaluators, sorted_priorities = self.get_evaluators(context, process_step)
 
-        # pre-process evaluators if needed
-        instantiated_evaluators = self.preprocess(context, instantiated_evaluators)
-
-        for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]:
-            grouped_evaluators.setdefault(evaluator.priority, []).append(evaluator)
-
-        # order the groups by priority, the higher first
-        sorted_priorities = sorted(grouped_evaluators.keys(), reverse=True)
-
-        # process
         iteration = 0
         while True:
             with context.push(process_step,
-                              {"iteration": iteration},
-                              desc=f"iteration #{iteration}",
-                              iteration=iteration) as iteration_context:
+                              {"step": process_step, "iteration": iteration},
+                              desc=f"iteration #{iteration}") as iteration_context:
                 simple_digest = return_values[:]
                 iteration_context.add_inputs(return_values=simple_digest)
 
@@ -348,13 +451,14 @@ class SheerkaExecute(BaseService):
                     evaluated_items = []
                     to_delete = []
                     for evaluator in grouped_evaluators[priority]:
-                        evaluator = self.preprocess(context, evaluator.__class__())  # fresh copy
+                        evaluator.reset()
 
                         sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})"
                         with iteration_context.push(process_step,
-                                                    {"iteration": iteration, "evaluator": evaluator.name},
-                                                    desc=sub_context_desc,
-                                                    logger=evaluator.verbose_log) as sub_context:
+                                                    {"step": process_step,
+                                                     "iteration": iteration,
+                                                     "evaluator": evaluator.name},
+                                                    desc=sub_context_desc) as sub_context:
                             sub_context.add_inputs(return_values=original_items)
 
                             # process evaluators that work on one simple return value at the time
@@ -365,6 +469,8 @@ class SheerkaExecute(BaseService):
                                     if evaluator.matches(sub_context, item):
 
                                         # init the evaluator is possible
+                                        # KSI. 20201102 : Evaluators are now instantiated at startup,
+                                        # Can we move this section into reset_evaluators()
                                         if hasattr(evaluator, "init_evaluator") and not evaluator.is_initialized:
                                             evaluator.init_evaluator(sub_context, original_items)
 
@@ -401,6 +507,7 @@ class SheerkaExecute(BaseService):
                             # process evaluators that work on all return values
                             else:
                                 if evaluator.matches(sub_context, original_items):
+
                                     results = evaluator.eval(sub_context, original_items)
                                     if results is None:
                                         continue
@@ -427,6 +534,8 @@ class SheerkaExecute(BaseService):
             # inc the iteration and continue
             iteration += 1
 
+        self.undo_preprocess()
+
         return return_values
 
     def execute(self, context, return_values, execution_steps):
@@ -441,40 +550,30 @@ class SheerkaExecute(BaseService):
         for step in execution_steps:
             copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values]
             with context.push(BuiltinConcepts.PROCESSING,
-                              {"step": step},
-                              step=step, iteration=0, desc=f"{step=}") as sub_context:
+                              {"step": step, "iteration": 0},
+                              desc=f"{step=}") as sub_context:
+
+                sub_context.add_inputs(return_values=copy)
 
                 if step == BuiltinConcepts.PARSING:
                     return_values = self.call_parsers(sub_context, return_values)
                 else:
                     return_values = self.call_evaluators(sub_context, return_values, step)
 
-                if copy != return_values:
+                has_changed = copy != return_values
+                if has_changed:
                     sub_context.log_result(return_values)
 
                 sub_context.add_values(return_values=return_values)
+                sub_context.add_values(has_changed=has_changed)
 
         return return_values
 
-    def preprocess(self, context, parsers_or_evaluators):
-        if not context.preprocess:
-            return parsers_or_evaluators
+    def undo_preprocess(self):
+        for item, var_name, value in self.old_values:
+            setattr(item, var_name, value)
 
-        if not hasattr(parsers_or_evaluators, "__iter__"):
-            single_one = True
-            parsers_or_evaluators = [parsers_or_evaluators]
-        else:
-            single_one = False
-
-        for preprocess in context.preprocess:
-            for e in parsers_or_evaluators:
-                if self.matches(e.name, preprocess.get_value("name")):
-                    for var_name in preprocess.values:
-                        if var_name == "name":
-                            continue
-                        if hasattr(e, var_name):
-                            setattr(e, var_name, preprocess.get_value(var_name))
-        return parsers_or_evaluators[0] if single_one else parsers_or_evaluators
+        self.old_values.clear()
 
     @staticmethod
     def matches(parser_or_evaluator_name, preprocessor_name):

src/core/sheerka/services/SheerkaHasAManager.py

@@ -1,5 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import ensure_concept
+from core.builtin_helpers import ensure_concept
 from core.sheerka.services.sheerka_service import BaseService
 
 
@@ -25,8 +25,8 @@ class SheerkaHasAManager(BaseService):
         context.log(f"Setting concept {concept_a} has a {concept_b}", who=self.NAME)
         ensure_concept(concept_a, concept_b)
 
-        if (BuiltinConcepts.HASA in concept_a.metadata.props and
-            concept_b in concept_a.metadata.props[BuiltinConcepts.HASA]):
+        if (BuiltinConcepts.HASA in concept_a.get_metadata().props and
+            concept_b in concept_a.get_metadata().props[BuiltinConcepts.HASA]):
             return self.sheerka.ret(
                 self.NAME,
                 False,
@@ -49,5 +49,5 @@ class SheerkaHasAManager(BaseService):
         """
 
         ensure_concept(concept_a, concept_b)
-        return (BuiltinConcepts.HASA in concept_a.metadata.props and
-                concept_b in concept_a.metadata.props[BuiltinConcepts.HASA])
+        return (BuiltinConcepts.HASA in concept_a.get_metadata().props and
+                concept_b in concept_a.get_metadata().props[BuiltinConcepts.HASA])

src/core/sheerka/services/SheerkaMemory.py

@@ -1,8 +1,10 @@
 from dataclasses import dataclass
 
+from cache.FastCache import FastCache
 from cache.ListIfNeededCache import ListIfNeededCache
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
+from core.global_symbols import CONTEXT_DISPOSED
 from core.sheerka.services.sheerka_service import BaseService, ServiceObj
 
 
@@ -13,48 +15,72 @@ class MemoryObject(ServiceObj):
 
 class SheerkaMemory(BaseService):
     NAME = "Memory"
+    GLOBAL = "global"
 
     SHORT_TERM_OBJECTS_ENTRY = "Memory:ShortTermMemoryObjects"
     OBJECTS_ENTRY = "Memory:Objects"
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
-        self.short_term_objects = ListIfNeededCache()
+        self.short_term_objects = FastCache()
         self.objects = ListIfNeededCache(default=lambda k: self.sheerka.sdp.get(self.OBJECTS_ENTRY, k))
         self.registration = {}
 
     def initialize(self):
         self.sheerka.bind_service_method(self.get_from_short_term_memory, False, visible=False)
         self.sheerka.bind_service_method(self.add_to_short_term_memory, True, visible=False)
+        self.sheerka.bind_service_method(self.remove_context, True, as_name="clear_short_term_memory", visible=False)
         self.sheerka.bind_service_method(self.add_to_memory, True, visible=False)
+        self.sheerka.bind_service_method(self.add_many_to_short_term_memory, True, visible=False)
         self.sheerka.bind_service_method(self.get_from_memory, False)
         self.sheerka.bind_service_method(self.register_object, True, visible=False)
         self.sheerka.bind_service_method(self.unregister_object, True, visible=False)
         self.sheerka.bind_service_method(self.add_registered_objects, True, visible=False)
         self.sheerka.bind_service_method(self.memory, False)
 
-        self.sheerka.cache_manager.register_cache(self.SHORT_TERM_OBJECTS_ENTRY, self.short_term_objects, persist=False)
         self.sheerka.cache_manager.register_cache(self.OBJECTS_ENTRY, self.objects, persist=True, use_ref=True)
 
+    def reset(self):
+        self.short_term_objects.clear()
+
+    def initialize_deferred(self, context, is_first_time):
+        self.sheerka.subscribe(CONTEXT_DISPOSED, self.remove_context)
+
     def get_from_short_term_memory(self, context, key):
         while True:
-            key_to_use = (str(context.id) if context else "") + ":" + key
-            if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use)) is not None:
-                return obj
+            try:
+                id_to_use = context.id if context else self.GLOBAL
+                return self.short_term_objects.cache[id_to_use][key]
+            except KeyError:
+                if context is None:
+                    return None
 
-            if context is None:
-                return None
+                context = context.get_parent()
 
-            context = context.get_parent()
+    def get_all_short_term_memory(self, context):
+        return self.short_term_objects.get(context.id)
 
-    def add_to_short_term_memory(self, context, key, concept):
+    def add_to_short_term_memory(self, context, key, value):
         if context:
             context.stm = True
-        key_to_use = (str(context.id) if context else "") + ":" + key
-        return self.sheerka.cache_manager.put(self.SHORT_TERM_OBJECTS_ENTRY, key_to_use, concept)
+            id_to_use = context.id
+        else:
+            id_to_use = SheerkaMemory.GLOBAL
+
+        if id_to_use in self.short_term_objects.cache:
+            self.short_term_objects.cache[id_to_use][key] = value
+        else:
+            self.short_term_objects.put(id_to_use, {key: value})
+
+    def add_many_to_short_term_memory(self, context, bag):
+        context.stm = True
+        self.short_term_objects.put(context.id if context else self.GLOBAL, bag)
 
     def remove_context(self, context):
-        self.short_term_objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))
+        try:
+            del self.short_term_objects.cache[context.id]
+        except KeyError:
+            pass
 
     def add_to_memory(self, context, key, concept):
         """
@@ -74,6 +100,11 @@ class SheerkaMemory(BaseService):
     def register_object(self, context, key, concept):
         """
         Before adding objects to memory, they first need to be registered
+        More:
+        We don't want to add all evaluated concept into memory
+        (because some of them may be ref to concept already in memory)
+        So we first register them, and add the end of sheerka.evaluate_user_input()
+        all remaining registered concepts will be added to memory
         :param context:
         :param key:
         :param concept:

src/core/sheerka/services/SheerkaModifyConcept.py

@@ -1,7 +1,8 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import ensure_concept
+from core.builtin_helpers import ensure_concept
+from core.concept import NotInit, freeze_concept_attrs, Concept
 from core.sheerka.services.sheerka_service import BaseService
-from parsers.BnfParser import BnfParser
+from parsers.BnfDefinitionParser import BnfDefinitionParser
 
 
 class SheerkaModifyConcept(BaseService):
@@ -47,6 +48,9 @@ class SheerkaModifyConcept(BaseService):
                     BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
                     body=concept))
 
+        # update attributes
+        freeze_concept_attrs(concept)
+
         self.sheerka.cache_manager.update_concept(old_version, concept)
 
         # TODO : update concept by first keyword : have a look at update_references() below
@@ -72,8 +76,8 @@ class SheerkaModifyConcept(BaseService):
         for concept_id in refs:
             concept = self.sheerka.get_by_id(concept_id)
 
-            if concept.bnf is not None:
-                BnfParser.update_recurse_id(context, concept_id, concept.bnf)
+            if concept.get_bnf() is not None:
+                BnfDefinitionParser.update_recurse_id(context, concept_id, concept.get_bnf())
 
             # remove the grammar entry so that it can be recreated
             self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
@@ -88,7 +92,9 @@ class SheerkaModifyConcept(BaseService):
         :return:
         """
         ensure_concept(concept)
-        concept.set_value(attribute, value)
+
+        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
+        concept.set_value(attr, value)
         return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
     def get_attr(self, concept, attribute):
@@ -103,6 +109,8 @@ class SheerkaModifyConcept(BaseService):
         if not self.sheerka.is_success(concept):
             return concept
 
-        if (value := concept.get_value(attribute)) == BuiltinConcepts.NOT_INITIALIZED:
+        attr = attribute.str_id if isinstance(attribute, Concept) else attribute
+
+        if (value := concept.get_value(attr)) == NotInit:
             return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute})
         return value

src/core/sheerka/services/SheerkaOut.py

@@ -0,0 +1,100 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.sheerka.services.sheerka_service import BaseService
+from core.utils import as_bag
+from out.ConsoleVisistor import ConsoleVisitor
+from out.DeveloperVisitor import DeveloperVisitor
+
+
+class SheerkaOut(BaseService):
+    NAME = "Out"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.out_visitors = [ConsoleVisitor()]
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.process_return_values, False)
+
+    def create_out_tree(self, context, obj):
+        return self.create_out_tree_recursive(context, {'__obj': obj}, DeveloperVisitor(self, set(), 0))
+
+    def create_out_tree_recursive(self, context, bag, visitor):
+        debugger = context.get_debugger(SheerkaOut.NAME, "create_out_tree")
+        debugger.debug_entering(bag=bag)
+
+        current_obj = bag["__obj"]
+        bag = self.update_bag(bag, visitor.list_recursion_depth)
+
+        valid_rules = self.sheerka.evaluate_format_rules(context, bag, visitor.already_seen).get(True, None)
+        res = None
+        if valid_rules:
+            if len(valid_rules) > 1:
+                # TODO manage when too many rules
+                pass
+
+            rule = valid_rules[0]
+            if rule.id in visitor.already_seen:
+                debugger.debug_log(f"Rule #{rule.id} already fired.")
+            else:
+                debugger.debug_log(f"Applying rule {rule}.")
+                visitor.already_seen.add(rule.id)
+
+                bag.update(as_bag(current_obj))  # update with the current obj attributes
+                visitor.visit(context, rule.compiled_action, bag)
+                res = visitor.get_result()
+
+        if res is None:
+            debugger.debug_log(f"No matching rule.")
+            res = current_obj
+
+        debugger.debug_var("out_tree", res)
+
+        return res
+
+    def process_return_values(self, context, ret):
+        with context.push(BuiltinConcepts.BEFORE_RENDERING,
+                          None,
+                          desc=f"step='{BuiltinConcepts.BEFORE_RENDERING}'") as sub_context:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+            # sub_context.deactivate_push()
+
+            out_tree = self.create_out_tree(sub_context, ret)
+
+            # sub_context.activate_push()
+
+        if out_tree:
+            for visitor in self.out_visitors:
+                visitor.visit(context, out_tree, None)
+                if hasattr(visitor, "finalize"):
+                    visitor.finalize()
+
+            return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+        return self.sheerka.ret(self.NAME, False, self.sheerka.new(BuiltinConcepts.NO_RESULT))
+
+    def update_bag(self, bag, depth):
+        obj = bag["__obj"]
+        bag["__tab"] = "  " * depth
+
+        if self.sheerka.isinstance(obj, BuiltinConcepts.RETURN_VALUE):
+            bag["__ret"] = obj
+            if self.sheerka.is_container(obj.body):
+                bag["__ret_container"] = obj.body
+                bag["__ret_value"] = self.simplify_list(obj.body.body)
+                bag["__ret_val"] = bag["__ret_value"]
+            else:
+                bag["__ret_value"] = self.simplify_list(obj.body)
+                bag["__ret_val"] = bag["__ret_value"]
+        elif isinstance(obj, list) and len(obj) > 0 and self.sheerka.isinstance(obj[0], BuiltinConcepts.RETURN_VALUE):
+            bag["__rets"] = obj
+
+        return bag
+
+    @staticmethod
+    def simplify_list(item):
+        try:
+            return item[0] if hasattr(item, "__len__") and len(item) == 1 else item
+        except KeyError:
+            return item  # Caution. it's a dict, not a list !

src/core/sheerka/services/SheerkaResultManager.py

@@ -1,5 +1,8 @@
+import ast
+
 from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.services.sheerka_service import BaseService
+from core.utils import as_bag
 
 
 class SheerkaResultConcept(BaseService):
@@ -14,10 +17,31 @@ class SheerkaResultConcept(BaseService):
         self.sheerka.bind_service_method(self.get_results_by_command, True)  # digest is recorded
         self.sheerka.bind_service_method(self.get_last_results, True)  # digest is recorded
         self.sheerka.bind_service_method(self.get_results, False)
+        self.sheerka.bind_service_method(self.get_execution_item, False)
 
-    def get_results_by_digest(self, context, digest, record_digest=True):
+    @staticmethod
+    def get_predicate(**kwargs):
+        if len(kwargs) == 0:
+            return None
+        res = []
+        if "filter" in kwargs:
+            res.append(kwargs["filter"])
+            kwargs.pop("filter")
+
+        for k, v in kwargs.items():
+            if k in ("depth", "recursion_depth"):
+                continue
+
+            if isinstance(v, str):
+                v = '"' + v.translate(str.maketrans({'"': r'\"'})) + '"'
+            res.append(f"{k} == {v}")
+        predicate = " and ".join(res)
+        return compile(ast.parse(predicate, mode="eval"), "<SheerkaResultManager.get_predicate>", mode="eval")
+
+    def get_results_by_digest(self, context, digest, filter=None, record_digest=True, **kwargs):
         """
         Gets the entire execution tree for the given event digest
+        :param filter:
         :param context:
         :param digest:
         :param record_digest:
@@ -26,27 +50,39 @@ class SheerkaResultConcept(BaseService):
         if digest is None:
             return None
 
+        if filter is not None:
+            kwargs["filter"] = filter
+
         try:
             result = self.sheerka.sdp.load_result(digest)
             event = self.sheerka.sdp.load_event(digest)
 
             if record_digest:
                 context.log(f"Recording digest '{digest}'")
-                self.sheerka.record(context, self.NAME, "digest", digest)
+                self.sheerka.record_var(context, self.NAME, "digest", digest)
+
+            explanation = self.sheerka.new(BuiltinConcepts.EXPLANATION,
+                                           digest=event.get_digest(),
+                                           command=event.message,
+                                           body=self.as_list(result, self.get_predicate(**kwargs)))
+
+            # add format instructions if applicable
+            if (depth := kwargs.get("depth", None)) is not None or \
+                (depth := kwargs.get("recursion_depth", None)) is not None:
+                explanation.set_format_instr(recursion_depth=depth, recurse_on="_children")
+
+            return explanation
 
-            return self.sheerka.new(BuiltinConcepts.EXPLANATION,
-                                    digest=event.get_digest(),
-                                    command=event.message,
-                                    body=self.as_list(result))
         except FileNotFoundError as ex:
             context.log_error(f"Digest {digest} is not found.", self.NAME, ex)
             return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": digest})
 
-    def get_results_by_command(self, context, command, record_digest=True):
+    def get_results_by_command(self, context, command, filter=None, record_digest=True, **kwargs):
         """
         Get the result of the command that starts with command
         :param context:
         :param command:
+        :param filter:
         :param record_digest:
         :return:
         """
@@ -59,7 +95,7 @@ class SheerkaResultConcept(BaseService):
             for event in self.sheerka.sdp.load_events(self.page_size, start):
                 consumed += 1
                 if event.message.startswith(command):
-                    return self.get_results_by_digest(context, event.get_digest(), record_digest)
+                    return self.get_results_by_digest(context, event.get_digest(), filter, record_digest, **kwargs)
 
             if consumed < self.page_size:
                 break
@@ -69,10 +105,11 @@ class SheerkaResultConcept(BaseService):
 
         return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"command": command})
 
-    def get_last_results(self, context, record_digest=True):
+    def get_last_results(self, context, filter=None, record_digest=True, **kwargs):
         """
         Gets the results of the last command
         :param context:
+        :param filter:
         :param record_digest:
         :return:
         """
@@ -84,7 +121,7 @@ class SheerkaResultConcept(BaseService):
             for event in self.sheerka.sdp.load_events(page_size, start):
                 consumed += 1
                 if self.sheerka.sdp.has_result(event.get_digest()):
-                    return self.get_results_by_digest(context, event.get_digest(), record_digest)
+                    return self.get_results_by_digest(context, event.get_digest(), filter, record_digest, **kwargs)
 
             if consumed < page_size:
                 break
@@ -97,27 +134,45 @@ class SheerkaResultConcept(BaseService):
 
         return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "last"})
 
-    def get_results(self, context):
+    def get_results(self, context, filter=None, **kwargs):
         """
         Use the last digest saved to get the execution results
         :param context:
+        :param filter:
         :return:
         """
 
-        digest = self.sheerka.load(self.NAME, "digest")
+        digest = self.sheerka.load_var(self.NAME, "digest")
         if digest is None:
             context.log("No recorded digest found.")
             return None
 
-        return self.get_results_by_digest(context, digest, False)
+        return self.get_results_by_digest(context, digest, filter, False, **kwargs)
+
+    def get_execution_item(self, context, item_id):
+        digest = self.sheerka.load_var(self.NAME, "digest")
+        if digest is None:
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body="no digest")
+
+        try:
+            result = self.sheerka.sdp.load_result(digest)
+            items = list(self.as_list(result, self.get_predicate(id=item_id)))
+
+            if len(items) == 0:
+                return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"id": item_id})
+
+            return items[0]
+
+        except FileNotFoundError as ex:
+            context.log_error(f"Digest {digest} is not found.", self.NAME, ex)
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": digest})
 
     @staticmethod
-    def as_list(execution_context):
-
+    def as_list(execution_context, predicate):
         def _yield_result(lst):
-
             for e in lst:
-                yield e
+                if predicate is None or eval(predicate, as_bag(e)):
+                    yield e
 
                 if e._children:
                     yield from _yield_result(e._children)

src/core/sheerka/services/SheerkaRuleManager.py

@@ -0,0 +1,642 @@
+import operator
+import re
+from dataclasses import dataclass
+from typing import Union
+
+from cache.Cache import Cache
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
+from core.builtin_helpers import parse_unrecognized, only_successful, ensure_rule
+from core.concept import Concept
+from core.global_symbols import RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT
+from core.rule import Rule
+from core.sheerka.services.sheerka_service import BaseService
+from core.tokenizer import Keywords, TokenKind, Token, IterParser
+from core.utils import index_tokens, COLORS, get_text_from_tokens
+from evaluators.ConceptEvaluator import ConceptEvaluator
+from evaluators.PythonEvaluator import PythonEvaluator
+from parsers.BaseNodeParser import SourceCodeWithConceptNode, ConceptNode, SourceCodeNode
+from parsers.PythonParser import PythonNode
+
+identifier_regex = re.compile(r"[\w _.]+")
+
+
+@dataclass
+class FormatRuleError:
+    pass
+
+
+@dataclass
+class BraceMismatch(FormatRuleError):
+    lbrace: Token
+
+
+@dataclass
+class UnexpectedEof(FormatRuleError):
+    message: str
+    token: Token = None
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, UnexpectedEof):
+            return False
+
+        return self.message == other.message and (other.token is None or other.token == self.token)
+
+    def __hash__(self):
+        return hash(self.message, self.token)
+
+
+@dataclass
+class FormatRuleSyntaxError(FormatRuleError):
+    message: str
+    token: Token
+
+
+@dataclass
+class FormatAstNode:
+    @staticmethod
+    def repr_value(items):
+        if items is None:
+            return ""
+
+        return ", ".join(repr(item) for item in items)
+
+
+@dataclass
+class FormatAstRawText(FormatAstNode):
+    text: str
+
+
+@dataclass
+class FormatAstVariable(FormatAstNode):
+    name: str
+    format: Union[str, None] = None
+    value: object = None
+    index: object = None
+
+
+@dataclass
+class FormatAstVariableNotFound(FormatAstNode):
+    name: str
+
+
+@dataclass
+class FormatAstGrid(FormatAstNode):
+    pass
+
+
+@dataclass
+class FormatAstList(FormatAstNode):
+    variable: str
+    items_prop: str = None  # where to search the list if variable does not resolve to an iterable
+    recurse_on: str = None
+    recursion_depth: int = 0
+    items: object = None
+
+
+@dataclass
+class FormatAstColor(FormatAstNode):
+    def __init__(self, color, format_ast):
+        self.color = color
+        self.format_ast = format_ast
+
+    def __repr__(self):
+        return f"{self.color}({self.format_ast})"
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, FormatAstColor):
+            return False
+
+        return self.color == other.color and self.format_ast == other.format_ast
+
+    def __hash__(self):
+        return hash((self.color, self.format_ast))
+
+
+@dataclass
+class FormatAstFunction(FormatAstNode):
+    name: str
+    args: list = None
+    kwargs: dict = None
+
+
+class FormatAstSequence(FormatAstNode):
+    def __init__(self, items):
+        self.items = items
+
+    def __repr__(self):
+        return "FormatAstSequence(" + self.repr_value(self.items) + ")"
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, FormatAstSequence):
+            return False
+
+        return self.items == other.items
+
+
+class FormatRuleParser(IterParser):
+
+    @staticmethod
+    def to_text(list_or_dict_of_tokens):
+        """
+        Works on list of list of tokens
+        or dict of list of tokens
+        :param list_or_dict_of_tokens:
+        :return:
+        """
+        get_text = get_text_from_tokens
+        if isinstance(list_or_dict_of_tokens, list):
+            return [get_text(i) for i in list_or_dict_of_tokens]
+        if isinstance(list_or_dict_of_tokens, dict):
+            return {k: get_text(v) for k, v in list_or_dict_of_tokens.items()}
+        raise NotImplementedError("")
+
+    def to_value(self, tokens):
+        """
+        Works on list of tokens
+        return string or numeric value of the tokens
+        :return:
+        """
+
+        value = get_text_from_tokens(tokens)
+        if value[0] in ("'", '"'):
+            return value[1:-1]
+
+        try:
+            return int(value)
+        except ValueError:
+            pass
+
+        try:
+            return float(value)
+        except ValueError:
+            self.error_sink = FormatRuleSyntaxError(f"'{value}' is not numeric", None)
+
+    def parse(self):
+        """
+        Parses a format rule
+        format ::= {variable'} | function(...) | rawtext
+        :return:
+        """
+
+        if self.source == "":
+            return FormatAstRawText("")
+
+        buffer = []
+        result = []
+        res = None
+        escaped = False
+
+        def _flush_buffer():
+            if len(buffer) > 0:
+                result.append(FormatAstRawText(get_text_from_tokens(buffer)))
+                buffer.clear()
+
+        while self.next_token(skip_whitespace=False):
+            if not escaped:
+                if self.token.type == TokenKind.IDENTIFIER and self.the_token_after().type == TokenKind.LPAR:
+                    _flush_buffer()
+                    res = self.parse_function(self.token)
+                elif self.token.type == TokenKind.LBRACE:
+                    _flush_buffer()
+                    res = self.parse_variable(self.token)
+                elif self.token.type == TokenKind.BACK_SLASH:
+                    escaped = True
+                else:
+                    buffer.append(self.token)
+            else:
+                escaped = False
+                buffer.append(self.token)
+
+            if self.error_sink:
+                break
+
+            if res:
+                result.append(res)
+                res = None
+
+        _flush_buffer()
+
+        return [] if len(result) == 0 else result[0] if len(result) == 1 else FormatAstSequence(result)
+
+    def parse_function(self, func_name):
+        self.next_token()
+        self.next_token()
+
+        if self.token.type == TokenKind.EOF:
+            self.error_sink = UnexpectedEof("while parsing function", func_name)
+            return None
+
+        param_buffer = []
+        args = []
+        kwargs = {}
+        get_text = get_text_from_tokens
+
+        def _process_parameters():
+            if len(param_buffer) == 0:
+                self.error_sink = FormatRuleSyntaxError("no parameter found", self.token)
+                return None
+            if (index := index_tokens(param_buffer, "=")) > 0:
+                kwargs[get_text(param_buffer[:index])] = param_buffer[index + 1:]
+            else:
+                args.append(param_buffer.copy())
+            param_buffer.clear()
+
+        while True:
+            if self.token.type == TokenKind.RPAR:
+                if len(param_buffer) > 0:
+                    _process_parameters()
+                break
+
+            elif self.token.type == TokenKind.COMMA:
+                _process_parameters()
+                if self.error_sink:
+                    break
+
+            else:
+                param_buffer.append(self.token)
+
+            if not self.next_token():
+                break
+
+        if self.error_sink:
+            return None
+
+        if self.token.type != TokenKind.RPAR:
+            self.error_sink = UnexpectedEof("while parsing function", func_name)
+            return None
+
+        if func_name.value in COLORS:
+            return self.return_color(func_name.value, args, kwargs)
+        elif func_name.value == "list":
+            return self.return_list(args, kwargs)
+
+        return FormatAstFunction(func_name.value, self.to_text(args), self.to_text(kwargs))
+
+    def parse_variable(self, lbrace):
+        self.next_token()
+
+        if self.token.type == TokenKind.EOF:
+            self.error_sink = UnexpectedEof("while parsing variable", lbrace)
+            return None
+
+        buffer = []
+        while True:
+            if self.token.type == TokenKind.RBRACE:
+                break
+            buffer.append(self.token)
+
+            if not self.next_token():
+                break
+
+        # if self.error_sink:
+        #     return None
+
+        if self.token.type != TokenKind.RBRACE:
+            self.error_sink = UnexpectedEof("while parsing variable", lbrace)
+            return None
+
+        if len(buffer) == 0:
+            self.error_sink = FormatRuleSyntaxError("variable name not found", None)
+            return None
+
+        variable = get_text_from_tokens(buffer)
+        try:
+            index = variable.index(":")
+            return FormatAstVariable(variable[:index], variable[index + 1:])
+        except ValueError:
+            return FormatAstVariable(variable)
+
+    def return_color(self, color, args, kwargs):
+        if len(kwargs) > 0:
+            self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None)
+            return None
+
+        if len(args) == 0:
+            return FormatAstColor(color, FormatAstRawText(""))
+
+        if len(args) > 1:
+            self.error_sink = FormatRuleSyntaxError("only one parameter supported", args[1][0])
+            return None
+
+        source = get_text_from_tokens(args[0])
+        if len(source) > 1 and source[0] in ("'", '"') and source[-1] in ("'", '"'):
+            source = source[1:-1]
+            parser = FormatRuleParser(source)
+            res = parser.parse()
+            self.error_sink = parser.error_sink
+            return FormatAstColor(color, res)
+        else:
+            try:
+                index = source.index(":")
+                variable, vformat = source[:index], source[index + 1:]
+            except ValueError:
+                variable, vformat = source, None
+
+            if not identifier_regex.fullmatch(variable):
+                self.error_sink = FormatRuleSyntaxError("Invalid identifier", None)
+                return None
+            return FormatAstColor(color, FormatAstVariable(variable, vformat))
+
+    def return_list(self, args, kwargs):
+        len_args = len(args)
+        if len_args < 1:
+            self.error_sink = FormatRuleSyntaxError("variable name not found", None)
+            return None
+
+        if len_args > 3:
+            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[3][0])
+            return None
+
+        variable_name = get_text_from_tokens(args[0])
+        recurse_on, recursion_depth, items_prop = None, 0, None
+
+        if len_args == 2:
+            recursion_depth = self.to_value(args[1])
+        elif len_args == 3:
+            recursion_depth = self.to_value(args[1])
+            recurse_on = self.to_value(args[2])
+
+        if "recurse_on" in kwargs:
+            recurse_on = self.to_value(kwargs["recurse_on"])
+
+        if "recursion_depth" in kwargs:
+            recursion_depth = self.to_value(kwargs["recursion_depth"])
+
+        if "items_prop" in kwargs:
+            items_prop = self.to_value(kwargs["items_prop"])
+
+        if self.error_sink:
+            return None
+
+        if not isinstance(recursion_depth, int):
+            self.error_sink = FormatRuleSyntaxError("'recursion_depth' must be an integer", None)
+            return None
+
+        return FormatAstList(variable_name, items_prop, recurse_on, recursion_depth)
+
+
+@dataclass()
+class RulePredicate:
+    source: str
+    evaluator: str
+    predicate: ReturnValueConcept
+    concept: Union[Concept, None]
+
+
+class SheerkaRuleManager(BaseService):
+    NAME = "RuleManager"
+    RULE_IDS = "Rules_Ids"
+    FORMAT_RULE_ENTRY = "RuleManager:FormatRules"
+    EXEC_RULE_ENTRY = "RuleManager:ExecRules"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.format_rule_cache = Cache(default=lambda k: self.sheerka.sdp.get(self.FORMAT_RULE_ENTRY, k))
+        self.exec_rule_cache = Cache(default=lambda k: self.sheerka.sdp.get(self.EXEC_RULE_ENTRY, k))
+
+        self._format_rules = None  # sorted by priority
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.create_new_rule, True, visible=False)
+        self.sheerka.bind_service_method(self.get_rule_by_id, False)
+        self.sheerka.bind_service_method(self.dump_desc_rule, False, as_name="desc_rule")
+        self.sheerka.bind_service_method(self.get_format_rules, False, visible=False)
+
+        self.sheerka.cache_manager.register_cache(self.FORMAT_RULE_ENTRY, self.format_rule_cache, True, True)
+        self.sheerka.cache_manager.register_cache(self.EXEC_RULE_ENTRY, self.exec_rule_cache, True, True)
+
+    def initialize_deferred(self, context, is_first_time):
+
+        if is_first_time:
+            # add builtin rules if it's the first initialization of Sheerka
+            self.init_builtin_rules(context)
+
+        # adds the other rules (when it's not the first time)
+        self.format_rule_cache.populate(lambda: self.sheerka.sdp.list(self.FORMAT_RULE_ENTRY), lambda rule: rule.id)
+        self.exec_rule_cache.populate(lambda: self.sheerka.sdp.list(self.EXEC_RULE_ENTRY), lambda rule: rule.id)
+
+        # compile all the rules
+        for rule_id in self.format_rule_cache:
+            rule = self.init_rule(context, self.format_rule_cache.get(rule_id))
+
+        # update rules priorities
+        self.update_rules_priorities(context)
+
+        self.sheerka.subscribe(RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities)
+
+    def update_rules_priorities(self, context):
+        """
+        Ask the SheerkaComparisonManager for the priorities
+        :return:
+        """
+        # get the priorities
+        rules_weights = self.sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE, RULE_COMPARISON_CONTEXT)
+
+        # compile all the rules
+        for rule_id in self.format_rule_cache:
+            rule = self.format_rule_cache.get(rule_id)
+            if rule.str_id in rules_weights:
+                rule.priority = rules_weights[rule.str_id]
+
+        self._format_rules = None
+
+    def init_rule(self, context, rule: Rule):
+        if rule.metadata.is_compiled:
+            return
+
+        if rule.compiled_predicate is None:
+            res = self.compile_when(context, self.NAME, rule.metadata.predicate)
+            if not isinstance(res, list):
+                rule.error_sink = [res.body]
+                return
+            rule.compiled_predicate = res
+
+        if rule.compiled_action is None:
+            res = self.compile_print(context, rule.metadata.action)
+            if not res.status:
+                rule.error_sink = [res.body]
+                return
+            rule.compiled_action = res.body
+
+        # rule.variables = self.get_variables()
+
+        rule.metadata.is_compiled = True
+        rule.metadata.is_enabled = True
+        return rule
+
+    def compile_when(self, context, name, source):
+        # parser_input = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
+        parsed = parse_unrecognized(context,
+                                    source,
+                                    parsers="all",
+                                    who=name,
+                                    prop=Keywords.WHEN,
+                                    filter_func=only_successful)
+
+        if not parsed.status:
+            return parsed
+
+        if self.sheerka.isinstance(parsed.body, BuiltinConcepts.ONLY_SUCCESSFUL):
+            parsed = parsed.body.body
+
+        return self.add_evaluators(source, parsed if hasattr(parsed, "__iter__") else [parsed])
+
+    def compile_print(self, context, source):
+        parser = FormatRuleParser(source)
+        parsed = parser.parse()
+        if parser.error_sink:
+            return self.sheerka.ret(self.NAME,
+                                    False,
+                                    self.sheerka.new(BuiltinConcepts.ERROR, body=[parser.error_sink]))
+        else:
+            return self.sheerka.ret(self.NAME, True, parsed)
+
+    def set_id_if_needed(self, rule: Rule):
+        """
+        Set the id for the concept if needed
+        :param rule:
+        :return:
+        """
+        if rule.metadata.id is not None:
+            return
+
+        rule.metadata.id = str(self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_KEYS_ENTRY, self.RULE_IDS))
+
+    def create_new_rule(self, context, rule):
+        """
+        Saves the new rule in DB
+        :param context:
+        :param rule:
+        :return:
+        """
+        sheerka = self.sheerka
+
+        # set id before saving in db
+        self.set_id_if_needed(rule)
+        if rule.compiled_predicate and rule.compiled_action:
+            rule.metadata.is_compiled = True
+            rule.metadata.is_enabled = True
+
+        # save it
+        if rule.metadata.action_type == "print":
+            self.sheerka.cache_manager.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule)
+            self._format_rules = None
+        else:
+            self.sheerka.cache_manager.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule)
+
+        # process the return if needed
+        ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_RULE, body=rule))
+        return ret
+
+    def init_builtin_rules(self, context):
+        # self.sheerka.init_log.debug("Initializing default rules")
+        rules = [
+            Rule("print", "Print return values", "__rets", "list(__rets)"),
+            Rule("print", "Print ReturnValue",
+                 "__ret",
+                 "\\ReturnValue(who={__ret.who}, status={__ret.status}, value={__ret.value})"),
+            Rule("print", "Failed ReturnValue in red",
+                 "__ret and not __ret.status",
+                 "red(__ret)"),
+            Rule("print", "List explanations",
+                 "isinstance(__ret_container, BuiltinConcepts.EXPLANATION)",
+                 "blue(__ret_container.digest) : {__ret_container.command}\nlist(__ret_container)"),
+            Rule("print", "Print ExecutionContext",
+                 "isinstance(__obj, ExecutionContext)",
+                 "[{id:3}] {__tab}{desc} ({status})"),
+            Rule("print", "Display formatted list",
+                 "isinstance(__ret_container, BuiltinConcepts.TO_LIST)",
+                 "list(__ret_container)"),
+        ]
+
+        for r in rules:
+            self.create_new_rule(context, r)
+
+        self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[2], RULE_COMPARISON_CONTEXT)
+        self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[3], RULE_COMPARISON_CONTEXT)
+        self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[5], RULE_COMPARISON_CONTEXT)
+        self.sheerka.set_is_greatest(context, BuiltinConcepts.PRECEDENCE, rules[0], RULE_COMPARISON_CONTEXT)
+
+    def get_rule_by_id(self, rule_id):
+        """
+        Looks in the caches for a specific rule id
+        :param rule_id:
+        :return:
+        """
+        if rule_id is None:
+            return None
+
+        rule = self.format_rule_cache.get(rule_id)
+        if rule:
+            return rule
+
+        rule = self.exec_rule_cache.get(rule_id)
+        if rule:
+            return rule
+
+        metadata = [("id", rule_id)]
+        return self.sheerka.new(BuiltinConcepts.UNKNOWN_RULE, body=metadata)
+
+    def dump_desc_rule(self, rules):
+        """
+        dumps the definition of a rule
+        :param rules:
+        :return:
+        """
+        ensure_rule(rules)
+
+        if not hasattr(rules, "__iter__"):
+            rules = [rules]
+
+        first = True
+        for rule in rules:
+            if not first:
+                self.sheerka.log.info("")
+            self.sheerka.log.info(f"id          : {rule.id}")
+            self.sheerka.log.info(f"name        : {rule.metadata.name}")
+            self.sheerka.log.info(f"type        : {rule.metadata.action_type}")
+            self.sheerka.log.info(f"predicate   : {rule.metadata.predicate}")
+            self.sheerka.log.info(f"action      : {rule.metadata.action}")
+            self.sheerka.log.info(f"compiled    : {rule.metadata.is_compiled}")
+            self.sheerka.log.info(f"enabled     : {rule.metadata.is_enabled}")
+
+    def get_format_rules(self):
+        if self._format_rules:
+            return self._format_rules
+
+        self._format_rules = sorted(self.format_rule_cache.get_all(), key=operator.attrgetter('priority'), reverse=True)
+        return self._format_rules
+
+    def add_evaluators(self, source, ret_vals):
+        """
+        Browse the ReturnValueConcepts to determine the evaluator to use
+        Returns a list of tuple (evaluator_name, return_value)
+        :param source:
+        :param ret_vals:
+        :return:
+        """
+        res = []
+        for r in ret_vals:
+            underlying = self.sheerka.objvalue(r)
+            if isinstance(underlying, PythonNode):
+                res.append(RulePredicate(source, PythonEvaluator.NAME, r, None))
+            elif isinstance(underlying, SourceCodeWithConceptNode):
+                res.append(RulePredicate(source, PythonEvaluator.NAME, r, None))
+            elif isinstance(underlying, SourceCodeNode):
+                res.append(RulePredicate(source, PythonEvaluator.NAME, r, None))
+            elif isinstance(underlying, Concept):
+                res.append(RulePredicate(source, ConceptEvaluator.NAME, r, underlying))
+            elif hasattr(underlying, "__iter__") and len(underlying) == 1 and isinstance(underlying[0], ConceptNode):
+                res.append(RulePredicate(source, ConceptEvaluator.NAME, r, underlying[0].concept))
+            else:
+                raise NotImplementedError(r)
+        return res

src/core/sheerka/services/SheerkaSetsManager.py

@@ -1,9 +1,9 @@
 import core.builtin_helpers
 from cache.Cache import Cache
 from cache.SetCache import SetCache
-from core.ast.nodes import python_to_concept
+from core.ast_helpers import UnreferencedVariablesVisitor
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts, ensure_concept, DEFINITION_TYPE_BNF
+from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF
 from core.sheerka.services.SheerkaModifyConcept import SheerkaModifyConcept
 from core.sheerka.services.sheerka_service import BaseService
 
@@ -41,9 +41,10 @@ class SheerkaSetsManager(BaseService):
         """
 
         context.log(f"Setting concept {concept} is a {concept_set}", who=self.NAME)
-        ensure_concept(concept, concept_set)
+        core.builtin_helpers.ensure_concept(concept, concept_set)
 
-        if BuiltinConcepts.ISA in concept.metadata.props and concept_set in concept.metadata.props[BuiltinConcepts.ISA]:
+        if BuiltinConcepts.ISA in concept.get_metadata().props and concept_set in concept.get_metadata().props[
+            BuiltinConcepts.ISA]:
             return self.sheerka.ret(
                 self.NAME,
                 False,
@@ -71,7 +72,7 @@ class SheerkaSetsManager(BaseService):
         """
 
         context.log(f"Adding concept {concept} to set {concept_set}", who=self.NAME)
-        ensure_concept(concept, concept_set)
+        core.builtin_helpers.ensure_concept(concept, concept_set)
 
         set_elements = self.sets.get(concept_set.id)
         if set_elements and concept.id in set_elements:
@@ -98,7 +99,7 @@ class SheerkaSetsManager(BaseService):
         """
 
         context.log(f"Adding concepts {concepts} to set {concept_set}", who=self.NAME)
-        ensure_concept(concept_set)
+        core.builtin_helpers.ensure_concept(concept_set)
         already_in_set = []
         for concept in concepts:
             res = self.add_concept_to_set(context, concept, concept_set)
@@ -124,7 +125,7 @@ class SheerkaSetsManager(BaseService):
         :return:
         """
 
-        ensure_concept(concept)
+        core.builtin_helpers.ensure_concept(concept)
 
         def _get_set_elements(sub_concept):
             if not self.isaset(context, sub_concept):
@@ -146,8 +147,8 @@ class SheerkaSetsManager(BaseService):
                 concepts.extend(other_concepts)
 
             # apply the where clause if any
-            if sub_concept.metadata.where:
-                new_condition = self._validate_where_clause(sub_concept)
+            if sub_concept.get_metadata().where:
+                new_condition = self._validate_where_clause(context, sub_concept)
                 if not new_condition:
                     return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=sub_concept)
 
@@ -179,7 +180,7 @@ class SheerkaSetsManager(BaseService):
         :return:
         """
 
-        ensure_concept(a, b)
+        core.builtin_helpers.ensure_concept(a, b)
 
         # TODO, first check the 'isa' property of a
         if not (a.id and b.id):
@@ -190,11 +191,11 @@ class SheerkaSetsManager(BaseService):
 
     def isa(self, a, b):
 
-        ensure_concept(a, b)
-        if BuiltinConcepts.ISA not in a.metadata.props:
+        core.builtin_helpers.ensure_concept(a, b)
+        if BuiltinConcepts.ISA not in a.get_metadata().props:
             return False
 
-        for c in a.metadata.props[BuiltinConcepts.ISA]:
+        for c in a.get_metadata().props[BuiltinConcepts.ISA]:
             if c == b:
                 return True
             if self.isa(self.sheerka.get_by_id(c.id), b):
@@ -216,7 +217,7 @@ class SheerkaSetsManager(BaseService):
 
         # KSI 20200629
         # To resolve infinite recursion between group concepts and BNF concepts
-        if concept.metadata.definition_type == DEFINITION_TYPE_BNF:
+        if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
             return False
 
         # check if it has a group
@@ -231,18 +232,18 @@ class SheerkaSetsManager(BaseService):
 
         return self.isaset(context, concept.body)
 
-    def _validate_where_clause(self, concept):
-        python_parser_result = [r for r in concept.compiled[ConceptParts.WHERE] if r.who == "parsers.Python"]
+    def _validate_where_clause(self, context, concept):
+        python_parser_result = [r for r in concept.get_compiled()[ConceptParts.WHERE] if r.who == "parsers.Python"]
         if not python_parser_result or not python_parser_result[0].status:
             return None
 
         ast_ = python_parser_result[0].body.body.ast_
-        ast_as_concepts = python_to_concept(ast_)
-        names = core.builtin_helpers.get_names(self.sheerka, ast_as_concepts)
-        if len(names) != 1 or names[0] != concept.metadata.body:
+        visitor = UnreferencedVariablesVisitor(context)
+        names = list(visitor.get_names(ast_))
+        if len(names) != 1 or names[0] != concept.get_metadata().body:
             return None
 
-        condition = concept.metadata.where.replace(concept.metadata.body, "sheerka.objvalue(x)")
+        condition = concept.get_metadata().where.replace(concept.get_metadata().body, "sheerka.objvalue(x)")
         expression = f"""
 result=[]
 for x in xx__concepts__xx:
@@ -277,7 +278,7 @@ for x in xx__concepts__xx:
             errors = []
             for element_id in ids:
                 concept = self.sheerka.get_by_id(element_id)
-                if len(concept.metadata.variables) == 0:
+                if len(concept.get_metadata().variables) == 0:
                     # The concepts are directly taken from Sheerka.get_by_id, so variable cannot be filled
                     # It's the reason why we only evaluate concept with no variable
                     evaluated = self.sheerka.evaluate_concept(sub_context, concept)

src/core/sheerka/services/SheerkaVariableManager.py

@@ -19,6 +19,9 @@ class Variable(ServiceObj):
     def get_key(self):
         return f"{self.who}|{self.key}"
 
+    def __str__(self):
+        return f"({self.who}){self.key}={self.value}"
+
 
 class SheerkaVariableManager(BaseService):
     NAME = "VariableManager"
@@ -26,18 +29,28 @@ class SheerkaVariableManager(BaseService):
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
+        self.bound = {
+            "sheerka.enable_process_return_values": "enable_process_return_values",
+            "sheerka.save_execution_context": "save_execution_context"
+        }
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.record, True)
-        self.sheerka.bind_service_method(self.load, False)
-        self.sheerka.bind_service_method(self.delete, True)
-        self.sheerka.bind_service_method(self.set, True)
-        self.sheerka.bind_service_method(self.get, False)
+        self.sheerka.bind_service_method(self.record_var, True, visible=False)
+        self.sheerka.bind_service_method(self.load_var, False, visible=False)
+        self.sheerka.bind_service_method(self.delete_var, True, visible=False)
+        self.sheerka.bind_service_method(self.set_var, True)
+        self.sheerka.bind_service_method(self.get_var, False)
+        self.sheerka.bind_service_method(self.list_vars, False)
 
-        cache = Cache(default=lambda k: self.sheerka.sdp.get(self.VARIABLES_ENTRY, k))
+        cache = Cache()
+        cache.populate(lambda: self.sheerka.sdp.list(self.VARIABLES_ENTRY), lambda var: var.get_key())
         self.sheerka.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True)
 
-    def record(self, context, who, key, value):
+        for variable in cache.get_all():
+            if variable.key in self.bound:
+                setattr(self.sheerka, self.bound[variable.key], variable.value)
+
+    def record_var(self, context, who, key, value):
         """
 
         :param context:
@@ -49,20 +62,33 @@ class SheerkaVariableManager(BaseService):
 
         variable = Variable(context.event.get_digest(), who, key, value, None)
         self.sheerka.cache_manager.put(self.VARIABLES_ENTRY, variable.get_key(), variable)
+
+        # TODO: manage credentials
+        if key in self.bound:
+            setattr(self.sheerka, self.bound[key], value)
+
         return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
-    def load(self, who, key):
+    def load_var(self, who, key):
         variable = self.sheerka.cache_manager.get(self.VARIABLES_ENTRY, who + "|" + key)
         if variable is None:
             return None
 
         return variable.value
 
-    def delete(self, context, who, key):
+    def delete_var(self, context, who, key):
         self.sheerka.cache_manager.delete(self.VARIABLES_ENTRY, who + "|" + key)
 
-    def set(self, context, key, value):
-        return self.record(context, context.event.user_id, key, value)
+    def set_var(self, context, key, value):
+        return self.record_var(context, context.event.user_id, key, value)
 
-    def get(self, context, key):
-        return self.load(context.event.user_id, key)
+    def get_var(self, context, key):
+        return self.load_var(context.event.user_id, key)
+
+    def list_vars(self, context, all_vars=False):
+        if all_vars:
+            res = [str(v) for v in self.sheerka.cache_manager.copy(self.VARIABLES_ENTRY).values()]
+        else:
+            res = [str(v) for v in self.sheerka.cache_manager.copy(self.VARIABLES_ENTRY).values() if
+                   v.who == context.event.user_id]
+        return res

src/core/sheerka/services/sheerka_service.py

@@ -10,6 +10,7 @@ class BaseService:
     """
     Base class for services
     """
+
     def __init__(self, sheerka):
         self.sheerka = sheerka
 
@@ -19,3 +20,13 @@ class BaseService:
         :return:
         """
         pass
+
+    def restore_values(self, *args):
+        """
+        Use Variable Manager to restore the state of a service
+        :param args:
+        :return:
+        """
+        for prop_name in args:
+            if (value := self.sheerka.load_var(self.NAME, prop_name)) is not None:
+                setattr(self, prop_name, value)