Implemented a first and basic version of a Rete rule engine

src/core/ast_helpers.py

@@ -92,6 +92,15 @@ class UnreferencedVariablesVisitor(UnreferencedNamesVisitor):
 
 
 class NamesWithAttributesVisitor(ast.NodeVisitor):
+    """
+    Looks for all atrtibutes for a given name
+    >>> ast_ = ast.parse("foo.bar.baz", "<src>", mode="exec")
+    >>> assert NamesWithAttributesVisitor().get_sequences(ast_, "foo") == [["foo", "bar", "baz"]]
+
+    It parses all expressions / statements
+    >>> ast_ = ast.parse("foo.bar.baz; one.two.three; foo.bar", "<src>", mode="exec")
+    >>> assert NamesWithAttributesVisitor().get_sequences(ast_, "foo") == [["foo", "bar", "baz"], ["foo", "bar"]]
+    """
 
     def __init__(self):
         self.sequences = []

src/core/builtin_concepts.py

@@ -1,6 +1,6 @@
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.concept import Concept, ConceptParts
-from core.error import ErrorObj
+from core.global_symbols import ErrorObj
 
 
 class UserInputConcept(Concept):
@@ -167,6 +167,37 @@ class ParserResultConcept(Concept):
         return parser.name if isinstance(parser, BaseParser) else str(parser)
 
 
+class RuleEvaluationResultConcept(Concept):
+    """
+    Result of the evaluation of a rule, using the Rete algorithm
+    """
+
+    ALL_ATTRIBUTES = ["rule"]
+
+    def __init__(self, rule=None, concept_id=None):
+        Concept.__init__(self,
+                         BuiltinConcepts.RULE_EVALUATION_RESULT,
+                         True,
+                         False,
+                         BuiltinConcepts.RULE_EVALUATION_RESULT,
+                         id=concept_id,
+                         bound_body="rule")
+        self.set_value("rule", rule)
+        self._metadata.is_evaluated = True
+
+    def __repr__(self):
+        return f"RuleEvaluationResult(rule={self.rule})"
+
+    def __eq__(self, other):
+        if not isinstance(other, RuleEvaluationResultConcept):
+            return False
+
+        return self.rule == other.rule
+
+    def __hash__(self):
+        return hash((self._metadata.name, self.rule))
+
+
 class InvalidReturnValueConcept(Concept, ErrorObj):
     """
     Error returned when an evaluator is not correctly coded

src/core/builtin_concepts_ids.py

@@ -33,6 +33,9 @@ class BuiltinConcepts:
     BEFORE_RENDERING = "__BEFORE_RENDERING"  # activate before the output is rendered
     RENDERING = "__RENDERING"  # rendering the response from sheerka
     AFTER_RENDERING = "__AFTER_RENDERING"  # rendering the response from sheerka
+    BEFORE_RULES_EVALUATION = "__BEFORE_RULES_EVALUATION"  # just before evaluating rules
+    RULES_EVALUATION = "__RULES_EVALUATION"  # evaluating rules
+    AFTER_RULES_EVALUATION = "__AFTER_RULES_EVALUATION"  # after evaluating rules
     EVALUATE_SOURCE = "__EVALUATE_SOURCE"  #
     EVALUATE_CONCEPT = "__EVALUATE_CONCEPT"  # a concept will be evaluated
     EVALUATING_CONCEPT = "__EVALUATING_CONCEPT"  # a concept will be evaluated
@@ -46,12 +49,12 @@ class BuiltinConcepts:
     EXEC_CODE = "__EXEC_CODE"  # to use when executing Python or other language compiled code
     TESTING = "__TESTING"
     EVALUATOR_PRE_PROCESS = "__EVALUATOR_PRE_PROCESS"  # used modify / tweak behaviour of evaluators
-    EVALUATING_RULES = "__EVALUATING_RULES"
 
     # builtin attributes
     ISA = "__ISA"  # when a concept is an instance of another one
     HASA = "__HASA"  # when a concept has/owns another concept
     AUTO_EVAL = "__AUTO_EVAL"  # when the concept must be auto evaluated
+    RECOGNIZED_BY = "__RECOGNIZED_BY"  # indicate how a concept was recognized
 
     # object
     USER_INPUT = "__USER_INPUT"  # represent an input from an user
@@ -64,6 +67,7 @@ class BuiltinConcepts:
     NEW_CONCEPT = "__NEW_CONCEPT"  # when a new concept is added
     UNKNOWN_PROPERTY = "__UNKNOWN_PROPERTY"  # when requesting for a unknown property
     PARSER_RESULT = "__PARSER_RESULT"
+    RULE_EVALUATION_RESULT = "__RULE_EVALUATION_RESULT"
     TOO_MANY_SUCCESS = "__TOO_MANY_SUCCESS"  # when expecting a limited number of successful return value
     TOO_MANY_ERRORS = "__TOO_MANY_ERRORS"  # when expecting a limited number of successful return value
     ONLY_SUCCESSFUL = "__ONLY_SUCCESSFUL"  # filter the result, only keep successful ones

src/core/builtin_helpers.py

@@ -13,6 +13,7 @@ from core.utils import as_bag
 from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
     RuleNode
 from parsers.BaseParser import ParsingError
+from parsers.PythonParser import PythonParser
 
 PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
 EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION,
@@ -235,7 +236,7 @@ def only_parsers_results(context, return_values):
     Filters the return_values and returns when the result is a ParserResult
     regardless of the status
 
-    So it filters errors
+    So it filters parsers in error (ERROR, NOT_FOR_ME, EMPTY...)
     :param context:
     :param return_values:
     :return:
@@ -332,7 +333,7 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun
 
 def parse_function(context, source, tokens=None, start=0):
     """
-    Helper function to parse what is supposed to be a function
+    Helper function that parses what is supposed to be a function
     :param context:
     :param source:
     :param tokens:
@@ -361,6 +362,34 @@ def parse_function(context, source, tokens=None, start=0):
     return res
 
 
+def parse_python(context, source, desc=None):
+    """
+    Helper function that parses what is known to be Python source code
+    :param context:
+    :param source:
+    :param desc: option description when creating the sub context
+    """
+    desc = desc or f"Compiling python '{source}'"
+    with context.push(BuiltinConcepts.PARSE_CODE,
+                      {"language": "Python", "source": source},
+                      desc) as sub_context:
+        parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
+        python_parser = PythonParser()
+        return python_parser.parse(sub_context, parser_input)
+
+
+def parse_expression(context, source, desc=None):
+    """
+    Helper function to parser expressions with AND, OR and NOT
+    """
+    desc = desc or f"Parsing expression '{source}'"
+    with context.push(BuiltinConcepts.PARSE_CODE, source, desc) as sub_context:
+        parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
+        from parsers.ExpressionParser import ExpressionParser
+        expr_parser = ExpressionParser()
+        return expr_parser.parse(sub_context, parser_input)
+
+
 def evaluate(context,
              source,
              evaluators="all",
@@ -472,12 +501,67 @@ def get_lexer_nodes(return_values, start, tokens):
     return lexer_nodes
 
 
-def ensure_evaluated(context, concept, eval_body=True):
+def get_lexer_nodes_using_positions(return_values, positions):
+    """
+    Transform all elements from return_values into lexer nodes
+    use positions to remap the exact positions
+    """
+    lexer_nodes = []
+    for ret_val, position in zip(return_values, positions):
+        if ret_val.who in ("parsers.Python", 'parsers.PythonWithConcepts'):
+
+            lexer_nodes.append(SourceCodeNode(position.start,
+                                              position.end,
+                                              position.tokens,
+                                              ret_val.body.source,
+                                              python_node=ret_val.body.body,
+                                              return_value=ret_val))
+
+        elif ret_val.who == "parsers.ExactConcept":
+            concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
+            for concept in concepts:
+                lexer_nodes.append(ConceptNode(concept,
+                                               position.start,
+                                               position.end,
+                                               position.tokens,
+                                               ret_val.body.source))
+
+        elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
+            nodes = [node for node in ret_val.body.body]
+            for node in nodes:
+                node.start = position.start
+                node.end = position.end
+
+            # but append the whole sequence if when it's a sequence
+            lexer_nodes.extend(nodes)
+
+        elif ret_val.who == "parsers.Rule":
+            rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
+            for rule in rules:
+                lexer_nodes.append(RuleNode(rule,
+                                            position.start,
+                                            position.end,
+                                            position.tokens, ret_val.body.source))
+
+        elif ret_val.who == "parsers.Function":
+            node = ret_val.body.body
+            node.start = position.start
+            node.end = position.end
+            lexer_nodes.append(node)
+
+        else:
+            raise NotImplementedError()
+
+    return lexer_nodes
+
+
+def ensure_evaluated(context, concept, eval_body=True, metadata=None):
     """
     Evaluate a concept is not already evaluated
     :param context:
     :param concept:
     :param eval_body:
+    :param metadata:
     :return:
     """
     if concept.get_metadata().is_evaluated:
@@ -485,13 +569,13 @@ def ensure_evaluated(context, concept, eval_body=True):
 
     # do not try to evaluate concept that are not fully initialized
     if concept.get_metadata().definition_type != DEFINITION_TYPE_BNF:
-        for var in concept.get_metadata().variables:
-            if var[1] is None and \
-                var[0] not in concept.get_compiled() and \
-                (var[0] not in concept.values() or concept.get_value(var[0]) == NotInit):
+        for var_name, var_default_value in concept.get_metadata().variables:
+            if var_default_value is None and \
+                var_name not in concept.get_compiled() and \
+                (var_name not in concept.values() or concept.get_value(var_name) == NotInit):
                 return concept
 
-    evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=eval_body)
+    evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=eval_body, metadata=metadata)
     return evaluated
 
 
@@ -731,3 +815,90 @@ def evaluate_object(bag, properties):
         bag = as_bag(obj)
 
     return obj
+
+
+def is_a_question(context, concept):
+    """
+    Returns True if the concept must be executed in the context of BuiltinConcepts.EVAL_QUESTION_REQUESTED
+    The only two ways that are currently supported are
+    * is_question() appears in the pre condition
+    * context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED) appears in the pre condition
+    :param context:
+    :param concept: concept to analyse
+    """
+    pre = concept.get_metadata().pre
+    if pre in (None, NotInit, ""):
+        return False
+
+    parser_input_service = context.sheerka.services[SheerkaExecute.NAME]
+    from parsers.ExpressionParser import ExpressionParser
+    parser = ExpressionParser()
+
+    res = parser.parse(context, parser_input_service.get_parser_input(pre))
+    if not res.status:
+        return False
+
+    node = res.body.body
+    from parsers.expressions import IsAQuestionVisitor
+    return IsAQuestionVisitor().is_a_question(node)
+
+
+def get_inner_body(context, concept):
+    """
+    For container concept, returns the body
+    """
+    if context.sheerka.isinstance(concept.body, BuiltinConcepts.ONLY_SUCCESSFUL):
+        return concept.body.body
+    else:
+        return concept.body
+
+
+class CreateObjectIdentifiers:
+    """
+    Class that creates unique identifiers for Concept or Rule objects
+    """
+
+    def __init__(self):
+        self.identifiers = {}
+        self.identifiers_key = {}
+
+    @staticmethod
+    def sanitize(identifier):
+        if identifier is None:
+            return ""
+
+        res = ""
+        for c in identifier:
+            res += c if c.isalnum() else "0"
+        return res
+
+    def get_identifier(self, obj, wrapper):
+        """
+        Get an identifier for a concept.
+        Make sure to return the same identifier if the same concept
+        Make sure to return a different identifier if same name but different concept
+
+        Internal function because I don't want identifiers, identifiers_key and python_ids_mappings
+        to be instance variables
+        I would like to keep this parser as stateless as possible
+        :param obj:
+        :param wrapper: string or char that will wrap the result (ex '__C__' or '__R__')
+        :return:
+        """
+        if id(obj) in self.identifiers:
+            return self.identifiers[id(obj)]
+
+        identifier = wrapper + self.sanitize(obj.key or obj.name)
+        if obj.id:
+            identifier += "__" + obj.id
+
+        if identifier in self.identifiers_key:
+            self.identifiers_key[identifier] += 1
+            identifier += f"_{self.identifiers_key[identifier]}"
+        else:
+            self.identifiers_key[identifier] = 0
+
+        identifier += wrapper
+
+        self.identifiers[id(obj)] = identifier
+        return identifier

src/core/concept.py

@@ -150,6 +150,7 @@ class Concept:
         self._bnf = None  # parsing expression
         self._original_definition_hash = None  # concept hash before any alteration of the metadata
         self._format = None  # how to print the concept
+        self._hints = {}  # extra processing information to help processing
 
     def __repr__(self):
         text = f"({self._metadata.id}){self._metadata.name}"
@@ -499,6 +500,15 @@ class Concept:
         return {k: v for k, v in self.values().items() if not k[0] == "#"}
         # return dict([(k, v) for k, v in self.values.items() if isinstance(k, str)])
 
+    def set_hint(self, name, value):
+        self._hints[name] = value
+
+    def get_hint(self, name):
+        try:
+            return self._hints[name]
+        except KeyError:
+            return None
+
     def auto_init(self):
         """
         Sometimes (for tests purposes)
@@ -533,10 +543,9 @@ class Concept:
         It quicker to implement than creating the actual property mechanism with @property
         And it removes the visibility from the other attributes/methods
         """
-        bag = self.variables()
-
-        for prop in ("id", "name", "key", "body"):
-            bag[prop] = getattr(self, prop)
+        bag = {prop: getattr(self, prop) for prop in ("id", "name", "key")}
+        bag.update(self.variables())
+        bag["body"] = getattr(self, "body")
         return bag
 
     def as_debug_bag(self, new_obj, recurse):

src/core/error.py

@@ -1,6 +0,0 @@
-class ErrorObj:
-    """
-    To indicate that somehow, the underlying object is (or has) an error
-    """
-    pass
-

src/core/global_symbols.py

@@ -1,9 +1,17 @@
 # events
-EVENT_CONCEPT_PRECEDENCE_MODIFIED = "evt_cpm"
-EVENT_RULE_PRECEDENCE_MODIFIED = "evt_rpm"
-EVENT_CONTEXT_DISPOSED = "evt_cd"
-EVENT_USER_INPUT_EVALUATED = "evt_uie"
-EVENT_CONCEPT_CREATED = "evt_cc"
+
+EVENT_CONCEPT_PRECEDENCE_MODIFIED = "evt_cp_m"
+EVENT_RULE_PRECEDENCE_MODIFIED = "evt_rp_m"
+EVENT_CONTEXT_DISPOSED = "evt_ctx_d"
+EVENT_USER_INPUT_EVALUATED = "evt_ui_e"
+EVENT_CONCEPT_CREATED = "evt_c_c"
+EVENT_CONCEPT_DELETED = "evt_c_d"
+EVENT_CONCEPT_ID_DELETED = "evt_c_id_d"
+EVENT_RULE_CREATED = "evt_r_c"
+EVENT_RULE_DELETED = "evt_r_d"
+EVENT_RULE_ID_DELETED = "evt_r_id_d"
+EVENT_ONTOLOGY_CREATED = "evt_o_c"
+EVENT_ONTOLOGY_DELETED = "evt_o_d"
 
 # comparison context
 RULE_COMPARISON_CONTEXT = "Rule"
@@ -40,3 +48,10 @@ class RemovedType(CustomType):
 NotInit = NotInitType()
 NotFound = NotFoundType()
 Removed = RemovedType()
+
+
+class ErrorObj:
+    """
+    To indicate that somehow, the underlying object is (or has) an error
+    """
+    pass

src/core/rule.py

@@ -5,6 +5,7 @@ import core.utils
 
 ACTION_TYPE_PRINT = "print"
 ACTION_TYPE_EXEC = "exec"
+ACTION_TYPE_TEST = "test"
 
 
 @dataclass
@@ -30,17 +31,23 @@ class Rule:
                  rule_id=None,
                  is_enabled=None):
         self.metadata = RuleMetadata(action_type, name, predicate, action, id=rule_id, is_enabled=is_enabled)
-        self.compiled_predicate = None
+        self.compiled_predicates = None
         self.compiled_action = None
         from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
         self.priority = priority if priority is not None else SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
         self.error_sink = None
 
+        # from SheerkaRete, not quite sure one when it will be used
+        self.rete_net = None
+        self.rete_p_nodes = []  # list of production nodes for this rule
+        self.rete_disjunctions = None  # list of list as it may be several interpretation for a rule
+
     def __repr__(self):
         rule_id = f"#{self.metadata.id}"
         if self.name:
             rule_id += f" ({self.metadata.name})"
-        return f"Rule({rule_id}, when '{self.metadata.predicate}' {self.metadata.action_type} '{self.metadata.action}', priority={self.priority})"
+        action_type = "print" if self.metadata.action_type == "print" else "then"
+        return f"Rule({rule_id}, when '{self.metadata.predicate}' {action_type} '{self.metadata.action}', priority={self.priority})"
 
     def __eq__(self, other):
         if id(other) == id(self):
@@ -69,8 +76,12 @@ class Rule:
                     self.priority,
                     self.id,
                     self.metadata.is_enabled)
-        copy.compiled_predicate = self.compiled_predicate
+
+        copy.compiled_predicates = self.compiled_predicates
         copy.compiled_action = self.compiled_action
+        copy.metadata.is_compiled = self.metadata.is_compiled
+        copy.metadata.id_is_unresolved = self.metadata.id_is_unresolved
+        # copy.error_sink = self.error_sink # Uncomment this line if necessary
 
         return copy
 
@@ -102,3 +113,6 @@ class Rule:
 
     def short_str(self):
         return f"Rule(#{self.metadata.id}, '{self.metadata.predicate}', priority={self.priority})"
+
+    def get_rete_disjunctions(self):
+        return self.rete_disjunctions

src/core/sheerka/ExecutionContext.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import pprint
 import time
 
@@ -87,9 +86,6 @@ class ExecutionContext:
         self.obj = obj
         self.concepts = concepts
 
-        self_debug, self.debug_mode = sheerka.get_context_debug_mode(self.id)
-        self.debug_enabled = self_debug is not None
-
     @property
     def elapsed(self):
         if self._start == 0:
@@ -184,10 +180,6 @@ class ExecutionContext:
         new.preprocess_evaluators = self.preprocess_evaluators
         new.protected_hints.update(self.protected_hints)
 
-        if new.debug_mode is None and self.debug_mode == "protected":
-            new.debug_mode = "protected"
-            new.debug_enabled = True
-
         self._children.append(new)
 
         return new
@@ -315,9 +307,10 @@ class ExecutionContext:
             to_str = self.return_value_to_str(r)
             self._logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
-    def get_debugger(self, who, method_name):
-        return self.sheerka.get_debugger(self, who, method_name)
+    def get_debugger(self, who, method_name, new_debug_id=True):
+        return self.sheerka.get_debugger(self, who, method_name, new_debug_id)
 
+    # TODO: TO REMOVE
     def debug(self, who, method_name, variable_name, text, is_error=False):
         activated = self.sheerka.debug_activated_for(who)
         if activated:
@@ -330,6 +323,7 @@ class ExecutionContext:
                 self.sheerka.debug(f"[{self._id:3}] {CCM[color]}{who}.{method_name}.{variable_name}: {CCM['reset']}")
                 self.sheerka.debug(str_text)
 
+    # TODO: TO REMOVE
     def debug_entering(self, who, method_name, **kwargs):
         if self.sheerka.debug_activated_for(who):
             str_text = pp.pformat(kwargs)
@@ -340,6 +334,7 @@ class ExecutionContext:
                 self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}Entering {who}.{method_name}:{CCM['reset']}")
                 self.sheerka.debug(f"[{self._id:3}] {str_text}")
 
+    # TODO: TO REMOVE
     def debug_log(self, who, text):
         if self.sheerka.debug_activated_for(who):
             self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}{text}{CCM['reset']}")
@@ -493,5 +488,3 @@ class ExecutionContext:
                 return False
 
         return False
-
-

src/core/sheerka/Sheerka.py

@@ -1,6 +1,7 @@
 import inspect
 import logging
 from dataclasses import dataclass
+from operator import attrgetter
 
 import core.builtin_helpers
 import core.utils
@@ -10,8 +11,7 @@ from cache.IncCache import IncCache
 from core.builtin_concepts import ErrorConcept, ReturnValueConcept, UnknownConcept
 from core.builtin_concepts_ids import BuiltinErrors, BuiltinConcepts
 from core.concept import Concept, ConceptParts, get_concept_attrs
-from core.error import ErrorObj
-from core.global_symbols import EVENT_USER_INPUT_EVALUATED, NotInit, NotFound
+from core.global_symbols import EVENT_USER_INPUT_EVALUATED, NotInit, NotFound, ErrorObj, EVENT_ONTOLOGY_CREATED
 from core.profiling import profile
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager, OntologyAlreadyExists
@@ -31,6 +31,23 @@ EXECUTE_STEPS = [
     BuiltinConcepts.AFTER_EVALUATION
 ]
 
+RULES_EVALUATE_STEPS = [
+    BuiltinConcepts.BEFORE_RULES_EVALUATION,
+    BuiltinConcepts.RULES_EVALUATION,
+    BuiltinConcepts.AFTER_RULES_EVALUATION,
+]
+
+RULES_EXECUTE_STEPS = [
+    BuiltinConcepts.BEFORE_EVALUATION,
+    BuiltinConcepts.EVALUATION,
+    BuiltinConcepts.AFTER_EVALUATION
+]
+
+# when a concept is instantiated via resolve or false_resolve
+# It indicate which parameter was used to recognize the concept
+RECOGNIZED_BY_ID = "by_id"
+RECOGNIZED_BY_NAME = "by_name"
+
 
 @dataclass
 class SheerkaMethod:
@@ -92,17 +109,25 @@ class Sheerka(Concept):
 
         self.save_execution_context = True
         self.enable_process_return_values = True
+        self.enable_process_rules = True
 
         self.methods_with_context = {"test_using_context"}  # only the names, the method is defined in sheerka_methods
         self.sheerka_methods = {
             "test": SheerkaMethod(self.test, False),
             "test_using_context": SheerkaMethod(self.test_using_context, False),
-            "test_dict": SheerkaMethod(self.test_dict, False)
+            "test_dict": SheerkaMethod(self.test_dict, False),
+            "test_error": SheerkaMethod(self.test_error, False),
         }
 
         self.locals = {}
         self.concepts_ids = None
 
+    def __copy__(self):
+        return self
+
+    def __deepcopy__(self, memodict={}):
+        return self
+
     @property
     def concepts_grammars(self):
         """
@@ -138,7 +163,7 @@ class Sheerka(Concept):
 
         setattr(self, bound_method.__name__, bound_method)
 
-    def initialize(self, root_folder: str = None, save_execution_context=None, enable_process_return_values=None):
+    def initialize(self, root_folder: str = None, **kwargs):
         """
         Starting Sheerka
         Loads the current configuration
@@ -149,11 +174,10 @@ class Sheerka(Concept):
         :return: ReturnValue(Success or Error)
         """
 
-        if save_execution_context is not None:
-            self.save_execution_context = save_execution_context
-
-        if enable_process_return_values is not None:
-            self.enable_process_return_values = enable_process_return_values
+        self.save_execution_context = kwargs.get("save_execution_context", self.save_execution_context)
+        self.enable_process_return_values = kwargs.get("enable_process_return_values",
+                                                       self.enable_process_return_values)
+        self.enable_process_rules = kwargs.get("enable_process_rules", self.enable_process_rules)
 
         try:
             self.during_initialisation = True
@@ -168,6 +192,7 @@ class Sheerka(Concept):
             self.get_builtin_evaluators()
             self.initialize_services()
             self.initialize_builtin_evaluators()
+            self.om.init_subscribers()
 
             event = Event("Initializing Sheerka.", user_id=self.name)
             self.om.save_event(event)
@@ -234,11 +259,12 @@ class Sheerka(Concept):
 
         core.utils.import_module_and_sub_module('core.sheerka.services')
         base_class = "core.sheerka.services.sheerka_service.BaseService"
-        for service in core.utils.get_sub_classes("core.sheerka.services", base_class):
-            instance = service(self)
-            if hasattr(instance, "initialize"):
-                instance.initialize()
-            self.services[service.NAME] = instance
+        services = [service(self) for service in core.utils.get_sub_classes("core.sheerka.services", base_class)]
+        services.sort(key=attrgetter("order"))
+        for service in services:
+            if hasattr(service, "initialize"):
+                service.initialize()
+            self.services[service.NAME] = service
 
     def initialize_services_deferred(self, context, is_first_time):
         """
@@ -325,7 +351,6 @@ class Sheerka(Concept):
         ontologies = self.om.current_sdp().load_ontologies()
         if not ontologies:
             return
-
         for ontology_name in list(reversed(ontologies))[1:]:
             self.om.push_ontology(ontology_name, False)
             self.initialize_services_deferred(context, False)
@@ -360,6 +385,10 @@ class Sheerka(Concept):
             ret = self.execute(execution_context, [user_input, reduce_requested], EXECUTE_STEPS)
             execution_context.add_values(return_values=ret)
 
+            # rule management
+            if self.enable_process_rules:
+                ret = self.execute_rules(execution_context, ret, RULES_EVALUATE_STEPS, RULES_EXECUTE_STEPS)
+
             if self.om.is_dirty:
                 self.om.commit(execution_context)
 
@@ -388,10 +417,14 @@ class Sheerka(Concept):
         :return:
         """
 
-        def new_instances(concepts):
+        def add_recognized_by(c, _recognized_by):
+            c.set_hint(BuiltinConcepts.RECOGNIZED_BY, _recognized_by)
+            return c
+
+        def new_instances(concepts, _recognized_by):
             if hasattr(concepts, "__iter__"):
-                return [self.new_from_template(c, c.key) for c in concepts]
-            return self.new_from_template(concepts, concepts.key)
+                return [add_recognized_by(self.new_from_template(c, c.key), _recognized_by) for c in concepts]
+            return add_recognized_by(self.new_from_template(concepts, concepts.key), _recognized_by)
 
         if concept is None:
             return None
@@ -421,10 +454,11 @@ class Sheerka(Concept):
                 if self.is_known(found := self.get_by_id(concept[1])):
                     instance = self.new_from_template(found, found.key)
                     instance._metadata.is_evaluated = True
+                    instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_ID)
                     return instance
             elif concept[0]:
                 if self.is_known(found := self.get_by_name(concept[0])):
-                    instances = new_instances(found)
+                    instances = new_instances(found, RECOGNIZED_BY_NAME)
                     core.builtin_helpers.set_is_evaluated(instances)
                     return instances
             else:
@@ -433,17 +467,22 @@ class Sheerka(Concept):
         # otherwise search in db
         if isinstance(concept, str):
             if self.is_known(found := self.get_by_name(concept)):
-                instances = new_instances(found)
+                instances = new_instances(found, RECOGNIZED_BY_NAME)
                 core.builtin_helpers.set_is_evaluated(instances, check_nb_variables=True)
                 return instances
 
         return None
 
     def fast_resolve(self, key, return_new=True):
-        def new_instances(concepts):
+        def add_recognized_by(c, _recognized_by):
+            c.set_hint(BuiltinConcepts.RECOGNIZED_BY, _recognized_by)
+            return c
+
+        def new_instances(concepts, _recognized_by):
             if hasattr(concepts, "__iter__"):
-                return [self.new_from_template(c, c.key) for c in concepts]
-            return self.new_from_template(concepts, concepts.key)
+                return [add_recognized_by(self.new_from_template(c, c.key), _recognized_by) for c in concepts]
+
+            return add_recognized_by(self.new_from_template(concepts, concepts.key), _recognized_by)
 
         if isinstance(key, Token):
             if key.type == TokenKind.RULE:  # do not recognize rules !!!
@@ -459,14 +498,17 @@ class Sheerka(Concept):
 
             if key[1]:
                 concept = self.om.get(self.CONCEPTS_BY_ID_ENTRY, key[1])
+                recognized_by = RECOGNIZED_BY_ID
             else:
                 concept = self.om.get(self.CONCEPTS_BY_NAME_ENTRY, key[0])
+                recognized_by = RECOGNIZED_BY_NAME
         else:
             concept = self.om.get(self.CONCEPTS_BY_NAME_ENTRY, key)
+            recognized_by = RECOGNIZED_BY_NAME
 
         if concept is NotFound:
             return None
-        return new_instances(concept) if return_new else concept
+        return new_instances(concept, recognized_by) if return_new else concept
 
     def new(self, concept_key, **kwargs):
         """
@@ -478,6 +520,8 @@ class Sheerka(Concept):
         """
         if isinstance(concept_key, tuple):
             concept_key, concept_id = concept_key[0], concept_key[1]
+        elif isinstance(concept_key, Concept):
+            concept_key, concept_id = concept_key.key, concept_key.id
         else:
             concept_id = None
 
@@ -547,12 +591,13 @@ class Sheerka(Concept):
         if name in self.om.current_sdp().load_ontologies():
             self.initialize_services_deferred(context, False)
 
-        self.om.save_ontologies()
+        self.om.save_ontologies_names()
+        self.publish(context, EVENT_ONTOLOGY_CREATED, name)
 
         return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
 
-    def pop_ontology(self):
-        ontology = self.om.pop_ontology()
+    def pop_ontology(self, context):
+        ontology = self.om.pop_ontology(context)
 
         self.om.reset_sheerka_state()
         for service in self.services.values():
@@ -561,7 +606,7 @@ class Sheerka(Concept):
             if hasattr(service, "reset_state"):
                 service.reset_state()
 
-        self.om.save_ontologies()
+        self.om.save_ontologies_names()
         return self.ret(self.name, True, self.new(BuiltinConcepts.ONTOLOGY_REMOVED, body=ontology))
 
     def get_ontology(self, context):
@@ -699,6 +744,13 @@ class Sheerka(Concept):
 
         return bool(obj)
 
+    @staticmethod
+    def is_error(obj):
+        """
+        opposite of is_success
+        """
+        return not Sheerka.is_success(obj)
+
     @staticmethod
     def is_known(obj):
         if not isinstance(obj, Concept):

src/core/sheerka/SheerkaOntologyManager.py

@@ -1,7 +1,10 @@
+from cache.Cache import Cache
 from cache.CacheManager import CacheManager
 from cache.DictionaryCache import DictionaryCache
+from cache.SetCache import SetCache
 from core.concept import copy_concepts_attrs, load_concepts_attrs
-from core.global_symbols import NotFound, Removed
+from core.global_symbols import NotFound, Removed, EVENT_CONCEPT_CREATED, EVENT_CONCEPT_DELETED, EVENT_RULE_CREATED, \
+    EVENT_RULE_DELETED, EVENT_CONCEPT_ID_DELETED, EVENT_RULE_ID_DELETED
 from core.utils import sheerka_deepcopy
 from sdp.sheerkaDataProvider import SheerkaDataProvider
 
@@ -88,6 +91,11 @@ class Ontology:
 
 class SheerkaOntologyManager:
     ROOT_ONTOLOGY_NAME = "__default__"
+    SELF_CACHE_MANAGER = "__ontology_manager__"  # cache to store SheerkaOntologyManager info
+    CONCEPTS_BY_ONTOLOGY_ENTRY = "ConceptsByOntologyEntry"
+    RULES_BY_ONTOLOGY_ENTRY = "RulesByOntologyEntry"
+    ONTOLOGY_BY_CONCEPT_ENTRY = "OntologyByConceptEntry"
+    ONTOLOGY_BY_RULE_ENTRY = "OntologyByRuleEntry"
 
     def __init__(self, sheerka, root_folder, cache_only):
         self.sheerka = sheerka
@@ -98,6 +106,20 @@ class SheerkaOntologyManager:
         ref_cache_manager = CacheManager(self.cache_only, sdp=SheerkaDataProvider(root_folder, self.sheerka))
         self.ontologies = [Ontology(self.ROOT_ONTOLOGY_NAME, ref_cache_manager, None)]
 
+        self_sdp = SheerkaDataProvider(root_folder, self.sheerka, self.SELF_CACHE_MANAGER)
+        self.self_cache_manager = CacheManager(self.cache_only, sdp=self_sdp)
+        cache = SetCache(max_size=None).auto_configure(self.CONCEPTS_BY_ONTOLOGY_ENTRY)
+        self.self_cache_manager.register_cache(self.CONCEPTS_BY_ONTOLOGY_ENTRY, cache)
+
+        cache = SetCache(max_size=None).auto_configure(self.RULES_BY_ONTOLOGY_ENTRY)
+        self.self_cache_manager.register_cache(self.RULES_BY_ONTOLOGY_ENTRY, cache)
+
+        cache = Cache(max_size=None).auto_configure(self.ONTOLOGY_BY_CONCEPT_ENTRY)
+        self.self_cache_manager.register_cache(self.ONTOLOGY_BY_CONCEPT_ENTRY, cache)
+
+        cache = Cache(max_size=None).auto_configure(self.ONTOLOGY_BY_RULE_ENTRY)
+        self.self_cache_manager.register_cache(self.ONTOLOGY_BY_RULE_ENTRY, cache)
+
     @property
     def ontologies_names(self):
         return [o.name for o in self.ontologies]
@@ -111,6 +133,12 @@ class SheerkaOntologyManager:
         self.frozen = False
         return self
 
+    def init_subscribers(self):
+        self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.on_concept_created)
+        self.sheerka.subscribe(EVENT_CONCEPT_DELETED, self.on_concept_deleted)
+        self.sheerka.subscribe(EVENT_RULE_CREATED, self.on_rule_created)
+        self.sheerka.subscribe(EVENT_RULE_DELETED, self.on_rule_deleted)
+
     def push_ontology(self, name, cache_only=None):
         """
         Add an ontology layer
@@ -138,7 +166,7 @@ class SheerkaOntologyManager:
         self.ontologies.insert(0, Ontology(name, cache_manager, alt_sdp))
         return self
 
-    def pop_ontology(self):
+    def pop_ontology(self, context):
         """
         Remove the top ontology layer
         """
@@ -148,6 +176,22 @@ class SheerkaOntologyManager:
         if len(self.ontologies) == 1:
             raise OntologyManagerCannotPopLatest()
 
+        # remove concepts and rules tracking for the ontology to pop
+        ontology_name = self.current_ontology().name
+        concepts = self.self_cache_manager.get(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name)
+        if concepts is not NotFound:
+            for concept in concepts:
+                self.sheerka.publish(context, EVENT_CONCEPT_ID_DELETED, concept)
+                self.self_cache_manager.delete(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept)
+            self.self_cache_manager.delete(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name)
+
+        rules = self.self_cache_manager.get(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name)
+        if rules is not NotFound:
+            for rule in rules:
+                self.sheerka.publish(context, EVENT_RULE_ID_DELETED, rule)
+                self.self_cache_manager.delete(self.ONTOLOGY_BY_RULE_ENTRY, rule)
+            self.self_cache_manager.delete(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name)
+
         return self.ontologies.pop(0)
 
     def add_ontology(self, ontology: Ontology):
@@ -179,7 +223,7 @@ class SheerkaOntologyManager:
 
         raise KeyError(name)
 
-    def save_ontologies(self):
+    def save_ontologies_names(self):
         self.current_sdp().save_ontologies(self.ontologies_names)
 
     # def load_ontologies(self):
@@ -446,6 +490,7 @@ class SheerkaOntologyManager:
         :param context:
         :return:
         """
+        self.self_cache_manager.commit(context)
         return self.current_cache_manager().commit(context)
 
     def clear(self, cache_name=None):
@@ -468,3 +513,21 @@ class SheerkaOntologyManager:
 
     def is_dirty(self):
         return self.current_cache_manager().is_dirty
+
+    def on_concept_created(self, context, concept):
+        self.self_cache_manager.put(self.CONCEPTS_BY_ONTOLOGY_ENTRY, self.current_ontology().name, concept.id)
+        self.self_cache_manager.put(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id, self.current_ontology().name)
+
+    def on_concept_deleted(self, context, concept):
+        ontology_name = self.self_cache_manager.get(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id)
+        self.self_cache_manager.delete(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name, concept.id)
+        self.self_cache_manager.delete(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id)
+
+    def on_rule_created(self, context, rule):
+        self.self_cache_manager.put(self.RULES_BY_ONTOLOGY_ENTRY, self.current_ontology().name, rule.id)
+        self.self_cache_manager.put(self.ONTOLOGY_BY_RULE_ENTRY, rule.id, self.current_ontology().name)
+
+    def on_rule_deleted(self, context, rule):
+        ontology_name = self.self_cache_manager.get(self.ONTOLOGY_BY_RULE_ENTRY, rule.id)
+        self.self_cache_manager.delete(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name, rule.id)
+        self.self_cache_manager.delete(self.ONTOLOGY_BY_RULE_ENTRY, rule.id)

src/core/sheerka/services/SheerkaAdmin.py

@@ -3,7 +3,7 @@ import time
 from os import path
 
 from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
-from core.builtin_helpers import ensure_concept
+from core.builtin_helpers import ensure_concept_or_rule
 from core.concept import Concept
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from core.sheerka.services.sheerka_service import BaseService
@@ -28,6 +28,7 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.extended_isinstance, False)
         self.sheerka.bind_service_method(self.is_container, False)
         self.sheerka.bind_service_method(self.format_rules, False)
+        self.sheerka.bind_service_method(self.exec_rules, False)
         self.sheerka.bind_service_method(self.admin_push_ontology, True, as_name="push_ontology")
         self.sheerka.bind_service_method(self.admin_pop_ontology, True, as_name="pop_ontology")
         self.sheerka.bind_service_method(self.ontologies, False)
@@ -127,24 +128,36 @@ class SheerkaAdmin(BaseService):
         concepts = sorted(self.sheerka.om.list(self.sheerka.CONCEPTS_BY_ID_ENTRY), key=lambda item: int(item.id))
         return self.sheerka.new(BuiltinConcepts.TO_LIST, body=concepts)
 
-    def desc(self, *concepts):
-        ensure_concept(*concepts)
+    def desc(self, *items):
+        ensure_concept_or_rule(*items)
         res = []
-        for c in concepts:
-            bag = {
-                "id": c.id,
-                "name": c.name,
-                "key": c.key,
-                "definition": c.get_metadata().definition,
-                "type": c.get_metadata().definition_type,
-                "body": c.get_metadata().body,
-                "where": c.get_metadata().where,
-                "pre": c.get_metadata().pre,
-                "post": c.get_metadata().post,
-                "ret": c.get_metadata().ret,
-                "vars": c.get_metadata().variables,
-                "props": c.get_metadata().props,
-            }
+        for item in items:
+            if isinstance(item, Concept):
+                bag = {
+                    "id": item.id,
+                    "name": item.name,
+                    "key": item.key,
+                    "definition": item.get_metadata().definition,
+                    "type": item.get_metadata().definition_type,
+                    "body": item.get_metadata().body,
+                    "where": item.get_metadata().where,
+                    "pre": item.get_metadata().pre,
+                    "post": item.get_metadata().post,
+                    "ret": item.get_metadata().ret,
+                    "vars": item.get_metadata().variables,
+                    "props": item.get_metadata().props,
+                }
+            else:
+                bag = {
+                    "id": item.id,
+                    "name": item.metadata.name,
+                    "type": item.metadata.action_type,
+                    "predicate": item.metadata.predicate,
+                    "action": item.metadata.action,
+                    "priority": item.priority,
+                    "compiled": item.metadata.is_compiled,
+                    "enabled": item.metadata.is_enabled,
+                }
             res.append(self.sheerka.new(BuiltinConcepts.TO_DICT, body=bag))
 
         return res[0] if len(res) == 1 else self.sheerka.new(BuiltinConcepts.TO_LIST, body=res)
@@ -152,6 +165,9 @@ class SheerkaAdmin(BaseService):
     def format_rules(self):
         return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_format_rules())
 
+    def exec_rules(self):
+        return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_exec_rules())
+
     def extended_isinstance(self, a, b):
         """
         switch between sheerka.isinstance and builtin.isinstance
@@ -180,8 +196,8 @@ class SheerkaAdmin(BaseService):
     def admin_push_ontology(self, context, name):
         return self.sheerka.push_ontology(context, name, False)
 
-    def admin_pop_ontology(self):
-        return self.sheerka.pop_ontology()
+    def admin_pop_ontology(self, context):
+        return self.sheerka.pop_ontology(context)
 
     def ontologies(self):
         ontologies = self.sheerka.om.ontologies_names

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -38,7 +38,7 @@ class SheerkaComparisonManager(BaseService):
     DEFAULT_COMPARISON_VALUE = 1
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=14)
 
     def initialize(self):
         cache = ListCache().auto_configure(self.COMPARISON_ENTRY)

src/core/sheerka/services/SheerkaConceptManager.py

@@ -12,8 +12,7 @@ from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, Built
 from core.builtin_helpers import ensure_concept, ensure_bnf
 from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \
     VARIABLE_PREFIX
-from core.error import ErrorObj
-from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound
+from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound, ErrorObj, EVENT_CONCEPT_DELETED
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer, TokenKind
 from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
@@ -100,7 +99,7 @@ class SheerkaConceptManager(BaseService):
     RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Resolved_Concepts_By_First_Keyword"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=11)
         self.forbidden_meta = {"is_builtin", "key", "id", "props", "variables"}
         self.allowed_meta = {attr for attr in vars(ConceptMetadata) if
                              not attr.startswith("_") and attr not in self.forbidden_meta}
@@ -147,7 +146,7 @@ class SheerkaConceptManager(BaseService):
             self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
 
         # initialize the dictionary of first tokens
-        self.sheerka.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init the cache with the values from sdp
+        self.sheerka.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None)  # to init the cache with the values from sdp
         concepts_by_first_keyword = self.sheerka.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
         res = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
         self.sheerka.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
@@ -353,6 +352,8 @@ class SheerkaConceptManager(BaseService):
         :param concept:
         :return:
         """
+        # TODO : resolve concept first
+
         sheerka = context.sheerka
         refs = self.sheerka.om.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
         if refs is not NotFound:
@@ -361,6 +362,7 @@ class SheerkaConceptManager(BaseService):
 
         try:
             sheerka.om.remove_concept(concept)
+            sheerka.publish(context, EVENT_CONCEPT_DELETED, concept)
             return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS))
         except ConceptNotFound as ex:
             return sheerka.ret(self.NAME, False, sheerka.err(ex))

src/core/sheerka/services/SheerkaDebugManager.py

@@ -91,6 +91,14 @@ class BaseDebugLogger:
             BaseDebugLogger.ids[hint] = 0
         return BaseDebugLogger.ids[hint]
 
+    @staticmethod
+    def current_id(hint):
+        if hint in BaseDebugLogger.ids:
+            return BaseDebugLogger.ids[hint]
+        else:
+            BaseDebugLogger.ids[hint] = 0
+            return 0
+
     def __init__(self, debug_manager, context, who, method_name, debug_id):
         pass
 
@@ -276,7 +284,7 @@ class SheerkaDebugManager(BaseService):
     children_activation_regex = re.compile(r"(\d+)\+")
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=1)
         self.activated = False  # is debug activated
         self.explicit = False  # No need to activate context debug when debug mode is on # to remove ?
         self.context_cache = set()  # debug for specific context # to remove ?
@@ -295,16 +303,6 @@ class SheerkaDebugManager(BaseService):
         ]
 
     def initialize(self):
-        # TO REMOVE ???
-        self.sheerka.bind_service_method(self.set_explicit, True)
-        self.sheerka.bind_service_method(self.activate_debug_for, True)
-        self.sheerka.bind_service_method(self.deactivate_debug_for, True)
-        self.sheerka.bind_service_method(self.debug_activated, False)
-        self.sheerka.bind_service_method(self.debug_activated_for, False)
-        self.sheerka.bind_service_method(self.get_context_debug_mode, False)
-        self.sheerka.bind_service_method(self.debug_rule_activated, False)
-        self.sheerka.bind_service_method(self.debug, False, visible=False)
-
         self.sheerka.bind_service_method(self.set_debug, True)
         self.sheerka.bind_service_method(self.inspect, False)
         self.sheerka.bind_service_method(self.get_debugger, False)
@@ -341,79 +339,13 @@ class SheerkaDebugManager(BaseService):
         self.sheerka.record_var(context, self.NAME, "activated", self.activated)
         return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
-    def set_explicit(self, context, value=True):
-        self.explicit = value
-        self.sheerka.record_var(context, self.NAME, "explicit", self.explicit)
-        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-
-    def activate_debug_for(self, context, debug_id, children=False):
-        """
-
-        :param context:
-        :param debug_id: if debug_id is str, activate variable cache, context_cache otherwise
-        :param children:
-        :return:
-        """
-        # preprocess
-        if isinstance(debug_id, str) and (m := self.children_activation_regex.match(debug_id)):
-            debug_id = int(m.group(1))
-            children = True
-
-        if isinstance(debug_id, str):
-            self.variable_cache.add(debug_id)
-            self.sheerka.record_var(context, self.NAME, "variable_cache", self.variable_cache)
-        else:
-            self.context_cache.add(debug_id)
-            if children:
-                self.context_cache.add(str(debug_id) + "+")
-            self.sheerka.record_var(context, self.NAME, "context_cache", self.context_cache)
-        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-
-    def deactivate_debug_for(self, context, debug_id, children=False):
-        if isinstance(debug_id, str):
-            self.variable_cache.discard(debug_id)
-            self.sheerka.record_var(context, self.NAME, "variable_cache", self.variable_cache)
-        else:
-            self.context_cache.discard(debug_id)
-            if children:
-                self.context_cache.discard(str(debug_id) + "+")
-            self.sheerka.record_var(context, self.NAME, "context_cache", self.context_cache)
-        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-
-    def debug_activated(self):
-        return self.activated
-
-    def debug_activated_for(self, debug_id):
-        if not self.activated:
-            return None
-
-        return debug_id in self.variable_cache
-
-    def debug_rule_activated(self, rule_id, context_id):
-        """
-
-        :param rule_id:
-        :param context_id:
-        :return:
-        """
-        key = f"{rule_id}|{context_id}"
-        return key in self.rules_cache
-
-    def get_context_debug_mode(self, context_id):
-        if not self.activated:
-            return None, None
-
-        debug_for_children = "protected" if str(context_id) + "+" in self.context_cache else None
-        debug_for_self = "private" if not self.explicit or context_id in self.context_cache else None
-
-        return debug_for_self, debug_for_children
-
     def debug(self, *args, **kwargs):
         print(*args, **kwargs)
 
-    def get_debugger(self, context, who, method_name):
+    def get_debugger(self, context, who, method_name, new_debug_id=True):
         if self.compute_debug(context, who, method_name):
-            debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id))
+            debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id)) if new_debug_id \
+                else ConsoleDebugLogger.current_id(context.event.get_digest() + str(context.id))
             return ConsoleDebugLogger(self, context, who, method_name, debug_id)
 
         return NullDebugLogger()

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -11,7 +11,8 @@ from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
 from core.utils import unstr_concept
 from parsers.BaseNodeParser import ConceptNode
-from parsers.ExpressionParser import ExpressionParser, TrueifyVisitor
+from parsers.ExpressionParser import ExpressionParser
+from parsers.expressions import TrueifyVisitor
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,

src/core/sheerka/services/SheerkaEvaluateRules.py

@@ -1,7 +1,9 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one
+from core.global_symbols import EVENT_RULE_CREATED, EVENT_RULE_DELETED, EVENT_RULE_ID_DELETED
 from core.sheerka.services.sheerka_service import BaseService
 from evaluators.ConceptEvaluator import ConceptEvaluator
+from sheerkarete.network import ReteNetwork
 
 DISABLED_RULES = "#disabled#"
 LOW_PRIORITY_RULES = "#low_priority#"
@@ -11,12 +13,18 @@ class SheerkaEvaluateRules(BaseService):
     NAME = "EvaluateRules"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        # order must be before RuleManager because of event subscription
+        super().__init__(sheerka, 4)
         self.evaluators_by_name = None
+        self.network = ReteNetwork()
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.evaluate_format_rules, False)
+        self.sheerka.bind_service_method(self.evaluate_format_rules, False, visible=False)
+        self.sheerka.bind_service_method(self.evaluate_exec_rules, False, visible=False)
         self.reset_evaluators()
+        self.sheerka.subscribe(EVENT_RULE_CREATED, self.on_rule_created)
+        self.sheerka.subscribe(EVENT_RULE_DELETED, self.on_rule_deleted)
+        self.sheerka.subscribe(EVENT_RULE_ID_DELETED, self.on_rule_deleted)
 
     def reset_evaluators(self):
         # instantiate evaluators, once for all, only keep when it's enabled
@@ -24,6 +32,20 @@ class SheerkaEvaluateRules(BaseService):
         evaluators = [e for e in evaluators if e.enabled]
         self.evaluators_by_name = {e.short_name: e for e in evaluators}
 
+    def evaluate_exec_rules(self, context, return_values):
+        # self.network.add_obj("__rets", return_values)
+        for ret in return_values:
+            self.network.add_obj("__ret", ret)
+
+        results = []  # list of return values, for activated rules
+        for match in self.network.matches:
+            for rule in match.pnode.rules:
+                body = context.sheerka.new(BuiltinConcepts.RULE_EVALUATION_RESULT, rule=rule)
+                return_value = context.sheerka.ret(self.NAME, True, body)
+                results.append(return_value)
+
+        return results
+
     def evaluate_format_rules(self, context, bag, disabled):
         return self.evaluate_rules(context, self.sheerka.get_format_rules(), bag, disabled)
 
@@ -37,7 +59,7 @@ class SheerkaEvaluateRules(BaseService):
         :param disabled: disabled rules (because they have already been fired or whatever)
         :return: { True : list of success, False :list of failed, '#disabled"': list of disabled...}
         """
-        with context.push(BuiltinConcepts.EVALUATING_RULES, bag, desc="Evaluating rules...") as sub_context:
+        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
             sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
             sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
             sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
@@ -81,7 +103,7 @@ class SheerkaEvaluateRules(BaseService):
         """
 
         results = []
-        for rule_predicate in rule.compiled_predicate:
+        for rule_predicate in rule.compiled_predicates:
 
             if rule_predicate.source in bag:
                 # simple case where the rule is an item of the bag. No need of complicate evaluation
@@ -94,15 +116,44 @@ class SheerkaEvaluateRules(BaseService):
                     rule_predicate.concept.get_metadata().is_evaluated = False
 
                 evaluator = self.evaluators_by_name[rule_predicate.evaluator]
-                results.append(evaluator.eval(context, rule_predicate.predicate))
+                res = evaluator.eval(context, rule_predicate.predicate)
+                if res.status and isinstance(res.body, bool) and res.body:
+                    # one successful value found. No need to look any further
+                    results = [res]
+                    break
+                else:
+                    results.append(res)
 
-        debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule")
+        debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule", new_debug_id=False)
         debugger.debug_rule(rule, results)
-        # if context.sheerka.debug_rule_activated(rule_id, context.id):
-        #     context.debug(SheerkaEvaluateRules.NAME, "evaluate_rules", f"result(#{rule_id})", results)
 
         return expect_one(context, results)
 
+    def remove_from_rete_memory(self, lst):
+        if lst is None:
+            return
+
+        for obj in lst:
+            self.network.remove_obj(obj)
+
+    def on_rule_created(self, context, rule):
+        """
+        When a new rule is added to the system, update the network
+        """
+        if rule.metadata.is_enabled and rule.rete_disjunctions:
+            self.network.add_rule(rule)
+
+    def on_rule_deleted(self, context, rule):
+        """
+        When a rule is deleted from the system, remove it from the network
+        """
+        if isinstance(rule, str):
+            rule = self.sheerka.get_rule_by_id(rule)
+            if not self.sheerka.is_known(rule):
+                return
+
+        self.network.remove_rule(rule)
+
     @staticmethod
     def get_debug_format(result):
         """

src/core/sheerka/services/SheerkaEventManager.py

@@ -11,7 +11,7 @@ class SheerkaEventManager(BaseService):
     NAME = "EventManager"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=2)
         self._lock = RLock()
         self.subscribers = {}
 

src/core/sheerka/services/SheerkaExecute.py

@@ -1,5 +1,4 @@
 import core.utils
-from cache.Cache import Cache
 from cache.FastCache import FastCache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.global_symbols import NotFound
@@ -16,6 +15,8 @@ EVALUATOR_STEPS = [
     BuiltinConcepts.BEFORE_RENDERING,
     BuiltinConcepts.RENDERING,
     BuiltinConcepts.AFTER_RENDERING,
+    BuiltinConcepts.BEFORE_RULES_EVALUATION,
+    BuiltinConcepts.AFTER_RULES_EVALUATION,
 ]
 
 
@@ -120,6 +121,10 @@ class ParserInput:
         return self.pos < self.end
 
     def the_token_after(self, skip_whitespace=True):
+        """
+        Returns the token after the current one
+        Never returns None (returns TokenKind.EOF instead)
+        """
         my_pos = self.pos + 1
         if my_pos >= self.end:
             return Token(TokenKind.EOF, "", -1, -1, -1)
@@ -167,7 +172,8 @@ class SheerkaExecute(BaseService):
     PARSERS_INPUTS_ENTRY = "Execute:ParserInput"  # entry for admin or internal variables
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        # order must be after SheerkaEvaluateRules because of self.rules_evaluation_service
+        super().__init__(sheerka, order=5)
         self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=20)
         self.instantiated_evaluators = None
         self.evaluators_by_name = None
@@ -191,12 +197,18 @@ class SheerkaExecute(BaseService):
         # Except 2 : we store the type of the parser, not its instance
         self.grouped_parsers_cache = {}
 
+        self.rules_eval_service = None
+
     def initialize(self):
-        self.sheerka.bind_service_method(self.execute, True)
+        self.sheerka.bind_service_method(self.execute, True, visible=False)
+        self.sheerka.bind_service_method(self.execute_rules, True, visible=False)
 
         self.reset_registered_evaluators()
         self.reset_registered_parsers()
 
+        from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
+        self.rules_eval_service = self.sheerka.services[SheerkaEvaluateRules.NAME]
+
     def reset_state(self):
         self.pi_cache.clear()
 
@@ -347,7 +359,7 @@ class SheerkaExecute(BaseService):
             if pi is NotFound:  # when CacheManager.cache_only is True
                 pi = ParserInput(text)
                 self.pi_cache.put(text, pi)
-            return pi
+            return ParserInput(text, pi.tokens)  # new instance, but no need to tokenize the text again
 
         key = text or core.utils.get_text_from_tokens(tokens)
         pi = ParserInput(key, tokens)
@@ -582,6 +594,55 @@ class SheerkaExecute(BaseService):
 
         return return_values
 
+    def execute_rules(self, context, return_values, rules_steps, evaluation_steps):
+        """"
+        Executes the execution rules until no match is found
+        :param context:
+        :param return_values: input return values
+        :param rules_steps: steps are configurable
+        :param evaluation_steps: steps are configurable
+        :return: out return_values
+        """
+        continue_execution = True
+        counter = 0
+        in_rete_memory = None
+        while continue_execution:
+            with context.push(BuiltinConcepts.PROCESSING, {"counter": counter}, desc=f"{counter=}") as sub_context:
+
+                # apply rule evaluation steps
+                for step in rules_steps:
+                    if step == BuiltinConcepts.RULES_EVALUATION:
+                        eval_res = self.rules_eval_service.evaluate_exec_rules(sub_context, return_values)
+                        if not eval_res:
+                            self.rules_eval_service.remove_from_rete_memory(return_values)
+                            continue_execution = False
+                            break
+                        else:
+                            in_rete_memory = return_values.copy()
+                            return_values = eval_res
+                    else:
+                        return_values = self.call_evaluators(sub_context, return_values, step)
+
+                if not continue_execution:
+                    break
+
+                # evaluate the result
+                return_values = [r.body.body.compiled_action for r in return_values]
+                while True:
+                    copy = return_values[:]
+                    for step in evaluation_steps:
+                        return_values = self.call_evaluators(sub_context, return_values, step)
+
+                    if copy == return_values[:]:
+                        break
+
+                # evaluation is done. Remove object in Rete memory
+                self.rules_eval_service.remove_from_rete_memory(in_rete_memory)
+
+                counter += 1
+
+        return return_values
+
     def undo_preprocess(self):
         for item, var_name, value in self.old_values:
             setattr(item, var_name, value)

src/core/sheerka/services/SheerkaHasAManager.py

@@ -7,7 +7,7 @@ class SheerkaHasAManager(BaseService):
     NAME = "HasAManager"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=22)
 
     def initialize(self):
         self.sheerka.bind_service_method(self.set_hasa, True)

src/core/sheerka/services/SheerkaIsAManager.py

@@ -15,7 +15,7 @@ class SheerkaIsAManager(BaseService):
     CONCEPTS_IN_GROUPS_ENTRY = "IsAManager:Concepts_In_Groups"  # cache for get_set_elements()
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=21)
 
     def initialize(self):
         self.sheerka.bind_service_method(self.set_isa, True)

src/core/sheerka/services/SheerkaMemory.py

@@ -20,7 +20,7 @@ class SheerkaMemory(BaseService):
     OBJECTS_ENTRY = "Memory:Objects"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=13)
         self.short_term_objects = FastCache()
         self.registration = {}
 
@@ -39,6 +39,8 @@ class SheerkaMemory(BaseService):
 
         cache = ListIfNeededCache().auto_configure(self.OBJECTS_ENTRY)
         self.sheerka.om.register_cache(self.OBJECTS_ENTRY, cache, persist=True, use_ref=True)
+        
+        self.sheerka.subscribe(EVENT_CONTEXT_DISPOSED, self.remove_context)
 
     def reset(self):
         self.short_term_objects.clear()
@@ -48,9 +50,6 @@ class SheerkaMemory(BaseService):
         self.short_term_objects.clear()
         self.registration.clear()
 
-    def initialize_deferred(self, context, is_first_time):
-        self.sheerka.subscribe(EVENT_CONTEXT_DISPOSED, self.remove_context)
-
     def get_from_short_term_memory(self, context, key):
         while True:
             try:

src/core/sheerka/services/SheerkaOut.py

@@ -31,6 +31,7 @@ class SheerkaOut(BaseService):
         if valid_rules:
             if len(valid_rules) > 1:
                 # TODO manage when too many rules
+                print("TODO: TOO MANY RULES !!!!!")
                 pass
 
             rule = valid_rules[0]

src/core/sheerka/services/SheerkaResultManager.py

@@ -37,11 +37,12 @@ class SheerkaResultManager(BaseService):
         self.sheerka.bind_service_method(self.get_last_created_concept, False, as_name="last_created_concept")
         self.sheerka.bind_service_method(self.get_last_error, False, as_name="last_err")
 
-    def initialize_deferred(self, context, is_first_time):
-        self.restore_values(*self.state_vars)
         self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.user_input_evaluated)
         self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.new_concept_created)
 
+    def initialize_deferred(self, context, is_first_time):
+        self.restore_values(*self.state_vars)
+
     def test_only_reset(self):
         self.executions_contexts_cache.clear()
         self.last_execution = None

src/core/sheerka/services/SheerkaRuleManager.py

@@ -1,28 +1,35 @@
 import operator
 import re
 from dataclasses import dataclass
-from typing import Union
+from typing import Union, Set, List
 
 from cache.Cache import Cache
 from cache.ListIfNeededCache import ListIfNeededCache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
-from core.builtin_helpers import parse_unrecognized, only_successful, ensure_rule
+from core.builtin_helpers import parse_unrecognized, is_a_question, parse_python, \
+    ensure_evaluated, expect_one, parse_expression
 from core.concept import Concept
-from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound
-from core.rule import Rule
-from core.sheerka.services.sheerka_service import BaseService
+from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound, ErrorObj, \
+    EVENT_RULE_CREATED, EVENT_RULE_DELETED
+from core.rule import Rule, ACTION_TYPE_PRINT
+from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
+from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError
 from core.tokenizer import Keywords, TokenKind, Token, IterParser
 from core.utils import index_tokens, COLORS, get_text_from_tokens
 from evaluators.ConceptEvaluator import ConceptEvaluator
-from evaluators.PythonEvaluator import PythonEvaluator
+from evaluators.PythonEvaluator import PythonEvaluator, Expando
 from parsers.BaseNodeParser import SourceCodeWithConceptNode, ConceptNode, SourceCodeNode
+from parsers.ExpressionParser import AndNode, ExpressionParser
 from parsers.PythonParser import PythonNode
+from sheerkarete.conditions import AndConditions
+
+CONCEPTS_ONLY_PARSERS = ["ExactConcept", "Bnf", "Sya", "Sequence"]
 
 identifier_regex = re.compile(r"[\w _.]+")
 
 
 @dataclass
-class FormatRuleError:
+class FormatRuleError(ErrorObj):
     pass
 
 
@@ -199,7 +206,7 @@ class FormatAstMulti(FormatAstNode):
             **kwargs)
 
 
-class FormatRuleParser(IterParser):
+class FormatRuleActionParser(IterParser):
 
     @staticmethod
     def to_text(list_or_dict_of_tokens):
@@ -242,7 +249,7 @@ class FormatRuleParser(IterParser):
 
     def parse(self):
         """
-        Parses a format rule
+        Parses the print part of the format rule
         format ::= {variable'} | function(...) | rawtext
         :return:
         """
@@ -394,7 +401,7 @@ class FormatRuleParser(IterParser):
         source = get_text_from_tokens(args[0])
         if len(source) > 1 and source[0] in ("'", '"') and source[-1] in ("'", '"'):
             source = source[1:-1]
-            parser = FormatRuleParser(source)
+            parser = FormatRuleActionParser(source)
             res = parser.parse()
             self.error_sink = parser.error_sink
             return FormatAstColor(color, res)
@@ -497,12 +504,139 @@ class FormatRuleParser(IterParser):
         return FormatAstMulti(get_text_from_tokens(args[0]))
 
 
+@dataclass
+class EmitPythonCodeException(Exception):
+    error: object
+
+
+class PythonCodeEmitter:
+
+    def __init__(self, context, text=None):
+        self.context = context
+        self.text = text or ""
+        self.var_counter = 0
+        self.variables = []
+
+    def add(self, text):
+        self.text += f" and {text}" if self.text else text
+        return self
+
+    def recognize(self, obj, as_name, root=True):
+        if isinstance(obj, str):
+            return self.recognize_str(obj, as_name)
+        elif isinstance(obj, (int, float)):
+            return self.recognize_int(obj, as_name)
+        elif isinstance(obj, Concept):
+            return self.recognize_concept(obj, as_name, root)
+        elif isinstance(obj, Expando):
+            return self.recognize_expando(obj, as_name, root)
+        else:
+            raise NotImplementedError()
+
+    def recognize_str(self, text, as_name):
+        if self.text:
+            self.text += " and "
+
+        if "'" in text and '"' in text:
+            self.text += f"{as_name} == '{text}'"
+        elif "'" in text:
+            self.text += f'{as_name} == "{text}"'
+        else:
+            self.text += f"{as_name} == '{text}'"
+        return self
+
+    def recognize_int(self, value, as_name):
+        if self.text:
+            self.text += " and "
+
+        self.text += f"{as_name} == {value}"
+        return self
+
+    def recognize_expando(self, value, as_name, root=True):
+        if self.text:
+            self.text += " and "
+
+        if not root:
+            as_name = self.add_variable(as_name)
+
+        self.text += f"isinstance({as_name}, Expando) and {as_name}.get_name() == '{value.get_name()}'"
+        return self
+
+    def recognize_concept(self, concept, as_name, root=True):
+        if self.text:
+            self.text += " and "
+
+        if not root:
+            as_name = self.add_variable(as_name)
+
+        if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
+            self.text += f"isinstance({as_name}, Concept) and {as_name}.name == '{concept.name}'"
+        elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
+            self.text += f"isinstance({as_name}, Concept) and {as_name}.id == '{concept.id}'"
+        else:
+            self.text += f"isinstance({as_name}, Concept) and {as_name}.key == '{concept.key}'"
+            if len(concept.get_metadata().variables) > 0:
+                # add variables constraints
+                evaluated = ensure_evaluated(self.context, concept, eval_body=False, metadata=["variables"])
+
+                if not self.context.sheerka.is_success(evaluated) and evaluated.key != concept.key:
+                    raise EmitPythonCodeException(evaluated)
+
+                for k, v in concept.variables().items():
+                    self.recognize(v, f"{as_name}.get_value('{k}')", root=False)
+
+        return self
+
+    def add_variable(self, target):
+        var_name = f"__x_{self.var_counter:02}__"
+        self.var_counter += 1
+        self.variables.append((var_name, target))
+        return var_name
+
+    def get_text(self):
+        if self.variables:
+            variables_as_str = '\n'.join([f"{k} = {v}" for k, v in self.variables])
+            return variables_as_str + "\n" + self.text
+
+        return self.text
+
+
+class NoConditionFound(ErrorObj):
+    def __eq__(self, other):
+        return isinstance(other, NoConditionFound)
+
+    def __hash__(self):
+        return 0
+
+
 @dataclass()
-class RulePredicate:
-    source: str
-    evaluator: str
-    predicate: ReturnValueConcept
-    concept: Union[Concept, None]
+class RuleCompiledPredicate:
+    """
+    The 'when' expression is parsed to have a ReturnValueConcept or a Concept that can then be evaluated
+    Depending on the evaluator, the 'predicate' attribute or the 'concept' attribute will be used
+    """
+    source: str  # what was compiled # DO NOT REMOVE
+    action: str  # sheerka action when the rule must be executed # can be removed
+
+    # when used as a list of predicate to iterate thru
+    evaluator: str  # evaluator to use when the rule will be evaluated
+    predicate: ReturnValueConcept  # compiled source as ReturnValue
+    concept: Union[Concept, None]  # compiled source as concept
+
+    variables: Set[str] = None  # TODO: set of required variables
+
+
+@dataclass
+class CompiledWhenResult:
+    """
+    For a given source to compile (a given 'when')
+    List of RuleCompiledPredicate found
+    and list of Rete Conditions
+
+    The two ways of evaluating a 'when' are used by Sheerka
+    """
+    compiled_predicates: List[RuleCompiledPredicate]
+    rete_disjunctions: List[AndConditions]
 
 
 class SheerkaRuleManager(BaseService):
@@ -513,15 +647,18 @@ class SheerkaRuleManager(BaseService):
     RULES_BY_NAME_ENTRY = "RuleManager:Rules_By_Name"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=12)
         self._format_rules = None  # sorted by priority
+        self._exec_rules = None  # sorted by priority
+        self.expression_parser = ExpressionParser()
 
     def initialize(self):
         self.sheerka.bind_service_method(self.create_new_rule, True, visible=False)
+        self.sheerka.bind_service_method(self.remove_rule, True)
         self.sheerka.bind_service_method(self.get_rule_by_id, False)
         self.sheerka.bind_service_method(self.get_rule_by_name, False)
-        self.sheerka.bind_service_method(self.dump_desc_rule, False, as_name="desc_rule")
         self.sheerka.bind_service_method(self.get_format_rules, False, visible=False)
+        self.sheerka.bind_service_method(self.get_exec_rules, False, visible=False)
         self.sheerka.bind_service_method(self.resolve_rule, False, visible=False)
 
         cache = Cache().auto_configure(self.FORMAT_RULE_ENTRY)
@@ -531,6 +668,8 @@ class SheerkaRuleManager(BaseService):
         cache = ListIfNeededCache().auto_configure(self.RULES_BY_NAME_ENTRY)
         self.sheerka.om.register_cache(self.RULES_BY_NAME_ENTRY, cache, True, True)
 
+        self.sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities)
+
     def initialize_deferred(self, context, is_first_time):
 
         if is_first_time:
@@ -551,15 +690,17 @@ class SheerkaRuleManager(BaseService):
 
         # compile all format the rules
         for rule_id, rule_def in self.sheerka.om.get_all(self.FORMAT_RULE_ENTRY, cache_only=True).items():
-            rule = self.init_rule(context, rule_def)
+            self.init_rule(context, rule_def)
+
+        for rule_id, rule_def in self.sheerka.om.get_all(self.EXEC_RULE_ENTRY, cache_only=True).items():
+            self.init_rule(context, rule_def)
 
         # update rules priorities
         self.update_rules_priorities(context)
 
-        self.sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities)
-
     def reset_state(self):
         self._format_rules = None
+        self._exec_rules = None
 
     def update_rules_priorities(self, context):
         """
@@ -575,50 +716,93 @@ class SheerkaRuleManager(BaseService):
                 rule.priority = rules_weights[rule.str_id]
 
         self._format_rules = None
+        self._exec_rules = None
 
     def init_rule(self, context, rule: Rule):
         if rule.metadata.is_compiled:
-            return
+            return rule
 
-        if rule.compiled_predicate is None:
-            res = self.compile_when(context, self.NAME, rule.metadata.predicate)
-            if not isinstance(res, list):
-                rule.error_sink = [res.body]
-                return
-            rule.compiled_predicate = res
+        if rule.compiled_predicates is None:
+            try:
+                compiled_result = self.compile_when(context, self.NAME, rule.metadata.predicate)
+                rule.compiled_predicates = compiled_result.compiled_predicates
+                rule.rete_disjunctions = compiled_result.rete_disjunctions
+            except FailedToCompileError as ex:
+                rule.compiled_predicates = None
+                rule.rete_disjunctions = None
+                rule.error_sink = {"when": ex.cause}
 
         if rule.compiled_action is None:
-            res = self.compile_print(context, rule.metadata.action)
-            if not res.status:
-                rule.error_sink = [res.body]
-                return
-            rule.compiled_action = res.body
+            compile_method = self.compile_print if rule.metadata.action_type == ACTION_TYPE_PRINT else self.compile_exec
 
-        # rule.variables = self.get_variables()
+            res = compile_method(context, rule.metadata.action)
+            if not res.status:
+                rule.compiled_action = None
+                if rule.error_sink is None:
+                    rule.error_sink = {rule.metadata.action_type: res.body}
+                else:
+                    rule.error_sink[rule.metadata.action_type] = res.body
+            else:
+                rule.compiled_action = res.body if rule.metadata.action_type == ACTION_TYPE_PRINT else res
 
         rule.metadata.is_compiled = True
-        rule.metadata.is_enabled = True
+        rule.metadata.is_enabled = rule.error_sink is None
+
         return rule
 
-    def compile_when(self, context, name, source):
-        # parser_input = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
-        parsed = parse_unrecognized(context,
-                                    source,
-                                    parsers="all",
-                                    who=name,
-                                    prop=Keywords.WHEN,
-                                    filter_func=only_successful)
+    def compile_when(self, context, who, source):
+        """
+        Compile the predicate
+        :param context:
+        :param who: service which requested the compilation
+        :param source: what to compile
+        """
 
-        if not parsed.status:
-            return parsed
+        # first, try to parse using expression parser
+        # -> Detect xxx and yyy or not zzz
 
-        if self.sheerka.isinstance(parsed.body, BuiltinConcepts.ONLY_SUCCESSFUL):
-            parsed = parsed.body.body
+        action = None
+        parsed = []
+        errors = []
+        all_rete_disjunctions = []
+        parsed_expr_ret = parse_expression(context, source)
+        if parsed_expr_ret.status:
+            conjunctions = parsed_expr_ret.body.body.parts if isinstance(parsed_expr_ret.body.body, AndNode) else \
+                [parsed_expr_ret.body.body]
 
-        return self.add_evaluators(source, parsed if hasattr(parsed, "__iter__") else [parsed])
+            # recognize __action == ''
+            if (action := self._recognized_action_definition(conjunctions[0].tokens)) is not None:
+                conjunctions = conjunctions[1:]
+
+            if len(conjunctions) == 0:
+                errors.append(NoConditionFound())
+            else:
+                # compile conditions
+                try:
+                    return_values, rete_disjunctions = self.expression_parser.compile_conjunctions(context,
+                                                                                                   conjunctions,
+                                                                                                   who)
+
+                    parsed.extend(return_values)
+                    all_rete_disjunctions.extend(rete_disjunctions)
+
+                except FailedToCompileError as ex:
+                    errors.append(ex.cause)
+
+        if len(parsed) == 0:
+            raise FailedToCompileError(errors)
+
+        try:
+            compiled_predicates = self.add_evaluators(context,
+                                                      source,
+                                                      action,
+                                                      parsed if hasattr(parsed, "__iter__") else [parsed])
+            return CompiledWhenResult(compiled_predicates, all_rete_disjunctions)
+        except EmitPythonCodeException as ex:
+            raise FailedToCompileError([ex.error])
 
     def compile_print(self, context, source):
-        parser = FormatRuleParser(source)
+        parser = FormatRuleActionParser(source)
         parsed = parser.parse()
         if parser.error_sink:
             return self.sheerka.ret(self.NAME,
@@ -627,6 +811,16 @@ class SheerkaRuleManager(BaseService):
         else:
             return self.sheerka.ret(self.NAME, True, parsed)
 
+    def compile_exec(self, context, source):
+        parsed = parse_unrecognized(context,
+                                    source,
+                                    parsers="all",
+                                    who=self.NAME,
+                                    prop=Keywords.THEN,
+                                    filter_func=expect_one)
+
+        return parsed
+
     def set_id_if_needed(self, rule: Rule):
         """
         Set the id for the concept if needed
@@ -649,25 +843,52 @@ class SheerkaRuleManager(BaseService):
 
         # set id before saving in db
         self.set_id_if_needed(rule)
-        if rule.compiled_predicate and rule.compiled_action:
+        if rule.compiled_predicates and rule.compiled_action:
             rule.metadata.is_compiled = True
             rule.metadata.is_enabled = True
 
         # save it
-        if rule.metadata.action_type == "print":
-            self.sheerka.om.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule)
+        if rule.metadata.action_type == ACTION_TYPE_PRINT:
+            sheerka.om.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule)
             self._format_rules = None
         else:
-            self.sheerka.om.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule)
+            sheerka.om.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule)
+            self._exec_rules = None
 
         # save by name if needed
         if rule.metadata.name:
             self.sheerka.om.put(self.RULES_BY_NAME_ENTRY, rule.metadata.name, rule)
 
+        # rule is created. publish the event
+        sheerka.publish(context, EVENT_RULE_CREATED, rule)
+
         # process the return if needed
         ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_RULE, body=rule))
         return ret
 
+    def remove_rule(self, context, rule):
+        """
+        Remove a rule
+        """
+        rule = self.resolve_rule(context, rule)
+        if rule is None:
+            return
+
+        # rule will be deleted. publish the event first, as the rule may not be available after
+        self.sheerka.publish(context, EVENT_RULE_DELETED, rule)
+
+        if rule.metadata.action_type == ACTION_TYPE_PRINT:
+            self.sheerka.om.delete(self.FORMAT_RULE_ENTRY, rule.metadata.id)
+            self._format_rules = None
+        else:
+            self.sheerka.om.delete(self.EXEC_RULE_ENTRY, rule.metadata.id)
+            self._exec_rules = None
+
+        if rule.metadata.name:
+            self.sheerka.om.delete(self.RULES_BY_NAME_ENTRY, rule.metadata.name)
+
+        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
     def init_builtin_rules(self, context):
         # self.sheerka.init_log.debug("Initializing default rules")
         rules = [
@@ -760,29 +981,6 @@ class SheerkaRuleManager(BaseService):
 
         return rule
 
-    def dump_desc_rule(self, rules):
-        """
-        dumps the definition of a rule
-        :param rules:
-        :return:
-        """
-        ensure_rule(rules)
-
-        if not hasattr(rules, "__iter__"):
-            rules = [rules]
-
-        first = True
-        for rule in rules:
-            if not first:
-                self.sheerka.log.info("")
-            self.sheerka.log.info(f"id          : {rule.id}")
-            self.sheerka.log.info(f"name        : {rule.metadata.name}")
-            self.sheerka.log.info(f"type        : {rule.metadata.action_type}")
-            self.sheerka.log.info(f"predicate   : {rule.metadata.predicate}")
-            self.sheerka.log.info(f"action      : {rule.metadata.action}")
-            self.sheerka.log.info(f"compiled    : {rule.metadata.is_compiled}")
-            self.sheerka.log.info(f"enabled     : {rule.metadata.is_enabled}")
-
     def get_format_rules(self):
         if self._format_rules:
             return self._format_rules
@@ -792,32 +990,56 @@ class SheerkaRuleManager(BaseService):
                                     reverse=True)
         return self._format_rules
 
-    def add_evaluators(self, source, ret_vals):
+    def get_exec_rules(self):
+        if self._exec_rules:
+            return self._exec_rules
+
+        self._exec_rules = sorted(self.sheerka.om.list(self.EXEC_RULE_ENTRY, cache_only=True),
+                                  key=operator.attrgetter('priority'),
+                                  reverse=True)
+        return self._exec_rules
+
+    def add_evaluators(self, context, source, action, ret_vals):
         """
         Browse the ReturnValueConcepts to determine the evaluator to use
-        Returns a list of tuple (evaluator_name, return_value)
+        Returns a list of RulePredicate, basically a tuple (evaluator_name, return_value)
+        :param context:
         :param source:
         :param ret_vals:
         :return:
         """
+
+        def get_rule_predicate_from_concept(c):
+            if is_a_question(context, c):
+                return RuleCompiledPredicate(source, action, ConceptEvaluator.NAME, r, c)
+            else:
+                to_parse = PythonCodeEmitter(context, "__ret.status").recognize_concept(c, "__ret.body").get_text()
+                return RuleCompiledPredicate(source, action, PythonEvaluator.NAME, parse_python(context, to_parse),
+                                             None)
+
         res = []
         for r in ret_vals:
             underlying = self.sheerka.objvalue(r)
             if isinstance(underlying, PythonNode):
-                res.append(RulePredicate(source, PythonEvaluator.NAME, r, None))
+                res.append(RuleCompiledPredicate(source, action, PythonEvaluator.NAME, r, None))
             elif isinstance(underlying, SourceCodeWithConceptNode):
-                res.append(RulePredicate(source, PythonEvaluator.NAME, r, None))
+                res.append(RuleCompiledPredicate(source, action, PythonEvaluator.NAME, r, None))
             elif isinstance(underlying, SourceCodeNode):
-                res.append(RulePredicate(source, PythonEvaluator.NAME, r, None))
+                res.append(RuleCompiledPredicate(source, action, PythonEvaluator.NAME, r, None))
             elif isinstance(underlying, Concept):
-                res.append(RulePredicate(source, ConceptEvaluator.NAME, r, underlying))
+                res.append(get_rule_predicate_from_concept(underlying))
             elif hasattr(underlying, "__iter__") and len(underlying) == 1 and isinstance(underlying[0], ConceptNode):
-                res.append(RulePredicate(source, ConceptEvaluator.NAME, r, underlying[0].concept))
+                res.append(get_rule_predicate_from_concept(underlying[0].concept))
             else:
                 raise NotImplementedError(r)
         return res
 
     def resolve_rule(self, context, obj):
+        """
+        Given obj, try to find the corresponding rule
+        :param context:
+        :param obj:
+        """
         if obj is None:
             return None
 
@@ -841,7 +1063,7 @@ class SheerkaRuleManager(BaseService):
                     (rule := self._inner_get_by_id(str(rule_id))) is not None:
                     return rule
             else:
-                return obj
+                return self._inner_get_by_id(obj.id)
 
         return None
 
@@ -855,3 +1077,41 @@ class SheerkaRuleManager(BaseService):
             return rule
 
         return None
+
+    @staticmethod
+    def _recognized_action_definition(tokens):
+        """
+        Tries to recognize the pattern __action = xxx in the tokens
+        """
+        iter_token = iter(tokens)
+        try:
+            token = next(iter_token)
+            if token.value != "__action":
+                return None
+            token = next(iter_token)
+            if token.type == TokenKind.WHITESPACE:
+                token = next(iter_token)
+            if token.type != TokenKind.EQUALSEQUALS:
+                return None
+            token = next(iter_token)
+            if token.type == TokenKind.WHITESPACE:
+                token = next(iter_token)
+            if token.type == TokenKind.STRING:
+                return token.strip_quote
+        except StopIteration:
+            pass
+
+        return None
+
+    @staticmethod
+    def _get_parsed_concept(context, return_value):
+        if not context.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT):
+            return None
+
+        if isinstance(return_value.body.body, Concept):
+            return return_value.body.body
+
+        if isinstance(return_value.body.body, ConceptNode):
+            return return_value.body.body.concept
+
+        return None

src/core/sheerka/services/SheerkaVariableManager.py

@@ -41,9 +41,9 @@ class SheerkaVariableManager(BaseService):
     INTERNAL_VARIABLES_ENTRY = "VariableManager:InternalVariables"  # internal to current process (can store lambda)
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=3)
         self.bound_variables = {
-            self.sheerka.name: {"enable_process_return_values", "save_execution_context"}
+            self.sheerka.name: {"enable_process_return_values", "save_execution_context", "enable_process_rules"}
         }
 
     def initialize(self):

src/core/sheerka/services/sheerka_service.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 
-from core.global_symbols import NotFound
+from core.global_symbols import NotFound, ErrorObj
 from core.utils import sheerka_deepcopy
 
 
@@ -14,8 +14,9 @@ class BaseService:
     Base class for services
     """
 
-    def __init__(self, sheerka):
+    def __init__(self, sheerka, order=999):
         self.sheerka = sheerka
+        self.order = order  # initialisation order. The lowest is initialized first
 
     def initialize(self):
         """
@@ -46,3 +47,8 @@ class BaseService:
         Store/record the value of an attribute
         """
         self.sheerka.record_var(context, self.NAME, var_name, getattr(self, var_name))
+
+
+@dataclass()
+class FailedToCompileError(Exception, ErrorObj):
+    cause: list

src/core/tokenizer.py

@@ -139,6 +139,7 @@ class LexerError(Exception):
 class Keywords(Enum):
     DEF = "def"
     CONCEPT = "concept"
+    RULE = "rule"
     FROM = "from"
     BNF = "bnf"
     AS = "as"
@@ -149,6 +150,7 @@ class Keywords(Enum):
     RET = "ret"
     WHEN = "when"
     PRINT = "print"
+    THEN = "then"
 
 
 class Tokenizer:
@@ -557,36 +559,3 @@ class IterParser:
             return token_after
         except StopIteration:
             return Token(TokenKind.EOF, -1, -1, -1, -1)
-
-
-# @dataclass
-# class PropDef:
-#     prop: str
-#     index: int
-#
-#
-# class SimpleExpressionParser(IterParser):
-#     def __init__(self, source):
-#         super().__init__(source)
-#         self.properties = []
-#
-#     def parse(self):
-#
-#         prop, index, key = None, None, None
-#         while self.next_token():
-#             if self.token.type == TokenKind.DOT:
-#                 self.properties.append(PropDef(prop, index, key))
-#                 prop, index, key = None, None, None
-#                 continue
-#
-#             if self.token.type == TokenKind.LBRACKET:
-#                 index = self.parse_index()
-#             elif self.token.type == TokenKind.LBRACE:
-#                 key = self.parse_key()
-#             else:
-#                 prop = self.token.value
-#
-#         if prop is not None:
-#             self.properties.append(PropDef(prop, index, key))
-#
-#     def parse_i

src/core/utils.py

@@ -3,12 +3,14 @@ import importlib
 import inspect
 import os
 import pkgutil
+import re
 from copy import deepcopy
 
-from pyparsing import *
+# from pyparsing import *
+from pyparsing import Literal, Word, nums, Combine, Optional, delimitedList, oneOf, alphas, Suppress
 
 from core.global_symbols import CustomType
-from core.tokenizer import TokenKind, Tokenizer
+from core.tokenizer import TokenKind, Tokenizer, Token
 
 COLORS = {
     "black",
@@ -249,7 +251,7 @@ def make_unique(lst, get_id=None):
     return list(_make_unique(lst, get_id))
 
 
-def product(a, b):
+def sheerka_product(a, b):
     """
     Kind of cartesian product between lists a and b
     knowing that a is also a list : a is a list of list !!!
@@ -569,7 +571,7 @@ def as_bag(obj, forced_properties=None):
     """
     Get the properties of an object (static and dynamic)
     :param obj:
-    :param forced_properties:
+    :param forced_properties: special mode where properties are given in parameter
     :return:
     """
 
@@ -638,6 +640,33 @@ def get_text_from_tokens(tokens, custom_switcher=None, tracker=None):
     return res
 
 
+def tokens_are_matching(tokens1, tokens2, skip_tokens=True):
+    def get_next(it):
+        try:
+            return next(it)
+        except StopIteration:
+            return Token(TokenKind.EOF, "", -1, -1, -1)
+
+    iter1 = iter(tokens1)
+    iter2 = iter(tokens2)
+
+    while True:
+        t1 = get_next(iter1)
+        t2 = get_next(iter2)
+
+        if skip_tokens:
+            if t1.type == TokenKind.WHITESPACE:
+                t1 = next(iter1)
+            if t2.type == TokenKind.WHITESPACE:
+                t2 = next(iter2)
+
+        if t1.type == TokenKind.EOF and t2.type == TokenKind.EOF:
+            return True
+
+        if t1.type != t2.type or t1.value != t2.value:
+            return False
+
+
 def dump_ast(node):
     dump = ast.dump(node)
     for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]: