Implemented a first and basic version of a Rete rule engine

src/core/ast_helpers.py

@@ -92,6 +92,15 @@ class UnreferencedVariablesVisitor(UnreferencedNamesVisitor):
 
 
 class NamesWithAttributesVisitor(ast.NodeVisitor):
+    """
+    Looks for all atrtibutes for a given name
+    >>> ast_ = ast.parse("foo.bar.baz", "<src>", mode="exec")
+    >>> assert NamesWithAttributesVisitor().get_sequences(ast_, "foo") == [["foo", "bar", "baz"]]
+
+    It parses all expressions / statements
+    >>> ast_ = ast.parse("foo.bar.baz; one.two.three; foo.bar", "<src>", mode="exec")
+    >>> assert NamesWithAttributesVisitor().get_sequences(ast_, "foo") == [["foo", "bar", "baz"], ["foo", "bar"]]
+    """
 
     def __init__(self):
         self.sequences = []

src/core/builtin_concepts.py

@@ -1,6 +1,6 @@
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.concept import Concept, ConceptParts
-from core.error import ErrorObj
+from core.global_symbols import ErrorObj
 
 
 class UserInputConcept(Concept):
@@ -167,6 +167,37 @@ class ParserResultConcept(Concept):
         return parser.name if isinstance(parser, BaseParser) else str(parser)
 
 
+class RuleEvaluationResultConcept(Concept):
+    """
+    Result of the evaluation of a rule, using the Rete algorithm
+    """
+
+    ALL_ATTRIBUTES = ["rule"]
+
+    def __init__(self, rule=None, concept_id=None):
+        Concept.__init__(self,
+                         BuiltinConcepts.RULE_EVALUATION_RESULT,
+                         True,
+                         False,
+                         BuiltinConcepts.RULE_EVALUATION_RESULT,
+                         id=concept_id,
+                         bound_body="rule")
+        self.set_value("rule", rule)
+        self._metadata.is_evaluated = True
+
+    def __repr__(self):
+        return f"RuleEvaluationResult(rule={self.rule})"
+
+    def __eq__(self, other):
+        if not isinstance(other, RuleEvaluationResultConcept):
+            return False
+
+        return self.rule == other.rule
+
+    def __hash__(self):
+        return hash((self._metadata.name, self.rule))
+
+
 class InvalidReturnValueConcept(Concept, ErrorObj):
     """
     Error returned when an evaluator is not correctly coded

src/core/builtin_concepts_ids.py

@@ -33,6 +33,9 @@ class BuiltinConcepts:
     BEFORE_RENDERING = "__BEFORE_RENDERING"  # activate before the output is rendered
     RENDERING = "__RENDERING"  # rendering the response from sheerka
     AFTER_RENDERING = "__AFTER_RENDERING"  # rendering the response from sheerka
+    BEFORE_RULES_EVALUATION = "__BEFORE_RULES_EVALUATION"  # just before evaluating rules
+    RULES_EVALUATION = "__RULES_EVALUATION"  # evaluating rules
+    AFTER_RULES_EVALUATION = "__AFTER_RULES_EVALUATION"  # after evaluating rules
     EVALUATE_SOURCE = "__EVALUATE_SOURCE"  #
     EVALUATE_CONCEPT = "__EVALUATE_CONCEPT"  # a concept will be evaluated
     EVALUATING_CONCEPT = "__EVALUATING_CONCEPT"  # a concept will be evaluated
@@ -46,12 +49,12 @@ class BuiltinConcepts:
     EXEC_CODE = "__EXEC_CODE"  # to use when executing Python or other language compiled code
     TESTING = "__TESTING"
     EVALUATOR_PRE_PROCESS = "__EVALUATOR_PRE_PROCESS"  # used modify / tweak behaviour of evaluators
-    EVALUATING_RULES = "__EVALUATING_RULES"
 
     # builtin attributes
     ISA = "__ISA"  # when a concept is an instance of another one
     HASA = "__HASA"  # when a concept has/owns another concept
     AUTO_EVAL = "__AUTO_EVAL"  # when the concept must be auto evaluated
+    RECOGNIZED_BY = "__RECOGNIZED_BY"  # indicate how a concept was recognized
 
     # object
     USER_INPUT = "__USER_INPUT"  # represent an input from an user
@@ -64,6 +67,7 @@ class BuiltinConcepts:
     NEW_CONCEPT = "__NEW_CONCEPT"  # when a new concept is added
     UNKNOWN_PROPERTY = "__UNKNOWN_PROPERTY"  # when requesting for a unknown property
     PARSER_RESULT = "__PARSER_RESULT"
+    RULE_EVALUATION_RESULT = "__RULE_EVALUATION_RESULT"
     TOO_MANY_SUCCESS = "__TOO_MANY_SUCCESS"  # when expecting a limited number of successful return value
     TOO_MANY_ERRORS = "__TOO_MANY_ERRORS"  # when expecting a limited number of successful return value
     ONLY_SUCCESSFUL = "__ONLY_SUCCESSFUL"  # filter the result, only keep successful ones

src/core/builtin_helpers.py

@@ -13,6 +13,7 @@ from core.utils import as_bag
 from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
     RuleNode
 from parsers.BaseParser import ParsingError
+from parsers.PythonParser import PythonParser
 
 PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
 EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION,
@@ -235,7 +236,7 @@ def only_parsers_results(context, return_values):
     Filters the return_values and returns when the result is a ParserResult
     regardless of the status
 
-    So it filters errors
+    So it filters parsers in error (ERROR, NOT_FOR_ME, EMPTY...)
     :param context:
     :param return_values:
     :return:
@@ -332,7 +333,7 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun
 
 def parse_function(context, source, tokens=None, start=0):
     """
-    Helper function to parse what is supposed to be a function
+    Helper function that parses what is supposed to be a function
     :param context:
     :param source:
     :param tokens:
@@ -361,6 +362,34 @@ def parse_function(context, source, tokens=None, start=0):
     return res
 
 
+def parse_python(context, source, desc=None):
+    """
+    Helper function that parses what is known to be Python source code
+    :param context:
+    :param source:
+    :param desc: option description when creating the sub context
+    """
+    desc = desc or f"Compiling python '{source}'"
+    with context.push(BuiltinConcepts.PARSE_CODE,
+                      {"language": "Python", "source": source},
+                      desc) as sub_context:
+        parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
+        python_parser = PythonParser()
+        return python_parser.parse(sub_context, parser_input)
+
+
+def parse_expression(context, source, desc=None):
+    """
+    Helper function to parser expressions with AND, OR and NOT
+    """
+    desc = desc or f"Parsing expression '{source}'"
+    with context.push(BuiltinConcepts.PARSE_CODE, source, desc) as sub_context:
+        parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
+        from parsers.ExpressionParser import ExpressionParser
+        expr_parser = ExpressionParser()
+        return expr_parser.parse(sub_context, parser_input)
+
+
 def evaluate(context,
              source,
              evaluators="all",
@@ -472,12 +501,67 @@ def get_lexer_nodes(return_values, start, tokens):
     return lexer_nodes
 
 
-def ensure_evaluated(context, concept, eval_body=True):
+def get_lexer_nodes_using_positions(return_values, positions):
+    """
+    Transform all elements from return_values into lexer nodes
+    use positions to remap the exact positions
+    """
+    lexer_nodes = []
+    for ret_val, position in zip(return_values, positions):
+        if ret_val.who in ("parsers.Python", 'parsers.PythonWithConcepts'):
+
+            lexer_nodes.append(SourceCodeNode(position.start,
+                                              position.end,
+                                              position.tokens,
+                                              ret_val.body.source,
+                                              python_node=ret_val.body.body,
+                                              return_value=ret_val))
+
+        elif ret_val.who == "parsers.ExactConcept":
+            concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
+            for concept in concepts:
+                lexer_nodes.append(ConceptNode(concept,
+                                               position.start,
+                                               position.end,
+                                               position.tokens,
+                                               ret_val.body.source))
+
+        elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
+            nodes = [node for node in ret_val.body.body]
+            for node in nodes:
+                node.start = position.start
+                node.end = position.end
+
+            # but append the whole sequence if when it's a sequence
+            lexer_nodes.extend(nodes)
+
+        elif ret_val.who == "parsers.Rule":
+            rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
+            for rule in rules:
+                lexer_nodes.append(RuleNode(rule,
+                                            position.start,
+                                            position.end,
+                                            position.tokens, ret_val.body.source))
+
+        elif ret_val.who == "parsers.Function":
+            node = ret_val.body.body
+            node.start = position.start
+            node.end = position.end
+            lexer_nodes.append(node)
+
+        else:
+            raise NotImplementedError()
+
+    return lexer_nodes
+
+
+def ensure_evaluated(context, concept, eval_body=True, metadata=None):
     """
     Evaluate a concept is not already evaluated
     :param context:
     :param concept:
     :param eval_body:
+    :param metadata:
     :return:
     """
     if concept.get_metadata().is_evaluated:
@@ -485,13 +569,13 @@ def ensure_evaluated(context, concept, eval_body=True):
 
     # do not try to evaluate concept that are not fully initialized
     if concept.get_metadata().definition_type != DEFINITION_TYPE_BNF:
-        for var in concept.get_metadata().variables:
-            if var[1] is None and \
-                var[0] not in concept.get_compiled() and \
-                (var[0] not in concept.values() or concept.get_value(var[0]) == NotInit):
+        for var_name, var_default_value in concept.get_metadata().variables:
+            if var_default_value is None and \
+                var_name not in concept.get_compiled() and \
+                (var_name not in concept.values() or concept.get_value(var_name) == NotInit):
                 return concept
 
-    evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=eval_body)
+    evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=eval_body, metadata=metadata)
     return evaluated
 
 
@@ -731,3 +815,90 @@ def evaluate_object(bag, properties):
         bag = as_bag(obj)
 
     return obj
+
+
+def is_a_question(context, concept):
+    """
+    Returns True if the concept must be executed in the context of BuiltinConcepts.EVAL_QUESTION_REQUESTED
+    The only two ways that are currently supported are
+    * is_question() appears in the pre condition
+    * context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED) appears in the pre condition
+    :param context:
+    :param concept: concept to analyse
+    """
+    pre = concept.get_metadata().pre
+    if pre in (None, NotInit, ""):
+        return False
+
+    parser_input_service = context.sheerka.services[SheerkaExecute.NAME]
+    from parsers.ExpressionParser import ExpressionParser
+    parser = ExpressionParser()
+
+    res = parser.parse(context, parser_input_service.get_parser_input(pre))
+    if not res.status:
+        return False
+
+    node = res.body.body
+    from parsers.expressions import IsAQuestionVisitor
+    return IsAQuestionVisitor().is_a_question(node)
+
+
+def get_inner_body(context, concept):
+    """
+    For container concept, returns the body
+    """
+    if context.sheerka.isinstance(concept.body, BuiltinConcepts.ONLY_SUCCESSFUL):
+        return concept.body.body
+    else:
+        return concept.body
+
+
+class CreateObjectIdentifiers:
+    """
+    Class that creates unique identifiers for Concept or Rule objects
+    """
+
+    def __init__(self):
+        self.identifiers = {}
+        self.identifiers_key = {}
+
+    @staticmethod
+    def sanitize(identifier):
+        if identifier is None:
+            return ""
+
+        res = ""
+        for c in identifier:
+            res += c if c.isalnum() else "0"
+        return res
+
+    def get_identifier(self, obj, wrapper):
+        """
+        Get an identifier for a concept.
+        Make sure to return the same identifier if the same concept
+        Make sure to return a different identifier if same name but different concept
+
+        Internal function because I don't want identifiers, identifiers_key and python_ids_mappings
+        to be instance variables
+        I would like to keep this parser as stateless as possible
+        :param obj:
+        :param wrapper: string or char that will wrap the result (ex '__C__' or '__R__')
+        :return:
+        """
+        if id(obj) in self.identifiers:
+            return self.identifiers[id(obj)]
+
+        identifier = wrapper + self.sanitize(obj.key or obj.name)
+        if obj.id:
+            identifier += "__" + obj.id
+
+        if identifier in self.identifiers_key:
+            self.identifiers_key[identifier] += 1
+            identifier += f"_{self.identifiers_key[identifier]}"
+        else:
+            self.identifiers_key[identifier] = 0
+
+        identifier += wrapper
+
+        self.identifiers[id(obj)] = identifier
+        return identifier

src/core/concept.py

@@ -150,6 +150,7 @@ class Concept:
         self._bnf = None  # parsing expression
         self._original_definition_hash = None  # concept hash before any alteration of the metadata
         self._format = None  # how to print the concept
+        self._hints = {}  # extra processing information to help processing
 
     def __repr__(self):
         text = f"({self._metadata.id}){self._metadata.name}"
@@ -499,6 +500,15 @@ class Concept:
         return {k: v for k, v in self.values().items() if not k[0] == "#"}
         # return dict([(k, v) for k, v in self.values.items() if isinstance(k, str)])
 
+    def set_hint(self, name, value):
+        self._hints[name] = value
+
+    def get_hint(self, name):
+        try:
+            return self._hints[name]
+        except KeyError:
+            return None
+
     def auto_init(self):
         """
         Sometimes (for tests purposes)
@@ -533,10 +543,9 @@ class Concept:
         It quicker to implement than creating the actual property mechanism with @property
         And it removes the visibility from the other attributes/methods
         """
-        bag = self.variables()
-
-        for prop in ("id", "name", "key", "body"):
-            bag[prop] = getattr(self, prop)
+        bag = {prop: getattr(self, prop) for prop in ("id", "name", "key")}
+        bag.update(self.variables())
+        bag["body"] = getattr(self, "body")
         return bag
 
     def as_debug_bag(self, new_obj, recurse):

src/core/error.py

@@ -1,6 +0,0 @@
-class ErrorObj:
-    """
-    To indicate that somehow, the underlying object is (or has) an error
-    """
-    pass
-

src/core/global_symbols.py

@@ -1,9 +1,17 @@
 # events
-EVENT_CONCEPT_PRECEDENCE_MODIFIED = "evt_cpm"
-EVENT_RULE_PRECEDENCE_MODIFIED = "evt_rpm"
-EVENT_CONTEXT_DISPOSED = "evt_cd"
-EVENT_USER_INPUT_EVALUATED = "evt_uie"
-EVENT_CONCEPT_CREATED = "evt_cc"
+
+EVENT_CONCEPT_PRECEDENCE_MODIFIED = "evt_cp_m"
+EVENT_RULE_PRECEDENCE_MODIFIED = "evt_rp_m"
+EVENT_CONTEXT_DISPOSED = "evt_ctx_d"
+EVENT_USER_INPUT_EVALUATED = "evt_ui_e"
+EVENT_CONCEPT_CREATED = "evt_c_c"
+EVENT_CONCEPT_DELETED = "evt_c_d"
+EVENT_CONCEPT_ID_DELETED = "evt_c_id_d"
+EVENT_RULE_CREATED = "evt_r_c"
+EVENT_RULE_DELETED = "evt_r_d"
+EVENT_RULE_ID_DELETED = "evt_r_id_d"
+EVENT_ONTOLOGY_CREATED = "evt_o_c"
+EVENT_ONTOLOGY_DELETED = "evt_o_d"
 
 # comparison context
 RULE_COMPARISON_CONTEXT = "Rule"
@@ -40,3 +48,10 @@ class RemovedType(CustomType):
 NotInit = NotInitType()
 NotFound = NotFoundType()
 Removed = RemovedType()
+
+
+class ErrorObj:
+    """
+    To indicate that somehow, the underlying object is (or has) an error
+    """
+    pass

src/core/rule.py

@@ -5,6 +5,7 @@ import core.utils
 
 ACTION_TYPE_PRINT = "print"
 ACTION_TYPE_EXEC = "exec"
+ACTION_TYPE_TEST = "test"
 
 
 @dataclass
@@ -30,17 +31,23 @@ class Rule:
                  rule_id=None,
                  is_enabled=None):
         self.metadata = RuleMetadata(action_type, name, predicate, action, id=rule_id, is_enabled=is_enabled)
-        self.compiled_predicate = None
+        self.compiled_predicates = None
         self.compiled_action = None
         from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
         self.priority = priority if priority is not None else SheerkaComparisonManager.DEFAULT_COMPARISON_VALUE
         self.error_sink = None
 
+        # from SheerkaRete, not quite sure one when it will be used
+        self.rete_net = None
+        self.rete_p_nodes = []  # list of production nodes for this rule
+        self.rete_disjunctions = None  # list of list as it may be several interpretation for a rule
+
     def __repr__(self):
         rule_id = f"#{self.metadata.id}"
         if self.name:
             rule_id += f" ({self.metadata.name})"
-        return f"Rule({rule_id}, when '{self.metadata.predicate}' {self.metadata.action_type} '{self.metadata.action}', priority={self.priority})"
+        action_type = "print" if self.metadata.action_type == "print" else "then"
+        return f"Rule({rule_id}, when '{self.metadata.predicate}' {action_type} '{self.metadata.action}', priority={self.priority})"
 
     def __eq__(self, other):
         if id(other) == id(self):
@@ -69,8 +76,12 @@ class Rule:
                     self.priority,
                     self.id,
                     self.metadata.is_enabled)
-        copy.compiled_predicate = self.compiled_predicate
+
+        copy.compiled_predicates = self.compiled_predicates
         copy.compiled_action = self.compiled_action
+        copy.metadata.is_compiled = self.metadata.is_compiled
+        copy.metadata.id_is_unresolved = self.metadata.id_is_unresolved
+        # copy.error_sink = self.error_sink # Uncomment this line if necessary
 
         return copy
 
@@ -102,3 +113,6 @@ class Rule:
 
     def short_str(self):
         return f"Rule(#{self.metadata.id}, '{self.metadata.predicate}', priority={self.priority})"
+
+    def get_rete_disjunctions(self):
+        return self.rete_disjunctions

src/core/sheerka/ExecutionContext.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import pprint
 import time
 
@@ -87,9 +86,6 @@ class ExecutionContext:
         self.obj = obj
         self.concepts = concepts
 
-        self_debug, self.debug_mode = sheerka.get_context_debug_mode(self.id)
-        self.debug_enabled = self_debug is not None
-
     @property
     def elapsed(self):
         if self._start == 0:
@@ -184,10 +180,6 @@ class ExecutionContext:
         new.preprocess_evaluators = self.preprocess_evaluators
         new.protected_hints.update(self.protected_hints)
 
-        if new.debug_mode is None and self.debug_mode == "protected":
-            new.debug_mode = "protected"
-            new.debug_enabled = True
-
         self._children.append(new)
 
         return new
@@ -315,9 +307,10 @@ class ExecutionContext:
             to_str = self.return_value_to_str(r)
             self._logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
-    def get_debugger(self, who, method_name):
-        return self.sheerka.get_debugger(self, who, method_name)
+    def get_debugger(self, who, method_name, new_debug_id=True):
+        return self.sheerka.get_debugger(self, who, method_name, new_debug_id)
 
+    # TODO: TO REMOVE
     def debug(self, who, method_name, variable_name, text, is_error=False):
         activated = self.sheerka.debug_activated_for(who)
         if activated:
@@ -330,6 +323,7 @@ class ExecutionContext:
                 self.sheerka.debug(f"[{self._id:3}] {CCM[color]}{who}.{method_name}.{variable_name}: {CCM['reset']}")
                 self.sheerka.debug(str_text)
 
+    # TODO: TO REMOVE
     def debug_entering(self, who, method_name, **kwargs):
         if self.sheerka.debug_activated_for(who):
             str_text = pp.pformat(kwargs)
@@ -340,6 +334,7 @@ class ExecutionContext:
                 self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}Entering {who}.{method_name}:{CCM['reset']}")
                 self.sheerka.debug(f"[{self._id:3}] {str_text}")
 
+    # TODO: TO REMOVE
     def debug_log(self, who, text):
         if self.sheerka.debug_activated_for(who):
             self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}{text}{CCM['reset']}")
@@ -493,5 +488,3 @@ class ExecutionContext:
                 return False
 
         return False
-
-

src/core/sheerka/Sheerka.py

@@ -1,6 +1,7 @@
 import inspect
 import logging
 from dataclasses import dataclass
+from operator import attrgetter
 
 import core.builtin_helpers
 import core.utils
@@ -10,8 +11,7 @@ from cache.IncCache import IncCache
 from core.builtin_concepts import ErrorConcept, ReturnValueConcept, UnknownConcept
 from core.builtin_concepts_ids import BuiltinErrors, BuiltinConcepts
 from core.concept import Concept, ConceptParts, get_concept_attrs
-from core.error import ErrorObj
-from core.global_symbols import EVENT_USER_INPUT_EVALUATED, NotInit, NotFound
+from core.global_symbols import EVENT_USER_INPUT_EVALUATED, NotInit, NotFound, ErrorObj, EVENT_ONTOLOGY_CREATED
 from core.profiling import profile
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager, OntologyAlreadyExists
@@ -31,6 +31,23 @@ EXECUTE_STEPS = [
     BuiltinConcepts.AFTER_EVALUATION
 ]
 
+RULES_EVALUATE_STEPS = [
+    BuiltinConcepts.BEFORE_RULES_EVALUATION,
+    BuiltinConcepts.RULES_EVALUATION,
+    BuiltinConcepts.AFTER_RULES_EVALUATION,
+]
+
+RULES_EXECUTE_STEPS = [
+    BuiltinConcepts.BEFORE_EVALUATION,
+    BuiltinConcepts.EVALUATION,
+    BuiltinConcepts.AFTER_EVALUATION
+]
+
+# when a concept is instantiated via resolve or false_resolve
+# It indicate which parameter was used to recognize the concept
+RECOGNIZED_BY_ID = "by_id"
+RECOGNIZED_BY_NAME = "by_name"
+
 
 @dataclass
 class SheerkaMethod:
@@ -92,17 +109,25 @@ class Sheerka(Concept):
 
         self.save_execution_context = True
         self.enable_process_return_values = True
+        self.enable_process_rules = True
 
         self.methods_with_context = {"test_using_context"}  # only the names, the method is defined in sheerka_methods
         self.sheerka_methods = {
             "test": SheerkaMethod(self.test, False),
             "test_using_context": SheerkaMethod(self.test_using_context, False),
-            "test_dict": SheerkaMethod(self.test_dict, False)
+            "test_dict": SheerkaMethod(self.test_dict, False),
+            "test_error": SheerkaMethod(self.test_error, False),
         }
 
         self.locals = {}
         self.concepts_ids = None
 
+    def __copy__(self):
+        return self
+
+    def __deepcopy__(self, memodict={}):
+        return self
+
     @property
     def concepts_grammars(self):
         """
@@ -138,7 +163,7 @@ class Sheerka(Concept):
 
         setattr(self, bound_method.__name__, bound_method)
 
-    def initialize(self, root_folder: str = None, save_execution_context=None, enable_process_return_values=None):
+    def initialize(self, root_folder: str = None, **kwargs):
         """
         Starting Sheerka
         Loads the current configuration
@@ -149,11 +174,10 @@ class Sheerka(Concept):
         :return: ReturnValue(Success or Error)
         """
 
-        if save_execution_context is not None:
-            self.save_execution_context = save_execution_context
-
-        if enable_process_return_values is not None:
-            self.enable_process_return_values = enable_process_return_values
+        self.save_execution_context = kwargs.get("save_execution_context", self.save_execution_context)
+        self.enable_process_return_values = kwargs.get("enable_process_return_values",
+                                                       self.enable_process_return_values)
+        self.enable_process_rules = kwargs.get("enable_process_rules", self.enable_process_rules)
 
         try:
             self.during_initialisation = True
@@ -168,6 +192,7 @@ class Sheerka(Concept):
             self.get_builtin_evaluators()
             self.initialize_services()
             self.initialize_builtin_evaluators()
+            self.om.init_subscribers()
 
             event = Event("Initializing Sheerka.", user_id=self.name)
             self.om.save_event(event)
@@ -234,11 +259,12 @@ class Sheerka(Concept):
 
         core.utils.import_module_and_sub_module('core.sheerka.services')
         base_class = "core.sheerka.services.sheerka_service.BaseService"
-        for service in core.utils.get_sub_classes("core.sheerka.services", base_class):
-            instance = service(self)
-            if hasattr(instance, "initialize"):
-                instance.initialize()
-            self.services[service.NAME] = instance
+        services = [service(self) for service in core.utils.get_sub_classes("core.sheerka.services", base_class)]
+        services.sort(key=attrgetter("order"))
+        for service in services:
+            if hasattr(service, "initialize"):
+                service.initialize()
+            self.services[service.NAME] = service
 
     def initialize_services_deferred(self, context, is_first_time):
         """
@@ -325,7 +351,6 @@ class Sheerka(Concept):
         ontologies = self.om.current_sdp().load_ontologies()
         if not ontologies:
             return
-
         for ontology_name in list(reversed(ontologies))[1:]:
             self.om.push_ontology(ontology_name, False)
             self.initialize_services_deferred(context, False)
@@ -360,6 +385,10 @@ class Sheerka(Concept):
             ret = self.execute(execution_context, [user_input, reduce_requested], EXECUTE_STEPS)
             execution_context.add_values(return_values=ret)
 
+            # rule management
+            if self.enable_process_rules:
+                ret = self.execute_rules(execution_context, ret, RULES_EVALUATE_STEPS, RULES_EXECUTE_STEPS)
+
             if self.om.is_dirty:
                 self.om.commit(execution_context)
 
@@ -388,10 +417,14 @@ class Sheerka(Concept):
         :return:
         """
 
-        def new_instances(concepts):
+        def add_recognized_by(c, _recognized_by):
+            c.set_hint(BuiltinConcepts.RECOGNIZED_BY, _recognized_by)
+            return c
+
+        def new_instances(concepts, _recognized_by):
             if hasattr(concepts, "__iter__"):
-                return [self.new_from_template(c, c.key) for c in concepts]
-            return self.new_from_template(concepts, concepts.key)
+                return [add_recognized_by(self.new_from_template(c, c.key), _recognized_by) for c in concepts]
+            return add_recognized_by(self.new_from_template(concepts, concepts.key), _recognized_by)
 
         if concept is None:
             return None
@@ -421,10 +454,11 @@ class Sheerka(Concept):
                 if self.is_known(found := self.get_by_id(concept[1])):
                     instance = self.new_from_template(found, found.key)
                     instance._metadata.is_evaluated = True
+                    instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_ID)
                     return instance
             elif concept[0]:
                 if self.is_known(found := self.get_by_name(concept[0])):
-                    instances = new_instances(found)
+                    instances = new_instances(found, RECOGNIZED_BY_NAME)
                     core.builtin_helpers.set_is_evaluated(instances)
                     return instances
             else:
@@ -433,17 +467,22 @@ class Sheerka(Concept):
         # otherwise search in db
         if isinstance(concept, str):
             if self.is_known(found := self.get_by_name(concept)):
-                instances = new_instances(found)
+                instances = new_instances(found, RECOGNIZED_BY_NAME)
                 core.builtin_helpers.set_is_evaluated(instances, check_nb_variables=True)
                 return instances
 
         return None
 
     def fast_resolve(self, key, return_new=True):
-        def new_instances(concepts):
+        def add_recognized_by(c, _recognized_by):
+            c.set_hint(BuiltinConcepts.RECOGNIZED_BY, _recognized_by)
+            return c
+
+        def new_instances(concepts, _recognized_by):
             if hasattr(concepts, "__iter__"):
-                return [self.new_from_template(c, c.key) for c in concepts]
-            return self.new_from_template(concepts, concepts.key)
+                return [add_recognized_by(self.new_from_template(c, c.key), _recognized_by) for c in concepts]
+
+            return add_recognized_by(self.new_from_template(concepts, concepts.key), _recognized_by)
 
         if isinstance(key, Token):
             if key.type == TokenKind.RULE:  # do not recognize rules !!!
@@ -459,14 +498,17 @@ class Sheerka(Concept):
 
             if key[1]:
                 concept = self.om.get(self.CONCEPTS_BY_ID_ENTRY, key[1])
+                recognized_by = RECOGNIZED_BY_ID
             else:
                 concept = self.om.get(self.CONCEPTS_BY_NAME_ENTRY, key[0])
+                recognized_by = RECOGNIZED_BY_NAME
         else:
             concept = self.om.get(self.CONCEPTS_BY_NAME_ENTRY, key)
+            recognized_by = RECOGNIZED_BY_NAME
 
         if concept is NotFound:
             return None
-        return new_instances(concept) if return_new else concept
+        return new_instances(concept, recognized_by) if return_new else concept
 
     def new(self, concept_key, **kwargs):
         """
@@ -478,6 +520,8 @@ class Sheerka(Concept):
         """
         if isinstance(concept_key, tuple):
             concept_key, concept_id = concept_key[0], concept_key[1]
+        elif isinstance(concept_key, Concept):
+            concept_key, concept_id = concept_key.key, concept_key.id
         else:
             concept_id = None
 
@@ -547,12 +591,13 @@ class Sheerka(Concept):
         if name in self.om.current_sdp().load_ontologies():
             self.initialize_services_deferred(context, False)
 
-        self.om.save_ontologies()
+        self.om.save_ontologies_names()
+        self.publish(context, EVENT_ONTOLOGY_CREATED, name)
 
         return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
 
-    def pop_ontology(self):
-        ontology = self.om.pop_ontology()
+    def pop_ontology(self, context):
+        ontology = self.om.pop_ontology(context)
 
         self.om.reset_sheerka_state()
         for service in self.services.values():
@@ -561,7 +606,7 @@ class Sheerka(Concept):
             if hasattr(service, "reset_state"):
                 service.reset_state()
 
-        self.om.save_ontologies()
+        self.om.save_ontologies_names()
         return self.ret(self.name, True, self.new(BuiltinConcepts.ONTOLOGY_REMOVED, body=ontology))
 
     def get_ontology(self, context):
@@ -699,6 +744,13 @@ class Sheerka(Concept):
 
         return bool(obj)
 
+    @staticmethod
+    def is_error(obj):
+        """
+        opposite of is_success
+        """
+        return not Sheerka.is_success(obj)
+
     @staticmethod
     def is_known(obj):
         if not isinstance(obj, Concept):

src/core/sheerka/SheerkaOntologyManager.py

@@ -1,7 +1,10 @@
+from cache.Cache import Cache
 from cache.CacheManager import CacheManager
 from cache.DictionaryCache import DictionaryCache
+from cache.SetCache import SetCache
 from core.concept import copy_concepts_attrs, load_concepts_attrs
-from core.global_symbols import NotFound, Removed
+from core.global_symbols import NotFound, Removed, EVENT_CONCEPT_CREATED, EVENT_CONCEPT_DELETED, EVENT_RULE_CREATED, \
+    EVENT_RULE_DELETED, EVENT_CONCEPT_ID_DELETED, EVENT_RULE_ID_DELETED
 from core.utils import sheerka_deepcopy
 from sdp.sheerkaDataProvider import SheerkaDataProvider
 
@@ -88,6 +91,11 @@ class Ontology:
 
 class SheerkaOntologyManager:
     ROOT_ONTOLOGY_NAME = "__default__"
+    SELF_CACHE_MANAGER = "__ontology_manager__"  # cache to store SheerkaOntologyManager info
+    CONCEPTS_BY_ONTOLOGY_ENTRY = "ConceptsByOntologyEntry"
+    RULES_BY_ONTOLOGY_ENTRY = "RulesByOntologyEntry"
+    ONTOLOGY_BY_CONCEPT_ENTRY = "OntologyByConceptEntry"
+    ONTOLOGY_BY_RULE_ENTRY = "OntologyByRuleEntry"
 
     def __init__(self, sheerka, root_folder, cache_only):
         self.sheerka = sheerka
@@ -98,6 +106,20 @@ class SheerkaOntologyManager:
         ref_cache_manager = CacheManager(self.cache_only, sdp=SheerkaDataProvider(root_folder, self.sheerka))
         self.ontologies = [Ontology(self.ROOT_ONTOLOGY_NAME, ref_cache_manager, None)]
 
+        self_sdp = SheerkaDataProvider(root_folder, self.sheerka, self.SELF_CACHE_MANAGER)
+        self.self_cache_manager = CacheManager(self.cache_only, sdp=self_sdp)
+        cache = SetCache(max_size=None).auto_configure(self.CONCEPTS_BY_ONTOLOGY_ENTRY)
+        self.self_cache_manager.register_cache(self.CONCEPTS_BY_ONTOLOGY_ENTRY, cache)
+
+        cache = SetCache(max_size=None).auto_configure(self.RULES_BY_ONTOLOGY_ENTRY)
+        self.self_cache_manager.register_cache(self.RULES_BY_ONTOLOGY_ENTRY, cache)
+
+        cache = Cache(max_size=None).auto_configure(self.ONTOLOGY_BY_CONCEPT_ENTRY)
+        self.self_cache_manager.register_cache(self.ONTOLOGY_BY_CONCEPT_ENTRY, cache)
+
+        cache = Cache(max_size=None).auto_configure(self.ONTOLOGY_BY_RULE_ENTRY)
+        self.self_cache_manager.register_cache(self.ONTOLOGY_BY_RULE_ENTRY, cache)
+
     @property
     def ontologies_names(self):
         return [o.name for o in self.ontologies]
@@ -111,6 +133,12 @@ class SheerkaOntologyManager:
         self.frozen = False
         return self
 
+    def init_subscribers(self):
+        self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.on_concept_created)
+        self.sheerka.subscribe(EVENT_CONCEPT_DELETED, self.on_concept_deleted)
+        self.sheerka.subscribe(EVENT_RULE_CREATED, self.on_rule_created)
+        self.sheerka.subscribe(EVENT_RULE_DELETED, self.on_rule_deleted)
+
     def push_ontology(self, name, cache_only=None):
         """
         Add an ontology layer
@@ -138,7 +166,7 @@ class SheerkaOntologyManager:
         self.ontologies.insert(0, Ontology(name, cache_manager, alt_sdp))
         return self
 
-    def pop_ontology(self):
+    def pop_ontology(self, context):
         """
         Remove the top ontology layer
         """
@@ -148,6 +176,22 @@ class SheerkaOntologyManager:
         if len(self.ontologies) == 1:
             raise OntologyManagerCannotPopLatest()
 
+        # remove concepts and rules tracking for the ontology to pop
+        ontology_name = self.current_ontology().name
+        concepts = self.self_cache_manager.get(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name)
+        if concepts is not NotFound:
+            for concept in concepts:
+                self.sheerka.publish(context, EVENT_CONCEPT_ID_DELETED, concept)
+                self.self_cache_manager.delete(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept)
+            self.self_cache_manager.delete(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name)
+
+        rules = self.self_cache_manager.get(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name)
+        if rules is not NotFound:
+            for rule in rules:
+                self.sheerka.publish(context, EVENT_RULE_ID_DELETED, rule)
+                self.self_cache_manager.delete(self.ONTOLOGY_BY_RULE_ENTRY, rule)
+            self.self_cache_manager.delete(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name)
+
         return self.ontologies.pop(0)
 
     def add_ontology(self, ontology: Ontology):
@@ -179,7 +223,7 @@ class SheerkaOntologyManager:
 
         raise KeyError(name)
 
-    def save_ontologies(self):
+    def save_ontologies_names(self):
         self.current_sdp().save_ontologies(self.ontologies_names)
 
     # def load_ontologies(self):
@@ -446,6 +490,7 @@ class SheerkaOntologyManager:
         :param context:
         :return:
         """
+        self.self_cache_manager.commit(context)
         return self.current_cache_manager().commit(context)
 
     def clear(self, cache_name=None):
@@ -468,3 +513,21 @@ class SheerkaOntologyManager:
 
     def is_dirty(self):
         return self.current_cache_manager().is_dirty
+
+    def on_concept_created(self, context, concept):
+        self.self_cache_manager.put(self.CONCEPTS_BY_ONTOLOGY_ENTRY, self.current_ontology().name, concept.id)
+        self.self_cache_manager.put(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id, self.current_ontology().name)
+
+    def on_concept_deleted(self, context, concept):
+        ontology_name = self.self_cache_manager.get(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id)
+        self.self_cache_manager.delete(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name, concept.id)
+        self.self_cache_manager.delete(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id)
+
+    def on_rule_created(self, context, rule):
+        self.self_cache_manager.put(self.RULES_BY_ONTOLOGY_ENTRY, self.current_ontology().name, rule.id)
+        self.self_cache_manager.put(self.ONTOLOGY_BY_RULE_ENTRY, rule.id, self.current_ontology().name)
+
+    def on_rule_deleted(self, context, rule):
+        ontology_name = self.self_cache_manager.get(self.ONTOLOGY_BY_RULE_ENTRY, rule.id)
+        self.self_cache_manager.delete(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name, rule.id)
+        self.self_cache_manager.delete(self.ONTOLOGY_BY_RULE_ENTRY, rule.id)

src/core/sheerka/services/SheerkaAdmin.py

@@ -3,7 +3,7 @@ import time
 from os import path
 
 from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
-from core.builtin_helpers import ensure_concept
+from core.builtin_helpers import ensure_concept_or_rule
 from core.concept import Concept
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from core.sheerka.services.sheerka_service import BaseService
@@ -28,6 +28,7 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.extended_isinstance, False)
         self.sheerka.bind_service_method(self.is_container, False)
         self.sheerka.bind_service_method(self.format_rules, False)
+        self.sheerka.bind_service_method(self.exec_rules, False)
         self.sheerka.bind_service_method(self.admin_push_ontology, True, as_name="push_ontology")
         self.sheerka.bind_service_method(self.admin_pop_ontology, True, as_name="pop_ontology")
         self.sheerka.bind_service_method(self.ontologies, False)
@@ -127,24 +128,36 @@ class SheerkaAdmin(BaseService):
         concepts = sorted(self.sheerka.om.list(self.sheerka.CONCEPTS_BY_ID_ENTRY), key=lambda item: int(item.id))
         return self.sheerka.new(BuiltinConcepts.TO_LIST, body=concepts)
 
-    def desc(self, *concepts):
-        ensure_concept(*concepts)
+    def desc(self, *items):
+        ensure_concept_or_rule(*items)
         res = []
-        for c in concepts:
-            bag = {
-                "id": c.id,
-                "name": c.name,
-                "key": c.key,
-                "definition": c.get_metadata().definition,
-                "type": c.get_metadata().definition_type,
-                "body": c.get_metadata().body,
-                "where": c.get_metadata().where,
-                "pre": c.get_metadata().pre,
-                "post": c.get_metadata().post,
-                "ret": c.get_metadata().ret,
-                "vars": c.get_metadata().variables,
-                "props": c.get_metadata().props,
-            }
+        for item in items:
+            if isinstance(item, Concept):
+                bag = {
+                    "id": item.id,
+                    "name": item.name,
+                    "key": item.key,
+                    "definition": item.get_metadata().definition,
+                    "type": item.get_metadata().definition_type,
+                    "body": item.get_metadata().body,
+                    "where": item.get_metadata().where,
+                    "pre": item.get_metadata().pre,
+                    "post": item.get_metadata().post,
+                    "ret": item.get_metadata().ret,
+                    "vars": item.get_metadata().variables,
+                    "props": item.get_metadata().props,
+                }
+            else:
+                bag = {
+                    "id": item.id,
+                    "name": item.metadata.name,
+                    "type": item.metadata.action_type,
+                    "predicate": item.metadata.predicate,
+                    "action": item.metadata.action,
+                    "priority": item.priority,
+                    "compiled": item.metadata.is_compiled,
+                    "enabled": item.metadata.is_enabled,
+                }
             res.append(self.sheerka.new(BuiltinConcepts.TO_DICT, body=bag))
 
         return res[0] if len(res) == 1 else self.sheerka.new(BuiltinConcepts.TO_LIST, body=res)
@@ -152,6 +165,9 @@ class SheerkaAdmin(BaseService):
     def format_rules(self):
         return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_format_rules())
 
+    def exec_rules(self):
+        return self.sheerka.new(BuiltinConcepts.TO_LIST, items=self.sheerka.get_exec_rules())
+
     def extended_isinstance(self, a, b):
         """
         switch between sheerka.isinstance and builtin.isinstance
@@ -180,8 +196,8 @@ class SheerkaAdmin(BaseService):
     def admin_push_ontology(self, context, name):
         return self.sheerka.push_ontology(context, name, False)
 
-    def admin_pop_ontology(self):
-        return self.sheerka.pop_ontology()
+    def admin_pop_ontology(self, context):
+        return self.sheerka.pop_ontology(context)
 
     def ontologies(self):
         ontologies = self.sheerka.om.ontologies_names

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -38,7 +38,7 @@ class SheerkaComparisonManager(BaseService):
     DEFAULT_COMPARISON_VALUE = 1
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=14)
 
     def initialize(self):
         cache = ListCache().auto_configure(self.COMPARISON_ENTRY)

src/core/sheerka/services/SheerkaConceptManager.py

@@ -12,8 +12,7 @@ from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, Built
 from core.builtin_helpers import ensure_concept, ensure_bnf
 from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \
     VARIABLE_PREFIX
-from core.error import ErrorObj
-from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound
+from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound, ErrorObj, EVENT_CONCEPT_DELETED
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer, TokenKind
 from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
@@ -100,7 +99,7 @@ class SheerkaConceptManager(BaseService):
     RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "ConceptManager:Resolved_Concepts_By_First_Keyword"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=11)
         self.forbidden_meta = {"is_builtin", "key", "id", "props", "variables"}
         self.allowed_meta = {attr for attr in vars(ConceptMetadata) if
                              not attr.startswith("_") and attr not in self.forbidden_meta}
@@ -147,7 +146,7 @@ class SheerkaConceptManager(BaseService):
             self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
 
         # initialize the dictionary of first tokens
-        self.sheerka.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init the cache with the values from sdp
+        self.sheerka.om.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None)  # to init the cache with the values from sdp
         concepts_by_first_keyword = self.sheerka.om.current_cache_manager().copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
         res = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
         self.sheerka.om.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
@@ -353,6 +352,8 @@ class SheerkaConceptManager(BaseService):
         :param concept:
         :return:
         """
+        # TODO : resolve concept first
+
         sheerka = context.sheerka
         refs = self.sheerka.om.get(self.CONCEPTS_REFERENCES_ENTRY, concept.id)
         if refs is not NotFound:
@@ -361,6 +362,7 @@ class SheerkaConceptManager(BaseService):
 
         try:
             sheerka.om.remove_concept(concept)
+            sheerka.publish(context, EVENT_CONCEPT_DELETED, concept)
             return sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.SUCCESS))
         except ConceptNotFound as ex:
             return sheerka.ret(self.NAME, False, sheerka.err(ex))

src/core/sheerka/services/SheerkaDebugManager.py

@@ -91,6 +91,14 @@ class BaseDebugLogger:
             BaseDebugLogger.ids[hint] = 0
         return BaseDebugLogger.ids[hint]
 
+    @staticmethod
+    def current_id(hint):
+        if hint in BaseDebugLogger.ids:
+            return BaseDebugLogger.ids[hint]
+        else:
+            BaseDebugLogger.ids[hint] = 0
+            return 0
+
     def __init__(self, debug_manager, context, who, method_name, debug_id):
         pass
 
@@ -276,7 +284,7 @@ class SheerkaDebugManager(BaseService):
     children_activation_regex = re.compile(r"(\d+)\+")
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=1)
         self.activated = False  # is debug activated
         self.explicit = False  # No need to activate context debug when debug mode is on # to remove ?
         self.context_cache = set()  # debug for specific context # to remove ?
@@ -295,16 +303,6 @@ class SheerkaDebugManager(BaseService):
         ]
 
     def initialize(self):
-        # TO REMOVE ???
-        self.sheerka.bind_service_method(self.set_explicit, True)
-        self.sheerka.bind_service_method(self.activate_debug_for, True)
-        self.sheerka.bind_service_method(self.deactivate_debug_for, True)
-        self.sheerka.bind_service_method(self.debug_activated, False)
-        self.sheerka.bind_service_method(self.debug_activated_for, False)
-        self.sheerka.bind_service_method(self.get_context_debug_mode, False)
-        self.sheerka.bind_service_method(self.debug_rule_activated, False)
-        self.sheerka.bind_service_method(self.debug, False, visible=False)
-
         self.sheerka.bind_service_method(self.set_debug, True)
         self.sheerka.bind_service_method(self.inspect, False)
         self.sheerka.bind_service_method(self.get_debugger, False)
@@ -341,79 +339,13 @@ class SheerkaDebugManager(BaseService):
         self.sheerka.record_var(context, self.NAME, "activated", self.activated)
         return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
-    def set_explicit(self, context, value=True):
-        self.explicit = value
-        self.sheerka.record_var(context, self.NAME, "explicit", self.explicit)
-        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-
-    def activate_debug_for(self, context, debug_id, children=False):
-        """
-
-        :param context:
-        :param debug_id: if debug_id is str, activate variable cache, context_cache otherwise
-        :param children:
-        :return:
-        """
-        # preprocess
-        if isinstance(debug_id, str) and (m := self.children_activation_regex.match(debug_id)):
-            debug_id = int(m.group(1))
-            children = True
-
-        if isinstance(debug_id, str):
-            self.variable_cache.add(debug_id)
-            self.sheerka.record_var(context, self.NAME, "variable_cache", self.variable_cache)
-        else:
-            self.context_cache.add(debug_id)
-            if children:
-                self.context_cache.add(str(debug_id) + "+")
-            self.sheerka.record_var(context, self.NAME, "context_cache", self.context_cache)
-        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-
-    def deactivate_debug_for(self, context, debug_id, children=False):
-        if isinstance(debug_id, str):
-            self.variable_cache.discard(debug_id)
-            self.sheerka.record_var(context, self.NAME, "variable_cache", self.variable_cache)
-        else:
-            self.context_cache.discard(debug_id)
-            if children:
-                self.context_cache.discard(str(debug_id) + "+")
-            self.sheerka.record_var(context, self.NAME, "context_cache", self.context_cache)
-        return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
-
-    def debug_activated(self):
-        return self.activated
-
-    def debug_activated_for(self, debug_id):
-        if not self.activated:
-            return None
-
-        return debug_id in self.variable_cache
-
-    def debug_rule_activated(self, rule_id, context_id):
-        """
-
-        :param rule_id:
-        :param context_id:
-        :return:
-        """
-        key = f"{rule_id}|{context_id}"
-        return key in self.rules_cache
-
-    def get_context_debug_mode(self, context_id):
-        if not self.activated:
-            return None, None
-
-        debug_for_children = "protected" if str(context_id) + "+" in self.context_cache else None
-        debug_for_self = "private" if not self.explicit or context_id in self.context_cache else None
-
-        return debug_for_self, debug_for_children
-
     def debug(self, *args, **kwargs):
         print(*args, **kwargs)
 
-    def get_debugger(self, context, who, method_name):
+    def get_debugger(self, context, who, method_name, new_debug_id=True):
         if self.compute_debug(context, who, method_name):
-            debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id))
+            debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id)) if new_debug_id \
+                else ConsoleDebugLogger.current_id(context.event.get_digest() + str(context.id))
             return ConsoleDebugLogger(self, context, who, method_name, debug_id)
 
         return NullDebugLogger()

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -11,7 +11,8 @@ from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
 from core.utils import unstr_concept
 from parsers.BaseNodeParser import ConceptNode
-from parsers.ExpressionParser import ExpressionParser, TrueifyVisitor
+from parsers.ExpressionParser import ExpressionParser
+from parsers.expressions import TrueifyVisitor
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,

src/core/sheerka/services/SheerkaEvaluateRules.py

@@ -1,7 +1,9 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one
+from core.global_symbols import EVENT_RULE_CREATED, EVENT_RULE_DELETED, EVENT_RULE_ID_DELETED
 from core.sheerka.services.sheerka_service import BaseService
 from evaluators.ConceptEvaluator import ConceptEvaluator
+from sheerkarete.network import ReteNetwork
 
 DISABLED_RULES = "#disabled#"
 LOW_PRIORITY_RULES = "#low_priority#"
@@ -11,12 +13,18 @@ class SheerkaEvaluateRules(BaseService):
     NAME = "EvaluateRules"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        # order must be before RuleManager because of event subscription
+        super().__init__(sheerka, 4)
         self.evaluators_by_name = None
+        self.network = ReteNetwork()
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.evaluate_format_rules, False)
+        self.sheerka.bind_service_method(self.evaluate_format_rules, False, visible=False)
+        self.sheerka.bind_service_method(self.evaluate_exec_rules, False, visible=False)
         self.reset_evaluators()
+        self.sheerka.subscribe(EVENT_RULE_CREATED, self.on_rule_created)
+        self.sheerka.subscribe(EVENT_RULE_DELETED, self.on_rule_deleted)
+        self.sheerka.subscribe(EVENT_RULE_ID_DELETED, self.on_rule_deleted)
 
     def reset_evaluators(self):
         # instantiate evaluators, once for all, only keep when it's enabled
@@ -24,6 +32,20 @@ class SheerkaEvaluateRules(BaseService):
         evaluators = [e for e in evaluators if e.enabled]
         self.evaluators_by_name = {e.short_name: e for e in evaluators}
 
+    def evaluate_exec_rules(self, context, return_values):
+        # self.network.add_obj("__rets", return_values)
+        for ret in return_values:
+            self.network.add_obj("__ret", ret)
+
+        results = []  # list of return values, for activated rules
+        for match in self.network.matches:
+            for rule in match.pnode.rules:
+                body = context.sheerka.new(BuiltinConcepts.RULE_EVALUATION_RESULT, rule=rule)
+                return_value = context.sheerka.ret(self.NAME, True, body)
+                results.append(return_value)
+
+        return results
+
     def evaluate_format_rules(self, context, bag, disabled):
         return self.evaluate_rules(context, self.sheerka.get_format_rules(), bag, disabled)
 
@@ -37,7 +59,7 @@ class SheerkaEvaluateRules(BaseService):
         :param disabled: disabled rules (because they have already been fired or whatever)
         :return: { True : list of success, False :list of failed, '#disabled"': list of disabled...}
         """
-        with context.push(BuiltinConcepts.EVALUATING_RULES, bag, desc="Evaluating rules...") as sub_context:
+        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
             sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
             sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
             sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
@@ -81,7 +103,7 @@ class SheerkaEvaluateRules(BaseService):
         """
 
         results = []
-        for rule_predicate in rule.compiled_predicate:
+        for rule_predicate in rule.compiled_predicates:
 
             if rule_predicate.source in bag:
                 # simple case where the rule is an item of the bag. No need of complicate evaluation
@@ -94,15 +116,44 @@ class SheerkaEvaluateRules(BaseService):
                     rule_predicate.concept.get_metadata().is_evaluated = False
 
                 evaluator = self.evaluators_by_name[rule_predicate.evaluator]
-                results.append(evaluator.eval(context, rule_predicate.predicate))
+                res = evaluator.eval(context, rule_predicate.predicate)
+                if res.status and isinstance(res.body, bool) and res.body:
+                    # one successful value found. No need to look any further
+                    results = [res]
+                    break
+                else:
+                    results.append(res)
 
-        debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule")
+        debugger = context.get_debugger(SheerkaEvaluateRules.NAME, "evaluate_rule", new_debug_id=False)
         debugger.debug_rule(rule, results)
-        # if context.sheerka.debug_rule_activated(rule_id, context.id):
-        #     context.debug(SheerkaEvaluateRules.NAME, "evaluate_rules", f"result(#{rule_id})", results)
 
         return expect_one(context, results)
 
+    def remove_from_rete_memory(self, lst):
+        if lst is None:
+            return
+
+        for obj in lst:
+            self.network.remove_obj(obj)
+
+    def on_rule_created(self, context, rule):
+        """
+        When a new rule is added to the system, update the network
+        """
+        if rule.metadata.is_enabled and rule.rete_disjunctions:
+            self.network.add_rule(rule)
+
+    def on_rule_deleted(self, context, rule):
+        """
+        When a rule is deleted from the system, remove it from the network
+        """
+        if isinstance(rule, str):
+            rule = self.sheerka.get_rule_by_id(rule)
+            if not self.sheerka.is_known(rule):
+                return
+
+        self.network.remove_rule(rule)
+
     @staticmethod
     def get_debug_format(result):
         """

src/core/sheerka/services/SheerkaEventManager.py

@@ -11,7 +11,7 @@ class SheerkaEventManager(BaseService):
     NAME = "EventManager"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=2)
         self._lock = RLock()
         self.subscribers = {}
 

src/core/sheerka/services/SheerkaExecute.py

@@ -1,5 +1,4 @@
 import core.utils
-from cache.Cache import Cache
 from cache.FastCache import FastCache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.global_symbols import NotFound
@@ -16,6 +15,8 @@ EVALUATOR_STEPS = [
     BuiltinConcepts.BEFORE_RENDERING,
     BuiltinConcepts.RENDERING,
     BuiltinConcepts.AFTER_RENDERING,
+    BuiltinConcepts.BEFORE_RULES_EVALUATION,
+    BuiltinConcepts.AFTER_RULES_EVALUATION,
 ]
 
 
@@ -120,6 +121,10 @@ class ParserInput:
         return self.pos < self.end
 
     def the_token_after(self, skip_whitespace=True):
+        """
+        Returns the token after the current one
+        Never returns None (returns TokenKind.EOF instead)
+        """
         my_pos = self.pos + 1
         if my_pos >= self.end:
             return Token(TokenKind.EOF, "", -1, -1, -1)
@@ -167,7 +172,8 @@ class SheerkaExecute(BaseService):
     PARSERS_INPUTS_ENTRY = "Execute:ParserInput"  # entry for admin or internal variables
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        # order must be after SheerkaEvaluateRules because of self.rules_evaluation_service
+        super().__init__(sheerka, order=5)
         self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=20)
         self.instantiated_evaluators = None
         self.evaluators_by_name = None
@@ -191,12 +197,18 @@ class SheerkaExecute(BaseService):
         # Except 2 : we store the type of the parser, not its instance
         self.grouped_parsers_cache = {}
 
+        self.rules_eval_service = None
+
     def initialize(self):
-        self.sheerka.bind_service_method(self.execute, True)
+        self.sheerka.bind_service_method(self.execute, True, visible=False)
+        self.sheerka.bind_service_method(self.execute_rules, True, visible=False)
 
         self.reset_registered_evaluators()
         self.reset_registered_parsers()
 
+        from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
+        self.rules_eval_service = self.sheerka.services[SheerkaEvaluateRules.NAME]
+
     def reset_state(self):
         self.pi_cache.clear()
 
@@ -347,7 +359,7 @@ class SheerkaExecute(BaseService):
             if pi is NotFound:  # when CacheManager.cache_only is True
                 pi = ParserInput(text)
                 self.pi_cache.put(text, pi)
-            return pi
+            return ParserInput(text, pi.tokens)  # new instance, but no need to tokenize the text again
 
         key = text or core.utils.get_text_from_tokens(tokens)
         pi = ParserInput(key, tokens)
@@ -582,6 +594,55 @@ class SheerkaExecute(BaseService):
 
         return return_values
 
+    def execute_rules(self, context, return_values, rules_steps, evaluation_steps):
+        """"
+        Executes the execution rules until no match is found
+        :param context:
+        :param return_values: input return values
+        :param rules_steps: steps are configurable
+        :param evaluation_steps: steps are configurable
+        :return: out return_values
+        """
+        continue_execution = True
+        counter = 0
+        in_rete_memory = None
+        while continue_execution:
+            with context.push(BuiltinConcepts.PROCESSING, {"counter": counter}, desc=f"{counter=}") as sub_context:
+
+                # apply rule evaluation steps
+                for step in rules_steps:
+                    if step == BuiltinConcepts.RULES_EVALUATION:
+                        eval_res = self.rules_eval_service.evaluate_exec_rules(sub_context, return_values)
+                        if not eval_res:
+                            self.rules_eval_service.remove_from_rete_memory(return_values)
+                            continue_execution = False
+                            break
+                        else:
+                            in_rete_memory = return_values.copy()
+                            return_values = eval_res
+                    else:
+                        return_values = self.call_evaluators(sub_context, return_values, step)
+
+                if not continue_execution:
+                    break
+
+                # evaluate the result
+                return_values = [r.body.body.compiled_action for r in return_values]
+                while True:
+                    copy = return_values[:]
+                    for step in evaluation_steps:
+                        return_values = self.call_evaluators(sub_context, return_values, step)
+
+                    if copy == return_values[:]:
+                        break
+
+                # evaluation is done. Remove object in Rete memory
+                self.rules_eval_service.remove_from_rete_memory(in_rete_memory)
+
+                counter += 1
+
+        return return_values
+
     def undo_preprocess(self):
         for item, var_name, value in self.old_values:
             setattr(item, var_name, value)

src/core/sheerka/services/SheerkaHasAManager.py

@@ -7,7 +7,7 @@ class SheerkaHasAManager(BaseService):
     NAME = "HasAManager"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=22)
 
     def initialize(self):
         self.sheerka.bind_service_method(self.set_hasa, True)

src/core/sheerka/services/SheerkaIsAManager.py

@@ -15,7 +15,7 @@ class SheerkaIsAManager(BaseService):
     CONCEPTS_IN_GROUPS_ENTRY = "IsAManager:Concepts_In_Groups"  # cache for get_set_elements()
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=21)
 
     def initialize(self):
         self.sheerka.bind_service_method(self.set_isa, True)

src/core/sheerka/services/SheerkaMemory.py

@@ -20,7 +20,7 @@ class SheerkaMemory(BaseService):
     OBJECTS_ENTRY = "Memory:Objects"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=13)
         self.short_term_objects = FastCache()
         self.registration = {}
 
@@ -39,6 +39,8 @@ class SheerkaMemory(BaseService):
 
         cache = ListIfNeededCache().auto_configure(self.OBJECTS_ENTRY)
         self.sheerka.om.register_cache(self.OBJECTS_ENTRY, cache, persist=True, use_ref=True)
+        
+        self.sheerka.subscribe(EVENT_CONTEXT_DISPOSED, self.remove_context)
 
     def reset(self):
         self.short_term_objects.clear()
@@ -48,9 +50,6 @@ class SheerkaMemory(BaseService):
         self.short_term_objects.clear()
         self.registration.clear()
 
-    def initialize_deferred(self, context, is_first_time):
-        self.sheerka.subscribe(EVENT_CONTEXT_DISPOSED, self.remove_context)
-
     def get_from_short_term_memory(self, context, key):
         while True:
             try:

src/core/sheerka/services/SheerkaOut.py

@@ -31,6 +31,7 @@ class SheerkaOut(BaseService):
         if valid_rules:
             if len(valid_rules) > 1:
                 # TODO manage when too many rules
+                print("TODO: TOO MANY RULES !!!!!")
                 pass
 
             rule = valid_rules[0]

src/core/sheerka/services/SheerkaResultManager.py

@@ -37,11 +37,12 @@ class SheerkaResultManager(BaseService):
         self.sheerka.bind_service_method(self.get_last_created_concept, False, as_name="last_created_concept")
         self.sheerka.bind_service_method(self.get_last_error, False, as_name="last_err")
 
-    def initialize_deferred(self, context, is_first_time):
-        self.restore_values(*self.state_vars)
         self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.user_input_evaluated)
         self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.new_concept_created)
 
+    def initialize_deferred(self, context, is_first_time):
+        self.restore_values(*self.state_vars)
+
     def test_only_reset(self):
         self.executions_contexts_cache.clear()
         self.last_execution = None

src/core/sheerka/services/SheerkaRuleManager.py

@@ -1,28 +1,35 @@
 import operator
 import re
 from dataclasses import dataclass
-from typing import Union
+from typing import Union, Set, List
 
 from cache.Cache import Cache
 from cache.ListIfNeededCache import ListIfNeededCache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
-from core.builtin_helpers import parse_unrecognized, only_successful, ensure_rule
+from core.builtin_helpers import parse_unrecognized, is_a_question, parse_python, \
+    ensure_evaluated, expect_one, parse_expression
 from core.concept import Concept
-from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound
-from core.rule import Rule
-from core.sheerka.services.sheerka_service import BaseService
+from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound, ErrorObj, \
+    EVENT_RULE_CREATED, EVENT_RULE_DELETED
+from core.rule import Rule, ACTION_TYPE_PRINT
+from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
+from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError
 from core.tokenizer import Keywords, TokenKind, Token, IterParser
 from core.utils import index_tokens, COLORS, get_text_from_tokens
 from evaluators.ConceptEvaluator import ConceptEvaluator
-from evaluators.PythonEvaluator import PythonEvaluator
+from evaluators.PythonEvaluator import PythonEvaluator, Expando
 from parsers.BaseNodeParser import SourceCodeWithConceptNode, ConceptNode, SourceCodeNode
+from parsers.ExpressionParser import AndNode, ExpressionParser
 from parsers.PythonParser import PythonNode
+from sheerkarete.conditions import AndConditions
+
+CONCEPTS_ONLY_PARSERS = ["ExactConcept", "Bnf", "Sya", "Sequence"]
 
 identifier_regex = re.compile(r"[\w _.]+")
 
 
 @dataclass
-class FormatRuleError:
+class FormatRuleError(ErrorObj):
     pass
 
 
@@ -199,7 +206,7 @@ class FormatAstMulti(FormatAstNode):
             **kwargs)
 
 
-class FormatRuleParser(IterParser):
+class FormatRuleActionParser(IterParser):
 
     @staticmethod
     def to_text(list_or_dict_of_tokens):
@@ -242,7 +249,7 @@ class FormatRuleParser(IterParser):
 
     def parse(self):
         """
-        Parses a format rule
+        Parses the print part of the format rule
         format ::= {variable'} | function(...) | rawtext
         :return:
         """
@@ -394,7 +401,7 @@ class FormatRuleParser(IterParser):
         source = get_text_from_tokens(args[0])
         if len(source) > 1 and source[0] in ("'", '"') and source[-1] in ("'", '"'):
             source = source[1:-1]
-            parser = FormatRuleParser(source)
+            parser = FormatRuleActionParser(source)
             res = parser.parse()
             self.error_sink = parser.error_sink
             return FormatAstColor(color, res)
@@ -497,12 +504,139 @@ class FormatRuleParser(IterParser):
         return FormatAstMulti(get_text_from_tokens(args[0]))
 
 
+@dataclass
+class EmitPythonCodeException(Exception):
+    error: object
+
+
+class PythonCodeEmitter:
+
+    def __init__(self, context, text=None):
+        self.context = context
+        self.text = text or ""
+        self.var_counter = 0
+        self.variables = []
+
+    def add(self, text):
+        self.text += f" and {text}" if self.text else text
+        return self
+
+    def recognize(self, obj, as_name, root=True):
+        if isinstance(obj, str):
+            return self.recognize_str(obj, as_name)
+        elif isinstance(obj, (int, float)):
+            return self.recognize_int(obj, as_name)
+        elif isinstance(obj, Concept):
+            return self.recognize_concept(obj, as_name, root)
+        elif isinstance(obj, Expando):
+            return self.recognize_expando(obj, as_name, root)
+        else:
+            raise NotImplementedError()
+
+    def recognize_str(self, text, as_name):
+        if self.text:
+            self.text += " and "
+
+        if "'" in text and '"' in text:
+            self.text += f"{as_name} == '{text}'"
+        elif "'" in text:
+            self.text += f'{as_name} == "{text}"'
+        else:
+            self.text += f"{as_name} == '{text}'"
+        return self
+
+    def recognize_int(self, value, as_name):
+        if self.text:
+            self.text += " and "
+
+        self.text += f"{as_name} == {value}"
+        return self
+
+    def recognize_expando(self, value, as_name, root=True):
+        if self.text:
+            self.text += " and "
+
+        if not root:
+            as_name = self.add_variable(as_name)
+
+        self.text += f"isinstance({as_name}, Expando) and {as_name}.get_name() == '{value.get_name()}'"
+        return self
+
+    def recognize_concept(self, concept, as_name, root=True):
+        if self.text:
+            self.text += " and "
+
+        if not root:
+            as_name = self.add_variable(as_name)
+
+        if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
+            self.text += f"isinstance({as_name}, Concept) and {as_name}.name == '{concept.name}'"
+        elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
+            self.text += f"isinstance({as_name}, Concept) and {as_name}.id == '{concept.id}'"
+        else:
+            self.text += f"isinstance({as_name}, Concept) and {as_name}.key == '{concept.key}'"
+            if len(concept.get_metadata().variables) > 0:
+                # add variables constraints
+                evaluated = ensure_evaluated(self.context, concept, eval_body=False, metadata=["variables"])
+
+                if not self.context.sheerka.is_success(evaluated) and evaluated.key != concept.key:
+                    raise EmitPythonCodeException(evaluated)
+
+                for k, v in concept.variables().items():
+                    self.recognize(v, f"{as_name}.get_value('{k}')", root=False)
+
+        return self
+
+    def add_variable(self, target):
+        var_name = f"__x_{self.var_counter:02}__"
+        self.var_counter += 1
+        self.variables.append((var_name, target))
+        return var_name
+
+    def get_text(self):
+        if self.variables:
+            variables_as_str = '\n'.join([f"{k} = {v}" for k, v in self.variables])
+            return variables_as_str + "\n" + self.text
+
+        return self.text
+
+
+class NoConditionFound(ErrorObj):
+    def __eq__(self, other):
+        return isinstance(other, NoConditionFound)
+
+    def __hash__(self):
+        return 0
+
+
 @dataclass()
-class RulePredicate:
-    source: str
-    evaluator: str
-    predicate: ReturnValueConcept
-    concept: Union[Concept, None]
+class RuleCompiledPredicate:
+    """
+    The 'when' expression is parsed to have a ReturnValueConcept or a Concept that can then be evaluated
+    Depending on the evaluator, the 'predicate' attribute or the 'concept' attribute will be used
+    """
+    source: str  # what was compiled # DO NOT REMOVE
+    action: str  # sheerka action when the rule must be executed # can be removed
+
+    # when used as a list of predicate to iterate thru
+    evaluator: str  # evaluator to use when the rule will be evaluated
+    predicate: ReturnValueConcept  # compiled source as ReturnValue
+    concept: Union[Concept, None]  # compiled source as concept
+
+    variables: Set[str] = None  # TODO: set of required variables
+
+
+@dataclass
+class CompiledWhenResult:
+    """
+    For a given source to compile (a given 'when')
+    List of RuleCompiledPredicate found
+    and list of Rete Conditions
+
+    The two ways of evaluating a 'when' are used by Sheerka
+    """
+    compiled_predicates: List[RuleCompiledPredicate]
+    rete_disjunctions: List[AndConditions]
 
 
 class SheerkaRuleManager(BaseService):
@@ -513,15 +647,18 @@ class SheerkaRuleManager(BaseService):
     RULES_BY_NAME_ENTRY = "RuleManager:Rules_By_Name"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=12)
         self._format_rules = None  # sorted by priority
+        self._exec_rules = None  # sorted by priority
+        self.expression_parser = ExpressionParser()
 
     def initialize(self):
         self.sheerka.bind_service_method(self.create_new_rule, True, visible=False)
+        self.sheerka.bind_service_method(self.remove_rule, True)
         self.sheerka.bind_service_method(self.get_rule_by_id, False)
         self.sheerka.bind_service_method(self.get_rule_by_name, False)
-        self.sheerka.bind_service_method(self.dump_desc_rule, False, as_name="desc_rule")
         self.sheerka.bind_service_method(self.get_format_rules, False, visible=False)
+        self.sheerka.bind_service_method(self.get_exec_rules, False, visible=False)
         self.sheerka.bind_service_method(self.resolve_rule, False, visible=False)
 
         cache = Cache().auto_configure(self.FORMAT_RULE_ENTRY)
@@ -531,6 +668,8 @@ class SheerkaRuleManager(BaseService):
         cache = ListIfNeededCache().auto_configure(self.RULES_BY_NAME_ENTRY)
         self.sheerka.om.register_cache(self.RULES_BY_NAME_ENTRY, cache, True, True)
 
+        self.sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities)
+
     def initialize_deferred(self, context, is_first_time):
 
         if is_first_time:
@@ -551,15 +690,17 @@ class SheerkaRuleManager(BaseService):
 
         # compile all format the rules
         for rule_id, rule_def in self.sheerka.om.get_all(self.FORMAT_RULE_ENTRY, cache_only=True).items():
-            rule = self.init_rule(context, rule_def)
+            self.init_rule(context, rule_def)
+
+        for rule_id, rule_def in self.sheerka.om.get_all(self.EXEC_RULE_ENTRY, cache_only=True).items():
+            self.init_rule(context, rule_def)
 
         # update rules priorities
         self.update_rules_priorities(context)
 
-        self.sheerka.subscribe(EVENT_RULE_PRECEDENCE_MODIFIED, self.update_rules_priorities)
-
     def reset_state(self):
         self._format_rules = None
+        self._exec_rules = None
 
     def update_rules_priorities(self, context):
         """
@@ -575,50 +716,93 @@ class SheerkaRuleManager(BaseService):
                 rule.priority = rules_weights[rule.str_id]
 
         self._format_rules = None
+        self._exec_rules = None
 
     def init_rule(self, context, rule: Rule):
         if rule.metadata.is_compiled:
-            return
+            return rule
 
-        if rule.compiled_predicate is None:
-            res = self.compile_when(context, self.NAME, rule.metadata.predicate)
-            if not isinstance(res, list):
-                rule.error_sink = [res.body]
-                return
-            rule.compiled_predicate = res
+        if rule.compiled_predicates is None:
+            try:
+                compiled_result = self.compile_when(context, self.NAME, rule.metadata.predicate)
+                rule.compiled_predicates = compiled_result.compiled_predicates
+                rule.rete_disjunctions = compiled_result.rete_disjunctions
+            except FailedToCompileError as ex:
+                rule.compiled_predicates = None
+                rule.rete_disjunctions = None
+                rule.error_sink = {"when": ex.cause}
 
         if rule.compiled_action is None:
-            res = self.compile_print(context, rule.metadata.action)
-            if not res.status:
-                rule.error_sink = [res.body]
-                return
-            rule.compiled_action = res.body
+            compile_method = self.compile_print if rule.metadata.action_type == ACTION_TYPE_PRINT else self.compile_exec
 
-        # rule.variables = self.get_variables()
+            res = compile_method(context, rule.metadata.action)
+            if not res.status:
+                rule.compiled_action = None
+                if rule.error_sink is None:
+                    rule.error_sink = {rule.metadata.action_type: res.body}
+                else:
+                    rule.error_sink[rule.metadata.action_type] = res.body
+            else:
+                rule.compiled_action = res.body if rule.metadata.action_type == ACTION_TYPE_PRINT else res
 
         rule.metadata.is_compiled = True
-        rule.metadata.is_enabled = True
+        rule.metadata.is_enabled = rule.error_sink is None
+
         return rule
 
-    def compile_when(self, context, name, source):
-        # parser_input = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
-        parsed = parse_unrecognized(context,
-                                    source,
-                                    parsers="all",
-                                    who=name,
-                                    prop=Keywords.WHEN,
-                                    filter_func=only_successful)
+    def compile_when(self, context, who, source):
+        """
+        Compile the predicate
+        :param context:
+        :param who: service which requested the compilation
+        :param source: what to compile
+        """
 
-        if not parsed.status:
-            return parsed
+        # first, try to parse using expression parser
+        # -> Detect xxx and yyy or not zzz
 
-        if self.sheerka.isinstance(parsed.body, BuiltinConcepts.ONLY_SUCCESSFUL):
-            parsed = parsed.body.body
+        action = None
+        parsed = []
+        errors = []
+        all_rete_disjunctions = []
+        parsed_expr_ret = parse_expression(context, source)
+        if parsed_expr_ret.status:
+            conjunctions = parsed_expr_ret.body.body.parts if isinstance(parsed_expr_ret.body.body, AndNode) else \
+                [parsed_expr_ret.body.body]
 
-        return self.add_evaluators(source, parsed if hasattr(parsed, "__iter__") else [parsed])
+            # recognize __action == ''
+            if (action := self._recognized_action_definition(conjunctions[0].tokens)) is not None:
+                conjunctions = conjunctions[1:]
+
+            if len(conjunctions) == 0:
+                errors.append(NoConditionFound())
+            else:
+                # compile conditions
+                try:
+                    return_values, rete_disjunctions = self.expression_parser.compile_conjunctions(context,
+                                                                                                   conjunctions,
+                                                                                                   who)
+
+                    parsed.extend(return_values)
+                    all_rete_disjunctions.extend(rete_disjunctions)
+
+                except FailedToCompileError as ex:
+                    errors.append(ex.cause)
+
+        if len(parsed) == 0:
+            raise FailedToCompileError(errors)
+
+        try:
+            compiled_predicates = self.add_evaluators(context,
+                                                      source,
+                                                      action,
+                                                      parsed if hasattr(parsed, "__iter__") else [parsed])
+            return CompiledWhenResult(compiled_predicates, all_rete_disjunctions)
+        except EmitPythonCodeException as ex:
+            raise FailedToCompileError([ex.error])
 
     def compile_print(self, context, source):
-        parser = FormatRuleParser(source)
+        parser = FormatRuleActionParser(source)
         parsed = parser.parse()
         if parser.error_sink:
             return self.sheerka.ret(self.NAME,
@@ -627,6 +811,16 @@ class SheerkaRuleManager(BaseService):
         else:
             return self.sheerka.ret(self.NAME, True, parsed)
 
+    def compile_exec(self, context, source):
+        parsed = parse_unrecognized(context,
+                                    source,
+                                    parsers="all",
+                                    who=self.NAME,
+                                    prop=Keywords.THEN,
+                                    filter_func=expect_one)
+
+        return parsed
+
     def set_id_if_needed(self, rule: Rule):
         """
         Set the id for the concept if needed
@@ -649,25 +843,52 @@ class SheerkaRuleManager(BaseService):
 
         # set id before saving in db
         self.set_id_if_needed(rule)
-        if rule.compiled_predicate and rule.compiled_action:
+        if rule.compiled_predicates and rule.compiled_action:
             rule.metadata.is_compiled = True
             rule.metadata.is_enabled = True
 
         # save it
-        if rule.metadata.action_type == "print":
-            self.sheerka.om.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule)
+        if rule.metadata.action_type == ACTION_TYPE_PRINT:
+            sheerka.om.put(self.FORMAT_RULE_ENTRY, rule.metadata.id, rule)
             self._format_rules = None
         else:
-            self.sheerka.om.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule)
+            sheerka.om.put(self.EXEC_RULE_ENTRY, rule.metadata.id, rule)
+            self._exec_rules = None
 
         # save by name if needed
         if rule.metadata.name:
             self.sheerka.om.put(self.RULES_BY_NAME_ENTRY, rule.metadata.name, rule)
 
+        # rule is created. publish the event
+        sheerka.publish(context, EVENT_RULE_CREATED, rule)
+
         # process the return if needed
         ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_RULE, body=rule))
         return ret
 
+    def remove_rule(self, context, rule):
+        """
+        Remove a rule
+        """
+        rule = self.resolve_rule(context, rule)
+        if rule is None:
+            return
+
+        # rule will be deleted. publish the event first, as the rule may not be available after
+        self.sheerka.publish(context, EVENT_RULE_DELETED, rule)
+
+        if rule.metadata.action_type == ACTION_TYPE_PRINT:
+            self.sheerka.om.delete(self.FORMAT_RULE_ENTRY, rule.metadata.id)
+            self._format_rules = None
+        else:
+            self.sheerka.om.delete(self.EXEC_RULE_ENTRY, rule.metadata.id)
+            self._exec_rules = None
+
+        if rule.metadata.name:
+            self.sheerka.om.delete(self.RULES_BY_NAME_ENTRY, rule.metadata.name)
+
+        return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+
     def init_builtin_rules(self, context):
         # self.sheerka.init_log.debug("Initializing default rules")
         rules = [
@@ -760,29 +981,6 @@ class SheerkaRuleManager(BaseService):
 
         return rule
 
-    def dump_desc_rule(self, rules):
-        """
-        dumps the definition of a rule
-        :param rules:
-        :return:
-        """
-        ensure_rule(rules)
-
-        if not hasattr(rules, "__iter__"):
-            rules = [rules]
-
-        first = True
-        for rule in rules:
-            if not first:
-                self.sheerka.log.info("")
-            self.sheerka.log.info(f"id          : {rule.id}")
-            self.sheerka.log.info(f"name        : {rule.metadata.name}")
-            self.sheerka.log.info(f"type        : {rule.metadata.action_type}")
-            self.sheerka.log.info(f"predicate   : {rule.metadata.predicate}")
-            self.sheerka.log.info(f"action      : {rule.metadata.action}")
-            self.sheerka.log.info(f"compiled    : {rule.metadata.is_compiled}")
-            self.sheerka.log.info(f"enabled     : {rule.metadata.is_enabled}")
-
     def get_format_rules(self):
         if self._format_rules:
             return self._format_rules
@@ -792,32 +990,56 @@ class SheerkaRuleManager(BaseService):
                                     reverse=True)
         return self._format_rules
 
-    def add_evaluators(self, source, ret_vals):
+    def get_exec_rules(self):
+        if self._exec_rules:
+            return self._exec_rules
+
+        self._exec_rules = sorted(self.sheerka.om.list(self.EXEC_RULE_ENTRY, cache_only=True),
+                                  key=operator.attrgetter('priority'),
+                                  reverse=True)
+        return self._exec_rules
+
+    def add_evaluators(self, context, source, action, ret_vals):
         """
         Browse the ReturnValueConcepts to determine the evaluator to use
-        Returns a list of tuple (evaluator_name, return_value)
+        Returns a list of RulePredicate, basically a tuple (evaluator_name, return_value)
+        :param context:
         :param source:
         :param ret_vals:
         :return:
         """
+
+        def get_rule_predicate_from_concept(c):
+            if is_a_question(context, c):
+                return RuleCompiledPredicate(source, action, ConceptEvaluator.NAME, r, c)
+            else:
+                to_parse = PythonCodeEmitter(context, "__ret.status").recognize_concept(c, "__ret.body").get_text()
+                return RuleCompiledPredicate(source, action, PythonEvaluator.NAME, parse_python(context, to_parse),
+                                             None)
+
         res = []
         for r in ret_vals:
             underlying = self.sheerka.objvalue(r)
             if isinstance(underlying, PythonNode):
-                res.append(RulePredicate(source, PythonEvaluator.NAME, r, None))
+                res.append(RuleCompiledPredicate(source, action, PythonEvaluator.NAME, r, None))
             elif isinstance(underlying, SourceCodeWithConceptNode):
-                res.append(RulePredicate(source, PythonEvaluator.NAME, r, None))
+                res.append(RuleCompiledPredicate(source, action, PythonEvaluator.NAME, r, None))
             elif isinstance(underlying, SourceCodeNode):
-                res.append(RulePredicate(source, PythonEvaluator.NAME, r, None))
+                res.append(RuleCompiledPredicate(source, action, PythonEvaluator.NAME, r, None))
             elif isinstance(underlying, Concept):
-                res.append(RulePredicate(source, ConceptEvaluator.NAME, r, underlying))
+                res.append(get_rule_predicate_from_concept(underlying))
             elif hasattr(underlying, "__iter__") and len(underlying) == 1 and isinstance(underlying[0], ConceptNode):
-                res.append(RulePredicate(source, ConceptEvaluator.NAME, r, underlying[0].concept))
+                res.append(get_rule_predicate_from_concept(underlying[0].concept))
             else:
                 raise NotImplementedError(r)
         return res
 
     def resolve_rule(self, context, obj):
+        """
+        Given obj, try to find the corresponding rule
+        :param context:
+        :param obj:
+        """
         if obj is None:
             return None
 
@@ -841,7 +1063,7 @@ class SheerkaRuleManager(BaseService):
                     (rule := self._inner_get_by_id(str(rule_id))) is not None:
                     return rule
             else:
-                return obj
+                return self._inner_get_by_id(obj.id)
 
         return None
 
@@ -855,3 +1077,41 @@ class SheerkaRuleManager(BaseService):
             return rule
 
         return None
+
+    @staticmethod
+    def _recognized_action_definition(tokens):
+        """
+        Tries to recognize the pattern __action = xxx in the tokens
+        """
+        iter_token = iter(tokens)
+        try:
+            token = next(iter_token)
+            if token.value != "__action":
+                return None
+            token = next(iter_token)
+            if token.type == TokenKind.WHITESPACE:
+                token = next(iter_token)
+            if token.type != TokenKind.EQUALSEQUALS:
+                return None
+            token = next(iter_token)
+            if token.type == TokenKind.WHITESPACE:
+                token = next(iter_token)
+            if token.type == TokenKind.STRING:
+                return token.strip_quote
+        except StopIteration:
+            pass
+
+        return None
+
+    @staticmethod
+    def _get_parsed_concept(context, return_value):
+        if not context.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT):
+            return None
+
+        if isinstance(return_value.body.body, Concept):
+            return return_value.body.body
+
+        if isinstance(return_value.body.body, ConceptNode):
+            return return_value.body.body.concept
+
+        return None

src/core/sheerka/services/SheerkaVariableManager.py

@@ -41,9 +41,9 @@ class SheerkaVariableManager(BaseService):
     INTERNAL_VARIABLES_ENTRY = "VariableManager:InternalVariables"  # internal to current process (can store lambda)
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=3)
         self.bound_variables = {
-            self.sheerka.name: {"enable_process_return_values", "save_execution_context"}
+            self.sheerka.name: {"enable_process_return_values", "save_execution_context", "enable_process_rules"}
         }
 
     def initialize(self):

src/core/sheerka/services/sheerka_service.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 
-from core.global_symbols import NotFound
+from core.global_symbols import NotFound, ErrorObj
 from core.utils import sheerka_deepcopy
 
 
@@ -14,8 +14,9 @@ class BaseService:
     Base class for services
     """
 
-    def __init__(self, sheerka):
+    def __init__(self, sheerka, order=999):
         self.sheerka = sheerka
+        self.order = order  # initialisation order. The lowest is initialized first
 
     def initialize(self):
         """
@@ -46,3 +47,8 @@ class BaseService:
         Store/record the value of an attribute
         """
         self.sheerka.record_var(context, self.NAME, var_name, getattr(self, var_name))
+
+
+@dataclass()
+class FailedToCompileError(Exception, ErrorObj):
+    cause: list

src/core/tokenizer.py

@@ -139,6 +139,7 @@ class LexerError(Exception):
 class Keywords(Enum):
     DEF = "def"
     CONCEPT = "concept"
+    RULE = "rule"
     FROM = "from"
     BNF = "bnf"
     AS = "as"
@@ -149,6 +150,7 @@ class Keywords(Enum):
     RET = "ret"
     WHEN = "when"
     PRINT = "print"
+    THEN = "then"
 
 
 class Tokenizer:
@@ -557,36 +559,3 @@ class IterParser:
             return token_after
         except StopIteration:
             return Token(TokenKind.EOF, -1, -1, -1, -1)
-
-
-# @dataclass
-# class PropDef:
-#     prop: str
-#     index: int
-#
-#
-# class SimpleExpressionParser(IterParser):
-#     def __init__(self, source):
-#         super().__init__(source)
-#         self.properties = []
-#
-#     def parse(self):
-#
-#         prop, index, key = None, None, None
-#         while self.next_token():
-#             if self.token.type == TokenKind.DOT:
-#                 self.properties.append(PropDef(prop, index, key))
-#                 prop, index, key = None, None, None
-#                 continue
-#
-#             if self.token.type == TokenKind.LBRACKET:
-#                 index = self.parse_index()
-#             elif self.token.type == TokenKind.LBRACE:
-#                 key = self.parse_key()
-#             else:
-#                 prop = self.token.value
-#
-#         if prop is not None:
-#             self.properties.append(PropDef(prop, index, key))
-#
-#     def parse_i

src/core/utils.py

@@ -3,12 +3,14 @@ import importlib
 import inspect
 import os
 import pkgutil
+import re
 from copy import deepcopy
 
-from pyparsing import *
+# from pyparsing import *
+from pyparsing import Literal, Word, nums, Combine, Optional, delimitedList, oneOf, alphas, Suppress
 
 from core.global_symbols import CustomType
-from core.tokenizer import TokenKind, Tokenizer
+from core.tokenizer import TokenKind, Tokenizer, Token
 
 COLORS = {
     "black",
@@ -249,7 +251,7 @@ def make_unique(lst, get_id=None):
     return list(_make_unique(lst, get_id))
 
 
-def product(a, b):
+def sheerka_product(a, b):
     """
     Kind of cartesian product between lists a and b
     knowing that a is also a list : a is a list of list !!!
@@ -569,7 +571,7 @@ def as_bag(obj, forced_properties=None):
     """
     Get the properties of an object (static and dynamic)
     :param obj:
-    :param forced_properties:
+    :param forced_properties: special mode where properties are given in parameter
     :return:
     """
 
@@ -638,6 +640,33 @@ def get_text_from_tokens(tokens, custom_switcher=None, tracker=None):
     return res
 
 
+def tokens_are_matching(tokens1, tokens2, skip_tokens=True):
+    def get_next(it):
+        try:
+            return next(it)
+        except StopIteration:
+            return Token(TokenKind.EOF, "", -1, -1, -1)
+
+    iter1 = iter(tokens1)
+    iter2 = iter(tokens2)
+
+    while True:
+        t1 = get_next(iter1)
+        t2 = get_next(iter2)
+
+        if skip_tokens:
+            if t1.type == TokenKind.WHITESPACE:
+                t1 = next(iter1)
+            if t2.type == TokenKind.WHITESPACE:
+                t2 = next(iter2)
+
+        if t1.type == TokenKind.EOF and t2.type == TokenKind.EOF:
+            return True
+
+        if t1.type != t2.type or t1.value != t2.value:
+            return False
+
+
 def dump_ast(node):
     dump = ast.dump(node)
     for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:

src/evaluators/DefConceptEvaluator.py

@@ -2,13 +2,13 @@ import core.utils
 from core.ast_helpers import UnreferencedVariablesVisitor
 from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
+from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import TokenKind, Tokenizer
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
-from parsers.BaseParser import NotInitializedNode
 from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
 from parsers.DefConceptParser import DefConceptNode, NameNode
-from parsers.PythonParser import PythonNode, get_python_node
+from parsers.PythonParser import get_python_node
 
 
 class ConceptOrRuleNameVisitor(ParsingExpressionVisitor):
@@ -70,7 +70,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
             part_ret_val = getattr(def_concept_node, prop)
 
             # put back the sources
-            if isinstance(part_ret_val, NotInitializedNode):
+            if part_ret_val is NotInit:
                 continue
             elif isinstance(part_ret_val, NameNode):
                 source = str(part_ret_val)
@@ -107,7 +107,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
         concept.init_key(key_source)
 
         # update the bnf definition if needed
-        if not isinstance(def_concept_node.definition, NotInitializedNode) and \
+        if def_concept_node.definition is not NotInit and \
             def_concept_node.definition_type == DEFINITION_TYPE_BNF:
             concept.set_bnf(def_concept_node.definition.value.value)
 

src/evaluators/DefRuleEvaluator.py

@@ -0,0 +1,56 @@
+import core.utils
+from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
+from core.global_symbols import NotInit
+from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
+from core.tokenizer import Keywords
+from evaluators.BaseEvaluator import OneReturnValueEvaluator
+from parsers.DefRuleParser import DefRuleNode, DefFormatRuleNode
+
+
+class DefRuleEvaluator(OneReturnValueEvaluator):
+    """
+    Used to store a new format rule
+    """
+    NAME = "DefRule"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
+
+    def matches(self, context, return_value):
+        return return_value.status and \
+               isinstance(return_value.value, ParserResultConcept) and \
+               isinstance(return_value.value.value, DefRuleNode)
+
+    def eval(self, context, return_value):
+        """
+        Creates a Rule out of a DefRule and saves it in db
+        :param context:
+        :param return_value:
+        :return:
+        """
+
+        context.log("Adding a new rule", self.name)
+        rule_definition = return_value.value.value
+        sheerka = context.sheerka
+
+        name = None if rule_definition.name is NotInit else str(rule_definition.name)
+        predicate = core.utils.get_text_from_tokens(rule_definition.tokens[Keywords.WHEN][1:])
+        if isinstance(rule_definition, DefFormatRuleNode):
+            action_type = ACTION_TYPE_PRINT
+            action = core.utils.get_text_from_tokens(rule_definition.tokens[Keywords.PRINT][1:])
+            compiled_action = rule_definition.print
+        else:
+            action_type = ACTION_TYPE_EXEC
+            action = core.utils.get_text_from_tokens(rule_definition.tokens[Keywords.THEN][1:])
+            compiled_action = rule_definition.then
+
+        rule = Rule(action_type, name, predicate, action)
+        rule.compiled_predicates = rule_definition.when
+        rule.rete_disjunctions = rule_definition.rete
+        rule.compiled_action = compiled_action
+
+        ret = sheerka.create_new_rule(context, rule)
+        if not ret.status:
+            error_cause = sheerka.objvalue(ret.body)
+            context.log(f"Failed to add new rule '{rule}'. Reason: {error_cause}", self.name)
+        return sheerka.ret(self.name, ret.status, ret.value, parents=[return_value])

src/evaluators/FormatRuleEvaluator.py

@@ -1,45 +0,0 @@
-import core.utils
-from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
-from core.rule import Rule
-from core.tokenizer import Keywords
-from evaluators.BaseEvaluator import OneReturnValueEvaluator
-from parsers.DefFormatRuleParser import FormatRuleNode
-
-
-class FormatRuleEvaluator(OneReturnValueEvaluator):
-    """
-    Used to store a new format rule
-    """
-    NAME = "FormatRule"
-
-    def __init__(self):
-        super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
-
-    def matches(self, context, return_value):
-        return return_value.status and \
-               isinstance(return_value.value, ParserResultConcept) and \
-               isinstance(return_value.value.value, FormatRuleNode)
-
-    def eval(self, context, return_value):
-        """
-        Creates a Rule out of a FormatRuleNode and saves it in db
-        :param context:
-        :param return_value:
-        :return:
-        """
-
-        context.log("Adding a new format rule", self.name)
-        format_rule_node = return_value.value.value
-        sheerka = context.sheerka
-
-        predicate = core.utils.get_text_from_tokens(format_rule_node.tokens[Keywords.WHEN][1:])
-        action = core.utils.get_text_from_tokens(format_rule_node.tokens[Keywords.PRINT][1:])
-        rule = Rule("print", None, predicate, action)
-        rule.compiled_predicate = format_rule_node.rule
-        rule.compiled_action = format_rule_node.format_ast
-
-        ret = sheerka.create_new_rule(context, rule)
-        if not ret.status:
-            error_cause = sheerka.objvalue(ret.body)
-            context.log(f"Failed to add new rule '{rule}'. Reason: {error_cause}", self.name)
-        return sheerka.ret(self.name, ret.status, ret.value, parents=[return_value])

src/evaluators/PythonEvaluator.py

@@ -36,12 +36,16 @@ def inject_context(context):
 
 
 class Expando:
-    def __init__(self, bag):
+    def __init__(self, name, bag):
+        self.__name = name
         for k, v in bag.items():
             setattr(self, k, v)
 
     def __repr__(self):
-        return f"{dir(self)}"
+        return f"{vars(self)}"
+
+    def get_name(self):
+        return self.__name
 
 
 @dataclass
@@ -89,12 +93,12 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
     def eval(self, context, return_value):
         sheerka = context.sheerka
-        node = get_python_node(return_value.value.value)
+        node = return_value.value.value.get_python_node()
 
         debugger = context.get_debugger(PythonEvaluator.NAME, "eval")
         debugger.debug_entering(node=node)
         exception_debugger = context.get_debugger("Exceptions", PythonEvaluator.NAME + ".eval")
-        get_trace_back = context.debug_enabled or exception_debugger.is_enabled()
+        get_trace_back = exception_debugger.is_enabled()
 
         context.log(f"Evaluating python node {node}.", self.name)
 
@@ -179,8 +183,6 @@ class PythonEvaluator(OneReturnValueEvaluator):
         """
         unreferenced_names_visitor = UnreferencedNamesVisitor(context)
         names = unreferenced_names_visitor.get_names(node.ast_)
-        if context.debug_enabled:
-            context.debug(self.NAME, "eval", "names", names)
         return self.get_globals_by_names(context, names, node, expression_only)
 
     def get_sheerka_method(self, context, name, expression_only):
@@ -200,6 +202,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
         my_globals = {
             "Concept": core.concept.Concept,
             "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts,
+            "Expando": Expando,
             "ExecutionContext": ExecutionContext,
             "in_context": context.in_context,
         }
@@ -218,14 +221,14 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 continue
 
             # support reference to sheerka
-            if name == "sheerka":
+            if name.lower() == "sheerka":
                 bag = {}
                 visitor = NamesWithAttributesVisitor()
                 for sequence in visitor.get_sequences(node.ast_, "sheerka"):
                     if (len(sequence) > 1 and
                         (method := self.get_sheerka_method(context, sequence[1], expression_only)) is not None):
                         bag[sequence[1]] = method
-                my_globals["sheerka"] = Expando(bag)
+                my_globals[name] = Expando("sheerka", bag)
                 continue
 
             # search in local variables. To remove when local variables will be merged with memory

src/out/DeveloperVisitor.py

@@ -62,7 +62,8 @@ class DeveloperVisitor:
 
             return format_ast.clone(value=res)
         except NameError as error:
-            context.debug("DeveloperVisitor", "visit_FormatAstList", "evaluate_expression", error, is_error=True)
+            self.debugger.debug_log(f"Error : {error}", is_error=True)
+            self.debugger.debug_var(f"Exception", error, is_error=True)
             return FormatAstVariableNotFound(format_ast.name)
 
     def visit_FormatAstSequence(self, context, format_ast, bag):

src/parsers/BaseCustomGrammarParser.py

@@ -62,6 +62,38 @@ class KeywordNotFound(CustomGrammarParserNode, ParsingError):
         return hash(self.keywords)
 
 
+@dataclass()
+class NameNode(CustomGrammarParserNode):
+
+    def get_name(self):
+        name = ""
+        first = True
+        for token in self.tokens:
+            if token.type == TokenKind.EOF:
+                break
+            if token.type == TokenKind.WHITESPACE:
+                continue
+            if not first:
+                name += " "
+
+            name += token.value[1:-1] if token.type == TokenKind.STRING else str(token.value)
+            first = False
+
+        return name
+
+    def __repr__(self):
+        return self.get_name()
+
+    def __eq__(self, other):
+        if not isinstance(other, NameNode):
+            return False
+
+        return self.get_name() == other.get_name()
+
+    def __hash__(self):
+        return hash(self.get_name())
+
+
 class BaseCustomGrammarParser(BaseParser):
     """
     Base class for sheerka specific grammars

src/parsers/BaseNodeParser.py

@@ -42,6 +42,9 @@ class LexerNode(Node):
     def to_short_str(self):
         raise NotImplementedError
 
+    def get_source_to_parse(self):
+        return self.source
+
 
 class UnrecognizedTokensNode(LexerNode):
     def __init__(self, start, end, tokens):
@@ -296,6 +299,12 @@ class SourceCodeNode(LexerNode):
     def to_short_str(self):
         return f"SCN('{self.source}')"
 
+    def get_python_node(self):
+        return self.python_node
+
+    def get_source_to_parse(self):
+        return self.python_node.source
+
 
 class SourceCodeWithConceptNode(LexerNode):
     """
@@ -409,6 +418,12 @@ class SourceCodeWithConceptNode(LexerNode):
     def to_short_str(self):
         return f"SCWC({self.first}" + ", ".join(n.to_short_str for n in self.nodes) + f"{self.last})"
 
+    def get_python_node(self):
+        return self.python_node
+
+    def get_source_to_parse(self):
+        return self.python_node.source
+
 
 @dataclass()
 class GrammarErrorNode(ParsingError):

src/parsers/BaseParser.py

@@ -1,33 +1,12 @@
-import logging
 from dataclasses import dataclass
 from typing import Union
 
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
 from core.concept import Concept
-from core.error import ErrorObj
+from core.global_symbols import ErrorObj
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka_logger import get_logger
-from core.tokenizer import TokenKind, Token, Tokenizer, LexerError
-
-
-# # keep a cache for the parser input
-# pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
-#
-#
-# def get_parser_input(text, tokens=None, length=None):
-#     """
-#     Returns new or existing parser input
-#     :param text:
-#     :param tokens:
-#     :param length:
-#     :return:
-#     """
-#     if tokens is None or pi_cache.has(text):
-#         return pi_cache.get(text)
-#     pi = ParserInput(text, tokens, length)
-#     pi_cache.put(text, pi)
-#     return pi
+from core.tokenizer import TokenKind, Token, LexerError
 
 
 @dataclass()
@@ -35,13 +14,6 @@ class Node:
     pass
 
 
-class NotInitializedNode(Node):
-    pass
-
-    def __repr__(self):
-        return "**N/A**"
-
-
 @dataclass()
 class ParsingError(Node, ErrorObj):
     pass
@@ -206,17 +178,6 @@ class BaseParser:
 
         return parser_input.value
 
-    # @staticmethod
-    # def manage_eof(lst, strip_eof):
-    #     if strip_eof:
-    #         if len(lst) and lst[-1].type == TokenKind.EOF:
-    #             lst.pop()
-    #         return lst
-    #
-    #     if len(lst) == 0 or not lst[-1].type == TokenKind.EOF:
-    #         lst.append(Token(TokenKind.EOF, "", -1, -1, -1))
-    #     return lst
-
     @staticmethod
     def get_tokens_boundaries(tokens):
         """

src/parsers/ComparisonParser.py

@@ -0,0 +1,185 @@
+from typing import Union, List
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
+from core.tokenizer import TokenKind, Token
+from core.utils import get_text_from_tokens
+from parsers.BaseParser import BaseParser
+from parsers.expressions import ComparisonNode, ParenthesisMismatchError, NameExprNode, ComparisonType, VariableNode
+
+
+class ComparisonParser(BaseParser):
+    """
+    Parses xxx (== | > | < | >= | <= | != | in | not in) yyy
+    Nothing else
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__("Expression", 60, False, yield_eof=True)
+
+    def parse(self, context, parser_input: Union[ParserInput, List[Token]]):
+        """
+        :param context:
+        :param parser_input:
+        :return:
+        """
+
+        if isinstance(parser_input, list):
+            parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens=parser_input)
+
+        elif not isinstance(parser_input, ParserInput):
+            return None
+
+        context.log(f"Parsing '{parser_input}' with ComparisonExpressionParser", self.name)
+        sheerka = context.sheerka
+
+        if parser_input.is_empty():
+            return context.sheerka.ret(self.name,
+                                       False,
+                                       sheerka.new(BuiltinConcepts.IS_EMPTY))
+
+        if not self.reset_parser(context, parser_input):
+            return self.sheerka.ret(
+                self.name,
+                False,
+                context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))
+
+        self.parser_input.next_token()
+
+        node = self.parse_compare()
+
+        value = self.get_return_value_body(context.sheerka, self.parser_input.as_text(), node, node)
+
+        ret = self.sheerka.ret(
+            self.name,
+            not self.has_error,
+            value)
+
+        return ret
+
+    def parse_compare(self):
+        start = self.parser_input.pos
+        left = self.parse_names()
+        if left is None:
+            return None
+
+        if (comp := self.eat_comparison()) is None:
+            return left
+
+        right = self.parse_names()
+        end = right.end if right else self.parser_input.pos
+        return ComparisonNode(start, end, self.parser_input.tokens[start: end + 1], comp, left, right)
+
+    def parse_names(self):
+
+        token = self.parser_input.token
+        if token.type == TokenKind.EOF:
+            return None
+
+        buffer = []
+        paren_count = 0
+        last_lparen = None
+        last_rparen = None
+        start = self.parser_input.pos
+        while (paren_count > 0 or not self.eat_comparison(False)) and token.type != TokenKind.EOF:
+            buffer.append(token)
+            if token.type == TokenKind.LPAR:
+                last_lparen = token
+                paren_count += 1
+            if token.type == TokenKind.RPAR:
+                last_rparen = token
+                paren_count -= 1
+            self.parser_input.next_token(False)
+            token = self.parser_input.token
+
+        if paren_count != 0:
+            pass
+
+        if paren_count > 0:
+            self.error_sink.append(ParenthesisMismatchError(last_lparen))
+            return None
+
+        if paren_count < 0:
+            self.error_sink.append(ParenthesisMismatchError(last_rparen))
+            return None
+
+        if buffer[-1].type == TokenKind.WHITESPACE:
+            buffer.pop()
+
+        end = start + len(buffer) - 1
+        return self.try_to_recognize(NameExprNode(start, end, buffer))
+
+    def eat_comparison(self, eat=True):
+        token = self.parser_input.token
+        if token.type == TokenKind.EQUALSEQUALS:
+            if eat:
+                self.parser_input.next_token()
+            return ComparisonType.EQUALS
+
+        if token.type == TokenKind.LESS:
+            if self.parser_input.the_token_after(False).type == TokenKind.EQUALS:
+                if eat:
+                    self.parser_input.next_token()
+                    self.parser_input.next_token()
+                return ComparisonType.LESS_THAN_OR_EQUALS
+            else:
+                if eat:
+                    self.parser_input.next_token()
+                return ComparisonType.LESS_THAN
+
+        if token.type == TokenKind.GREATER:
+            if self.parser_input.the_token_after(False).type == TokenKind.EQUALS:
+                if eat:
+                    self.parser_input.next_token()
+                    self.parser_input.next_token()
+                return ComparisonType.GREATER_THAN_OR_EQUALS
+            else:
+                if eat:
+                    self.parser_input.next_token()
+                return ComparisonType.GREATER_THAN
+
+        if token.type == TokenKind.IDENTIFIER and token.value == "not":
+            if self.parser_input.the_token_after(True).value == "in":
+                if eat:
+                    self.parser_input.next_token()
+                    self.parser_input.next_token()
+                return ComparisonType.NOT_IN
+
+        if token.type == TokenKind.IDENTIFIER and token.value == "in":
+            if eat:
+                self.parser_input.next_token()
+            return ComparisonType.IN
+
+        if token.type == TokenKind.EMARK and self.parser_input.the_token_after(False).type == TokenKind.EQUALS:
+            if eat:
+                self.parser_input.next_token()
+                self.parser_input.next_token()
+            return ComparisonType.NOT_EQUAlS
+
+        return None
+
+    @staticmethod
+    def try_to_recognize(expr: NameExprNode):
+        not_a_variable = False
+        expect_dot = False
+        for t in expr.tokens:
+            if expect_dot and t.type != TokenKind.DOT:
+                not_a_variable = True
+            if t.type == TokenKind.DOT:
+                break  # Only interested in the root part
+            elif t.type == TokenKind.WHITESPACE:
+                expect_dot = True
+            elif t.type == TokenKind.LPAR:
+                pass  # try to recognize function
+            elif not str(t.value).isidentifier():
+                not_a_variable = True
+
+        if not_a_variable:
+            return expr
+
+        full_name = get_text_from_tokens(expr.tokens)
+        split = full_name.split(".")
+        if len(split) == 1:
+            return VariableNode(expr.start, expr.end, expr.tokens, split[0])
+        else:
+            return VariableNode(expr.start, expr.end, expr.tokens, split[0], *split[1:])

src/parsers/DefConceptParser.py

@@ -1,13 +1,14 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 
 import core.builtin_helpers
 import core.utils
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
-from core.concept import ConceptParts, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
+from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
+from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
 from core.tokenizer import TokenKind, Keywords
-from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, SyntaxErrorNode
-from parsers.BaseParser import Node, ParsingError, NotInitializedNode, UnexpectedTokenParsingError
+from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, SyntaxErrorNode, NameNode, CustomGrammarParserNode
+from parsers.BaseParser import ParsingError, UnexpectedTokenParsingError
 from parsers.BnfDefinitionParser import BnfDefinitionParser
 
 
@@ -17,15 +18,7 @@ class ParsingException(Exception):
 
 
 @dataclass()
-class DefConceptParsingResult(Node):
-    """
-    Base node for all default parser nodes
-    """
-    tokens: list = field(compare=False, repr=False)
-
-
-@dataclass()
-class DefConceptParsingError(DefConceptParsingResult, ParsingError):
+class DefConceptParsingError(CustomGrammarParserNode, ParsingError):
     pass
 
 
@@ -38,63 +31,21 @@ class CannotHandleParsingError(DefConceptParsingError):
 
 
 @dataclass()
-class NameNode(DefConceptParsingResult):
-
-    def get_name(self):
-        name = ""
-        first = True
-        for token in self.tokens:
-            if token.type == TokenKind.EOF:
-                break
-            if token.type == TokenKind.WHITESPACE:
-                continue
-            if not first:
-                name += " "
-
-            name += token.value[1:-1] if token.type == TokenKind.STRING else str(token.value)
-            first = False
-
-        return name
-
-    def __repr__(self):
-        return self.get_name()
-
-    def __eq__(self, other):
-        if not isinstance(other, NameNode):
-            return False
-
-        return self.get_name() == other.get_name()
-
-    def __hash__(self):
-        return hash(self.get_name())
-
-
-@dataclass()
-class DefConceptNode(DefConceptParsingResult):
-    name: NameNode = NotInitializedNode()
-    where: ReturnValueConcept = NotInitializedNode()
-    pre: ReturnValueConcept = NotInitializedNode()
-    post: ReturnValueConcept = NotInitializedNode()
-    body: ReturnValueConcept = NotInitializedNode()
-    ret: ReturnValueConcept = NotInitializedNode()
-    definition: ReturnValueConcept = NotInitializedNode()
+class DefConceptNode(CustomGrammarParserNode):
+    name: NameNode = NotInit
+    where: ReturnValueConcept = NotInit
+    pre: ReturnValueConcept = NotInit
+    post: ReturnValueConcept = NotInit
+    body: ReturnValueConcept = NotInit
+    ret: ReturnValueConcept = NotInit
+    definition: ReturnValueConcept = NotInit
     definition_type: str = None
 
-    def get_asts(self):
-        asts = {}
-        for part_key in ConceptParts:
-            prop_value = getattr(self, part_key.value)
-            if isinstance(prop_value, ReturnValueConcept) and \
-                isinstance(prop_value.body, ParserResultConcept) and \
-                hasattr(prop_value.body.body, "ast_"):
-                asts[part_key] = prop_value
-        return asts
-
 
 @dataclass()
-class IsaConceptNode(DefConceptParsingResult):
-    concept: NameNode = NotInitializedNode()
-    set: NameNode = NotInitializedNode()
+class IsaConceptNode(CustomGrammarParserNode):
+    concept: NameNode = NotInit
+    set: NameNode = NotInit
 
 
 class DefConceptParser(BaseCustomGrammarParser):
@@ -201,12 +152,12 @@ class DefConceptParser(BaseCustomGrammarParser):
 
     def get_concept_definition(self, current_concept_def, parts):
         if Keywords.FROM not in parts:
-            return None, NotInitializedNode()
+            return None, NotInit
 
         tokens = parts[Keywords.FROM]
         if len(tokens) == 1:
             self.add_error(SyntaxErrorNode([], f"Empty '{tokens[0].value}' declaration."), False)
-            return None, NotInitializedNode()
+            return None, NotInit
 
         if tokens[1].value == Keywords.BNF.value:
             return self.get_concept_bnf_definition(current_concept_def, core.utils.strip_tokens(tokens[2:]))
@@ -216,7 +167,7 @@ class DefConceptParser(BaseCustomGrammarParser):
     def get_concept_bnf_definition(self, current_concept_def, tokens):
         if len(tokens) == 0:
             self.add_error(SyntaxErrorNode([], "Empty 'bnf' declaration"), False)
-            return None, NotInitializedNode()
+            return None, NotInit
 
         if tokens[0].type == TokenKind.COLON:
             tokens = self.get_body(tokens[1:])
@@ -233,7 +184,7 @@ class DefConceptParser(BaseCustomGrammarParser):
 
             if not parsing_result.status:
                 self.add_error(parsing_result.value)
-                return None, NotInitializedNode()
+                return None, NotInit
 
             return DEFINITION_TYPE_BNF, parsing_result
 
@@ -243,7 +194,7 @@ class DefConceptParser(BaseCustomGrammarParser):
         tokens = core.utils.strip_tokens(tokens[start:])
         if len(tokens) == 0:
             self.add_error(SyntaxErrorNode([], f"Empty 'from' declaration."), False)
-            return None, NotInitializedNode()
+            return None, NotInit
 
         if tokens[0].type == TokenKind.COLON:
             tokens = self.get_body(tokens[1:])
@@ -252,7 +203,7 @@ class DefConceptParser(BaseCustomGrammarParser):
 
     def get_ast(self, keyword, parts):
         if keyword not in parts:
-            return NotInitializedNode()
+            return NotInit
 
         tokens = parts[keyword]
         if len(tokens) == 1:

src/parsers/DefFormatRuleParser.py

@@ -1,122 +0,0 @@
-from dataclasses import dataclass
-
-import core.utils
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
-from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatAstNode
-from core.tokenizer import Keywords
-from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, KeywordNotFound
-from parsers.BaseParser import BaseParser, Node
-
-
-@dataclass
-class FormatRuleNode(Node):
-    tokens: dict
-    rule: ReturnValueConcept = None
-    format_ast: FormatAstNode = None
-
-
-class DefFormatRuleParser(BaseCustomGrammarParser):
-    """
-    Class that will parse formatting rules definitions
-    eg:   when xxx print yyy
-    where xxx will be evaluated in the context of BuiltinConcepts.EVAL_QUESTION_REQUESTED
-    and yyy is a internal way to describe a format (yet another one)
-    """
-
-    KEYWORDS = [Keywords.WHEN, Keywords.PRINT]
-    KEYWORDS_VALUES = [k.value for k in KEYWORDS]
-
-    def __init__(self, **kwargs):
-        BaseCustomGrammarParser.__init__(self, "DefFormatRule", 60)
-
-    def parse(self, context, parser_input: ParserInput):
-        """
-
-        :param context:
-        :param parser_input:
-        :return:
-        """
-
-        if not isinstance(parser_input, ParserInput):
-            return None
-
-        if parser_input.from_tokens:
-            ret = context.sheerka.ret(
-                self.name,
-                False,
-                context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input))
-            self.log_result(context, parser_input, ret)
-            return ret
-
-        context.log(f"Parsing '{parser_input}' with FunctionParser", self.name)
-        sheerka = context.sheerka
-
-        if parser_input.is_empty():
-            return sheerka.ret(self.name,
-                               False,
-                               sheerka.new(BuiltinConcepts.IS_EMPTY))
-
-        if not self.reset_parser(context, parser_input):
-            return self.sheerka.ret(self.name,
-                                    False,
-                                    context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))
-
-        self.parser_input.next_token()
-        rule = self.parse_rule()
-        body = self.get_return_value_body(sheerka, parser_input.as_text(), rule, rule)
-        ret = sheerka.ret(self.name, not self.has_error, body)
-
-        self.log_result(context, parser_input.as_text(), ret)
-        return ret
-
-    def parse_rule(self):
-        parts = self.get_parts(self.KEYWORDS_VALUES, strip_tokens=True)
-        if parts is None:
-            return None
-
-        node = FormatRuleNode(parts)
-        try:
-            res = self.get_when(parts[Keywords.WHEN])
-            if res is None:
-                return node
-            node.rule = res
-
-            parsed = self.get_print(parts[Keywords.PRINT])
-            if parsed is None:
-                return node
-            node.format_ast = parsed
-        except KeyError as e:
-            self.add_error(KeywordNotFound([], [e.args[0].value]))
-            return None
-
-        return node
-
-    def get_when(self, tokens):
-        """
-        Validate the when part of the rule.
-        :param tokens:
-        :return:
-        """
-        source = core.utils.get_text_from_tokens(core.utils.strip_tokens(tokens[1:]))
-        res = self.sheerka.services[SheerkaRuleManager.NAME].compile_when(self.context, self.name, source)
-
-        if not isinstance(res, list):
-            self.add_error(res.value)
-            return None
-
-        return res
-
-    def get_print(self, tokens):
-        """
-        Validate the print part
-        :param tokens:
-        :return:
-        """
-        source = core.utils.get_text_from_tokens(core.utils.strip_tokens(tokens[1:]))
-        res = self.sheerka.services[SheerkaRuleManager.NAME].compile_print(self.context, source)
-        if not res.status:
-            self.add_error(res.value)
-            return None
-
-        return res.body

src/parsers/DefRuleParser.py

@@ -0,0 +1,231 @@
+from dataclasses import dataclass
+from typing import List
+
+import core.utils
+from core.builtin_concepts import ReturnValueConcept
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.global_symbols import NotInit
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatAstNode, RuleCompiledPredicate
+from core.sheerka.services.sheerka_service import FailedToCompileError
+from core.tokenizer import Keywords, TokenKind
+from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, NameNode, KeywordNotFound, SyntaxErrorNode
+from parsers.BaseParser import Node, UnexpectedEofParsingError
+from sheerkarete.conditions import Condition
+
+
+@dataclass()
+class DefRuleNode(Node):
+    tokens: dict
+    name: NameNode = NotInit
+    when: List[RuleCompiledPredicate] = NotInit
+    rete: List[List[Condition]] = NotInit
+
+
+@dataclass()
+class DefExecRuleNode(DefRuleNode):
+    then: ReturnValueConcept = NotInit
+
+
+@dataclass
+class DefFormatRuleNode(DefRuleNode):
+    print: FormatAstNode = NotInit
+
+
+class DefRuleParser(BaseCustomGrammarParser):
+    DEF_KEYWORDS = [Keywords.RULE, Keywords.AS]
+    DEF_KEYWORDS_VALUES = [k.value for k in DEF_KEYWORDS]
+
+    RULE_KEYWORDS = [Keywords.WHEN, Keywords.THEN, Keywords.PRINT]
+    RULE_KEYWORDS_VALUES = [k.value for k in RULE_KEYWORDS]
+
+    def __init__(self, **kwargs):
+        BaseCustomGrammarParser.__init__(self, "DefRule", 60)
+
+    def parse(self, context, parser_input: ParserInput):
+        if not isinstance(parser_input, ParserInput):
+            return None
+
+        # rule parser can only manage string text
+        if parser_input.from_tokens:
+            ret = context.sheerka.ret(
+                self.name,
+                False,
+                context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input))
+            self.log_result(context, parser_input, ret)
+            return ret
+
+        context.log(f"Parsing '{parser_input}' with DefRuleParser", self.name)
+        sheerka = context.sheerka
+
+        if parser_input.is_empty():
+            return sheerka.ret(self.name,
+                               False,
+                               sheerka.new(BuiltinConcepts.IS_EMPTY))
+
+        if not self.reset_parser(context, parser_input):
+            return self.sheerka.ret(self.name,
+                                    False,
+                                    context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))
+
+        self.parser_input.next_token()
+        node = self.parse_def_rule()
+
+        body = self.get_return_value_body(sheerka, parser_input.as_text(), node, node)
+        ret = sheerka.ret(self.name, not self.has_error, body)
+
+        self.log_result(context, parser_input.as_text(), ret)
+        return ret
+
+    def parse_def_rule(self):
+        token = self.parser_input.token
+        if token.value == Keywords.DEF.value:
+            return self.parse_rule_name()
+        elif token.value in (Keywords.WHEN.value, Keywords.PRINT.value):
+            return self.parse_rule()
+        else:
+            self.add_error(KeywordNotFound([], [Keywords.WHEN.value]))
+            return None
+
+    def parse_rule_name(self):
+        """
+        Parses def rule xxx as yyyy
+        """
+        self.parser_input.next_token()  # eat def
+        token = self.parser_input.token
+        if token.value != Keywords.RULE.value:
+            self.add_error(KeywordNotFound([token], [Keywords.RULE.value]))
+            return None
+
+        buffer = []
+        while self.parser_input.next_token(skip_whitespace=False):
+            token = self.parser_input.token
+            if token.value == Keywords.AS.value:
+                break
+            else:
+                buffer.append(token)
+        else:  # 'as' keyword not found
+            self.add_error(KeywordNotFound([], [Keywords.AS.value]))
+            return None
+
+        if not self.parser_input.next_token():  # eat as
+            self.add_error(UnexpectedEofParsingError("While parsing 'when'."))
+            return None
+
+        rule = self.parse_rule()
+
+        name_node = self.get_concept_name(buffer)
+        if name_node is None:
+            return rule
+
+        rule.name = name_node
+        return rule
+
+    def parse_rule(self):
+        """"
+        Parses 'when xxx then yyy'
+        or 'when xxx print yyy'
+        """
+        parts = self.get_parts(self.RULE_KEYWORDS_VALUES, strip_tokens=True)
+        if Keywords.THEN in parts and Keywords.PRINT in parts:
+            self.add_error(SyntaxErrorNode([], "Cannot have both 'print' and 'then' keywords"))
+            return None
+
+        if Keywords.THEN not in parts and Keywords.PRINT not in parts:
+            self.add_error(KeywordNotFound([], [Keywords.THEN.value, Keywords.PRINT.value]))
+            return None
+
+        return self.parse_format_rule(parts) if Keywords.PRINT in parts else self.parse_exec_rule(parts)
+
+    def parse_exec_rule(self, parts):
+        node = DefExecRuleNode(parts)
+        try:
+            compiled_result = self.get_when(parts[Keywords.WHEN])
+            if compiled_result is None:
+                return node
+            node.when = compiled_result.compiled_predicates
+            node.rete = compiled_result.rete_disjunctions
+
+            parsed = self.get_then(parts[Keywords.THEN])
+            if parsed is None:
+                return node
+            node.then = parsed
+        except KeyError as e:
+            self.add_error(KeywordNotFound([], [e.args[0].value]))
+            return None
+
+        return node
+
+    def parse_format_rule(self, parts):
+        node = DefFormatRuleNode(parts)
+        try:
+            compiled_result = self.get_when(parts[Keywords.WHEN])
+            if compiled_result is None:
+                return node
+            node.when = compiled_result.compiled_predicates
+            node.rete = compiled_result.rete_disjunctions
+
+            parsed = self.get_print(parts[Keywords.PRINT])
+            if parsed is None:
+                return node
+            node.print = parsed
+        except KeyError as e:
+            self.add_error(KeywordNotFound([], [e.args[0].value]))
+            return None
+
+        return node
+
+    def get_when(self, tokens):
+        """
+        Validate the when part of the rule.
+        :param tokens:
+        :return:
+        """
+        source = core.utils.get_text_from_tokens(core.utils.strip_tokens(tokens[1:]))
+        try:
+            rule_manager_service = self.sheerka.services[SheerkaRuleManager.NAME]
+            compiled_result = rule_manager_service.compile_when(self.context, self.name, source)
+        except FailedToCompileError as ex:
+            for c in ex.cause:
+                self.add_error(c)
+            return None
+
+        return compiled_result
+
+    def get_then(self, tokens):
+        source = core.utils.get_text_from_tokens(core.utils.strip_tokens(tokens[1:]))
+        res = self.sheerka.services[SheerkaRuleManager.NAME].compile_exec(self.context, source)
+
+        if not res.status:
+            self.add_error(res.value)
+            return None
+
+        return res
+
+    def get_print(self, tokens):
+        """
+        Validate the print part
+        :param tokens:
+        :return:
+        """
+        source = core.utils.get_text_from_tokens(core.utils.strip_tokens(tokens[1:]))
+        res = self.sheerka.services[SheerkaRuleManager.NAME].compile_print(self.context, source)
+        if not res.status:
+            self.add_error(res.value)
+            return None
+
+        return res.body
+
+    def get_concept_name(self, tokens):
+        name_tokens = core.utils.strip_tokens(tokens)
+        if len(name_tokens) == 0:
+            self.add_error(SyntaxErrorNode([], "Name is mandatory"))
+            return None
+
+        for token in name_tokens:
+            if token.type == TokenKind.NEWLINE:
+                self.add_error(SyntaxErrorNode([token], "Newline are not allowed in name."))
+                return None
+
+        name_node = NameNode(name_tokens)  # skip the first token
+        return name_node

src/parsers/ExpressionParser.py

@@ -1,183 +1,64 @@
-from dataclasses import dataclass
-from typing import List, Tuple, Callable
+from itertools import product
 
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
+from core.builtin_helpers import only_successful, parse_unrecognized, get_inner_body, parse_python, \
+    get_lexer_nodes_using_positions
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import TokenKind, Token
-from parsers.BaseParser import Node, BaseParser, UnexpectedTokenParsingError, UnexpectedEofParsingError, ParsingError
+from core.sheerka.services.sheerka_service import FailedToCompileError
+from core.tokenizer import TokenKind, Tokenizer, Keywords
+from core.utils import get_text_from_tokens
+from parsers.BaseNodeParser import UnrecognizedTokensNode
+from parsers.BaseParser import BaseParser, UnexpectedTokenParsingError, UnexpectedEofParsingError
+from parsers.PythonWithConceptsParser import PythonWithConceptsParser
+from parsers.expressions import ParenthesisNode, OrNode, AndNode, NotNode, LeftPartNotFoundError, \
+    ParenthesisMismatchError, NameExprNode, ExprNode, VariableNode, ComparisonNode
+from sheerkarete.common import V
+from sheerkarete.conditions import Condition, AndConditions
 
 
-class ExprNode(Node):
-    """
-    Base ExprNode
-    eval() must be overridden
-    """
+class ReteConditionsEmitter:
 
-    def eval(self, obj):
-        return True
+    def __init__(self, context):
+        from parsers.ComparisonParser import ComparisonParser
+        self.context = context
+        self.comparison_parser = ComparisonParser()
+        self.var_counter = 0
+        self.variables = {}
 
+    def add_variable(self, target):
+        var_name = f"__x_{self.var_counter:02}__"
+        self.var_counter += 1
+        self.variables[target] = var_name
+        return var_name
 
-@dataclass()
-class LeftPartNotFoundError(ParsingError):
-    """
-    When the expression starts with 'or' or 'and'
-    """
-    pass
+    def init_variable_if_needed(self, node, res):
+        if node.name not in self.variables:
+            var_name = self.add_variable(node.name)
+            res.append(Condition(V(var_name), "__name__", node.name))
 
+        return V(self.variables[node.name])
 
-class NameExprNode(ExprNode):
-    def __init__(self, tokens):
-        self.tokens = tokens
-        self.value = "".join([t.str_value for t in self.tokens])
+    def get_conditions(self, expr_nodes):
+        conditions = []
+        for expr_node in expr_nodes:
+            parsed_ret = self.comparison_parser.parse(self.context, expr_node.tokens)
+            if not parsed_ret.status:
+                raise FailedToCompileError(parsed_ret.body)
+            tree = parsed_ret.body.body
 
-    def eval(self, obj):
-        return self.value
+            if isinstance(tree, VariableNode):
+                var_name = self.init_variable_if_needed(tree, conditions)
+                if tree.attributes_str is not None:
+                    conditions.append(Condition(var_name, tree.attributes_str, True))
 
-    def __repr__(self):
-        return f"NameExprNode('{self.value}')"
+            elif isinstance(tree, ComparisonNode):
+                if isinstance(tree.left, VariableNode):
+                    left = self.init_variable_if_needed(tree.left, conditions)
+                    attr = tree.left.attributes_str or "__self__"
+                    right = eval(get_text_from_tokens(tree.right.tokens))
+                    conditions.append(Condition(left, attr, right))
 
-    def __str__(self):
-        return self.value
-
-
-@dataclass
-class PropertyEqualsNode(ExprNode):
-    prop: str
-    value: object
-
-    def eval(self, obj):
-        if hasattr(obj, self.prop):
-            return str(getattr(obj, self.prop)) == self.value
-
-        return False
-
-
-@dataclass()
-class PropertyContainsNode(ExprNode):
-    prop: str
-    value: object
-
-    def eval(self, obj):
-        if hasattr(obj, self.prop):
-            return self.value in str(getattr(obj, self.prop))
-
-        return False
-
-
-@dataclass
-class PropertyEqualsSequenceNode(ExprNode):
-    """
-    To use when the test must be done across parent and child
-    """
-    props: List[str]
-    values: List[object]
-
-    def eval(self, obj):
-        index = len(self.props) - 1
-
-        while True:
-            if not hasattr(obj, self.props[index]) or getattr(obj, self.props[index]) != self.values[index]:
-                return False
-
-            if index == 0:
-                break
-
-            index -= 1
-            obj = obj.get_parent() if hasattr(obj, "get_parent") else obj.parent
-            if obj is None:
-                return False
-
-        return True
-
-
-@dataclass()
-class IsaNode(ExprNode):
-    """
-    To use to replicate instanceof, sheerka.instanceof,
-    """
-    obj_class: object
-
-    def eval(self, obj):
-        if isinstance(self.obj_class, type):
-            return isinstance(obj, self.obj_class)
-
-        if isinstance(self.obj_class, (BuiltinConcepts, str)):
-            return isinstance(obj, Concept) and str(self.obj_class) == obj.key
-
-        return False
-
-
-@dataclass()
-class LambdaNode(ExprNode):
-    """
-    Generic expression to ease the tests
-    """
-    lambda_exp: Callable[[object], bool]
-
-    def eval(self, obj):
-        try:
-            return self.lambda_exp(obj)
-        except Exception:
-            pass
-
-
-@dataclass(init=False)
-class AndNode(ExprNode):
-    parts: Tuple[ExprNode]
-
-    def __init__(self, *parts: ExprNode):
-        self.parts = parts
-
-    def eval(self, obj):
-        res = self.parts[0].eval(obj) and self.parts[1].eval(obj)
-        for part in self.parts[2:]:
-            res &= part.eval(obj)
-        return res
-
-    def __repr__(self):
-        return f"AndNode(" + ", ".join([repr(p) for p in self.parts]) + ")"
-
-    def __str__(self):
-        return " and ".join([str(p) for p in self.parts])
-
-
-@dataclass(init=False)
-class OrNode(ExprNode):
-    parts: Tuple[ExprNode]
-
-    def __init__(self, *parts: ExprNode):
-        self.parts = parts
-
-    def eval(self, obj):
-        res = self.parts[0].eval(obj) or self.parts[1].eval(obj)
-        for part in self.parts[2:]:
-            res |= part.eval(obj)
-        return res
-
-    def __repr__(self):
-        return f"OrNode(" + ", ".join([repr(p) for p in self.parts]) + ")"
-
-    def __str__(self):
-        return " or ".join([str(p) for p in self.parts])
-
-
-@dataclass()
-class NotNode(ExprNode):
-    node: ExprNode
-
-    def eval(self, obj):
-        return not self.node.eval(obj)
-
-
-class FalseNode(ExprNode):
-    def eval(self, obj):
-        return False
-
-
-class TrueNode(ExprNode):
-    def eval(self, obj):
-        return True
+        return [AndConditions(conditions)]
 
 
 class ExpressionParser(BaseParser):
@@ -191,6 +72,15 @@ class ExpressionParser(BaseParser):
 
     def __init__(self, **kwargs):
         super().__init__("Expression", 50, False, yield_eof=True)
+        self.and_tokens = list(Tokenizer(" and ", yield_eof=False))
+        self.and_not_tokens = list(Tokenizer(" and not ", yield_eof=False))
+        self.not_tokens = list(Tokenizer("not ", yield_eof=False))
+
+    @staticmethod
+    def clean_parenthesis_nodes(nodes):
+        for i, node in enumerate(nodes):
+            if isinstance(node, ParenthesisNode):
+                nodes[i] = node.node
 
     def parse(self, context, parser_input: ParserInput):
         """
@@ -232,6 +122,7 @@ class ExpressionParser(BaseParser):
         return ret
 
     def parse_or(self):
+        start = self.parser_input.pos
         expr = self.parse_and()
         token = self.parser_input.token
         if token.type != TokenKind.IDENTIFIER or token.value != "or":
@@ -243,14 +134,19 @@ class ExpressionParser(BaseParser):
             expr = self.parse_and()
             if expr is None:
                 self.add_error(UnexpectedEofParsingError("When parsing 'or'"))
-                return OrNode(*parts)
+                end = self.parser_input.pos
+                self.clean_parenthesis_nodes(parts)
+                return OrNode(start, end, self.parser_input.tokens[start: end + 1], *parts)
             parts.append(expr)
             token = self.parser_input.token
 
-        return OrNode(*parts)
+        end = parts[-1].end
+        self.clean_parenthesis_nodes(parts)
+        return OrNode(start, end, self.parser_input.tokens[start: end + 1], *parts)
 
     def parse_and(self):
-        expr = self.parse_names()
+        start = self.parser_input.pos
+        expr = self.parse_not()
         token = self.parser_input.token
         if token.type != TokenKind.IDENTIFIER or token.value != "and":
             return expr
@@ -258,27 +154,46 @@ class ExpressionParser(BaseParser):
         parts = [expr]
         while token.type == TokenKind.IDENTIFIER and token.value == "and":
             self.parser_input.next_token()
-            expr = self.parse_names()
+            expr = self.parse_not()
             if expr is None:
                 self.add_error(UnexpectedEofParsingError("When parsing 'and'"))
-                return AndNode(*parts)
+                end = self.parser_input.pos
+                self.clean_parenthesis_nodes(parts)
+                return AndNode(start, end, self.parser_input.tokens[start: end + 1], *parts)
             parts.append(expr)
             token = self.parser_input.token
 
-        return AndNode(*parts)
+        end = parts[-1].end
+        self.clean_parenthesis_nodes(parts)
+        return AndNode(start, end, self.parser_input.tokens[start: end + 1], *parts)
+
+    def parse_not(self):
+        token = self.parser_input.token
+        start = self.parser_input.pos
+        if token.type == TokenKind.IDENTIFIER and token.value == "not":
+            self.parser_input.next_token()
+            parsed = self.parse_not()
+            node = parsed.node if isinstance(parsed, ParenthesisNode) else parsed
+            return NotNode(start,
+                           parsed.end,
+                           self.parser_input.tokens[start: parsed.end + 1],
+                           node)
+        else:
+            return self.parse_names()
 
     def parse_names(self):
 
         def stop():
             return token.type == TokenKind.EOF or \
                    paren_count == 0 and token.type == TokenKind.RPAR or \
-                   token.type == TokenKind.IDENTIFIER and token.value in ("and", "or")
+                   token.type == TokenKind.IDENTIFIER and token.value in ("and", "or", "not")
 
         token = self.parser_input.token
         if token.type == TokenKind.EOF:
             return None
 
         if token.type == TokenKind.LPAR:
+            start = self.parser_input.pos
             self.parser_input.next_token()
             expr = self.parse_or()
             token = self.parser_input.token
@@ -286,14 +201,18 @@ class ExpressionParser(BaseParser):
                 self.error_sink.append(
                     UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.RPAR]))
                 return expr
+            end = self.parser_input.pos
             self.parser_input.next_token()
-            return expr
+            return ParenthesisNode(start, end, None, expr)
 
         buffer = []
         paren_count = 0
+        last_paren = None
+        start = self.parser_input.pos
         while not stop():
             buffer.append(token)
             if token.type == TokenKind.LPAR:
+                last_paren = token
                 paren_count += 1
             if token.type == TokenKind.RPAR:
                 paren_count -= 1
@@ -305,65 +224,106 @@ class ExpressionParser(BaseParser):
                 self.error_sink.append(LeftPartNotFoundError())
             return None
 
+        if paren_count != 0:
+            self.error_sink.append(ParenthesisMismatchError(last_paren))
+            return None
+
         if buffer[-1].type == TokenKind.WHITESPACE:
             buffer.pop()
 
-        return NameExprNode(buffer)
+        end = start + len(buffer) - 1
+        return NameExprNode(start, end, buffer)
 
+    def compile_conjunctions(self, context, conjunctions, who):
+        """
+        Transform a list of conjunctions (AND and OR) into one or multiple CompiledExpr
+        :param context:
+        :param conjunctions: list of ExprNode
+        :param who: service that calls the method
+        :returns: List Of CompiledExpr
+        May throw FailedToRecognized if a conjunction cannot be parsed
+        """
+        recognized = []
+        for conjunction in conjunctions:
+            # try to recognize conjunction, one by one
+            # negative conjunction can be a concept starting with 'not'
+            parsed_ret = parse_unrecognized(
+                context,
+                conjunction.get_value(),  # we remove the 'NOT' part when needed to ease the recognition
+                parsers="all",
+                who=who,
+                prop=Keywords.WHEN,
+                filter_func=only_successful)
 
-class ExpressionVisitor:
-    """
-    Pyhtonic implementation of visitors for ExprNode
-    """
+            if parsed_ret.status:
+                recognized.append(get_inner_body(context, parsed_ret.body))
+            else:
+                raise FailedToCompileError(parsed_ret.body)
 
-    def visit(self, expr_node):
-        name = expr_node.__class__.__name__
+        # for each conjunction, we have a list of recognized concepts (or python node)
+        # we need a cartesian product of the results
+        # Explanation for later
+        # conjunction[0] : 'x is a y' that can be resolved with two concepts c:|1001: and c:|1002:
+        # conjunction[1] : 'y is an z' that can also be resolved with two concepts (c:|1003: and c:|1004)
+        # so to understand the full question 'x is a y and y is an z'
+        # we can have c:|1001: then c:|1003:
+        # or          c:|1001: then c:|1004:
+        # or          c:|1002: then c:|1003:
+        # or          c:|1002: then c:|1004:
+        # if one of this combination works, it means that the question 'x is a y and y is an z' was matched
+        # hence the cartesian product
+        product_of_recognized = list(product(*recognized))
 
-        method = 'visit_' + name
-        visitor = getattr(self, method, self.generic_visit)
-        return visitor(expr_node)
+        return_values = []
+        for recognized_conjunctions in product_of_recognized:
+            if len(recognized_conjunctions) == 1 and not isinstance(conjunctions[0], NotNode):
+                return_values.append(recognized_conjunctions[0])
+            elif len(recognized_conjunctions) == 1 and recognized_conjunctions[0].who == "parsers.Python":
+                # it is a negated python Node. Need to parse again
+                ret = parse_python(context, source=str(conjunctions[0]))
+                if ret.status:
+                    return_values.append(ret)
+                else:
+                    # find a way to track the failure
+                    pass
+            else:
+                # complex result. Use PythonWithNode
+                lexer_nodes = get_lexer_nodes_using_positions(recognized_conjunctions,
+                                                              self._get_positions(conjunctions))
 
-    def generic_visit(self, expr_node):
-        """Called if no explicit visitor function exists for a node."""
-        for field, value in expr_node.__dict__.items():
-            if isinstance(value, (list, tuple)):
-                for item in value:
-                    if isinstance(item, ExprNode):
-                        self.visit(item)
-            elif isinstance(value, ExprNode):
-                self.visit(value)
+                # put back the 'and' / 'not' node
+                for i in range(len(lexer_nodes) - 1, 0, -1):
+                    end = lexer_nodes[i].start - 1
+                    start = lexer_nodes[i - 1].end + 1
+                    if isinstance(conjunctions[i], NotNode):
+                        lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_not_tokens))
+                    else:
+                        lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_tokens))
 
+                # add the starting 'not' if needed
+                # and reindex the following positions
+                if isinstance(conjunctions[0], NotNode):
+                    lexer_nodes[0].start = 2
+                    lexer_nodes.insert(0, UnrecognizedTokensNode(0, 1, self.not_tokens))
 
-class TrueifyVisitor(ExpressionVisitor):
-    """
-    Visit an ExprNode
-    replace all the nodes containing a variable to 'trueify' with True
-    The node containing both variables to trueify and to skip are skipped
-    """
+                python_with_concept_node_ret = PythonWithConceptsParser().parse_nodes(context, lexer_nodes)
+                if not python_with_concept_node_ret.status:
+                    # find a way to track the failure
+                    pass
+                return_values.append(python_with_concept_node_ret)
 
-    def __init__(self, to_trueify, to_skip):
-        self.to_trueify = to_trueify
-        self.to_skip = to_skip
+        rete_cond_emitter = ReteConditionsEmitter(context)
+        rete_disjunctions = rete_cond_emitter.get_conditions(conjunctions)
 
-    def visit_AndNode(self, expr_node):
-        parts = []
-        for part in expr_node.parts:
-            parts.append(self.visit(part))
-        return AndNode(*parts)
+        return return_values, rete_disjunctions
 
-    def visit_OrNode(self, expr_node):
-        parts = []
-        for part in expr_node.parts:
-            parts.append(self.visit(part))
-        return OrNode(*parts)
-
-    def visit_NameExprNode(self, expr_node):
-        return_true = False
-        for t in expr_node.tokens:
-            if t.type == TokenKind.IDENTIFIER:
-                if t.value in self.to_skip:
-                    return expr_node
-                if t.value in self.to_trueify:
-                    return_true = True
-
-        return NameExprNode([Token(TokenKind.IDENTIFIER, "True", -1, -1, -1)]) if return_true else expr_node
+    @staticmethod
+    def _get_positions(expr_nodes):
+        """
+        simply manage NotNodes to address the fact that the 'not' part in removed
+        """
+        for expr in expr_nodes:
+            if isinstance(expr, NotNode):
+                yield ExprNode(expr.start + 2, expr.end, expr.tokens[2:])
+            else:
+                yield expr

src/parsers/FunctionParser.py

@@ -44,9 +44,15 @@ class NamesNode(FunctionParserNode):
         return "".join([t.str_value for t in self.tokens])
 
     def to_unrecognized(self):
+        """
+        UnrecognizedTokensNode with all tokens
+        """
         return UnrecognizedTokensNode(self.start, self.end, self.tokens).fix_source()
 
     def to_str_unrecognized(self):
+        """
+        UnrecognizedTokensNode with one token, which is a string token of all the tokens
+        """
         token = Token(TokenKind.STRING,
                       "'" + self.str_value() + "'",
                       self.tokens[0].index,
@@ -342,6 +348,7 @@ class FunctionParser(BaseParser):
 
         res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())]
 
+        # try to recognize every parameter, one by one
         for param in function_node.parameters:
             if isinstance(param.value, NamesNode):
                 # try to recognize concepts

src/parsers/PythonParser.py

@@ -35,8 +35,9 @@ class ConceptDetectedError(ParsingError):
 
 class PythonNode(Node):
 
-    def __init__(self, source, ast_=None, objects=None):
-        self.source = source
+    def __init__(self, source, ast_=None, original_source=None, objects=None):
+        self.source = source  # what was parsed
+        self.original_source = original_source or source  # to remember source before concept id replacement
         self.ast_ = ast_  # if ast_ else ast.parse(source, mode="eval") if source else None
         self.objects = objects or {}  # when objects (mainly concepts or rules) are recognized in the expression
         self.compiled = None
@@ -64,6 +65,9 @@ class PythonNode(Node):
         if self.source != other.source:
             return False
 
+        if self.original_source != other.original_source:
+            return False
+
         if self.ast_ and other.ast_:
             self_dump = self.get_dump(self.ast_)
             other_dump = self.get_dump(other.ast_)
@@ -74,6 +78,9 @@ class PythonNode(Node):
     def __hash__(self):
         return hash((self.source, self.ast_.hash))
 
+    def get_python_node(self):
+        return self
+
     @staticmethod
     def get_dump(ast_):
         if not ast_:
@@ -156,7 +163,7 @@ class PythonParser(BaseParser):
                     BuiltinConcepts.PARSER_RESULT,
                     parser=self,
                     source=parser_input.as_text(),
-                    body=PythonNode(source_code, tree, tracker),
+                    body=PythonNode(source_code, tree, objects=tracker),
                     try_parsed=None))
 
         self.log_result(context, parser_input.as_text(), ret)

src/parsers/PythonWithConceptsParser.py

@@ -1,9 +1,8 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.sheerka.services.SheerkaExecute import SheerkaExecute
+from core.builtin_helpers import parse_python, CreateObjectIdentifiers
 from parsers.BaseNodeParser import ConceptNode, RuleNode
 from parsers.BaseNodeParser import SourceCodeWithConceptNode
 from parsers.BaseParser import BaseParser
-from parsers.PythonParser import PythonParser
 from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
 
 unrecognized_nodes_parser = UnrecognizedNodeParser()
@@ -13,16 +12,6 @@ class PythonWithConceptsParser(BaseParser):
     def __init__(self, **kwargs):
         super().__init__("PythonWithConcepts", 20)
 
-    @staticmethod
-    def sanitize(identifier):
-        if identifier is None:
-            return ""
-
-        res = ""
-        for c in identifier:
-            res += c if c.isalnum() else "0"
-        return res
-
     @staticmethod
     def get_nodes(nodes):
         for node in nodes:
@@ -46,73 +35,47 @@ class PythonWithConceptsParser(BaseParser):
 
         source = ""
         to_parse = ""
-        identifiers = {}
-        identifiers_key = {}
         python_ids_mappings = {}
-
-        def _get_identifier(c, wrapper):
-            """
-            Get an identifier for a concept.
-            Make sure to return the same identifier if the same concept
-            Make sure to return a different identifier if same name but different concept
-
-            Internal function because I don't want identifiers, identifiers_key and python_ids_mappings
-            to be instance variables
-            I would like to keep this parser as stateless as possible
-            :param c:
-            :return:
-            """
-            if id(c) in identifiers:
-                return identifiers[id(c)]
-
-            identifier = wrapper + self.sanitize(c.key or c.name)
-            if c.id:
-                identifier += "__" + c.id
-
-            if identifier in identifiers_key:
-                identifiers_key[identifier] += 1
-                identifier += f"_{identifiers_key[identifier]}"
-            else:
-                identifiers_key[identifier] = 0
-
-            identifier += wrapper
-
-            identifiers[id(c)] = identifier
-            return identifier
+        last_token_index = 0
+        ids_manager = CreateObjectIdentifiers()
 
         for node in self.get_nodes(nodes):
             if isinstance(node, ConceptNode):
-                source += node.source
-                if to_parse:
+                if node.start != last_token_index + 1 and source:  # put back missing whitespace
+                    source += " "
                     to_parse += " "
+
+                source += node.source
                 concept = node.concept
-                python_id = _get_identifier(concept, "__C__")
+                python_id = ids_manager.get_identifier(concept, "__C__")
                 to_parse += python_id
                 python_ids_mappings[python_id] = concept
+                last_token_index = node.end
 
             elif isinstance(node, RuleNode):
-                source += node.source
-                if to_parse:
+                if node.start != last_token_index + 1 and source:  # put back missing whitespace
+                    source += " "
                     to_parse += " "
+
+                source += node.source
                 rule = node.rule
-                python_id = _get_identifier(rule, "__R__")
+                python_id = ids_manager.get_identifier(rule, "__R__")
                 to_parse += python_id
                 python_ids_mappings[python_id] = rule
+                last_token_index = node.end
 
             else:
                 source += node.source
-                to_parse += node.source
+                to_parse += node.get_source_to_parse()
+                last_token_index = node.end
+                if hasattr(node, "get_python_node"):
+                    python_ids_mappings.update(node.get_python_node().objects)
 
-        with context.push(BuiltinConcepts.PARSE_CODE,
-                          {"language": "Python", "source": to_parse},
-                          "Trying Python for '" + to_parse + "'") as sub_context:
-            parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(to_parse)
-            python_parser = PythonParser()
-            result = python_parser.parse(sub_context, parser_input)
+        result = parse_python(context, to_parse, "Trying Python for '" + to_parse + "'")
 
         if result.status:
             python_node = result.body.body
-            python_node.source = source
+            python_node.original_source = source
             python_node.objects = python_ids_mappings
 
             return sheerka.ret(

src/parsers/UnrecognizedNodeParser.py

@@ -53,7 +53,7 @@ class UnrecognizedNodeParser(BaseParser):
                 if not res.status:
                     self.add_error(res.body)
                 else:
-                    sequences_found = core.utils.product(sequences_found, [res.body])
+                    sequences_found = core.utils.sheerka_product(sequences_found, [res.body])
 
             elif isinstance(node, UnrecognizedTokensNode):
                 res = parse_unrecognized(context, node.source, PARSERS)
@@ -62,16 +62,16 @@ class UnrecognizedNodeParser(BaseParser):
                     lexer_nodes = get_lexer_nodes(res.body.body, node.start, node.tokens)
                     if lexer_nodes:
                         # make lexer_nodes is not empty (for example, some Python result are discarded)
-                        sequences_found = core.utils.product(sequences_found, lexer_nodes)
+                        sequences_found = core.utils.sheerka_product(sequences_found, lexer_nodes)
                     else:
-                        sequences_found = core.utils.product(sequences_found, [node])
+                        sequences_found = core.utils.sheerka_product(sequences_found, [node])
                         has_unrecognized = True
                 else:
-                    sequences_found = core.utils.product(sequences_found, [node])
+                    sequences_found = core.utils.sheerka_product(sequences_found, [node])
                     has_unrecognized = True
 
             elif isinstance(node, SourceCodeNode):
-                sequences_found = core.utils.product(sequences_found, [node])
+                sequences_found = core.utils.sheerka_product(sequences_found, [node])
                 has_unrecognized = True  # to let PythonWithConceptParser validate the code
 
             elif isinstance(node, SourceCodeWithConceptNode):
@@ -82,7 +82,7 @@ class UnrecognizedNodeParser(BaseParser):
                         break
                     else:
                         node.nodes[i] = res.body
-                sequences_found = core.utils.product(sequences_found, [node])
+                sequences_found = core.utils.sheerka_product(sequences_found, [node])
                 has_unrecognized = True  # to let PythonWithConceptParser validate the code
 
             else:  # cannot happen as of today :-)

src/parsers/expressions.py

@@ -0,0 +1,414 @@
+from dataclasses import dataclass
+from typing import List, Tuple
+
+from core.tokenizer import Token, TokenKind, Tokenizer
+from core.utils import tokens_are_matching
+from parsers.BaseParser import Node, ParsingError
+
+
+class ComparisonType:
+    EQUALS = "EQ"
+    NOT_EQUAlS = "NOT_EQ"
+    LESS_THAN = "LT"
+    LESS_THAN_OR_EQUALS = "LTE"
+    GREATER_THAN = "GT"
+    GREATER_THAN_OR_EQUALS = "GTE"
+    IN = "IN"
+    NOT_IN = "NOT_IN"
+
+
+@dataclass()
+class LeftPartNotFoundError(ParsingError):
+    """
+    When the expression starts with 'or' or 'and'
+    """
+    pass
+
+
+@dataclass()
+class ParenthesisMismatchError(ParsingError):
+    token: Token
+
+
+@dataclass
+class ExprNode(Node):
+    """
+    Base ExprNode
+    eval() must be overridden
+    """
+    start: int  # index of the first token
+    end: int  # index of the last token
+    tokens: List[Token]
+
+    def eval(self, obj):
+        return True
+
+    def __eq__(self, other):
+        if not isinstance(other, ExprNode):
+            return False
+
+        if self.start != other.start or self.end != other.end:
+            return False
+
+        if other.tokens is not None and other.tokens != self.tokens:
+            return False
+
+        return True
+
+    def __hash__(self):
+        return hash((self.start, self.end))
+
+
+class NameExprNode(ExprNode):
+    def __init__(self, start, end, tokens):
+        super().__init__(start, end, tokens)
+        self.tokens = tokens
+        self.value = "".join([t.str_value for t in self.tokens])
+
+    def eval(self, obj):
+        return self.value
+
+    def get_value(self):
+        return self.value
+
+    def __repr__(self):
+        return f"NameExprNode('{self.value}')"
+
+    def __str__(self):
+        return self.value
+
+    def __eq__(self, other):
+        if not isinstance(other, NameExprNode):
+            return False
+
+        return super().__eq__(other)
+
+    def __hash__(self):
+        return super().__hash__()
+
+
+@dataclass(init=False)
+class AndNode(ExprNode):
+    parts: Tuple[ExprNode]
+
+    def __init__(self, start, end, tokens, *parts: ExprNode):
+        super().__init__(start, end, tokens)
+        self.parts = parts
+
+    def eval(self, obj):
+        res = self.parts[0].eval(obj) and self.parts[1].eval(obj)
+        for part in self.parts[2:]:
+            res &= part.eval(obj)
+        return res
+
+    def __repr__(self):
+        return f"AndNode(start={self.start}, end={self.end}, " + ", ".join([repr(p) for p in self.parts]) + ")"
+
+    def __str__(self):
+        return " and ".join([str(p) for p in self.parts])
+
+    def __eq__(self, other):
+        if not isinstance(other, AndNode):
+            return False
+
+        if self.start != other.start or self.end != other.end:
+            return False
+
+        if other.tokens is not None and other.tokens != self.tokens:
+            return False
+
+        return self.parts == other.parts
+
+    def __hash__(self):
+        return hash((self.start, self.end, self.parts))
+
+
+@dataclass(init=False)
+class OrNode(ExprNode):
+    parts: Tuple[ExprNode]
+
+    def __init__(self, start, end, tokens, *parts: ExprNode):
+        super().__init__(start, end, tokens)
+        self.parts = parts
+
+    def eval(self, obj):
+        res = self.parts[0].eval(obj) or self.parts[1].eval(obj)
+        for part in self.parts[2:]:
+            res |= part.eval(obj)
+        return res
+
+    def __repr__(self):
+        return f"OrNode(start={self.start}, end={self.end}, " + ", ".join([repr(p) for p in self.parts]) + ")"
+
+    def __str__(self):
+        return " or ".join([str(p) for p in self.parts])
+
+    def __eq__(self, other):
+        if not isinstance(other, OrNode):
+            return False
+
+        if self.start != other.start or self.end != other.end:
+            return False
+
+        if other.tokens is not None and other.tokens != self.tokens:
+            return False
+
+        return self.parts == other.parts
+
+    def __hash__(self):
+        return hash((self.start, self.end, self.parts))
+
+
+@dataclass()
+class NotNode(ExprNode):
+    node: ExprNode
+
+    def eval(self, obj):
+        return not self.node.eval(obj)
+
+    def get_value(self):
+        return self.node.get_value()
+
+    def __repr__(self):
+        return f"NotNode(start={self.start}, end={self.end}, {self.node!r})"
+
+    def __str__(self):
+        return f"not {self.node}"
+
+    def __eq__(self, other):
+        if not isinstance(other, NotNode):
+            return False
+
+        if self.start != other.start or self.end != other.end:
+            return False
+
+        if other.tokens is not None and other.tokens != self.tokens:
+            return False
+
+        return self.node == other.node
+
+    def __hash__(self):
+        return hash((self.start, self.end, self.node))
+
+
+@dataclass()
+class ParenthesisNode(ExprNode):
+    """
+    Contains the boundaries of an expression inside parenthesis
+    Need it, just to keep track of the boundaries of the parenthesis
+    """
+    node: ExprNode
+
+    def __eq__(self, other):
+        if not isinstance(other, ParenthesisNode):
+            return False
+
+        if self.start != other.start or self.end != other.end:
+            return False
+
+        if other.tokens is not None and other.tokens != self.tokens:
+            return False
+
+        return self.node == other.node
+
+    def __hash__(self):
+        return hash((self.start, self.end, self.node))
+
+
+class VariableNode(ExprNode):
+    def __init__(self, start, end, tokens, name, *attributes):
+        super().__init__(start, end, tokens)
+        self.name = name.strip()
+        self.attributes = [attr.strip() for attr in attributes]
+        if len(self.attributes) > 0:
+            self.attributes_str = ".".join(self.attributes)
+        else:
+            self.attributes_str = None
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, VariableNode):
+            return False
+
+        return self.name == other.name and self.attributes == other.attributes
+
+    def __hash__(self):
+        return hash((self.name, self.attributes))
+
+    def __repr__(self):
+        prefix = f"VariableNode(start={self.start}, end={self.end}, '{self.name}"
+        if len(self.attributes) > 0:
+            return prefix + "." + ".".join(self.attributes) + "')"
+        else:
+            return prefix + "')"
+
+    def __str__(self):
+        if self.attributes:
+            return self.name + "." + ".".join(self.attributes)
+        else:
+            return self.name
+
+
+@dataclass
+class ComparisonNode(ExprNode):
+    comp: str
+    left: ExprNode
+    right: ExprNode
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, ComparisonNode):
+            return False
+
+        return (self.comp == other.comp and
+                self.left == other.left and
+                self.right == other.right)
+
+    def __hash__(self):
+        return hash((self.comp, self.left, self.right))
+
+    def __repr__(self):
+        return f"ComparisonNode(start={self.start}, end={self.end}, {self.left!r} {self.comp} {self.right!r})"
+
+    def __str__(self):
+        return f"{self.left} {self.comp} {self.right}"
+
+
+class ExpressionVisitor:
+    """
+    Pyhtonic implementation of visitors for ExprNode
+    """
+
+    def visit(self, expr_node):
+        name = expr_node.__class__.__name__
+
+        method = 'visit_' + name
+        visitor = getattr(self, method, self.generic_visit)
+        return visitor(expr_node)
+
+    def generic_visit(self, expr_node):
+        """Called if no explicit visitor function exists for a node."""
+        for field, value in expr_node.__dict__.items():
+            if isinstance(value, (list, tuple)):
+                for item in value:
+                    if isinstance(item, ExprNode):
+                        self.visit(item)
+            elif isinstance(value, ExprNode):
+                self.visit(value)
+
+
+class TrueifyVisitor(ExpressionVisitor):
+    """
+    Visit an ExprNode
+    replace all the nodes containing a variable to 'trueify' with True
+    The node containing both variables to trueify and to skip are skipped
+    """
+
+    def __init__(self, to_trueify, to_skip):
+        self.to_trueify = to_trueify
+        self.to_skip = to_skip
+
+    def visit_AndNode(self, expr_node):
+        parts = []
+        for part in expr_node.parts:
+            parts.append(self.visit(part))
+        return AndNode(expr_node.start, expr_node.end, expr_node.tokens, *parts)
+
+    def visit_OrNode(self, expr_node):
+        parts = []
+        for part in expr_node.parts:
+            parts.append(self.visit(part))
+        return OrNode(expr_node.start, expr_node.end, expr_node.tokens, *parts)
+
+    def visit_NameExprNode(self, expr_node):
+        return_true = False
+        for t in expr_node.tokens:
+            if t.type == TokenKind.IDENTIFIER:
+                if t.value in self.to_skip:
+                    return expr_node
+                if t.value in self.to_trueify:
+                    return_true = True
+
+        return NameExprNode(expr_node.start,
+                            expr_node.end,
+                            [Token(TokenKind.IDENTIFIER, "True", -1, -1, -1)]) if return_true else expr_node
+
+
+is_question_tokens = list(Tokenizer("is_question()"))
+eval_question_requested_in_context = list(Tokenizer("context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)"))
+
+
+class IsAQuestionVisitor(ExpressionVisitor):
+    """
+    visit an expression and return True if is_question or context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+    if found.
+    """
+
+    def visit_NameExprNode(self, expr_node):
+        if tokens_are_matching(expr_node.tokens, is_question_tokens) or \
+            tokens_are_matching(expr_node.tokens, eval_question_requested_in_context):
+            return True
+        return None
+
+    def visit_AndNode(self, expr_node):
+        """
+          AND | True  | False | None
+        ------+-------+-------+----------
+        False | False | False | False
+        True  | True  | False | True
+        None  | True  | False | None
+        """
+        res = self.visit(expr_node.parts[0])
+        if isinstance(res, bool) and not res:
+            return res
+
+        for part in expr_node.parts[1:]:
+            visited = self.visit(part)
+            if isinstance(visited, bool):
+                if not visited:
+                    return visited
+                else:
+                    res = visited
+
+        return res
+
+    def visit_OrNode(self, expr_node):
+        """
+           OR | True  | False | None
+        ------+-------+-------+----------
+        True  | True  | True  | True
+        False | True  | False | False
+        None  | True  | False | None
+        """
+        res = self.visit(expr_node.parts[0])
+        if isinstance(res, bool) and res:
+            return res
+
+        for part in expr_node.parts[1:]:
+            visited = self.visit(part)
+            if isinstance(visited, bool):
+                if visited:
+                    return visited
+                else:
+                    res = visited
+
+        return res
+
+    def visit_NotNode(self, expr_node):
+        """
+              | NOT
+        ------+-------
+        False | True
+        True  | False
+        None  | None
+        """
+        visited = self.visit(expr_node.node)
+        return None if visited is None else not visited
+
+    def is_a_question(self, expr_node):
+        res = self.visit(expr_node)
+        return isinstance(res, bool) and res

src/sdp/sheerkaDataProvider.py

@@ -567,7 +567,8 @@ class SheerkaDataProvider:
         text = self.io.read_text(ontology_file)
         return text.split("\n")
 
-    def test_only_destroy_refs(self):
-        current_sdp_refs_folder = self.io.path_join(self.RefFolder, self.name)
+    def test_only_destroy_refs(self, name=None):
+        name = name or self.name
+        current_sdp_refs_folder = self.io.path_join(self.RefFolder, name)
         if path.exists(current_sdp_refs_folder):
             shutil.rmtree(current_sdp_refs_folder)

src/sheerkarete/alpha.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import List
+    from sheerkarete.join_node import JoinNode
+    from sheerkarete.common import WME
+
+
+class AlphaMemory:
+
+    def __init__(self, key, condition, items=None, successors=None):
+        """
+        Stores a set of WMEs (items). If activating an activated wme does not
+        exist, then it addes it. It also right activates all of its successors,
+        which correspond to beta nodes.
+        :param items: set of WMEs that match this memory
+        :param successors: list of JoinNode
+        """
+        self.key = key
+        self.condition = condition
+        self.items: List[WME] = items if items else []
+        self.successors: List[JoinNode] = successors if successors else []
+        self.reference_count = 0
+
+    def activation(self, wme: WME) -> None:
+        """
+        Adds the WME to the alpha memory and then right activates the children
+        in the beta network. Note, these are activated in reversed order to
+        prevent duplicate matches.
+        """
+        if not self.condition.test(wme):
+            return
+
+        self.items.append(wme)
+        wme.amems.append(self)
+        for child in reversed(self.successors):
+            child.right_activation(wme)

src/sheerkarete/beta.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+from sheerkarete.common import ReteToken
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import Any
+    from typing import List
+    from typing import Dict
+    from typing import Optional
+    from sheerkarete.common import V
+    from sheerkarete.common import WME
+
+
+class ReteNode:
+    """
+    Base BetaNode class, tracks parent and children.
+    """
+
+    def __init__(self, children=None, parent=None, **kwargs):
+        super().__init__(**kwargs)
+        self.children: List[ReteNode] = children if children else []
+        self.parent: Optional[ReteNode] = parent
+
+    def find_nearest_ancestor_with_same_amem(self, amem):
+        return None
+
+
+class BetaMemory(ReteNode):
+    """
+    A memory node for the beta network. Contains items (tokens) and a list of
+    `all_children`, which is used in conjunction with `children` to support
+    left unlinking.
+    """
+
+    def __init__(self, items: Optional[List[ReteToken]] = None, **kwargs):
+        """
+        Similar to alpha memory, but items is a set of tokens instead of wmes.
+        """
+        super().__init__(**kwargs)
+        self.items: List[ReteToken] = items if items else []
+        self.all_children: List[ReteNode] = []
+
+    def find_nearest_ancestor_with_same_amem(self, amem):
+        return self.parent.find_nearest_ancestor_with_same_amem(amem)
+
+    def left_activation(self, token, wme: Optional[WME] = None, binding: Optional[Dict[V, Any]] = None):
+        """
+        Creates a new token based on the incoming token/wme, adds it to the
+        memory (items) then activates the children with the token.
+        """
+        new_token = ReteToken(token, wme, node=self, binding=binding)
+        self.items.append(new_token)
+        for child in self.children:
+            child.left_activation(new_token)

src/sheerkarete/bind_node.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+import inspect
+
+from sheerkarete.beta import ReteNode
+from sheerkarete.common import V
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import Any
+    from typing import Dict
+    from sheerkarete.network import ReteNetwork
+
+    
+class BindNode(ReteNode):
+    """
+    A beta network class. This class stores a code snipit, with variables in
+    it. It gets all the bindings from the incoming token, updates them with the
+    current bindings, binds the result to the target variable (to), then
+    activates its children with the updated bindings.
+    """
+
+    def __init__(self, children, parent, func, to, rete: ReteNetwork):
+        """
+        :type children:
+        :type parent: BetaNode
+        :type to: str
+        """
+        super().__init__(children=children, parent=parent)
+        self.func = func
+        self.func_args = inspect.getfullargspec(self.func)[0]
+        self.bind = to
+        self._rete_net = rete
+
+    def get_function_result(self, binding: Dict[V, Any]):
+        """
+        Given a binding that maps variables to values, this instantiates the
+        arguments for the function and executes it.
+        """
+        args = {arg: self._rete_net if arg == 'network' else
+                self._rete_net.facts[binding[V(arg)]] if
+                binding[V(arg)] in self._rete_net.facts else
+                binding[V(arg)] for arg in self.func_args}
+        return self.func(**args)
+
+    def left_activation(self, token, wme, binding):
+        """
+        Copies and updates the bindings with the results of the function
+        execution. It then left_activates children with this binding.
+        """
+        result = self.get_function_result(binding)
+
+        if self.bind in binding:
+            if binding[self.bind] != result:
+                return
+        else:
+            binding = binding.copy()
+            binding[self.bind] = result
+
+        for child in self.children:
+            child.left_activation(token, wme, binding)

src/sheerkarete/common.py

@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import Hashable
+    from typing import List
+    from typing import Optional
+    from sheerkarete.alpha import AlphaMemory
+    from sheerkarete.pnode import PNode
+
+
+@dataclass(eq=True, frozen=True)
+class V:
+    """
+    A variable for pattern matching.
+    """
+    __slots__ = ['name']
+    name: str
+
+    def __repr__(self):
+        return "V({})".format(self.name)
+
+
+@dataclass(eq=True, frozen=True)
+class Match:
+    pnode: PNode
+    token: ReteToken
+
+    def fire(self):
+        print(f"rule {self.pnode.rule} if fired with {self.token=}")
+        # return self.pnode.rule.fire(self.token)
+
+
+class WME:
+    """
+    This is essentially a fact, it has no variables in it. A working memory is
+    essentially comprised of a collection of these elements.
+    """
+    __slots__ = ['identifier', 'attribute', 'value', 'amems', 'tokens',
+                 'negative_join_results']
+
+    def __init__(self, identifier: str, attribute: Hashable, value: Hashable) -> None:
+        """
+        Identifier, attribute, and value can be any kind of object except V
+        objects (i.e., variables).
+        """
+        if (isinstance(identifier, V) or isinstance(attribute, V) or
+            isinstance(value, V)):
+            raise ValueError("WMEs cannot have variables (V objects).")
+
+        self.identifier = identifier
+        self.attribute = attribute
+        self.value = value
+        from sheerkarete.alpha import AlphaMemory
+        self.amems: List[AlphaMemory] = []  # the ones containing this WME
+        self.tokens: List[ReteToken] = []  # the ones containing this WME
+        self.negative_join_results: List[NegativeJoinResult] = []
+
+    def __hash__(self):
+        return hash((self.identifier, self.attribute, self.value))
+
+    def __repr__(self):
+        return "(%s ^%s %s)" % (self.identifier, self.attribute, self.value)
+
+    def __eq__(self, other: object) -> bool:
+        """
+        :type other: WME
+        """
+        if not isinstance(other, WME):
+            return False
+        return self.identifier == other.identifier and \
+               self.attribute == other.attribute and \
+               self.value == other.value
+
+
+class ReteToken:
+    """
+    Tokens represent matches within the alpha and beta memories. The parent
+    corresponds to the match that was extended to create the current token.
+    """
+    __slots__ = ['parent', 'wme', 'node', 'children', 'join_results',
+                 'ncc_results', 'owner', 'binding']
+
+    def __init__(self, parent, wme, node=None, binding=None):
+        """
+        :type wme: WME
+        :type parent: Token
+        :type binding: dict
+        """
+        self.parent = parent
+        self.wme = wme
+        # points to memory this token is in
+        self.node = node
+        # the ones with parent = this token
+        self.children: List[ReteToken] = []
+        # used only on tokens in negative nodes
+        self.join_results: List[NegativeJoinResult] = []
+        self.ncc_results: List[ReteToken] = []
+        # Ncc
+        self.owner: Optional[ReteToken] = None
+        self.binding = binding if binding else {}  # {"$x": "B1"}
+
+        if self.parent:
+            self.parent.children.append(self)
+        if self.wme:
+            self.wme.tokens.append(self)
+
+    def __repr__(self) -> str:
+        return "<Token %s>" % self.wmes
+
+    def __eq__(self, other: object) -> bool:
+        return (isinstance(other, ReteToken) and self.parent == other.parent and
+                self.wme == other.wme)
+
+    def is_root(self) -> bool:
+        return not self.parent and not self.wme
+
+    @property
+    def wmes(self) -> List[Optional[WME]]:
+        ret = [self.wme]
+        t = self
+        while t.parent and not t.parent.is_root():
+            t = t.parent
+            ret.insert(0, t.wme)
+        return ret
+
+    def delete_descendents_of_token(self) -> None:
+        """
+        Helper function to delete all the descendent tokens.
+        """
+        for t in self.children:
+            t.delete_token_and_descendents()
+
+    def delete_token_and_descendents(self) -> None:
+        """
+        Deletes a token and its descendents, but has special cases that make
+        this difficult to understand in isolation.
+
+        TODO:
+            - Add optimization for right unlinking (pg 87 of Doorenbois
+              thesis).
+
+        :type token: Token
+        """
+        from sheerkarete.ncc_node import NccNode
+        from sheerkarete.ncc_node import NccPartnerNode
+        from sheerkarete.negative_node import NegativeNode
+        from sheerkarete.beta import BetaMemory
+        from sheerkarete.join_node import JoinNode
+
+        for child in self.children:
+            child.delete_token_and_descendents()
+
+        if (isinstance(self.node, BetaMemory) and not
+        isinstance(self.node, NccPartnerNode)):
+            self.node.items.remove(self)
+
+        if self.wme:
+            self.wme.tokens.remove(self)
+        if self.parent:
+            self.parent.children.remove(self)
+
+        if isinstance(self.node, BetaMemory):
+            if not self.node.items:
+                for bmchild in self.node.children:
+                    if (isinstance(bmchild, JoinNode) and bmchild in
+                        bmchild.amem.successors):
+                        bmchild.amem.successors.remove(bmchild)
+
+        if isinstance(self.node, NegativeNode):
+            if not self.node.items:
+                self.node.amem.successors.remove(self.node)
+            for jr in self.join_results:
+                jr.wme.negative_join_results.remove(jr)
+
+        if isinstance(self.node, NccNode):
+            for result_tok in self.ncc_results:
+                if result_tok.wme:
+                    result_tok.wme.tokens.remove(result_tok)
+                if result_tok.parent:
+                    result_tok.parent.children.remove(result_tok)
+
+        elif isinstance(self.node, NccPartnerNode):
+            self.owner.ncc_results.remove(self)
+            if not self.owner.ncc_results and self.node.ncc_node:
+                for bchild in self.node.ncc_node.children:
+                    bchild.left_activation(self.owner, None,
+                                           self.owner.binding)
+
+
+@dataclass(eq=True, frozen=True)
+class NegativeJoinResult:
+    """
+    A new class to store the result of a negative join. Similar to a token, it
+    is owned by a token.
+    """
+    __slots__ = ['owner', 'wme']
+    owner: ReteToken
+    wme: WME

src/sheerkarete/conditions.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Callable, Union
+
+from sheerkarete.common import V
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import List
+    from typing import Tuple
+    from typing import Hashable
+    from sheerkarete.common import WME
+
+
+@dataclass(eq=True, frozen=True)
+class AndConditions:
+    """
+    List of Conditions semantically bound by and ANDs
+    To avoid manipulating list of list
+    """
+    conditions: List[Union[Condition, Tuple]]
+
+
+class ConditionalList(tuple):
+    """
+    A conditional that consists of a list of other conditionals.
+    """
+
+    def __new__(cls, *args):
+        return super().__new__(cls, args)
+
+    def __repr__(self):
+        return "{}{}".format(self.__class__.__name__, super().__repr__())
+
+    def __hash__(self):
+        return hash((self.__class__.__name__, tuple(self)))
+
+
+@dataclass(eq=True, frozen=True)
+class Condition:
+    """
+    Triplet identifier attribute value to match
+    """
+    __slots__ = ['identifier', 'attribute', 'value']
+    identifier: Hashable
+    attribute: Hashable
+    value: Hashable
+
+    def __repr__(self):
+        return "(%s ^%s %s)" % (self.identifier, self.attribute, self.value)
+
+    def __str__(self):
+        return f"{self.identifier}.{self.attribute} = {self.value}"
+
+    def get_key(self):
+        id_test = '*'
+        attr_test = '*'
+        value_test = '*'
+
+        if not isinstance(self.identifier, V):
+            id_test = self.identifier
+        if not isinstance(self.attribute, V):
+            attr_test = self.attribute
+        if not isinstance(self.value, V):
+            value_test = self.value
+
+        return id_test, attr_test, value_test
+
+    @property
+    def vars(self) -> List[Tuple[str, V]]:
+        """
+        Returns a list of tuples (field, var) that contains the slot names as a
+        string and the variable object it maps to.
+        """
+        return [(field, getattr(self, field))
+                for field in ('identifier', 'attribute', 'value')
+                if isinstance(getattr(self, field), V)]
+
+    def contain(self, v: V) -> str:
+        """
+        Checks if a variable is in the condition. Returns as string with the
+        name of the field if it is, otherwise an empty string.
+        """
+        assert isinstance(v, V)
+
+        for f in ['identifier', 'attribute', 'value']:
+            _v = getattr(self, f)
+            if _v == v:
+                return f
+        return ""
+
+    def test(self, w: WME) -> bool:
+        """
+        Checks if a pattern matches a working memory element.
+        """
+        vars_values = {}
+        for f in ['identifier', 'attribute', 'value']:
+            self_value = getattr(self, f)
+            wme_value = getattr(w, f)
+            if isinstance(self_value, V):
+                if self_value in vars_values and vars_values[self_value] != wme_value:
+                    return False
+                vars_values[self_value] = wme_value
+            elif self_value != wme_value:
+                return False
+        return True
+
+    def __hash__(self):
+        return hash(('cond', self.identifier, self.attribute, self.value))
+
+
+class NotEqualsCondition(Condition):
+    """
+    Test not equals of the value
+    """
+
+    def test(self, w: WME) -> bool:
+        """
+        Checks if a pattern matches a working memory element.
+        """
+        for f in ['identifier', 'attribute']:
+            v = getattr(self, f)
+            if isinstance(v, V):
+                continue
+            if v != getattr(w, f):
+                return False
+
+        v = getattr(self, "value")
+        return isinstance(v, V) or getattr(w, "value") != v
+
+    def get_key(self):
+        id_test = '*'
+        attr_test = '*'
+
+        if not isinstance(self.identifier, V):
+            id_test = self.identifier
+        if not isinstance(self.attribute, V):
+            attr_test = self.attribute
+
+        return id_test, attr_test, "*"
+
+    def __hash__(self):
+        return hash(('ne-cond', self.identifier, self.attribute, self.value))
+
+    def __repr__(self):
+        return "(%s ^%s != %s)" % (self.identifier, self.attribute, self.value)
+
+    def __str__(self):
+        return f"{self.identifier}.{self.attribute} != {self.value}"
+
+
+class NegatedCondition(Condition):
+    """
+    A negated pattern.
+    """
+
+    def __repr__(self):
+        return "-(%s ^%s %s)" % (self.identifier, self.attribute, self.value)
+
+    def __hash__(self):
+        return hash(('neg', self.identifier, self.attribute, self.value))
+
+
+class NegatedConjunctiveConditions(ConditionalList):
+    """
+    A negated conjunction of conditions.
+    """
+
+    def __repr__(self):
+        return "-{}".format(super(NegatedConjunctiveConditions, self).__repr__())
+
+    @property
+    def number_of_conditions(self) -> int:
+        return len(self)
+
+    def __hash__(self):
+        return hash(('ncc', tuple(self)))
+
+
+@dataclass(eq=True, frozen=True)
+class FilterCondition:
+    """
+    This is a test, it includes a function that might include variables.
+    When employed in rete, the variable bindings are passed in as keyword args
+    and the function is exectued. The function must return a boolean and if it
+    evaluates to true, then the condition matches otherwise it does not.
+    """
+    __slots__ = ['func']
+    func: Callable
+
+    def __repr__(self):
+        return "Filter({})".format(repr(self.func))
+
+    def __hash__(self):
+        return hash(('filter', self.func))
+
+
+@dataclass(eq=True, frozen=True)
+class BindCondition:
+    """
+    Similar to Filter, but binds the result of a function execution to a new
+    variable.
+    """
+    __slots__ = ['func', 'to']
+    func: Callable
+    to: V
+
+    def __repr__(self):
+        return "Bind({},{})".format(repr(self.func), repr(self.to))
+
+    def __hash__(self):
+        return hash(('bind', self.func, self.to))

src/sheerkarete/filter_node.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import inspect
+from typing import TYPE_CHECKING
+
+from sheerkarete.beta import ReteNode
+from sheerkarete.common import V
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import List
+    from typing import Callable
+    from sheerkarete.network import ReteNetwork
+
+
+class FilterNode(ReteNode):
+    """
+    A beta network node. Takes a function, passes variables in as kwargs, and
+    executes it. If the code evaluates to True (boolean), then it activates the
+    children with the token/wme.
+    """
+
+    def __init__(self, children: List[ReteNode],
+                 parent: ReteNode,
+                 func: Callable,
+                 rete: ReteNetwork):
+        super().__init__(children=children, parent=parent)
+        self.func = func
+        self.func_args = inspect.getfullargspec(self.func)[0]
+        self._rete_net = rete
+
+    def get_function_result(self, token, wme, binding):
+        args = {arg: self._rete_net if arg == 'network' else
+                self._rete_net.facts[binding[V(arg)]] if
+                binding[V(arg)] in self._rete_net.facts else
+                binding[V(arg)] for arg in self.func_args}
+
+        return self.func(**args)
+
+    def left_activation(self, token, wme, binding):
+        """
+        :type binding: dict
+        :type wme: WME
+        :type token: Token
+        """
+        result = self.get_function_result(token, wme, binding)
+        if bool(result):
+            for child in self.children:
+                child.left_activation(token, wme, binding)

src/sheerkarete/join_node.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Dict, Any
+
+from sheerkarete.alpha import AlphaMemory
+from sheerkarete.beta import ReteNode
+from sheerkarete.common import V, WME, ReteToken
+
+if TYPE_CHECKING:  # pragma: no cover
+    from sheerkarete.conditions import Condition
+
+
+class JoinNode(ReteNode):
+    """
+    A beta network class. Does the heavly lifting of joining tokens from a beta
+    memory with wmes from an alpha memory.
+
+    This class has an alpha memory connected to its right side, which triggers
+    right_activations.
+
+    The parent constitutes the left side (another node in the beta network),
+    which triggers left_activations.
+
+    When the JoinNode is right activated, it checks the incoming wme against
+    all the tokens in the parent (on the left side), using the tests. For
+    every match, updated bindings are created and the children are activated.
+
+    When the JoinNode is left activated, it checks the incoming token against
+    the wmes from the alpha memory instead (essentially the opposite direction
+    as above). Similarly, for matches, updated bindings are created and
+    children are activated.
+    """
+
+    def __init__(self, amem: AlphaMemory, condition: Condition, **kwargs):
+        super().__init__(**kwargs)
+        self.amem: AlphaMemory = amem
+        self.condition = condition
+        self.nearest_ancestor_with_same_amem = None
+        self.vars = [(v, field) for field, v in self.condition.vars if
+                     isinstance(v, V)]
+
+    @property
+    def amem_recently_nonempty(self) -> bool:
+        return len(self.amem.items) == 1
+
+    @property
+    def parent_recently_nonempty(self) -> bool:
+        return len(self.parent.items) == 1
+
+    @property
+    def right_unlinked(self) -> bool:
+        return len(self.parent.items) == 0
+
+    @property
+    def left_unlinked(self) -> bool:
+        return len(self.amem.items) == 0
+
+    def update_nearest_ancestor_with_same_amem(self):
+        ancestor = self.parent.find_nearest_ancestor_with_same_amem(self.amem)
+        self.nearest_ancestor_with_same_amem = ancestor
+
+    def find_nearest_ancestor_with_same_amem(self, amem: AlphaMemory):
+        if self.amem == amem:
+            return self
+        return self.parent.find_nearest_ancestor_with_same_amem(amem)
+
+    def relink_to_alpha_memory(self):
+        """
+        When there is not parent to the join, we remove the children (successor) or the alpha memory
+        This procedure put back the link between an amem and the join node
+        """
+        ancestor = self.nearest_ancestor_with_same_amem
+        while ancestor and ancestor.right_unlinked:
+            ancestor = ancestor.nearest_ancestor_with_same_amem
+        if ancestor:
+            try:
+                loc = self.amem.successors.index(ancestor)
+            except ValueError:
+                loc = -1
+            self.amem.successors.insert(loc + 1, self)
+        else:
+            self.amem.successors.insert(0, self)
+
+    def relink_to_beta_memory(self):
+        self.parent.children.append(self)
+
+    def right_activation(self, wme: WME) -> None:
+        """
+        Called when an element is added to the respective alpha memory.
+        We look every tokens from the parent and check if it validates the condition brought by the amem
+        """
+        if self.amem_recently_nonempty:
+            self.relink_to_beta_memory()
+            if not self.parent.items:
+                try:
+                    self.amem.successors.remove(self)
+                except ValueError:
+                    pass
+        for token in self.parent.items:
+            if self.perform_join_test(token, wme):
+                binding = self.make_binding(token, wme)
+                for child in self.children:
+                    child.left_activation(token, wme, binding)
+
+    def left_activation(self, token: ReteToken) -> None:
+        """
+        Called when an element is added to the parent beta node.
+        """
+
+        if self.parent_recently_nonempty:
+            self.relink_to_alpha_memory()
+            if not self.amem.items:
+                self.parent.children.remove(self)
+
+        for wme in self.amem.items:
+            if self.perform_join_test(token, wme):
+                binding = self.make_binding(token, wme)
+                for child in self.children:
+                    child.left_activation(token, wme, binding=binding)
+
+    def perform_join_test(self, token: ReteToken, wme: WME) -> bool:
+        """
+        Test if the token and wme are compatible.
+        """
+        for v, field in self.vars:
+            if v in token.binding and getattr(wme, field) != token.binding[v]:
+                return False
+        return True
+
+    def make_binding(self, token: ReteToken, wme: WME) -> Dict[V, Any]:
+        """
+        Makes updated bindings that result from joining token and wme.
+        """
+        new_binding = {v: getattr(wme, field) for v, field in self.vars}
+        if new_binding:
+            binding = token.binding.copy()
+            binding.update(new_binding)
+            return binding
+        else:
+            return token.binding

src/sheerkarete/ncc_node.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sheerkarete.alpha import AlphaMemory
+from sheerkarete.beta import BetaMemory
+from sheerkarete.common import ReteToken
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import Optional
+    from typing import List
+    from typing import Dict
+    from typing import Any
+    from sheerkarete.beta import ReteNode
+    from sheerkarete.common import V
+    from sheerkarete.common import WME
+
+
+class NccNode(BetaMemory):
+    """
+    A beta network class for negated conjunctive conditions (ncc).
+
+    This has a memory of tokens (items) and a partner node. On left_activation
+    (from parent), the node adds results from its partner's result buffer to
+    the newly created token's ncc_results list, and sets the owner of the
+    result to the new token. If the new token does not have a any results in
+    the ncc_results list, then it activates all the children.
+    """
+
+    def __init__(self, partner: NccPartnerNode = None, **kwargs):
+        super().__init__(**kwargs)
+        self.partner = partner
+
+    def find_nearest_ancestor_with_same_amem(self, amem: AlphaMemory):
+        return self.partner.parent.find_nearest_ancestor_with_same_amem(amem)
+
+    def left_activation(self, token: ReteToken, wme: WME, binding: Dict[V, Any]):
+        new_token = ReteToken(token, wme, self, binding)
+        self.items.append(new_token)
+        for result in self.partner.new_result_buffer:
+            self.partner.new_result_buffer.remove(result)
+            new_token.ncc_results.append(result)
+            result.owner = new_token
+        if not new_token.ncc_results:
+            for child in self.children:
+                child.left_activation(new_token, None, binding)
+
+
+class NccPartnerNode:
+    """
+    The partner node for negated conjunctive conditions node.
+
+    Takes the associated ncc node, the number of conditions, and a buffer of
+    any new results.
+    """
+    def __init__(self, parent: Optional[ReteNode] = None,
+                 ncc_node: Optional[NccNode] = None,
+                 number_of_conditions: int = 0,
+                 new_result_buffer: Optional[List[ReteToken]] = None):
+        self.parent = parent
+        self.ncc_node = ncc_node
+        self.number_of_conditions = number_of_conditions
+        self.new_result_buffer = new_result_buffer if new_result_buffer else []
+
+    def left_activation(self, token: ReteToken, wme: WME, binding: Dict[V, Any]):
+        new_result = ReteToken(token, wme, self, binding)
+        owners_t = token
+        owners_w = wme
+        for i in range(self.number_of_conditions):
+            owners_w = owners_t.wme
+            owners_t = owners_t.parent
+        for token in self.ncc_node.items:
+            if token.parent == owners_t and token.wme == owners_w:
+                token.ncc_results.append(new_result)
+                new_result.owner = token
+                token.delete_descendents_of_token()
+                break
+        else:
+            self.new_result_buffer.append(new_result)

src/sheerkarete/negative_node.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+from sheerkarete.alpha import AlphaMemory
+from sheerkarete.beta import BetaMemory
+from sheerkarete.join_node import JoinNode
+from sheerkarete.common import ReteToken, NegativeJoinResult
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import Dict
+    from typing import Any
+    from sheerkarete.common import V
+    from sheerkarete.common import WME
+
+
+class NegativeNode(BetaMemory, JoinNode):
+    """
+    A beta network class that only passes on tokens when there is no match. The
+    left activation is called by the parent beta node.  The right activation is
+    called from the alpha network (amem).  Test are similar to those that
+    appear in JoinNode
+    """
+
+    def find_nearest_ancestor_with_same_amem(self, amem: AlphaMemory):
+        if self.amem == amem:
+            return self
+        return self.parent.find_nearest_ancestor_with_same_amem(amem)
+
+    @property
+    def right_unlinked(self) -> bool:
+        return len(self.items) == 0
+
+    def left_activation(self, token: ReteToken, wme: WME, binding: Dict[V, Any]):
+        if not self.items:
+            self.relink_to_alpha_memory()
+
+        new_token = ReteToken(parent=token, wme=wme, node=self, binding=binding)
+        self.items.append(new_token)
+
+        for wme in self.amem.items:
+            if self.perform_join_test(new_token, wme):
+                jr = NegativeJoinResult(new_token, wme)
+                new_token.join_results.append(jr)
+                wme.negative_join_results.append(jr)
+
+        if not new_token.join_results:
+            for child in self.children:
+                child.left_activation(new_token, None, binding)
+
+    def right_activation(self, wme: WME):
+        for token in self.items:
+            if self.perform_join_test(token, wme):
+                if not token.join_results:
+                    # TODO: TEST THIS - Chris ?? KSI - 2021-02-05 ???
+                    token.delete_descendents_of_token()
+                    # t.delete_token_and_descendents()
+                jr = NegativeJoinResult(token, wme)
+                token.join_results.append(jr)
+                wme.negative_join_results.append(jr)

src/sheerkarete/network.py

@@ -0,0 +1,479 @@
+from __future__ import annotations
+
+from itertools import product
+from typing import TYPE_CHECKING, Generator, Union
+
+from core.concept import Concept
+from core.global_symbols import NotInit
+from core.rule import Rule
+from core.utils import as_bag
+from sheerkarete.alpha import AlphaMemory
+from sheerkarete.beta import ReteNode, BetaMemory
+from sheerkarete.bind_node import BindNode
+from sheerkarete.common import WME, Match, V
+from sheerkarete.conditions import Condition, NegatedCondition, NegatedConjunctiveConditions, FilterCondition, \
+    BindCondition
+from sheerkarete.filter_node import FilterNode
+from sheerkarete.join_node import JoinNode
+from sheerkarete.ncc_node import NccNode, NccPartnerNode
+from sheerkarete.negative_node import NegativeNode
+from sheerkarete.pnode import PNode
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import Dict
+    from typing import Tuple
+    from typing import List
+    from typing import Set
+    from typing import Hashable
+
+FACT_ID = "##fact_id##"
+
+
+class ReteNetwork:
+    """
+    A Rete Network to store all the facts and productions to compute matches.
+    """
+
+    def __init__(self):
+        self.alpha_hash: Dict[Tuple[Hashable, Hashable, Hashable], List[AlphaMemory]] = {}
+        self.beta_root = ReteNode()
+        self.pnodes: List[PNode] = []  # list of all production nodes
+        self.rules: Set[Rule] = set()  # set of all know rules
+        self.working_memory: Set[WME] = set()
+
+        self.fact_counter: int = 0
+        self.facts: Dict[str, object] = {}
+
+        self.attributes_by_id = {}  # keep track of requested conditions attributes, for a given id
+        self.default_attributes = set()  # keep track of requested attributes, when the id is not given
+
+    @property
+    def matches(self) -> Generator[Match, None, None]:
+        for pnode in self.pnodes:
+            for t in pnode.activations:
+                yield Match(pnode, t)
+
+    def build_or_share_alpha_memory(self, condition):
+        """
+        :type condition: Condition
+        :rtype: AlphaMemory
+        """
+
+        key = condition.get_key()
+
+        # return existing alpha memory if it exists
+        if key in self.alpha_hash:
+            for amem in self.alpha_hash[key]:
+                if amem.condition == condition:
+                    return amem
+
+        # or create a new one
+        amem = AlphaMemory(key, condition)
+        self.alpha_hash.setdefault(key, []).append(amem)
+
+        # fire already created WME if needed
+        for w in self.working_memory:
+            if condition.test(w):
+                amem.activation(w)
+
+        return amem
+
+    def build_or_share_beta_memory(self, parent):
+        """
+        Create or reuse a BetaMemory
+        """
+        # search for an existing one
+        for child in parent.children:
+            # if isinstance(child, BetaMemory):  # Don't include subclasses
+            if type(child) == BetaMemory:
+                return child
+
+        node = BetaMemory(parent=parent)
+        parent.children.append(node)
+        self.update_new_node_with_matches_from_above(node)
+        return node
+
+    def build_or_share_join_node(self, parent: BetaMemory, amem: AlphaMemory, condition: Condition) -> JoinNode:
+        """
+        Creates or reuse a JoinNode
+        :param parent: parent beta memory
+        :param amem: parent alpha memory
+        :param condition: condition for the join
+        :returns:
+        """
+
+        # search for already created join node
+        for child in parent.all_children:
+            if type(child) == JoinNode and child.amem == amem and child.condition == condition:
+                return child
+
+        node = JoinNode(children=[], parent=parent, amem=amem, condition=condition)
+
+        parent.children.append(node)
+        parent.all_children.append(node)
+
+        amem.successors.append(node)
+        amem.reference_count += 1
+
+        node.update_nearest_ancestor_with_same_amem()
+
+        # little optimisation. No need to bind if there is no wme in parent
+        if not parent.items:
+            amem.successors.remove(node)
+        elif not amem.items:
+            parent.children.remove(node)
+
+        return node
+
+    def build_or_share_negative_node(self,
+                                     parent: JoinNode,
+                                     amem: AlphaMemory,
+                                     condition: NegatedCondition) -> NegativeNode:
+        # search for already created join node
+        for child in parent.children:
+            if isinstance(child, NegativeNode) and child.amem == amem and child.condition == condition:
+                return child
+
+        node = NegativeNode(parent=parent, amem=amem, condition=condition)
+        parent.children.append(node)
+
+        amem.successors.append(node)
+        amem.reference_count += 1
+
+        node.update_nearest_ancestor_with_same_amem()
+        self.update_new_node_with_matches_from_above(node)
+
+        # little optimisation. No need to bind if there is no wme in parent
+        if not node.items:
+            amem.successors.remove(node)
+
+        return node
+
+    def build_or_share_ncc_nodes(self,
+                                 parent: JoinNode,
+                                 ncc: NegatedConjunctiveConditions,
+                                 earlier_conds: List[Condition]) -> NccNode:
+
+        # search for already created join node
+        bottom_of_subnetwork = self.build_or_share_network_for_conditions(parent, ncc, earlier_conds)
+        for child in parent.children:
+            if isinstance(child, NccNode) and child.partner.parent == bottom_of_subnetwork:
+                return child
+
+        ncc_partner = NccPartnerNode(parent=bottom_of_subnetwork)
+        ncc_node = NccNode(partner=ncc_partner, children=[], parent=parent)
+        ncc_partner.ncc_node = ncc_node
+        parent.children.insert(0, ncc_node)
+        bottom_of_subnetwork.children.append(ncc_partner)
+        ncc_partner.number_of_conditions = ncc.number_of_conditions
+        self.update_new_node_with_matches_from_above(ncc_node)
+        self.update_new_node_with_matches_from_above(ncc_partner)
+        return ncc_node
+
+    def build_or_share_filter_node(self,
+                                   parent: ReteNode,
+                                   f: FilterCondition) -> FilterNode:
+        # search for already created join node
+        for child in parent.children:
+            if isinstance(child, FilterNode) and child.func == f.func:
+                return child
+
+        node = FilterNode([], parent, f.func, self)
+        parent.children.append(node)
+        return node
+
+    def build_or_share_bind_node(self,
+                                 parent: ReteNode,
+                                 b: BindCondition) -> BindNode:
+        # search for already created join node
+        for child in parent.children:
+            if isinstance(child, BindNode) and child.func == b.func and child.bind == b.to:
+                return child
+
+        node = BindNode([], parent, b.func, b.to, self)
+        parent.children.append(node)
+
+        return node
+
+    def build_or_share_p_node(self, parent: JoinNode, rule: Rule) -> Union[PNode, None]:
+        """
+        Create or reuse a production node
+        :param parent: parent join node
+        :param rule: rule that will be fired on activation
+        :return: returns None if the PNode already exists
+        """
+        for child in parent.children:
+            if isinstance(child, PNode):
+                child.rules.append(rule)
+                rule.rete_p_nodes.append(child)
+                return None
+
+        node = PNode(rule=rule, parent=parent)
+        parent.children.append(node)
+        self.update_new_node_with_matches_from_above(node)
+        rule.rete_p_nodes.append(node)
+        return node
+
+    def build_or_share_network_for_conditions(self, parent, conditions, earlier_conditions) -> ReteNode:
+        current_node = parent
+        conds_higher_up = earlier_conditions
+
+        # Explanation on vars_ids_mappings
+        # conditions = [Condition(V("x"), "__name__", "fact_name"),
+        #               Condition(V("x"), "attr1", "value1"),
+        #               Condition(V("x"), "attr1", "value1")]
+        # V(x) actually refers to a object named 'fact_name'
+        # self.conditions_attributes_by_id must be updated accordingly
+        vars_ids_mappings = {}
+
+        for cond in conditions:
+            # update requested attributes for a fact
+            if isinstance(cond, Condition):
+                # Manage list of requested attributes
+                if isinstance(cond.identifier, V) and cond.attribute == "__name__":
+                    vars_ids_mappings[cond.identifier] = cond.value
+
+                identifier = vars_ids_mappings[cond.identifier] if cond.identifier in vars_ids_mappings else \
+                    cond.identifier if not isinstance(cond.identifier, V) else \
+                        None
+                if identifier:
+                    attr = "*" if isinstance(cond.attribute, V) else cond.attribute
+                    self.attributes_by_id.setdefault(identifier, []).append(attr)
+                elif not isinstance(cond.attribute, V):
+                    self.default_attributes.add(cond.attribute)
+
+            # create the alpha memory (if needed), beta memory and join node
+            if isinstance(cond, Condition) and not isinstance(cond, NegatedCondition):
+                am = self.build_or_share_alpha_memory(cond)
+                current_node = self.build_or_share_beta_memory(current_node)
+                current_node = self.build_or_share_join_node(current_node, am, cond)
+
+            elif isinstance(cond, NegatedCondition):
+                am = self.build_or_share_alpha_memory(cond)
+                current_node = self.build_or_share_negative_node(current_node, am, cond)
+
+            elif isinstance(cond, NegatedConjunctiveConditions):
+                current_node = self.build_or_share_ncc_nodes(current_node, cond, conds_higher_up)
+
+            elif isinstance(cond, FilterCondition):
+                current_node = self.build_or_share_filter_node(current_node, cond)
+
+            elif isinstance(cond, BindCondition):
+                current_node = self.build_or_share_bind_node(current_node, cond)
+
+            conds_higher_up.append(cond)
+
+        return current_node
+
+    def get_rete_conditions(self, rule):
+        """
+        Gets the conditions from a rule
+        It's in fact the list of disjunctions
+        Not sure yet which component will hold this functionality
+        """
+        if hasattr(rule, "get_rete_disjunctions"):
+            return rule.get_rete_disjunctions()
+
+        raise NotImplementedError("")
+
+    def add_rule(self, rule: Rule):
+
+        if rule.id is None:
+            raise ValueError("Rule has no id, cannot add")
+
+        if not rule.metadata.is_enabled or not rule.metadata.is_compiled:
+            return
+
+        if rule.rete_net:
+            raise ValueError("Rule is already added")
+
+        rule.rete_net = self
+        self.rules.add(rule)
+
+        for full_condition in self.get_rete_conditions(rule):
+            conditions = full_condition.conditions
+            current_node = self.build_or_share_network_for_conditions(self.beta_root, conditions, [])
+            p_node = self.build_or_share_p_node(current_node, rule)
+            if p_node is not None:
+                self.pnodes.append(p_node)
+
+    def remove_rule(self, rule: Rule):
+        """
+        Removes a pnode from the network
+        """
+        if rule.rete_net is None:
+            return
+
+        # Remove production
+        self.rules.remove(rule)
+
+        for pnode in rule.rete_p_nodes:
+            pnode.rules.remove(rule)
+            if len(pnode.rules) == 0:
+                self.delete_node_and_any_unused_ancestors(pnode)
+                self.pnodes.remove(pnode)
+
+        rule.p_nodes = []
+
+    def add_wme(self, wme: WME) -> None:
+        if wme in self.working_memory:
+            return
+
+        keys = product([wme.identifier, '*'],
+                       [wme.attribute, '*'],
+                       [wme.value, '*'])
+
+        for key in keys:
+            if key in self.alpha_hash:
+                for amem in reversed(self.alpha_hash[key]):
+                    amem.activation(wme)
+
+        self.working_memory.add(wme)
+
+    def remove_wme(self, wme: WME) -> None:
+        for stored_wme in self.working_memory:
+            if wme == stored_wme:
+                wme = stored_wme
+                break
+
+        for am in wme.amems:
+            am.items.remove(wme)
+            if not am.items:
+                for node in am.successors:
+                    if isinstance(node, JoinNode) and not isinstance(node, NegativeNode):
+                        node.parent.children.remove(node)
+
+        while wme.tokens:
+            t = wme.tokens[0]
+            t.delete_token_and_descendents()
+
+        for jr in wme.negative_join_results:
+            jr.owner.join_results.remove(jr)
+            if not jr.owner.join_results:
+                if jr.owner.node and jr.owner.node.children is not None:
+                    for child in jr.owner.node.children:
+                        child.left_activation(jr.owner, None, jr.owner.binding)
+
+        self.working_memory.remove(wme)
+
+    def remove_wme_by_fact_id(self, identifier: str) -> None:
+        to_remove = [wme for wme in self.working_memory if wme.identifier ==
+                     identifier or wme.identifier.startswith(identifier + ".")]
+        for wme in to_remove:
+            self.remove_wme(wme)
+
+    def add_obj(self, name, obj, use_bag=False, root=True):
+        """
+        Adds a new object to the working memory
+        """
+
+        def inner_add_vme(ident, attr, val):
+            if val is NotInit:
+                pass
+            elif attr != "self" and isinstance(val, Concept):
+                new_name = f"{ident}.{attr}"
+                self.add_wme(WME(ident, attr, new_name))
+                self.add_obj(new_name, val, use_bag=True, root=False)
+            else:
+                self.add_wme(WME(ident, attr, val))
+
+        if root:
+            if hasattr(obj, FACT_ID):
+                raise ValueError("Object already has an id, cannot add")
+
+            fact_id = f"f-{self.fact_counter:05}"
+            setattr(obj, FACT_ID, fact_id)
+            self.facts[fact_id] = obj
+            self.fact_counter += 1
+        else:
+            fact_id = name
+
+        requested_attributes = "*" if use_bag else \
+            self.attributes_by_id[name] if name in self.attributes_by_id else \
+                self.default_attributes
+
+        for attribute in requested_attributes:
+            if attribute == "*":
+                bag = as_bag(obj)
+                for k, v in bag.items():
+                    inner_add_vme(fact_id, k, v)
+            elif attribute == "__name__":
+                self.add_wme(WME(fact_id, "__name__", name))
+            else:
+                try:
+                    value = getattr(obj, attribute)
+                    inner_add_vme(fact_id, attribute, value)
+                except AttributeError:
+                    pass
+
+    def remove_obj(self, obj):
+        if not hasattr(obj, FACT_ID) or (fact_id := getattr(obj, FACT_ID)) not in self.facts:
+            raise ValueError("Fact has no id or does not exist in network.")
+
+        self.remove_wme_by_fact_id(fact_id)
+        del self.facts[fact_id]
+        delattr(obj, FACT_ID)
+
+    def update_new_node_with_matches_from_above(self, new_node: ReteNode) -> None:
+        parent = new_node.parent
+        if parent == self.beta_root:
+            new_node.left_activation(None, None, {})
+        elif isinstance(parent, BetaMemory) and not isinstance(parent, (NccNode, NegativeNode)):
+            for tok in parent.items:
+                new_node.left_activation(token=tok)
+        elif isinstance(parent, JoinNode) and not isinstance(parent, NegativeNode):
+            saved_list_of_children = parent.children
+            parent.children = [new_node]
+            for item in parent.amem.items:
+                parent.right_activation(item)
+            parent.children = saved_list_of_children
+        elif isinstance(parent, NegativeNode):
+            for token in parent.items:
+                if not token.join_results:
+                    new_node.left_activation(token, None, token.binding)
+        elif isinstance(parent, NccNode):
+            for token in parent.items:
+                if not token.ncc_results:
+                    new_node.left_activation(token, None, token.binding)
+        elif isinstance(parent, (BindNode, FilterNode)):
+            saved_list_of_children = parent.children
+            parent.children = [new_node]
+            self.update_new_node_with_matches_from_above(parent)
+            parent.children = saved_list_of_children
+
+    def delete_alpha_memory(self, amem: AlphaMemory):
+        del self.alpha_hash[amem.key]
+
+    def delete_node_and_any_unused_ancestors(self, node: ReteNode):
+        if isinstance(node, NccNode):
+            self.delete_node_and_any_unused_ancestors(node.partner)
+
+        if isinstance(node, BetaMemory):
+            for item in node.items:
+                item.delete_token_and_descendents()
+
+        if isinstance(node, NccPartnerNode):
+            for item in node.new_result_buffer:
+                item.delete_token_and_descendents()
+
+        if isinstance(node, JoinNode) and not isinstance(node, NegativeNode):
+            if not node.right_unlinked:
+                node.amem.successors.remove(node)
+
+            node.amem.reference_count -= 1
+
+            if node.amem.reference_count == 0:
+                self.delete_alpha_memory(node.amem)
+
+            if not node.left_unlinked:
+                node.parent.children.remove(node)
+
+            node.parent.all_children.remove(node)
+
+            if not node.parent.all_children:
+                self.delete_node_and_any_unused_ancestors(node.parent)
+
+        elif node.parent:
+            node.parent.children.remove(node)
+            if not node.parent.children:
+                self.delete_node_and_any_unused_ancestors(node.parent)

src/sheerkarete/pnode.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from core.rule import Rule
+from sheerkarete.beta import BetaMemory
+from sheerkarete.common import ReteToken
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import List
+    from typing import Dict
+    from typing import Any
+    from sheerkarete.common import WME
+    from sheerkarete.common import V
+
+
+class PNode(BetaMemory):
+    """
+    A beta network node that stores the matches for productions.
+    """
+
+    def __init__(self, rule: Rule, **kwargs):
+        super(PNode, self).__init__(**kwargs)
+        self.rules = [rule]
+        self.new: List[ReteToken] = []
+
+    def left_activation(self, token: ReteToken, wme: WME, binding: Dict[V, Any]):
+        new_token = ReteToken(token, wme, node=self, binding=binding)
+        self.items.append(new_token)
+        self.new.append(new_token)
+
+    def pop_new_token(self):
+        if self.new:
+            return self.new.pop()
+
+    def new_activations(self):
+        while self.new:
+            t = self.new.pop()
+            yield t
+
+    @property
+    def activations(self):
+        for t in self.items:
+            yield t