Added first version of DebugManager. Implemented draft of the rule engine

src/evaluators/BaseEvaluator.py

@@ -10,11 +10,12 @@ class BaseEvaluator:
     PREFIX = "evaluators."
 
     def __init__(self, name, steps, priority: int, enabled=True):
-        self.log = get_logger(self.PREFIX + self.__class__.__name__)
-        self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__)
-        self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__)
+        # self.log = get_logger(self.PREFIX + self.__class__.__name__)
+        # self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__)
+        # self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__)
 
         self.name = self.PREFIX + name
+        self.short_name = name
         self.steps = steps
         self.priority = priority
         self.enabled = enabled
@@ -22,6 +23,9 @@ class BaseEvaluator:
     def __repr__(self):
         return f"{self.name} ({self.priority})"
 
+    def reset(self):
+        pass
+
 
 class OneReturnValueEvaluator(BaseEvaluator):
     """
@@ -50,3 +54,5 @@ class AllReturnValuesEvaluator(BaseEvaluator):
     def eval(self, context: ExecutionContext, return_values):
         pass
 
+    def reset(self):
+        self.eaten.clear()

src/evaluators/ConceptEvaluator.py

@@ -45,7 +45,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
         # Why ?
         # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
         # The body should be 'property_a', and not a concept called 'a'
-        if context.obj and concept.name in context.obj.values:
+        if context.obj and concept.name in context.obj.values():
             value = context.obj.get_value(concept.name)
             context.log(f"{concept.name} is a property. Returning value '{value}'.", self.name)
 
@@ -63,7 +63,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
                 evaluated,
                 parents=[return_value])
 
-        if self.return_body and ConceptParts.BODY in evaluated.compiled:
+        if self.return_body and ConceptParts.BODY in evaluated.get_compiled():
             return sheerka.ret(self.name, True, evaluated.body, parents=[return_value])
         else:
             return sheerka.ret(self.name, True, evaluated, parents=[return_value])

src/evaluators/DefConceptEvaluator.py

@@ -1,7 +1,6 @@
 import core.utils
-from core.ast.nodes import python_to_concept
+from core.ast_helpers import UnreferencedVariablesVisitor
 from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
-from core.builtin_helpers import get_names
 from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import TokenKind, Tokenizer
@@ -35,16 +34,18 @@ class ConceptOrRuleNameVisitor(ParsingExpressionVisitor):
             self.names.add(node.rule_name)
 
 
-class AddConceptEvaluator(OneReturnValueEvaluator):
+class DefConceptEvaluator(OneReturnValueEvaluator):
     """
     Used to add a new concept
     """
-    NAME = "AddNewConcept"
+    NAME = "DefConcept"
 
     def __init__(self):
         super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
 
     def matches(self, context, return_value):
+        debugger = context.get_debugger(self.NAME, "matches")
+        debugger.debug_entering(return_value=return_value)
         return return_value.status and \
                isinstance(return_value.value, ParserResultConcept) and \
                isinstance(return_value.value.value, DefConceptNode)
@@ -54,11 +55,14 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         def_concept_node = return_value.value.value
         sheerka = context.sheerka
 
+        debugger = context.get_debugger(self.NAME, "eval")
+        debugger.debug_entering(def_concept=def_concept_node)
+
         # validate the node
         variables_found = set()
 
-        concept = Concept(def_concept_node.name)
-        concept.metadata.definition_type = def_concept_node.definition_type
+        concept = Concept(str(def_concept_node.name))
+        concept.get_metadata().definition_type = def_concept_node.definition_type
         name_to_use = self.get_name_to_use(def_concept_node)
 
         for prop in ("definition", "where", "pre", "post", "body", "ret"):
@@ -75,14 +79,14 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
                                                                            ParserInput) else part_ret_val.value.source
             else:
                 raise Exception("Unexpected")
-            setattr(concept.metadata, prop, source)
+            setattr(concept.get_metadata(), prop, source)
 
             # Do not try to resolve variables from itself
-            if prop == "definition" and concept.metadata.definition_type == DEFINITION_TYPE_DEF:
+            if prop == "definition" and concept.get_metadata().definition_type == DEFINITION_TYPE_DEF:
                 continue
 
             # try to find what can be a property
-            for p in self.get_variables(sheerka, part_ret_val, name_to_use):
+            for p in self.get_variables(context, part_ret_val, name_to_use):
                 variables_found.add(p)
 
         # add variables by order of appearance when possible
@@ -93,7 +97,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         # add the remaining properties
         # They mainly come from BNF definition
         for p in variables_found:
-            if p not in concept.values:
+            if p not in concept.values():
                 concept.def_var(p, None)
 
         # initialize the key
@@ -105,7 +109,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         # update the bnf definition if needed
         if not isinstance(def_concept_node.definition, NotInitializedNode) and \
             def_concept_node.definition_type == DEFINITION_TYPE_BNF:
-            concept.bnf = def_concept_node.definition.value.value
+            concept.set_bnf(def_concept_node.definition.value.value)
 
         ret = sheerka.create_new_concept(context, concept)
         if not ret.status:
@@ -119,7 +123,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         return [part.str_value for part in core.utils.strip_tokens(source.tokens, True)]
 
     @staticmethod
-    def get_variables(sheerka, ret_value, concept_name):
+    def get_variables(context, ret_value, concept_name):
         """
         Try to find out the variables
         This function can only be a draft, as there may be tons of different situations
@@ -149,8 +153,8 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode):
             if len(concept_name) > 1:
                 python_node = ret_value.value.value
-                as_concept_node = python_to_concept(python_node.ast_)
-                names = get_names(sheerka, as_concept_node)
+                visitor = UnreferencedVariablesVisitor(context)
+                names = visitor.get_names(python_node.ast_)
                 variables = filter(lambda x: x in concept_name, names)
                 return set(variables)
 

src/evaluators/FormatRuleEvaluator.py

@@ -0,0 +1,46 @@
+import core.utils
+from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
+from core.rule import Rule
+from core.tokenizer import Keywords
+from evaluators.BaseEvaluator import OneReturnValueEvaluator
+from parsers.BaseParser import BaseParser
+from parsers.FormatRuleParser import FormatRuleNode
+
+
+class FormatRuleEvaluator(OneReturnValueEvaluator):
+    """
+    Used to store a new format rule
+    """
+    NAME = "FormatRule"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
+
+    def matches(self, context, return_value):
+        return return_value.status and \
+               isinstance(return_value.value, ParserResultConcept) and \
+               isinstance(return_value.value.value, FormatRuleNode)
+
+    def eval(self, context, return_value):
+        """
+        Creates a Rule out of a FormatRuleNode and saves it in db
+        :param context:
+        :param return_value:
+        :return:
+        """
+
+        context.log("Adding a new format rule", self.name)
+        format_rule_node = return_value.value.value
+        sheerka = context.sheerka
+
+        predicate = core.utils.get_text_from_tokens(format_rule_node.tokens[Keywords.WHEN][1:])
+        action = core.utils.get_text_from_tokens(format_rule_node.tokens[Keywords.PRINT][1:])
+        rule = Rule("print", None, predicate, action)
+        rule.compiled_predicate = format_rule_node.rule
+        rule.compiled_action = format_rule_node.format_ast
+
+        ret = sheerka.create_new_rule(context, rule)
+        if not ret.status:
+            error_cause = sheerka.objvalue(ret.body)
+            context.log(f"Failed to add new rule '{rule}'. Reason: {error_cause}", self.name)
+        return sheerka.ret(self.name, ret.status, ret.value, parents=[return_value])

src/evaluators/LexerNodeEvaluator.py

@@ -16,6 +16,10 @@ class LexerNodeEvaluator(OneReturnValueEvaluator):
         self.identifiers = {}  # cache for already created identifier (the key is id(concept))
         self.identifiers_key = {}  # number of identifiers with the same root (prefix)
 
+    def reset(self):
+        self.identifiers.clear()
+        self.identifiers_key.clear()
+
     def matches(self, context, return_value):
         if not return_value.status:
             return False

src/evaluators/MultipleErrorsEvaluator.py

@@ -15,6 +15,10 @@ class MultipleErrorsEvaluator(AllReturnValuesEvaluator):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 30)
         self.return_values_in_error = []
 
+    def reset(self):
+        super().reset()
+        self.return_values_in_error.clear()
+
     def matches(self, context, return_values):
         nb_evaluators_in_error = 0
         to_process = False
@@ -47,5 +51,5 @@ class MultipleErrorsEvaluator(AllReturnValuesEvaluator):
         return sheerka.ret(
             self.name,
             False,
-            sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=self.return_values_in_error),
+            sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=self.return_values_in_error.copy()),
             parents=self.eaten)

src/evaluators/MutipleSameSuccessEvaluator.py

@@ -21,6 +21,10 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 50)
         self.success = []
 
+    def reset(self):
+        super().reset()
+        self.success.clear()
+
     def matches(self, context, return_values):
         nb_successful_evaluators = 0
         only_parsers_in_error = True

src/evaluators/OneErrorEvaluator.py

@@ -15,6 +15,10 @@ class OneErrorEvaluator(AllReturnValuesEvaluator):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 30)
         self.return_value_in_error = None
 
+    def reset(self):
+        super().reset()
+        self.return_value_in_error = None
+
     def matches(self, context, return_values):
         nb_evaluators_in_error = 0
         to_process = False

src/evaluators/OneSuccessEvaluator.py

@@ -16,6 +16,12 @@ class OneSuccessEvaluator(AllReturnValuesEvaluator):
     def __init__(self):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 60)  # before MultipleSameSuccess
         self.successful_return_value = None
+        self.value_to_return = None
+
+    def reset(self):
+        super().reset()
+        self.successful_return_value = None
+        self.value_to_return = None
 
     def matches(self, context, return_values):
         nb_successful_evaluators = 0
@@ -30,6 +36,7 @@ class OneSuccessEvaluator(AllReturnValuesEvaluator):
             elif ret.status and ret.who.startswith(self.PREFIX):
                 nb_successful_evaluators += 1
                 self.successful_return_value = ret
+                self.value_to_return = ret if context.sheerka.is_container(ret) else ret.body
                 self.eaten.append(ret)
             elif not ret.status:
                 self.eaten.append(ret)
@@ -41,4 +48,4 @@ class OneSuccessEvaluator(AllReturnValuesEvaluator):
         context.log(f"{self.successful_return_value}", who=self)
 
         sheerka = context.sheerka
-        return sheerka.ret(self.name, True, self.successful_return_value.value, parents=self.eaten)
+        return sheerka.ret(self.name, True, self.value_to_return, parents=self.eaten)

src/evaluators/PostExecutionEvaluator.py

@@ -1,5 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
+from core.concept import Concept, NotInit
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 
 
@@ -25,11 +25,31 @@ class PostExecutionEvaluator(OneReturnValueEvaluator):
         return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.AUTO_EVAL))
 
     def eval(self, context, return_value):
-        # only support the rule for the COMMANDS
-        #body = return_value.body.body
-        body = context.sheerka.objvalue(return_value)
+        # only support the rule for the COMMANDS ??
         return context.sheerka.ret(
             self.name,
             True,
-            body if body != BuiltinConcepts.NOT_INITIALIZED else return_value.body,
+            self.custom_obj_value(context, return_value.body),
             parents=[return_value])
+
+    @staticmethod
+    def custom_obj_value(context, obj):
+        """
+        get concept inner value.
+        You cannot use sheerka.objvalue() as it does not handle container
+        And...
+        Do not try to make it manage it, it won't work ;-)
+        :param context:
+        :param obj:
+        :return:
+        """
+        if context.sheerka.is_container(obj):
+            return obj
+
+        if isinstance(obj, Concept):
+            if isinstance(obj.body, Concept):
+                return PostExecutionEvaluator.custom_obj_value(context, obj.body)
+            else:
+                return obj.body if obj.body != NotInit else obj
+
+        return obj

src/evaluators/PrepareEvalBodyEvaluator.py

@@ -13,6 +13,9 @@ class PrepareEvalBodyEvaluator(OneReturnValueEvaluator):
         super().__init__(self.NAME, [BuiltinConcepts.BEFORE_PARSING], 90)
         self.text = None
 
+    def reset(self):
+        self.text = None
+
     def matches(self, context, return_value):
         if not (return_value.status and
                 context.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) and

src/evaluators/PrepareEvalQuestionEvaluator.py

@@ -13,6 +13,9 @@ class PrepareEvalQuestionEvaluator(OneReturnValueEvaluator):
         super().__init__(self.NAME, [BuiltinConcepts.BEFORE_PARSING], 90)
         self.question = None
 
+    def reset(self):
+        self.question = None
+
     def matches(self, context, return_value):
         if not (return_value.status and
                 context.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) and

src/evaluators/PythonEvaluator.py

@@ -3,12 +3,14 @@ import copy
 import traceback
 from dataclasses import dataclass, field
 
-import core.ast.nodes
+import core.builtin_helpers
 import core.utils
-from core.ast.visitors import UnreferencedNamesVisitor
+from core.ast_helpers import UnreferencedNamesVisitor, NamesWithAttributesVisitor
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
 from core.concept import ConceptParts, Concept, NotInit
-from core.sheerka.services.SheerkaFilter import Pipe
+from core.rule import Rule
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.tokenizer import Token, TokenKind
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.PythonParser import PythonNode
 
@@ -47,6 +49,20 @@ class PythonEvalError:
     traceback: str = field(repr=False)
     concepts: dict = field(repr=False)
 
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, PythonEvalError):
+            return False
+
+        return isinstance(self.error, type(other.error)) and \
+               self.traceback == other.traceback and \
+               self.concepts == other.concepts
+
+    def __hash__(self):
+        return hash(self.error)
+
 
 class PythonEvaluator(OneReturnValueEvaluator):
     NAME = "Python"
@@ -54,37 +70,31 @@ class PythonEvaluator(OneReturnValueEvaluator):
     """
     Evaluate a Python node, ie, evaluate some Python code
     """
+    isinstance = None
 
     def __init__(self):
         super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
-        self.globals = {}
+
+    @staticmethod
+    def initialize(sheerka):
+        from core.sheerka.services.SheerkaAdmin import SheerkaAdmin
+        PythonEvaluator.isinstance = sheerka.services[SheerkaAdmin.NAME].extended_isinstance
 
     def matches(self, context, return_value):
         if not return_value.status or not isinstance(return_value.value, ParserResultConcept):
             return False
         body = return_value.value.value
-        return isinstance(body, PythonNode) or (
-            hasattr(body, "python_node") and isinstance(body.python_node, PythonNode))
-        # return return_value.status and \
-        #        isinstance(return_value.value, ParserResultConcept) and \
-        #        isinstance(return_value.value.value, PythonNode)
+        return isinstance(body, PythonNode) or hasattr(body, "python_node")
 
     def eval(self, context, return_value):
         sheerka = context.sheerka
         node = return_value.value.value if isinstance(return_value.value.value, PythonNode) else \
             return_value.value.value.python_node
 
+        debugger = context.get_debugger(PythonEvaluator.NAME, "eval")
+        debugger.debug_entering(node=node)
         context.log(f"Evaluating python node {node}.", self.name)
 
-        # Do not evaluate if the ast refers to a concept (leave it to ConceptEvaluator)
-        # TODO: Remove this section when this check will be implemented in the AFTER_PARSING step
-        if isinstance(node.ast_, ast.Expression) and isinstance(node.ast_.body, ast.Name):
-            c = context.sheerka.resolve(node.ast_.body.id)
-            if c is not None:
-                context.log("It's a simple concept. Not for me.", self.name)
-                not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
-                return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
-
         # If we evaluate a Concept metadata which is NOT the body ex (pre, post, where...)
         # We need to disable the function that may alter the state
         # It's a poor way to have source code security check
@@ -104,11 +114,12 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         # get globals
         my_globals = self.get_globals(context, node, expression_only)
-        context.log(f"globals={my_globals}", self.name)
+
+        debugger.debug_var("globals", my_globals)
 
         all_possible_globals = self.get_all_possible_globals(context, my_globals)
-        concepts_entries = None
-        evaluated = BuiltinConcepts.NOT_INITIALIZED
+        concepts_entries = None  # entries in globals_ that refers to Concept objects
+        evaluated = NotInit
         errors = []
         expect_success = context.in_context(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
         for globals_ in all_possible_globals:
@@ -116,8 +127,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 # eval
                 if isinstance(node.ast_, ast.Expression):
                     context.log("Evaluating using 'eval'.", self.name)
-                    compiled = compile(node.ast_, "<string>", "eval")
-                    evaluated = eval(compiled, globals_, sheerka.locals)
+                    evaluated = eval(node.get_compiled(), globals_, sheerka.locals)
                 else:
                     context.log("Evaluating using 'exec'.", self.name)
                     evaluated = self.exec_with_return(node.ast_, globals_, sheerka.locals)
@@ -128,143 +138,125 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 if concepts_entries is None:
                     concepts_entries = self.get_concepts_entries_from_globals(my_globals)
                 errors.append(PythonEvalError(ex,
-                                              traceback.format_exc(),
+                                              traceback.format_exc() if context.debug_enabled else None,
                                               self.get_concepts_values_from_globals(globals_, concepts_entries)))
 
-        if evaluated == BuiltinConcepts.NOT_INITIALIZED:
+        if evaluated == NotInit:
             if len(errors) == 1:
                 context.log_error(errors[0].error, who=self.name, exc=errors[0].traceback)
-                one_error = sheerka.new(BuiltinConcepts.ERROR, body=errors[0])
-                return sheerka.ret(self.name, False, one_error, parents=[return_value])
+                return sheerka.ret(self.name, False, sheerka.err(errors[0]), parents=[return_value])
 
             if len(errors) > 1:
                 for eval_error in errors:
                     context.log_error(eval_error.error, who=self.name, exc=eval_error.traceback)
-                too_many_errors = sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=errors)
-                return sheerka.ret(self.name, False, too_many_errors, parents=[return_value])
+                return sheerka.ret(self.name, False, sheerka.err(errors), parents=[return_value])
 
         context.log(f"{evaluated=}", self.name)
+        debugger.debug_var("ret", evaluated)
         return sheerka.ret(self.name, True, evaluated, parents=[return_value])
 
     def get_globals(self, context, node, expression_only):
         """
-        Creates the global variables for python source code evaluation
+        Creates the globals variables
         :param context:
         :param node:
-        :param expression_only: most of the commands are refused
+        :param expression_only:
         :return:
         """
+        unreferenced_names_visitor = UnreferencedNamesVisitor(context)
+        names = unreferenced_names_visitor.get_names(node.ast_)
+        if context.debug_enabled:
+            context.debug(self.NAME, "eval", "names", names)
+        return self.get_globals_by_names(context, names, node, expression_only)
+
+    def get_sheerka_method(self, context, name, expression_only):
+        try:
+            method = context.sheerka.sheerka_methods[name]
+            context.log(f"Resolving '{name}'. It's a sheerka method.", self.name)
+            if expression_only and method.has_side_effect:
+                context.log(f"...but with side effect when {expression_only=}. Discarding.", self.name)
+                return None
+            else:
+                return inject_context(context)(method.method) if name in context.sheerka.methods_with_context \
+                    else method.method
+        except KeyError:
+            return None
+
+    def get_globals_by_names(self, context, names, node, expression_only):
         my_globals = {
             "Concept": core.concept.Concept,
             "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts,
+            "ExecutionContext": ExecutionContext,
             "in_context": context.in_context,
         }
 
-        if expression_only:
-            # disable some builtin
-            for statement in TO_DISABLED:
-                my_globals[statement] = None
-
-        # has to be the first, to allow override
-        method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context, expression_only)
-
-        self.update_globals_with_context(my_globals, context)
-        already_know = set(my_globals.keys())
-        self.update_globals_with_node(my_globals, context, node, already_know)
-
-        if self.globals:  # when extra values are given. Add them
-            my_globals.update(self.globals)
-
-        my_globals["sheerka"] = Expando(method_from_sheerka)  # it's the last, so I cannot be overridden
-        return my_globals
-
-    @staticmethod
-    def update_globals_with_sheerka_methods(my_locals, context, expression_only):
-        methods_from_sheerka = {}
-
-        # Add all the methods as a direct access
-        for method_name, method in context.sheerka.sheerka_methods.items():
-            if expression_only and method.has_side_effect:
+        for name in names:
+            if name in my_globals:
                 continue
 
-            if method_name in context.sheerka.methods_with_context:
-                my_locals[method_name] = inject_context(context)(method.method)
-            else:
-                my_locals[method_name] = method.method
-            methods_from_sheerka[method_name] = my_locals[method_name]
-
-        # Add pipeable functions
-        for func_name, function in context.sheerka.sheerka_pipeables.items():
-            if expression_only and function.has_side_effect:
+            if expression_only and name in TO_DISABLED:
+                my_globals[name] = None
                 continue
 
-            my_locals[func_name] = Pipe(function.method, context)
-
-        return methods_from_sheerka  # to allow access using prefix "sheerka."
-
-    def update_globals_with_context(self, my_globals, context):
-        """
-        Update globals with the current object being evaluated (and its variables)
-        :param my_globals:
-        :param context:
-        :return:
-        """
-        if context.obj:
-            context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name)
-
-            for prop_name in context.obj.variables():
-                value = context.obj.get_value(prop_name)
-                if value != NotInit:
-                    my_globals[prop_name] = value
-            my_globals["self"] = context.obj
-
-    def update_globals_with_node(self, my_globals, context, node, already_known):
-        """
-        Try to find concepts using the names that appear in the AST of the node.
-        :param my_globals: dictionary to update
-        :param context:
-        :param node:
-        :param already_known: if the name is in this list, do no try to instantiate it again
-        :return:
-        """
-        node_concept = core.ast.nodes.python_to_concept(node.ast_)
-        unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)
-        unreferenced_names_visitor.visit(node_concept)
-
-        for name in unreferenced_names_visitor.names:
-            context.log(f"Resolving '{name}'.", self.name)
-
-            # get the concept
-            if name in node.concepts:
-                # use it, even if it already in already_known
-                # This concept take precedence other the outer world
-                context.log(f"Using value from node.", self.name)
-                concept = self.resolve_concept(context, node.concepts[name])
-            elif name in already_known:
-                context.log(f"Already known. Skipping.", self.name)
-                continue
-            elif (concept := context.get_from_short_term_memory(name)) is not None:
-                context.log(f"Using from STM known.", self.name)
-            else:
-                context.log(f"Instantiating new concept with {name}.", self.name)
-                concept = self.resolve_concept(context, name)
-
-            if concept is None:
-                context.log(f"Concept '{name}' is not found or cannot be instantiated. Skipping.", self.name)
+            # need to add it manually to avoid conflict with sheerka.isinstance
+            if name == "isinstance":
+                my_globals["isinstance"] = PythonEvaluator.isinstance
                 continue
 
-            # evaluate it if needed
-            if concept.metadata.is_evaluated:
-                context.log(f"Concept {name} is already evaluated.", self.name)
-            else:
-                context.log(f"Evaluating '{concept}'", self.name)
-                evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True)
-                if not context.sheerka.is_success(evaluated) and evaluated.key != concept.key:
-                    context.log(f"Error while evaluating '{name}'. Skipping.", self.name)
+            # support reference to sheerka
+            if name == "sheerka":
+                bag = {}
+                visitor = NamesWithAttributesVisitor()
+                for sequence in visitor.get_sequences(node.ast_, "sheerka"):
+                    if (len(sequence) > 1 and
+                        (method := self.get_sheerka_method(context, sequence[1], expression_only)) is not None):
+                        bag[sequence[1]] = method
+                my_globals["sheerka"] = Expando(bag)
+                continue
+
+            # search in short term memory
+            if (obj := context.get_from_short_term_memory(name)) is not None:
+                context.log(f"Resolving '{name}'. Using value found in STM.", self.name)
+                my_globals[name] = obj
+                continue
+
+            # search in sheerka methods
+            if (method := self.get_sheerka_method(context, name, expression_only)) is not None:
+                my_globals[name] = method
+                continue
+
+            # search in context.obj (to replace by short time memory ?)
+            if context.obj:
+                if name == "self":
+                    my_globals["self"] = context.obj
                     continue
-                concept = evaluated
 
-            my_globals[name] = concept
+                try:
+                    attribute = context.obj.variables()[name]
+                    if attribute != NotInit:
+                        my_globals[name] = attribute
+                        continue
+                    context.log(f"Resolving '{name}'. It's obj attribute (obj={context.obj}).", self.name)
+                except KeyError:
+                    pass
+
+            # search in current node (if the name was found during the parsing)
+            if name in node.objects:
+                context.log(f"Resolving '{name}'. Using value from node.", self.name)
+                obj = self.resolve_object(context, node.objects[name])
+
+            # at last, try to instantiate a new concept
+            else:
+                context.log(f"Resolving '{name}'. Instantiating new concept.", self.name)
+                obj = self.resolve_object(context, name)
+
+            if obj is None:
+                context.log(f"...'{name}' is not found or cannot be instantiated. Skipping.", self.name)
+                continue
+
+            my_globals[name] = obj
+
+        return my_globals
 
     @staticmethod
     def get_all_possible_globals(context, my_globals):
@@ -309,31 +301,37 @@ class PythonEvaluator(OneReturnValueEvaluator):
         return {name: my_globals[name] for name in names}
 
     @staticmethod
-    def resolve_concept(context, concept_hint):
+    def resolve_object(context, name):
         """
         Try to find a concept by its name, id or the pattern c:key|id:
         :param context:
-        :param concept_hint:
+        :param name:
         :return:
         """
-        if isinstance(concept_hint, Concept):
-            return concept_hint
+        if isinstance(name, Rule):
+            return name
 
-        concept = context.sheerka.resolve(concept_hint)
+        if isinstance(name, Concept):
+            name = core.builtin_helpers.ensure_evaluated(context, name)
+            return name
+
+        if isinstance(name, Token) and name.type == TokenKind.RULE:
+            rule = context.sheerka.get_rule_by_id(name.value[1])  # TODO: need a resolve function for the rules
+            return rule if isinstance(rule, Rule) else None
+
+        if isinstance(name, tuple):
+            raise Exception()
+
+        # try to resolve by name
+        concept = context.sheerka.fast_resolve(name)
         if concept is None:
             return None
-        new_instance = context.sheerka.new_from_template(concept, concept.key)
-        if isinstance(concept_hint, tuple):
-            # It's means that it was requested by PythonParser which have found a concept token (c:xxx:)
-            # So a concept was explicitly required, not its value
-            # We mark the concept as already evaluated, so it's body will not be evaluated
-            new_instance.metadata.is_evaluated = True
-        if len(concept.metadata.variables) > 0:
-            # In this situation, it means that we are dealing with the concept and not its instantiation
-            # So do not try to evaluate it
-            new_instance.metadata.is_evaluated = True
 
-        return new_instance
+        if hasattr(concept, "__iter__"):
+            raise NotImplementedError("Too many concepts")
+
+        concept = core.builtin_helpers.ensure_evaluated(context, concept)
+        return concept
 
     @staticmethod
     def expr_to_expression(expr):
@@ -343,7 +341,8 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         return result
 
-    def exec_with_return(self, code_ast, my_globals, my_locals):
+    @staticmethod
+    def exec_with_return(code_ast, my_globals, my_locals):
 
         init_ast = copy.deepcopy(code_ast)
         init_ast.body = code_ast.body[:-1]
@@ -353,6 +352,8 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         exec(compile(init_ast, "<ast>", "exec"), my_globals, my_locals)
         if type(last_ast.body[0]) == ast.Expr:
-            return eval(compile(self.expr_to_expression(last_ast.body[0]), "<ast>", "eval"), my_globals, my_locals)
+            return eval(compile(PythonEvaluator.expr_to_expression(last_ast.body[0]), "<ast>", "eval"),
+                        my_globals,
+                        my_locals)
         else:
             exec(compile(last_ast, "<ast>", "exec"), my_globals, my_locals)

src/evaluators/ResolveAmbiguityEvaluator.py

@@ -15,6 +15,10 @@ class ResolveAmbiguityEvaluator(AllReturnValuesEvaluator):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_PARSING], 50)
         self.sources = None
 
+    def reset(self):
+        super().reset()
+        self.sources = None
+
     def matches(self, context, return_values):
         # first, arrange return_values by sources.
         # If they share the same source, that means that there are multiple results for one ParserInput

src/evaluators/RetEvaluator.py

@@ -18,14 +18,14 @@
 #     def matches(self, context, return_value):
 #         return return_value.status and \
 #                isinstance(return_value.value, Concept) and \
-#                return_value.value.metadata.ret is not None
+#                return_value.value.get_metadata().ret is not None
 #
 #     def eval(self, context, return_value):
 #         sheerka = context.sheerka
 #         concept = return_value.value
 #         context.log(f"Processing ret value for concept {concept}.", self.name)
 #
-#         if not concept.metadata.is_evaluated:
+#         if not concept.get_metadata().is_evaluated:
 #             evaluated = ensure_evaluated(context, concept)
 #             if evaluated.key != concept.key:
 #                 context.log(f"Failed to evaluate concept '{concept}'")

src/evaluators/ReturnBodyEvaluator.py

@@ -1,5 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
+from core.concept import Concept, NotInit
 from evaluators.BaseEvaluator import AllReturnValuesEvaluator
 
 
@@ -23,7 +23,10 @@ class ReturnBodyEvaluator(AllReturnValuesEvaluator):
         result = []
 
         for ret_val in return_values:
-            if ret_val.status and isinstance(ret_val.body, Concept) and ret_val.body.body != BuiltinConcepts.NOT_INITIALIZED:
+            if ret_val.status and \
+                    isinstance(ret_val.body, Concept) and \
+                    not sheerka.is_container(ret_val.body) and \
+                    ret_val.body.body != NotInit:
                 context.log(f"Evaluating {ret_val.body}", who=self)
                 result.append(sheerka.ret(self.name, True, ret_val.body.body, parents=[ret_val]))
             elif ret_val.status and sheerka.isaset(context, ret_val.body):

src/evaluators/RuleEvaluator.py

@@ -0,0 +1,47 @@
+from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
+from core.rule import Rule, ACTION_TYPE_DEFERRED
+from evaluators.BaseEvaluator import OneReturnValueEvaluator
+
+
+class RuleEvaluator(OneReturnValueEvaluator):
+    """
+    This first version will only returns the rule found by the parser
+    """
+    NAME = "Rule"
+
+    def __init__(self):
+        """
+
+        """
+        super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
+
+    def matches(self, context, return_value):
+        if not return_value.status:
+            return False
+
+        if not isinstance(return_value.value, ParserResultConcept):
+            return False
+
+        value = return_value.value.value
+        return isinstance(value, Rule) or isinstance(value, list) and len(value) > 0 and isinstance(value[0], Rule)
+
+    def eval(self, context, return_value):
+        sheerka = context.sheerka
+        rules = return_value.value.value
+
+        resolved = []
+        success = True
+        for r in rules:
+            # Browse the rules to find possible deferred rules
+            if r.metadata.action_type == ACTION_TYPE_DEFERRED:
+                rule_id = sheerka.get_from_short_term_memory(context, r.id)
+                rule = sheerka.get_rule_by_id(str(rule_id or r.id))
+                resolved.append(rule)
+                success &= isinstance(rule, Rule)
+            else:
+                resolved.append(r)
+
+        return context.sheerka.ret(self.name,
+                                   success,
+                                   resolved if len(resolved) > 1 else resolved[0],
+                                   parents=[return_value])

src/evaluators/TooManySuccessEvaluator.py

@@ -20,6 +20,10 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 60)
         self.success = []
 
+    def reset(self):
+        super().reset()
+        self.success.clear()
+
     def matches(self, context, return_values):
         to_process = False
 
@@ -39,10 +43,10 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
 
     def eval(self, context, return_values):
         sheerka = context.sheerka
-        if self.verbose_log.isEnabledFor(logging.DEBUG):
-            for s in self.success:
-                context.log(s, self.name)
-                context.log(f"value={sheerka.value(s.value)}", self.name)
+        # if self.verbose_log.isEnabledFor(logging.DEBUG):
+        #     for s in self.success:
+        #         context.log(s, self.name)
+        #         context.log(f"value={sheerka.value(s.value)}", self.name)
 
         same_success = core.builtin_helpers.is_same_success(context, self.success)
         if same_success is None:
@@ -50,7 +54,7 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
 
         if not same_success:
             context.log(f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name)
-            too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success)
+            too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success.copy())
             return sheerka.ret(self.name, False, too_many_success, parents=self.eaten)
 
         context.log(f"Values are the same. Nothing to do.", self.name)

src/evaluators/UpdateFunctionsParametersEvaluator.py

@@ -28,6 +28,9 @@ class UpdateFunctionsParametersEvaluator(OneReturnValueEvaluator):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 79)
         self.enabled = False
 
+    def reset(self):
+        self.enabled = False
+
     def matches(self, context, return_value):
         """
         True if the return value is the successful result of PythonEvaluator
@@ -88,14 +91,14 @@ class UpdateFunctionsParametersEvaluator(OneReturnValueEvaluator):
     @staticmethod
     def get_function(node):
         if len(node.children) == 2 and \
-                isinstance(node.children[0], Name) and \
-                isinstance(node.children[1], PythonNode) and \
-                node.children[1].type == "trailer" and \
-                len(node.children[1].children) >= 2 and \
-                isinstance(node.children[1].children[0], Operator) and \
-                node.children[1].children[0].value == "(" and \
-                isinstance(node.children[1].children[-1], Operator) and \
-                node.children[1].children[-1].value == ")":
+            isinstance(node.children[0], Name) and \
+            isinstance(node.children[1], PythonNode) and \
+            node.children[1].type == "trailer" and \
+            len(node.children[1].children) >= 2 and \
+            isinstance(node.children[1].children[0], Operator) and \
+            node.children[1].children[0].value == "(" and \
+            isinstance(node.children[1].children[-1], Operator) and \
+            node.children[1].children[-1].value == ")":
             name = node.children[0].value
             if len(node.children[1].children) == 2:
                 params = []