Fixed #29: Parsers: Implement parsing memoization

Fixed #77 : Parser: ShortTermMemoryParser should be called separately
Fixed #78 : Remove VariableNode usage
Fixed #79 : ConceptManager: Implement compile caching
Fixed #80 : SheerkaExecute : parsers_key is not correctly computed
Fixed #81 : ValidateConceptEvaluator : Validate concept's where and pre clauses right after the parsing
Fixed #82 : SheerkaIsAManager: isa() failed when the set as a body
Fixed #83 : ValidateConceptEvaluator : Support BNF and SYA Concepts
Fixed #84 : ExpressionParser: Implement the parser as a standard parser
Fixed #85 : Services: Give order to services
Fixed #86 : cannot manage smart_get_attr(the short, color)

sheerka_backup/default.sb

@@ -1,6 +1,6 @@
 # question
 push_ontology("english")
-def concept q from q ? as question(q) pre is_question() auto_eval True
+def concept q from q ? as question(q) auto_eval True
 set_is_lesser(__PRECEDENCE, q, 'Sya')
 
 def concept the x where isinstance(x, Concept) ret memory(x)

sheerka_backup/numbers.sb

@@ -93,5 +93,5 @@ set_is_greater_than(__PRECEDENCE, multiplied, minus, 'Sya')
 set_is_greater_than(__PRECEDENCE, divided, minus, 'Sya')
 
 def concept quantity
-def concept quantify x from bnf number x as set_attr(x, quantity, number) ret x
+def concept quantify x from bnf number x where not isa(x, number) as set_attr(x, quantity, number) ret x
 def concept how many x pre is_question() as get_attr(memory(x), quantity)

src/core/builtin_concepts.py

@@ -15,7 +15,7 @@ class UserInputConcept(Concept):
                          BuiltinConcepts.USER_INPUT, bound_body="text")
         self.set_value("text", text)
         self.set_value("user_name", user_name)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"({self.id}){self.name}: '{self.body}'"
@@ -33,7 +33,7 @@ class ErrorConcept(Concept, ErrorObj):
                          id=concept_id,
                          bound_body="error")
         self.set_value("error", error)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"({self.id}){self.name}: {self.body}"
@@ -49,7 +49,7 @@ class UnknownConcept(Concept, ErrorObj):
                          False,
                          BuiltinConcepts.UNKNOWN_CONCEPT, bound_body="concept_ref")
         self.set_value("concept_ref", concept_ref)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"({self.id}){self.name}: {self.body}"
@@ -75,7 +75,7 @@ class ReturnValueConcept(Concept):
         self.set_value("status", status)
         self.set_value("value", value)
         self.set_value("parents", parents)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"ReturnValue(who={self.who}, status={self.status}, value={self.value})"
@@ -110,7 +110,7 @@ class UnknownPropertyConcept(Concept, ErrorObj):
                          bound_body="property_name")
         self.set_value("property_name", property_name)
         self.set_value("concept", concept)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"UnknownProperty(property={self.property_name}, concept={self.concept})"
@@ -136,7 +136,7 @@ class ParserResultConcept(Concept):
         self.set_value("tokens", tokens)
         self.set_value("value", value)
         self.set_value("try_parsed", try_parsed)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         text = f"ParserResult(parser={self.parser}"
@@ -179,7 +179,7 @@ class RuleEvaluationResultConcept(Concept):
                          id=concept_id,
                          bound_body="rule")
         self.set_value("rule", rule)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"RuleEvaluationResult(rule={self.rule})"
@@ -212,7 +212,7 @@ class InvalidReturnValueConcept(Concept, ErrorObj):
             bound_body="return_value")
         self.set_value("return_value", return_value)
         self.set_value("evaluator", evaluator)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
 
 class ConceptEvalError(Concept, ErrorObj):
@@ -227,7 +227,7 @@ class ConceptEvalError(Concept, ErrorObj):
         self.set_value("error", error)
         self.set_value("concept", concept)
         self.set_value("property_name", property_name)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"ConceptEvalError(error={self.error}, concept={self.concept}, property={self.property_name})"
@@ -244,16 +244,19 @@ class ListConcept(Concept):
                          BuiltinConcepts.LIST,
                          bound_body="items")
         self.set_value("items", items or [])
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def append(self, obj):
         self.body.append(obj)
 
 
 class FilteredConcept(Concept):
-    ALL_ATTRIBUTES = ["filtered", "iterable", "predicate"]
+    ALL_ATTRIBUTES = ["filtered", "iterable", "predicate", "reason"]
+    # To explain the reason why it's filtered, you can either
+    #  provide the original list (iterable) and the predicate
+    #  provide the reason (It may be a CONDITION_FAILED concept)
 
-    def __init__(self, filtered=None, iterable=None, predicate=None):
+    def __init__(self, filtered=None, iterable=None, predicate=None, reason=None):
         Concept.__init__(self,
                          BuiltinConcepts.FILTERED,
                          True,
@@ -263,7 +266,8 @@ class FilteredConcept(Concept):
         self.set_value("filtered", filtered)
         self.set_value("iterable", iterable)
         self.set_value("predicate", predicate)
-        self._metadata.is_evaluated = True
+        self.set_value("reason", reason)
+        self._hints.is_evaluated = True
 
 
 class ConceptAlreadyInSet(Concept, ErrorObj):
@@ -278,7 +282,7 @@ class ConceptAlreadyInSet(Concept, ErrorObj):
                          bound_body="concept")
         self.set_value("concept", concept)
         self.set_value("concept_set", concept_set)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"ConceptAlreadyInSet(concept={self.concept}, concept_set={self.concept_set})"
@@ -297,7 +301,7 @@ class PropertyAlreadyDefined(Concept, ErrorObj):
         self.set_value("property_name", property_name)
         self.set_value("property_value", property_value)
         self.set_value("concept", concept)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"PropertyAlreadyDefined(property={self.property_name}, value={self.property_value}, concept={self.concept})"
@@ -317,7 +321,7 @@ class ConditionFailed(Concept, ErrorObj):
         self.set_value("concept", concept)
         self.set_value("prop", prop)
         self.set_value("reason", reason)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"ConditionFailed(condition='{self.body}', concept='{self.concept}', prop='{self.prop}')"
@@ -335,7 +339,7 @@ class NotForMeConcept(Concept):  # Not considered as an error ?
                          bound_body="source")
         self.set_value("source", source)
         self.set_value("reason", reason)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         return f"NotForMeConcept(source={self.body}, reason={self.get_value('reason')})"
@@ -356,7 +360,7 @@ class ExplanationConcept(Concept):
         self.set_value("title", title)  # a title to the explanation
         self.set_value("instructions", instructions)  # instructions for SheerkaPrint
         self.set_value("execution_result", execution_result)  # list of results
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
 
 class PythonSecurityError(Concept, ErrorObj):
@@ -375,7 +379,7 @@ class PythonSecurityError(Concept, ErrorObj):
         self.set_value("line", line)  # line number
         self.set_value("column", column)  # column number
         self.set_value("source_code", source_code)  # code being executed
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
 
 class NotFoundConcept(Concept, ErrorObj):
@@ -407,7 +411,7 @@ class ToListConcept(Concept):
         self.set_value("recursion_depth", recursion_depth)  # recursion depth when showing children
         self.set_value("recurse_on", recurse_on)  # which sub items should we display
         self.set_value("tab", tab)  # customise tab (content and length)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
 
 class NewConceptConcept(Concept):
@@ -422,7 +426,7 @@ class NewConceptConcept(Concept):
                          bound_body="concept")
 
         self.set_value("concept", concept)
-        self._metadata.is_evaluated = True
+        self._hints.is_evaluated = True
 
     def __repr__(self):
         if self.concept:

src/core/builtin_concepts_ids.py

@@ -1,12 +1,9 @@
 class BuiltinConcepts:
     """
     List of builtin concepts that do no need any specific implementation
-    Please note that the value of the enum is informal. It is not used in the system
-    For example, the concept 'NODE' DOES NOT have the key, the id or whatever 200
-    The key if the name of the concept
+    Please note that the value of the enum is informal.
+    The key is the name of the concept
     The id is a sequential number given just before the concept is saved in sdp
-
-    The values of the enum is not used the code
     """
     SHEERKA = "__SHEERKA"
 
@@ -19,6 +16,7 @@ class BuiltinConcepts:
     REDUCE_REQUESTED = "__REDUCE_REQUESTED"  # remove meaningless error when possible
     EVAL_UNTIL_SUCCESS_REQUESTED = "__EVAL_UNTIL_SUCCESS_REQUESTED"  # PythonEvaluator tries combination until True is found
     EVAL_QUESTION_REQUESTED = "__EVAL_QUESTION_REQUESTED"  # the user input must be treated as question
+    VALIDATION_ONLY_REQUESTED = "__VALIDATION_ONLY_REQUESTED"  # Validation mode activated. Never evaluate the body
 
     # possible actions during sheerka.execute() or sheerka.evaluate_rules()
     INIT_SHEERKA = "__INIT_SHEERKA"  #
@@ -54,7 +52,6 @@ class BuiltinConcepts:
     ISA = "__ISA"  # when a concept is an instance of another one
     HASA = "__HASA"  # when a concept has/owns another concept
     AUTO_EVAL = "__AUTO_EVAL"  # when the concept must be auto evaluated
-    RECOGNIZED_BY = "__RECOGNIZED_BY"  # indicate how a concept was recognized
 
     # object
     USER_INPUT = "__USER_INPUT"  # represent an input from an user
@@ -102,6 +99,7 @@ class BuiltinConcepts:
     UNKNOWN_RULE = "__UNKNOWN_RULE"
     ONTOLOGY_ALREADY_DEFINED = "__ONTOLOGY_ALREADY_DEFINED"
     ONTOLOGY_REMOVED = "__ONTOLOGY_REMOVED"
+    METHOD_ACCESS_ERROR = "__METHOD_ACCESS_ERROR"
 
     NODE = "__NODE"
     GENERIC_NODE = "__GENERIC_NODE"
@@ -174,7 +172,8 @@ BuiltinErrors = [
     BuiltinConcepts.NOT_FOUND,
     BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR,
     BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR,
-    BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED
+    BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED,
+    BuiltinConcepts.METHOD_ACCESS_ERROR
 ]
 
 BuiltinContainers = [

src/core/builtin_helpers.py

@@ -5,17 +5,17 @@ from cache.Cache import Cache
 from core.ast_helpers import ast_to_props
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value
-from core.global_symbols import NotInit, NotFound, CURRENT_OBJ
+from core.global_symbols import NotInit, NotFound, INIT_AST_PARSERS, DEFAULT_EVALUATORS
 from core.rule import Rule
-from core.tokenizer import Tokenizer
+from core.tokenizer import Tokenizer, TokenKind
 from core.utils import as_bag
 from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
-    RuleNode, VariableNode
+    RuleNode, LexerNode
 from parsers.BaseParser import ParsingError
 
 PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
-EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION,
-                            BuiltinConcepts.AFTER_EVALUATION]
+EVAL_ONLY_STEPS = [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION]
+EVAL_STEPS = PARSE_STEPS + EVAL_ONLY_STEPS
 PARSERS = ["EmptyString", "ShortTermMemory", "Sequence", "Bnf", "Sya", "Python"]
 
 
@@ -48,7 +48,7 @@ def is_same_success(context, return_values):
         if not ret_val.status:
             raise Exception("Status is false")
 
-        if isinstance(ret_val.body, Concept) and not ret_val.body.get_metadata().is_evaluated:
+        if isinstance(ret_val.body, Concept) and not ret_val.body.get_hints().is_evaluated:
             raise Exception("Concept is not evaluated")
 
         return context.sheerka.objvalue(ret_val)
@@ -223,7 +223,10 @@ def resolve_ambiguity(context, concepts):
             remaining_concepts.extend(by_complexity[complexity])
         else:
             for c in by_complexity[complexity]:
-                evaluated = context.sheerka.evaluate_concept(context, c, metadata=[ConceptParts.PRE])
+                evaluated = context.sheerka.evaluate_concept(context, c,
+                                                             eval_body=False,
+                                                             validation_only=True,
+                                                             metadata=[ConceptParts.PRE])
                 if context.sheerka.is_success(evaluated) or evaluated.key == c.key:
                     remaining_concepts.append(c)
 
@@ -316,7 +319,8 @@ def only_parsers_results(context, return_values):
 
 def evaluate_from_source(context,
                          source,
-                         evaluators="all",
+                         parsers=INIT_AST_PARSERS,
+                         evaluators=DEFAULT_EVALUATORS,
                          desc=None,
                          eval_body=True,
                          eval_where=True,
@@ -327,13 +331,14 @@ def evaluate_from_source(context,
 
     :param context:
     :param source:
+    :param parsers:
     :param evaluators:
     :param desc:
     :param eval_body:
     :param eval_where:
     :param is_question:
     :param expect_success:
-    :param stm: short term memories entries
+    :param stm: short term memories entries AKA current namespace
     :return:
     """
 
@@ -362,12 +367,11 @@ def evaluate_from_source(context,
             for k, v in stm.items():
                 sub_context.add_to_short_term_memory(k, v)
 
-        # disable all evaluators but the requested ones
+        if parsers != "all":
+            sub_context.preprocess_parsers = parsers
+
         if evaluators != "all":
-            from evaluators.BaseEvaluator import BaseEvaluator
-            sub_context.add_preprocess(BaseEvaluator.PREFIX + "*", enabled=False)
-            for evaluator in evaluators:
-                sub_context.add_preprocess(BaseEvaluator.PREFIX + evaluator, enabled=True)
+            sub_context.preprocess_evaluators = evaluators
 
         user_input = sheerka.ret(context.who, True, sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
         ret = sheerka.execute(sub_context, [user_input], EVAL_STEPS)
@@ -376,6 +380,52 @@ def evaluate_from_source(context,
         return ret
 
 
+def evaluate_return_values(context,
+                           source,
+                           return_values,
+                           evaluators=DEFAULT_EVALUATORS,
+                           desc=None,
+                           eval_body=True,
+                           eval_where=True,
+                           is_question=False,
+                           expect_success=False,
+                           stm=None):
+    sheerka = context.sheerka
+    desc = desc or f"Eval '{source}' using return values"
+    hints_to_reset = {
+        BuiltinConcepts.EVAL_BODY_REQUESTED,
+        BuiltinConcepts.EVAL_WHERE_REQUESTED,
+        BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED,
+        BuiltinConcepts.EVAL_QUESTION_REQUESTED,
+    }
+    with context.push(BuiltinConcepts.EVALUATE_SOURCE, source, desc=desc, reset_hints=hints_to_reset) as sub_context:
+        if eval_body:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+
+        if eval_where:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+
+        if expect_success:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+
+        if is_question:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+
+        if stm:
+            for k, v in stm.items():
+                sub_context.add_to_short_term_memory(k, v)
+
+        if evaluators != "all":
+            sub_context.preprocess_evaluators = evaluators
+
+        sub_context.add_inputs(return_values=return_values)
+        res = sheerka.execute(sub_context, return_values.copy(), EVAL_ONLY_STEPS)
+        one_r = expect_one(context, res)
+        sub_context.add_values(return_values=one_r)
+
+        return one_r
+
+
 def get_lexer_nodes(return_values, start, tokens):
     """
     Transform all elements from return_values into lexer nodes (ConceptNode, UnrecognizedTokensNode, SourceCodeNode...)
@@ -386,10 +436,14 @@ def get_lexer_nodes(return_values, start, tokens):
     :param tokens:
     :return: list of list (list of concept node sequence)
     """
+    from evaluators.BaseEvaluator import BaseEvaluator
 
     lexer_nodes = []
     for ret_val in return_values:
-        if ret_val.who == "parsers.Python":
+        # To manage AFTER_PARSING evaluators
+        who = ret_val.parents[0].who if ret_val.who.startswith(BaseEvaluator.PREFIX) else ret_val.who
+
+        if who == "parsers.Python":
 
             if ret_val.body.source.strip().isidentifier():
                 # Discard SourceCodeNode which seems to be a concept name
@@ -404,27 +458,29 @@ def get_lexer_nodes(return_values, start, tokens):
                                                python_node=ret_val.body.body,
                                                return_value=ret_val)])
 
-        elif ret_val.who == "parsers.ExactConcept":
+        elif who == "parsers.ExactConcept":
             concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
             end = start + len(tokens) - 1
             for concept in concepts:
                 lexer_nodes.append([ConceptNode(concept, start, end, tokens, ret_val.body.source)])
 
-        elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
-            nodes = [node for node in ret_val.body.body]
+        elif who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
+            nodes = [node.clone() for node in ret_val.body.body]
             for node in nodes:
                 node.start += start
                 node.end += start
                 if isinstance(node, ConceptNode):
                     for k, v in node.concept.get_compiled().items():
-                        if hasattr(v, "start"):
+                        if isinstance(v, LexerNode):
+                            v = v.clone()
                             v.start += start
                             v.end += start
+                            node.concept.get_compiled()[k] = v
 
             # but append the whole sequence if when it's a sequence
             lexer_nodes.append(nodes)
 
-        elif ret_val.who == "parsers.Rule":
+        elif who == "parsers.Rule":
             rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
             end = start + len(tokens) - 1
             for rule in rules:
@@ -447,9 +503,14 @@ def get_lexer_nodes_using_positions(return_values, positions):
     :return:
     """
 
+    from evaluators.BaseEvaluator import BaseEvaluator
+
     lexer_nodes = []
     for ret_val, position in zip(return_values, positions):
-        if ret_val.who in ("parsers.Python", 'parsers.PythonWithConcepts'):
+        # To manage AFTER_PARSING evaluators
+        who = ret_val.parents[0].who if ret_val.who.startswith(BaseEvaluator.PREFIX) else ret_val.who
+
+        if who in ("parsers.Python", 'parsers.PythonWithConcepts'):
 
             lexer_nodes.append(SourceCodeNode(position.start,
                                               position.end,
@@ -458,7 +519,7 @@ def get_lexer_nodes_using_positions(return_values, positions):
                                               python_node=ret_val.body.body,
                                               return_value=ret_val))
 
-        elif ret_val.who == "parsers.ExactConcept":
+        elif who == "parsers.ExactConcept":
             concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
             for concept in concepts:
                 lexer_nodes.append(ConceptNode(concept,
@@ -467,21 +528,23 @@ def get_lexer_nodes_using_positions(return_values, positions):
                                                position.tokens,
                                                ret_val.body.source))
 
-        elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
-            nodes = [node for node in ret_val.body.body]
+        elif who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"):
+            nodes = [node.clone() for node in ret_val.body.body]
             for node in nodes:
                 node.start = position.start
                 node.end = position.end
                 if isinstance(node, ConceptNode):
                     for k, v in node.concept.get_compiled().items():
-                        if hasattr(v, "start"):
+                        if isinstance(v, LexerNode):
+                            v = v.clone()
                             v.start += position.start
                             v.end += position.start
+                            node.concept.get_compiled()[k] = v
 
             # but append the whole sequence if when it's a sequence
             lexer_nodes.extend(nodes)
 
-        elif ret_val.who == "parsers.Rule":
+        elif who == "parsers.Rule":
             rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
             for rule in rules:
                 lexer_nodes.append(RuleNode(rule,
@@ -489,7 +552,7 @@ def get_lexer_nodes_using_positions(return_values, positions):
                                             position.end,
                                             position.tokens, ret_val.body.source))
 
-        elif ret_val.who == "parsers.Function":
+        elif who == "parsers.Function":
             node = ret_val.body.body
             node.start = position.start
             node.end = position.end
@@ -510,8 +573,10 @@ def ensure_evaluated(context, concept, eval_body=True, metadata=None):
     :param metadata:
     :return:
     """
-    if concept.get_metadata().is_evaluated:
-        return concept
+    if concept.get_hints().is_evaluated:
+        from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
+        return SheerkaEvaluateConcept.apply_ret(concept,
+                                                eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
 
     # do not try to evaluate concept that are not fully initialized
     if concept.get_metadata().definition_type != DEFINITION_TYPE_BNF:
@@ -533,19 +598,6 @@ def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers
     :param parsers:
     :return:
     """
-
-    # first look into short term memory to see if the unrecognized is not a variable of the current object
-    if (current_obj := context.sheerka.get_from_short_term_memory(context, CURRENT_OBJ)) is not NotFound:
-        if isinstance(current_obj, Concept):
-            source = unrecognized_tokens_node.source
-            if source in current_obj.get_compiled() or source in current_obj.variables():
-                return [[VariableNode(current_obj,
-                                      source,
-                                      unrecognized_tokens_node.start,
-                                      unrecognized_tokens_node.end,
-                                      unrecognized_tokens_node.tokens,
-                                      unrecognized_tokens_node.source)]]
-
     res = context.sheerka.parse_unrecognized(context, unrecognized_tokens_node.source, parsers)
     res = only_parsers_results(context, res)
 
@@ -630,7 +682,7 @@ def update_compiled(context, concept, errors, parsers=None):
             if _get_source(concept.get_compiled(), name) != name:
                 break
         else:
-            concept.get_metadata().is_evaluated = True
+            concept.get_hints().is_evaluated = True
 
 
 def add_to_ret_val(sheerka, context, return_values, concept_key):
@@ -665,10 +717,10 @@ def set_is_evaluated(concepts, check_nb_variables=False):
     if hasattr(concepts, "__iter__"):
         for c in concepts:
             if not check_nb_variables or check_nb_variables and len(c.get_metadata().variables) > 0:
-                c.get_metadata().is_evaluated = True
+                c.get_hints().is_evaluated = True
     else:
         if not check_nb_variables or check_nb_variables and len(concepts.get_metadata().variables) > 0:
-            concepts.get_metadata().is_evaluated = True
+            concepts.get_hints().is_evaluated = True
 
 
 def ensure_concept(*concepts):
@@ -701,7 +753,7 @@ def ensure_concept_or_rule(*items):
             raise TypeError(f"'{items}' must be a concept or rule")
 
 
-def ensure_bnf(context, concept, parser_name="BaseNodeParser", update_bnf_for_cached_concept=True):
+def ensure_bnf(context, concept, parser_name="BaseNodeParser"):
     if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf():
         from parsers.BnfDefinitionParser import BnfDefinitionParser
         regex_parser = BnfDefinitionParser()
@@ -719,8 +771,6 @@ def ensure_bnf(context, concept, parser_name="BaseNodeParser", update_bnf_for_ca
                 raise Exception(bnf_parsing_ret_val.value)
 
             concept.set_bnf(bnf_parsing_ret_val.body.body)
-            if concept.id and update_bnf_for_cached_concept:
-                context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf())  # update bnf in cache
 
 
 expressions_cache = Cache()
@@ -828,7 +878,24 @@ def get_possible_variables_from_concept(context, concept):
     concept_name = [t.str_value for t in Tokenizer(concept.name, yield_eof=False)]
     names = [v_value or v_name for v_name, v_value in concept.get_metadata().variables if v_name in concept_name]
     possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names)
-    return set(possible_vars)
+    to_keep = set()
+    for var in possible_vars:
+        tokens = Tokenizer(var, yield_eof=False)
+        for t in tokens:
+            if t.type in (TokenKind.IDENTIFIER, TokenKind.KEYWORD):
+                to_keep.add(var)
+    return to_keep
+
+
+def is_only_successful(sheerka, return_value):
+    """
+
+    :param sheerka:
+    :param return_value
+    :return:
+    """
+    return sheerka.isinstance(return_value, BuiltinConcepts.RETURN_VALUE) and \
+           sheerka.isinstance(return_value.body, BuiltinConcepts.ONLY_SUCCESSFUL)
 
 
 class CreateObjectIdentifiers:

src/core/concept.py

@@ -40,6 +40,20 @@ def concept_part_value(c):
     return c[1:-1]
 
 
+@dataclass
+class ConceptHints:
+    is_evaluated: bool = False  # True is the concept is evaluated by sheerka.eval_concept()
+    need_validation: bool = False  # True if the properties of the concept need to be validated
+    recognized_by: str = None
+    use_copy: bool = False
+
+    def copy(self):
+        return ConceptHints(self.is_evaluated,
+                            self.need_validation,
+                            self.recognized_by,
+                            self.use_copy)
+
+
 @dataclass
 class ConceptMetadata:
     name: str
@@ -57,8 +71,6 @@ class ConceptMetadata:
     id: str  # unique identifier for a concept. The id will never be modified (but the key can)
     props: dict  # hashmap of properties, values
     variables: list  # list of concept variables(tuple), with their default values
-    is_evaluated: bool = False  # True is the concept is evaluated by sheerka.eval_concept()
-    need_validation = False  # True if the properties of the concept need to be validated
     full_serialization: bool = False  # If True, the full object will be serialized, rather than just the diff
     all_attributes: List[str] = None  # list of instance attributes
 
@@ -158,7 +170,7 @@ class Concept:
         self._bnf = None  # parsing expression
         self._original_definition_hash = None  # concept hash before any alteration of the metadata
         self._format = None  # how to print the concept
-        self._hints = {}  # extra processing information to help processing
+        self._hints = ConceptHints()
 
     def __repr__(self):
         text = f"({self._metadata.id}){self._metadata.name}"
@@ -201,7 +213,7 @@ class Concept:
             return False
 
         for name, value in self_values.items():
-            if value == self: # not very resilient...
+            if value == self:  # not very resilient...
                 continue
 
             if value != other.get_value(name):
@@ -253,6 +265,9 @@ class Concept:
     def get_metadata(self):
         return self._metadata
 
+    def get_hints(self) -> ConceptHints:
+        return self._hints
+
     def get_compiled(self):
         return self._compiled
 
@@ -381,13 +396,15 @@ class Concept:
                     setattr(self._metadata, prop, as_dict[prop])
         return self
 
-    def update_from(self, other, update_value=True):
+    def update_from(self, other, update_value=True, update_hint=False, update_compiled=False):
         """
         Update self using the properties of another concept
         This method is to mimic the class to instance pattern
         'other' is the class, the template, and 'self' is a new instance
         :param other:
         :param update_value:
+        :param update_hint:
+        :param update_compiled:
         :return:
         """
         if other is None:
@@ -405,14 +422,23 @@ class Concept:
             else:
                 setattr(self._metadata, prop, getattr(other.get_metadata(), prop))
 
-        # # update metadata
-        # self.from_dict(other.to_dict())
-
         # update values
         if update_value:
             for k, v in other.values().items():
                 self.set_value(k, v)
 
+        if update_hint:
+            self._hints = other.get_hints().copy()
+
+        if update_compiled:
+            for k, v in other.get_compiled().items():
+                if isinstance(v, Concept) and v.get_hints().use_copy:
+                    copied = v.copy()
+                    copied.get_hints().use_copy = False
+                    self._compiled[k] = copied
+                else:
+                    self._compiled[k] = v
+
         # update bnf definition
         self._bnf = other.get_bnf()
 
@@ -510,15 +536,6 @@ class Concept:
     def variables(self):
         return {k: v for k, v in self.values().items() if not k[0] == "#"}
 
-    def set_hint(self, name, value):
-        self._hints[name] = value
-
-    def get_hint(self, name):
-        try:
-            return self._hints[name]
-        except KeyError:
-            return None
-
     def auto_init(self):
         """
         Sometimes (for tests purposes)
@@ -527,7 +544,7 @@ class Concept:
         :return:
         """
 
-        if self._metadata.is_evaluated:
+        if self.get_hints().is_evaluated:
             return self
 
         for metadata in AllConceptParts:
@@ -538,7 +555,7 @@ class Concept:
         for var, value in self._metadata.variables:
             self.set_value(var, value)
 
-        self._metadata.is_evaluated = True
+        self.get_hints().is_evaluated = True
         return self
 
     def freeze_definition_hash(self):
@@ -596,6 +613,19 @@ class Concept:
 
         return self._format.get(key, None)
 
+    def copy(self):
+        """
+        A copy is not a clone, as not all the attributes are copied
+        :return:
+        """
+        if self._metadata.is_unique:
+            return self
+
+        return Concept().update_from(self, update_hint=True, update_compiled=True)
+
+    def get_concept(self):
+        return self
+
 
 @dataclass()
 class DoNotResolve:

src/core/global_symbols.py

@@ -84,3 +84,16 @@ class SyaAssociativity(Enum):
 
 SHEERKA_BACKUP_FOLDER = "SHEERKA_BACKUP_FOLDER"
 SHEERKA_BACKUP_FILE = "SHEERKA_BACKUP_FILE"
+
+INIT_AST_PARSERS = ["ExactConcept",
+                    "Rule",
+                    "Sequence",
+                    "Sya",
+                    "Python",
+                    "Bnf",
+                    "UnrecognizedNode",
+                    "PythonWithConcepts"]
+
+INIT_AST_QUESTION_PARSERS = ["Expression"]
+
+DEFAULT_EVALUATORS = ["Python", "Concept", "LexerNode", "ValidateConcept", "ResolveAmbiguity"]

src/core/sheerka/ExecutionContext.py

@@ -74,6 +74,7 @@ class ExecutionContext:
         self.preprocess_evaluators = None
         self.preprocess = None
         self.stm = False  # True if the context has short term memory entries
+        self.possible_variables = None  # when concepts must be considered as variables
 
         self.private_hints = set()
         self.protected_hints = set()
@@ -191,6 +192,7 @@ class ExecutionContext:
         new.preprocess_parsers = self.preprocess_parsers
         new.preprocess_evaluators = self.preprocess_evaluators
         new.protected_hints.update(protected_hints)
+        new.possible_variables = self.possible_variables
 
         self._children.append(new)
 
@@ -266,6 +268,9 @@ class ExecutionContext:
         """
         return self.sheerka.get_from_short_term_memory(self, key)
 
+    def debug_short_term_memory(self):
+        return self.sheerka.get_all_short_term_memory(self, recursive=True)
+
     def get_concept(self, key):
         # search in obj
         if isinstance(self.obj, Concept):

src/core/sheerka/Sheerka.py

@@ -116,6 +116,7 @@ class Sheerka(Concept):
             "test_dict": SheerkaMethod(self.test_dict, False),
             "test_error": SheerkaMethod(self.test_error, False),
             "is_sheerka": SheerkaMethod(self.is_sheerka, False),
+            "objvalue": SheerkaMethod(self.objvalue, False),
         }
 
         self.concepts_ids = None
@@ -409,7 +410,7 @@ class Sheerka(Concept):
         """
 
         def add_recognized_by(c, _recognized_by):
-            c.set_hint(BuiltinConcepts.RECOGNIZED_BY, _recognized_by)
+            c.get_hints().recognized_by = _recognized_by
             return c
 
         def new_instances(concepts, _recognized_by):
@@ -444,8 +445,8 @@ class Sheerka(Concept):
             if concept[1]:
                 if self.is_known(found := self.get_by_id(concept[1])):
                     instance = self.new_from_template(found, found.key)
-                    instance._metadata.is_evaluated = True
-                    instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_ID)
+                    instance.get_hints().is_evaluated = True
+                    instance.get_hints().recognized_by = RECOGNIZED_BY_ID
                     return instance
             elif concept[0]:
                 if self.is_known(found := self.get_by_name(concept[0])):
@@ -466,7 +467,7 @@ class Sheerka(Concept):
 
     def fast_resolve(self, key, return_new=True):
         def add_recognized_by(c, _recognized_by):
-            c.set_hint(BuiltinConcepts.RECOGNIZED_BY, _recognized_by)
+            c.get_hints().recognized_by = _recognized_by
             return c
 
         def new_instances(concepts, _recognized_by):
@@ -557,7 +558,7 @@ class Sheerka(Concept):
                 return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept)
 
         # TODO : add the concept to the list of known concepts (self.instances)
-        concept._metadata.is_evaluated = True  # because we have manually set the variables
+        concept.get_hints().is_evaluated = True  # because we have manually set the variables
         return concept
 
     def push_ontology(self, context, name, cache_only=False):
@@ -720,6 +721,9 @@ class Sheerka(Concept):
                 else:
                     return [_obj]
 
+            if self.isinstance(_obj, BuiltinConcepts.FILTERED):
+                return inner_get_errors(_obj.reason)
+
             if isinstance(_obj, Concept) and _obj.body != NotInit:
                 return inner_get_errors(_obj.body)
 
@@ -831,7 +835,7 @@ class Sheerka(Concept):
 
         unknown_concept = UnknownConcept()  # don't use new() for prevent circular reference
         unknown_concept.set_value(ConceptParts.BODY, metadata)
-        unknown_concept._metadata.is_evaluated = True
+        unknown_concept.get_hints().is_evaluated = True
         return unknown_concept
 
     @staticmethod

src/core/sheerka/services/SheerkaAdmin.py

@@ -6,6 +6,7 @@ from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
 from core.builtin_helpers import ensure_concept_or_rule, ensure_concept
 from core.concept import Concept
 from core.global_symbols import SHEERKA_BACKUP_FOLDER
+from core.sheerka.services.SheerkaExecute import SheerkaExecute
 from core.sheerka.services.SheerkaHistoryManager import SheerkaHistoryManager
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from core.sheerka.services.sheerka_service import BaseService
@@ -26,6 +27,8 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.restore, True)
         self.sheerka.bind_service_method(self.concepts, False)
         self.sheerka.bind_service_method(self.desc, False)
+        self.sheerka.bind_service_method(self.desc_evaluators, False)
+        self.sheerka.bind_service_method(self.desc_parsers, False)
         self.sheerka.bind_service_method(self.extended_isinstance, False)
         self.sheerka.bind_service_method(self.is_container, False)
         self.sheerka.bind_service_method(self.format_rules, False)
@@ -35,7 +38,6 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.ontologies, False)
         self.sheerka.bind_service_method(self.in_memory, False)
         self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
-        self.sheerka.bind_service_method(self.admin_history, False, as_name="history")
         self.sheerka.bind_service_method(self.sdp, False)
         self.sheerka.bind_service_method(self.atomic_def, False)
 
@@ -165,7 +167,27 @@ class SheerkaAdmin(BaseService):
         concepts = sorted(self.sheerka.om.list(self.sheerka.CONCEPTS_BY_ID_ENTRY), key=lambda item: int(item.id))
         return self.sheerka.new(BuiltinConcepts.TO_LIST, body=concepts)
 
+    def desc_evaluators(self):
+        evaluators = {k: sorted(v[0].items(), reverse=True)
+                      for k, v in self.sheerka.services[SheerkaExecute.NAME].grouped_evaluators_cache.items()}
+        return self.sheerka.new(BuiltinConcepts.TO_DICT, body=evaluators)
+
+    def desc_parsers(self):
+        res = {}
+        for k, v in self.sheerka.services[SheerkaExecute.NAME].grouped_parsers_cache.items():
+            parsers = {k1: [p.__name__ for p in v1] for k1, v1 in v[0].items()}
+            sorted_parsers = sorted(parsers.items(), reverse=True)
+            res[k] = sorted_parsers
+        return self.sheerka.new(BuiltinConcepts.TO_DICT, body=res)
+
     def desc(self, *items):
+        if len(items) == 1 and isinstance(items[0], str):
+            name = items[0].strip().lower()
+            if name == "parsers":
+                return self.desc_parsers()
+            elif name == "evaluators":
+                return self.desc_evaluators()
+
         ensure_concept_or_rule(*items)
         res = []
         for item in items:

src/core/sheerka/services/SheerkaConceptManager.py

@@ -14,7 +14,7 @@ from core.builtin_helpers import ensure_concept, ensure_bnf
 from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \
     VARIABLE_PREFIX
 from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound, ErrorObj, EVENT_CONCEPT_DELETED, NoFirstToken, \
-    EVENT_CONCEPT_MODIFIED
+    EVENT_CONCEPT_MODIFIED, CONCEPT_COMPARISON_CONTEXT
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer, TokenKind
 from parsers.BnfNodeParser import RegExDef
@@ -132,6 +132,7 @@ class SheerkaConceptManager(BaseService):
         self.sheerka.bind_service_method(self.get_concepts_by_first_regex, False, visible=False)
         self.sheerka.bind_service_method(self.get_concepts_bnf_definitions, False, visible=False)
         self.sheerka.bind_service_method(self.clear_bnf_definition, True, visible=False)
+        self.sheerka.bind_service_method(self.set_precedence, True)
 
         register_concept_cache = self.sheerka.om.register_concept_cache
 
@@ -194,7 +195,7 @@ class SheerkaConceptManager(BaseService):
 
             if key in BuiltinUnique:
                 concept.get_metadata().is_unique = True
-                concept.get_metadata().is_evaluated = True
+                concept.get_hints().is_evaluated = True
 
             from_db = self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key)
             if from_db is NotFound:
@@ -728,6 +729,39 @@ class SheerkaConceptManager(BaseService):
         else:
             self.sheerka.om.clear(self.CONCEPTS_BNF_DEFINITIONS_ENTRY)
 
+    def set_precedence(self, context, *concepts):
+        """
+        Set the precedence order when parsing concept with SyaNodeParser
+        The first concept in the list have the highest priority
+        :param context:
+        :param concepts:
+        :return:
+        """
+
+        if len(concepts) < 2:
+            return self.sheerka.err("Not enough elements")
+
+        as_iterable = iter(concepts)
+        first = next(as_iterable)
+        ensure_concept(first)
+
+        try:
+            while True:
+                second = next(as_iterable)
+                ret = self.sheerka.set_is_greater_than(context,
+                                                       BuiltinConcepts.PRECEDENCE,
+                                                       first,
+                                                       second,
+                                                       CONCEPT_COMPARISON_CONTEXT)
+                if not ret.status:
+                    return ret
+
+                first = second
+        except StopIteration:
+            pass
+
+        return self.sheerka.new(BuiltinConcepts.SUCCESS)
+
     @staticmethod
     def _name_has_changed(to_add):
         if to_add is None or "meta" not in to_add:
@@ -821,7 +855,7 @@ class SheerkaConceptManager(BaseService):
         concept.get_metadata().key = None
         if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF:
             concept.set_bnf(None)
-            ensure_bnf(context, concept, update_bnf_for_cached_concept=False)
+            ensure_bnf(context, concept)
 
         concept.init_key()
 

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -1,18 +1,21 @@
 from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import expect_one, only_successful, evaluate_from_source, ensure_concept
+from core.builtin_helpers import expect_one, only_successful, ensure_concept, is_only_successful, ensure_bnf
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \
     concept_part_value
-from core.global_symbols import NotInit, CURRENT_OBJ
+from core.global_symbols import NotInit, CURRENT_OBJ, INIT_AST_PARSERS, INIT_AST_QUESTION_PARSERS
 from core.rule import Rule
-from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
-from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka.services.sheerka_service import BaseService
+from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
+from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
+from core.sheerka.services.SheerkaRuleManager import PythonConditionExprVisitor
+from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, ChickenAndEggException
 from core.tokenizer import Tokenizer
 from core.utils import unstr_concept
 from parsers.BaseExpressionParser import TrueifyVisitor
 from parsers.BaseNodeParser import ConceptNode
+from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
+from parsers.ExpressionParser import ExpressionParser
 from parsers.LogicalOperatorParser import LogicalOperatorParser
 
 CONCEPT_EVALUATION_STEPS = [
@@ -21,11 +24,6 @@ CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.AFTER_EVALUATION]
 
 
-@dataclass
-class ChickenAndEggException(Exception):
-    error: Concept
-
-
 @dataclass
 class ConceptEvalException(Exception):
     error: Concept
@@ -37,7 +35,7 @@ class WhereClauseDef:
     clause: str  # original where clause
     trueified: str  # modified where clause (where unresolvable variables are removed)
     prop: str  # variable to test
-    compiled: object  # trueified where clause Python compiled
+    conditions: list  # compiled trueified
 
 
 class SheerkaEvaluateConcept(BaseService):
@@ -45,6 +43,7 @@ class SheerkaEvaluateConcept(BaseService):
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
+        self.rule_evaluator = None
 
     def initialize(self):
         self.sheerka.bind_service_method(self.evaluate_concept, True)
@@ -52,6 +51,9 @@ class SheerkaEvaluateConcept(BaseService):
         self.sheerka.bind_service_method(self.call_concept, False, as_name="evaluate_question")
         self.sheerka.bind_service_method(self.set_auto_eval, True)
 
+    def initialize_deferred(self, context, is_first_time):
+        self.rule_evaluator = self.sheerka.services[SheerkaEvaluateRules.NAME]
+
     @staticmethod
     def infinite_recursion_detected(context, concept):
         """
@@ -86,14 +88,20 @@ class SheerkaEvaluateConcept(BaseService):
         return None
 
     @staticmethod
-    def apply_ret(concept):
+    def apply_ret(concept, eval_body=True):
         """
         Check if a concept has its RET part defined
         If True, returns it
         :param concept:
+        :param eval_body: Do not return the ret is eval body is not requested
         :return:
         """
-        return concept.get_value(ConceptParts.RET) if ConceptParts.RET in concept.values else concept
+        if (eval_body and
+            ConceptParts.RET in concept.values() and
+            (ret_value := concept.get_value(ConceptParts.RET)) != NotInit):
+            return ret_value
+        else:
+            return concept
 
     @staticmethod
     def get_needed_metadata(concept, concept_part, check_vars, check_body):
@@ -153,20 +161,37 @@ class SheerkaEvaluateConcept(BaseService):
         to_trueify = [v[0] for v in concept.get_metadata().variables if v[0] != var_name]
         trueified_where = str(TrueifyVisitor(to_trueify, [var_name]).visit(expr))
 
-        tokens = [t.str_value for t in Tokenizer(trueified_where)]
-        if var_name in tokens:
-            compiled = None
-            try:
-                compiled = compile(trueified_where, "<where clause>", "eval")
-            except Exception:
-                pass
-            return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, compiled)
-        else:
+        try:
+            parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(trueified_where)
+            parsed = ExpressionParser(auto_compile=False).parse(context, parser_input)
+            python_visitor = PythonConditionExprVisitor(context)
+            conditions = python_visitor.get_conditions(parsed.body.body)
+            return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, conditions)
+        except FailedToCompileError:
+            # TODO: manage invalid where clause
             return None
 
-    def get_recursive_definitions(self, concept, return_values):
+        # tokens = [t.str_value for t in Tokenizer(trueified_where)]
+        # if var_name in tokens:
+        #     compiled = None
+        #     try:
+        #         compiled = compile(trueified_where, "<where clause>", "eval")
+        #     except Exception:
+        #         pass
+        #     return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, compiled)
+        # else:
+        #     return None
+
+    @staticmethod
+    def get_recursive_definitions(context, concept, return_values):
         """
         Returns the name of the parsers that will resolve to a recursive evaluation
+        For example, when getting ast for Concept("a and b", body="a and b")
+            Chances are that we will end up with two parsers
+              - Python parser for 'a and b'
+              - ExactConcept parser that point to the concept itself
+        The ExactConcept will be returned (to be removed from the list as it's a cyclic reference to itself)
+        :param context:
         :param concept:
         :param return_values:
         :return:
@@ -176,8 +201,9 @@ class SheerkaEvaluateConcept(BaseService):
             # During evaluation, inner variables take precedence other concepts
             # So there won't be any cyclic reference, the variable will be picked
             return
+
         for parser in [r.body for r in return_values if
-                       r.status and self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]:
+                       r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]:
             parsed = parser.body if isinstance(parser.body, list) else [parser.body]
             for parsed_item in parsed:
                 if isinstance(parsed_item, Concept) and parsed_item.id == concept.id:
@@ -185,6 +211,85 @@ class SheerkaEvaluateConcept(BaseService):
                 elif isinstance(parsed_item, ConceptNode) and parsed_item.concept.id == concept.id:
                     yield parser.parser
 
+    @staticmethod
+    def get_asts(context, who, source, concept, part_key, possible_variables):
+        """
+        Get the return_value_concept or the concept for a given source
+        :param context:
+        :param who:
+        :param concept:
+        :param part_key: Concept part (#body#, #pre#, #where#...) being compiled
+        :param source: string or parser input, it does not matter
+        :param possible_variables: concepts that must be considered as variables
+        :return:
+        """
+
+        def parse_token_concept(s):
+            """
+            The source is a direct reference / call to another concept
+            :param s: source
+            :return:
+            """
+            if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None):
+                return context.sheerka.fast_resolve(identifier)
+            return None
+
+        def get_return_value(current_context, c, s, p):
+            """
+
+            :param current_context:
+            :param c: concept
+            :param s: source
+            :param p: part of the concept being parsed
+            :return:
+            """
+            parsers_to_use = INIT_AST_QUESTION_PARSERS if p in [ConceptParts.WHERE,
+                                                                ConceptParts.PRE] else INIT_AST_PARSERS
+            while True:
+                return_value = current_context.sheerka.parse_unrecognized(current_context,
+                                                                          s,
+                                                                          parsers=parsers_to_use,
+                                                                          who=who,
+                                                                          prop=p,
+                                                                          filter_func=only_successful,
+                                                                          possible_variables=possible_variables)
+
+                if not return_value.status:
+                    if current_context.preprocess:
+                        raise ChickenAndEggException(context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c}))
+                    else:
+                        raise FailedToCompileError([return_value.body])
+
+                return_value = return_value.body.body if is_only_successful(context.sheerka, return_value) else \
+                    [return_value]
+
+                if c is None:
+                    # No concept provided, we cannot look for recursive definition
+                    return return_value
+
+                recursive_parsers = list(SheerkaEvaluateConcept.get_recursive_definitions(context, c, return_value))
+
+                if len(recursive_parsers) == 0:
+                    return return_value
+
+                desc = f"Removing parsers {recursive_parsers}"
+                current_context = current_context.push(context.action, context.action_context, desc=desc)
+                for recursive_parser in recursive_parsers:
+                    current_context.add_preprocess(recursive_parser.name, enabled=False)
+
+        as_str = source.as_text() if isinstance(source, ParserInput) else source
+
+        if as_str.strip() == "":
+            return DoNotResolve(as_str)
+        else:
+            if concept_found := parse_token_concept(as_str):
+                # the  compiled can be a reference to another concept...
+                context.log(f"Recognized concept '{concept_found}'", SheerkaEvaluateConcept.NAME)
+                return concept_found
+            else:
+                # ...or a list of ReturnValueConcept to resolve
+                return get_return_value(context, concept, source, part_key)
+
     def apply_where_clause(self, context, where_clause_def, return_values):
         """
         Apply intermediate where clause when evaluating concept variables
@@ -195,26 +300,53 @@ class SheerkaEvaluateConcept(BaseService):
         """
         ret = []
         valid_return_values = [r for r in return_values if r.status]
+        with context.push(BuiltinConcepts.VALIDATING_CONCEPT,
+                          {"concept": where_clause_def.concept, "attr": ConceptParts.WHERE},
+                          desc=f"Apply where clause on '{where_clause_def.prop}'") as sub_context:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+            # sub_context.sheerka.add_many_to_short_term_memory(sub_context, objects)
+
         for r in valid_return_values:
-            if where_clause_def.compiled:
-                try:
-                    if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}):
-                        ret.append(r)
-                except NameError:
-                    ret.append(r)  # it cannot be solved unitary, let's give a chance to the global where condition
+            sub_context.add_to_short_term_memory(where_clause_def.prop, r.body)
+            res = self.rule_evaluator.evaluate_conditions(sub_context,
+                                                          where_clause_def.conditions,
+                                                          {where_clause_def.prop: r.body})
+            if len(res) == 0:
+                # case where missing variables were detected
+                # This means that the 'where' clause can only be evaluated after all the parts are evaluated
+                ret.append(r)
             else:
-                # it means that the where condition is an expression that needs to be executed
-                evaluation_res = evaluate_from_source(context,
-                                                      where_clause_def.trueified,
-                                                      desc=f"Apply where clause on '{where_clause_def.prop}'",
-                                                      expect_success=True,
-                                                      is_question=True,
-                                                      stm={where_clause_def.prop: r.body})
-                one_res = expect_one(context, evaluation_res)
+                one_res = expect_one(context, res)
                 if one_res.status:
                     value = context.sheerka.objvalue(one_res)
                     if isinstance(value, bool) and value:
                         ret.append(r)
+            # value = context.sheerka.objvalue(res.body)
+            # if res.status and isinstance(bool, value) and value:
+            #     ret.append(r)
+
+            # if where_clause_def.compiled:
+            #     try:
+            #         if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}):
+            #             ret.append(r)
+            #     except NameError:
+            #         ret.append(r)  # it cannot be solved unitary, let's give a chance to the global where condition
+            # else:
+            #     # it means that the where condition is an expression that needs to be executed
+            #     evaluation_res = evaluate_from_source(context,
+            #                                           where_clause_def.trueified,
+            #                                           desc=f"Apply where clause on '{where_clause_def.prop}'",
+            #                                           expect_success=True,
+            #                                           is_question=True,
+            #                                           stm={where_clause_def.prop: r.body})
+            #     one_res = expect_one(context, evaluation_res)
+            #     if one_res.status:
+            #         value = context.sheerka.objvalue(one_res)
+            #         if isinstance(value, bool) and value:
+            #             ret.append(r)
 
         if len(ret) > 0:
             return ret
@@ -261,57 +393,19 @@ class SheerkaEvaluateConcept(BaseService):
         :return:
         """
 
-        def is_only_successful(r):
-            """
+        # for BNF concepts, concepts are sometimes considered as variables
+        # example :
+        # def concept a from bnf 'bar' | 'baz'
+        # def concept foo a as 'foo' a where a == 'baz'
+        # In the second concept (foo) as is a still a concept, but also a variable in the where clause
 
-            :param r: return_value
-            :return:
-            """
-            return context.sheerka.isinstance(r, BuiltinConcepts.RETURN_VALUE) and \
-                   context.sheerka.isinstance(r.body, BuiltinConcepts.ONLY_SUCCESSFUL)
-
-        def parse_token_concept(s):
-            """
-
-            :param s: source
-            :return:
-            """
-            if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None):
-                return self.sheerka.fast_resolve(identifier)
-            return None
-
-        def get_return_value(current_context, c, s, p):
-            """
-
-            :param current_context:
-            :param c: concept
-            :param s: source
-            :param p: part of the concept being parsed
-            :return:
-            """
-            while True:
-                return_value = current_context.sheerka.parse_unrecognized(current_context,
-                                                                          s,
-                                                                          parsers="all",
-                                                                          prop=p,
-                                                                          filter_func=only_successful)
-
-                if not return_value.status:
-                    if current_context.preprocess:
-                        raise ChickenAndEggException(self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c}))
-                    else:
-                        raise Exception(f"Failed to build '{s}'. But it doesn't seems to be recursion")
-
-                return_value = return_value.body.body if is_only_successful(return_value) else [return_value]
-                recursive_parsers = list(self.get_recursive_definitions(c, return_value))
-
-                if len(recursive_parsers) == 0:
-                    return return_value
-
-                desc = f"Removing parsers {recursive_parsers}"
-                current_context = current_context.push(context.action, context.action_context, desc=desc)
-                for recursive_parser in recursive_parsers:
-                    current_context.add_preprocess(recursive_parser.name, enabled=False)
+        ensure_bnf(context, concept)
+        if concept.get_bnf():
+            visitor = BnfNodeConceptExpressionVisitor()
+            visitor.visit(concept.get_bnf())
+            possible_variables = [c.name if isinstance(c, Concept) else c for c in visitor.references]
+        else:
+            possible_variables = None
 
         for part_key in AllConceptParts:
             if part_key in concept.get_compiled():
@@ -324,16 +418,12 @@ class SheerkaEvaluateConcept(BaseService):
             if not isinstance(source, str):
                 raise Exception("Invalid concept init. metadata must be a string")
 
-            if source.strip() == "":
-                concept.get_compiled()[part_key] = DoNotResolve(source)
-            else:
-                if concept_found := parse_token_concept(source):
-                    # the  compiled can be a reference to another concept...
-                    context.log(f"Recognized concept '{concept_found}'", self.NAME)
-                    concept.get_compiled()[part_key] = concept_found
-                else:
-                    # ...or a list of ReturnValueConcept to resolve
-                    concept.get_compiled()[part_key] = get_return_value(context, concept, source, part_key)
+            concept.get_compiled()[part_key] = self.get_asts(context,
+                                                             self.NAME,
+                                                             source,
+                                                             concept,
+                                                             part_key,
+                                                             possible_variables)
 
         for var_name, default_value in concept.get_metadata().variables:
             if var_name in concept.get_compiled():
@@ -345,22 +435,12 @@ class SheerkaEvaluateConcept(BaseService):
             if not isinstance(default_value, str):
                 raise Exception("Invalid concept init. variable metadata must be a string")
 
-            if default_value.strip() == "":
-                concept.get_compiled()[var_name] = DoNotResolve(default_value)
-            else:
-                if concept_found := parse_token_concept(default_value):
-                    # the  compiled can be a reference to another concept...
-                    context.log(f"Recognized concept '{concept_found}'", self.NAME)
-                    concept.get_compiled()[var_name] = concept_found
-                else:
-                    # ...or a list of ReturnValueConcept to resolve
-                    concept.get_compiled()[var_name] = get_return_value(context, concept, default_value, var_name)
-
-        # Updates the cache of concepts when possible
-        # This piece of code is not used, a the compile part is removed by sheerka.new_from_template()
-        service = context.sheerka.services[SheerkaConceptManager.NAME]
-        if service.has_id(concept.id):
-            self.sheerka.get_by_id(concept.id).set_compiled(concept.get_compiled())
+            concept.get_compiled()[var_name] = self.get_asts(context,
+                                                             self.NAME,
+                                                             default_value,
+                                                             concept,
+                                                             var_name,
+                                                             possible_variables)
 
     def resolve(self,
                 context,
@@ -368,6 +448,7 @@ class SheerkaEvaluateConcept(BaseService):
                 current_prop,
                 current_concept,
                 force_evaluation,
+                forbid_methods_with_side_effect,
                 where_clause_def):
         """
         Resolve a variable or a Concept
@@ -375,7 +456,8 @@ class SheerkaEvaluateConcept(BaseService):
         :param to_resolve: Concept or list of ReturnValueConcept to resolve
         :param current_prop: current property or ConceptPart
         :param current_concept: current concept
-        :param force_evaluation: Force body evaluation
+        :param force_evaluation: force body evaluation
+        :param forbid_methods_with_side_effect: Do not call methods with side effect when EVAL_BODY_REQUESTED
         :param where_clause_def: intermediate where clause for variables
         :return:
         """
@@ -412,6 +494,9 @@ class SheerkaEvaluateConcept(BaseService):
             if force_evaluation:
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
+            if forbid_methods_with_side_effect:
+                sub_context.protected_hints.add(BuiltinConcepts.VALIDATION_ONLY_REQUESTED)
+
             if current_prop in (ConceptParts.WHERE, ConceptParts.PRE):
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
 
@@ -440,7 +525,7 @@ class SheerkaEvaluateConcept(BaseService):
                         value = current_concept.get_value(var[0])
                         if value != NotInit:
                             sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0]))
-                use_copy = [r for r in to_resolve] if hasattr(to_resolve, "__iter__") else to_resolve
+                use_copy = to_resolve.copy() if isinstance(to_resolve, list) else to_resolve
                 r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS)
 
                 if where_clause_def:
@@ -469,6 +554,7 @@ class SheerkaEvaluateConcept(BaseService):
                      current_prop,
                      current_concept,
                      force_evaluation,
+                     forbid_methods_with_side_effect,
                      where_clause_def):
         """When dealing with a list, there are two possibilities"""
         # It may be a list of ReturnValueConcept to execute (always the case for metadata)
@@ -483,6 +569,7 @@ class SheerkaEvaluateConcept(BaseService):
                                 current_prop,
                                 current_concept,
                                 force_evaluation,
+                                forbid_methods_with_side_effect,
                                 where_clause_def)
 
         res = []
@@ -499,6 +586,7 @@ class SheerkaEvaluateConcept(BaseService):
                              current_prop,
                              current_concept,
                              force_evaluation,
+                             forbid_methods_with_side_effect,
                              where_clause_def)
             if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
                 return r
@@ -506,19 +594,27 @@ class SheerkaEvaluateConcept(BaseService):
 
         return res
 
-    def evaluate_concept(self, context, concept: Concept, eval_body=False, metadata=None):
+    def evaluate_concept(self, context, concept: Concept, eval_body=False, validation_only=False, metadata=None):
         """
         Evaluation a concept
         ie : resolve its body
         :param context:
         :param concept:
         :param eval_body:
+        :param validation_only: When set, the body is never evaluated, whatever eval_body or EVAL_BODY_REQUESTED
         :param metadata: list of metadata to evaluate ('pre', 'post'...)
         :return: value of the evaluation or error
         """
 
-        if concept.get_metadata().is_evaluated:
-            return concept
+        failed_to_evaluate_body = False
+
+        if concept.get_hints().is_evaluated:
+            return self.apply_ret(concept, eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
+
+        # if concept.get_hints().use_copy:
+        #     raise Exception("Use copy")
+        #     concept = concept.copy()
+        #     concept.get_hints().use_copy = False
 
         # I cannot use cache because of concept like 'number'.
         # They don't have variables, but their values change every time they are instantiated
@@ -526,9 +622,9 @@ class SheerkaEvaluateConcept(BaseService):
         # need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)
         # if need_body and len(concept.get_metadata().variables) == 0 and context.sheerka.has_id(concept.id):
         #     from_cache = context.sheerka.get_by_id(concept.id)
-        #     if from_cache.get_metadata().is_evaluated:
+        #     if from_cache.get_hints().is_evaluated:
         #         concept.set_value(ConceptParts.BODY, from_cache.body)
-        #         concept.get_metadata().is_evaluated = True
+        #         concept.get_hints().is_evaluated = True
         #         return concept
 
         desc = f"Evaluating concept {concept}"
@@ -538,6 +634,10 @@ class SheerkaEvaluateConcept(BaseService):
                 # ask for body evaluation
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
+            if validation_only:
+                # Never eval the body
+                sub_context.protected_hints.add(BuiltinConcepts.VALIDATION_ONLY_REQUESTED)
+
             # auto evaluate commands
             if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
@@ -563,10 +663,23 @@ class SheerkaEvaluateConcept(BaseService):
 
                         if isinstance(prop_ast, list):
                             # Do not send the current concept for the properties
-                            resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, w_clause)
+                            resolved = self.resolve_list(
+                                sub_context,
+                                prop_ast,
+                                var_name,
+                                None,
+                                True,
+                                not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
+                                w_clause)
                         else:
                             # Do not send the current concept for the properties
-                            resolved = self.resolve(sub_context, prop_ast, var_name, None, True, w_clause)
+                            resolved = self.resolve(sub_context,
+                                                    prop_ast,
+                                                    var_name,
+                                                    None,
+                                                    True,
+                                                    not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
+                                                    w_clause)
 
                         if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
                             resolved.set_value("concept", concept)  # since current concept was not sent
@@ -592,11 +705,25 @@ class SheerkaEvaluateConcept(BaseService):
                     force_concept_eval = False if part_key == ConceptParts.BODY else True
 
                     # resolve
-                    resolved = self.resolve(sub_context, metadata_ast, part_key, concept, force_concept_eval, None)
+                    resolved = self.resolve(sub_context,
+                                            metadata_ast,
+                                            part_key,
+                                            concept,
+                                            force_concept_eval,
+                                            not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
+                                            None)
 
                     # 'FATAL' error is detected, let's stop
                     if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
-                        return resolved
+                        if not (part_key == ConceptParts.BODY and
+                                self.sheerka.has_error(context, resolved, body=BuiltinConcepts.METHOD_ACCESS_ERROR) and
+                                sub_context.in_context(BuiltinConcepts.VALIDATION_ONLY_REQUESTED)):
+                            return resolved
+                        else:
+                            # BuiltinConcepts.METHOD_ACCESS_ERROR is returned only when the access to side effect
+                            # method is forbidden (during validation or ast initialisation)
+                            resolved = NotInit
+                            failed_to_evaluate_body = True
 
                     concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved)
 
@@ -614,8 +741,8 @@ class SheerkaEvaluateConcept(BaseService):
 
             concept.init_key()  # Necessary for old unit tests. To remove someday
 
-            if ConceptParts.BODY in all_metadata_to_eval:
-                concept.get_metadata().is_evaluated = True
+            if ConceptParts.BODY in all_metadata_to_eval and not failed_to_evaluate_body:
+                concept.get_hints().is_evaluated = True
 
             # # update the cache for concepts with no variables
             # Cannot use cache. See the comment at the beginning of this method
@@ -626,10 +753,7 @@ class SheerkaEvaluateConcept(BaseService):
                 self.sheerka.register_object(sub_context, concept.name, concept)
 
             # manage RET metadata
-            if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values():
-                return concept.get_value(ConceptParts.RET)
-            else:
-                return concept
+            return self.apply_ret(concept, sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
 
     def call_concept(self, context, concept, *args, **kwargs):
         """
@@ -656,7 +780,7 @@ class SheerkaEvaluateConcept(BaseService):
         needed, variables, body = self.get_needed_metadata(concept, ConceptParts.PRE, True, True)
         to_eval.extend(needed)
 
-        if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.get_metadata().need_validation:
+        if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.get_hints().need_validation:
             # What are the cases where we do not need a validation ?
             # see test_sheerka_non_reg::test_i_can_evaluate_bnf_concept_with_where_clause()
             # res = sheerka.evaluate_user_input("foobar")

src/core/sheerka/services/SheerkaEvaluateRules.py

@@ -111,13 +111,23 @@ class SheerkaEvaluateRules(BaseService):
 
         return expect_one(context, results)
 
-    def evaluate_conditions(self, context, conditions, bag):
+    def evaluate_conditions(self, context, conditions, bag, missing_vars=None):
+        """
+        Evaluate the conditions
+        :param context:
+        :param conditions:
+        :param bag: variables that are supposed to be in short term memory
+        :param missing_vars: if initialized to a set, keeps tracks of the missing variables
+        :return:
+        """
         bag_variables = set(bag.keys())
 
         results = []
         for compiled_condition in conditions:
 
             if compiled_condition.variables.intersection(bag_variables) != compiled_condition.variables:
+                if isinstance(missing_vars, set):
+                    missing_vars.update(compiled_condition.variables - bag_variables)
                 continue
 
             if compiled_condition.not_variables.intersection(bag_variables):
@@ -131,11 +141,12 @@ class SheerkaEvaluateRules(BaseService):
 
                 # do not forget to reset the 'is_evaluated' in the case of a concept
                 for concept in compiled_condition.concepts_to_reset:
-                    concept.get_metadata().is_evaluated = False
+                    concept.get_hints().is_evaluated = False
 
                 evaluator = self.evaluators_by_name[compiled_condition.evaluator_type]
                 res = evaluator.eval(context, compiled_condition.return_value)
-                if res.status and isinstance(res.body, bool) and res.body:
+                value = context.sheerka.objvalue(res.body)
+                if res.status and isinstance(value, bool) and value:
                     # one successful value found. No need to look any further
                     results = [res]  # don't we care about the other failing results ?
                     break

src/core/sheerka/services/SheerkaExecute.py

@@ -2,7 +2,7 @@ import core.utils
 from cache.FastCache import FastCache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.concept import ConceptParts
-from core.global_symbols import NotFound, NO_MATCH
+from core.global_symbols import NotFound, NO_MATCH, EVENT_CONCEPT_CREATED, EVENT_CONCEPT_MODIFIED, EVENT_CONCEPT_DELETED
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer, TokenKind, Token, Keywords
 
@@ -15,6 +15,8 @@ ALL_STEPS = PARSE_AND_EVAL_STEPS + [BuiltinConcepts.BEFORE_RENDERING,
                                     BuiltinConcepts.AFTER_RENDERING,
                                     BuiltinConcepts.BEFORE_RULES_EVALUATION,
                                     BuiltinConcepts.AFTER_RULES_EVALUATION]
+STM_PARSER_NAME = "ShortTermMemory"
+DEFAULT = "__default"
 
 
 class ParserInput:
@@ -190,17 +192,20 @@ class SheerkaExecute(BaseService):
         # order must be after SheerkaEvaluateRules because of self.rules_evaluation_service
         # order must be after ConceptManager because it needs concept bnf definitions
         super().__init__(sheerka, order=15)
-        self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=20)
+        self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=200)
+        self.parsers_cache = FastCache(max_size=2000)
+
         self.instantiated_evaluators = None
         self.evaluators_by_name = None
 
         self.instantiated_parsers = None
         self.parsers_by_name = None
         self.preprocessed_items_old_values = []
+        self.question_parsers = []  # parsers to use when BuiltinConcepts.EVAL_QUESTION_REQUESTED is set
 
         # cache for all preregistered evaluator combination
-        # the key is the concatenation of the step and the name of evaluators in the group
-        # ex : BEFORE_EVALUATION|Python|Sya|Bnf
+        # the key is a tuple with the name of the step the names of the evaluators in the group
+        # ex : (BEFORE_EVALUATION, "Python|Sya|Bnf")
         # The value is a tuple,
         # The first entry is the grouped evaluators
         # ex : {60 : [PythonEvaluator(), SyaEvaluator()], 50: [BnfEvaluator()]}
@@ -209,7 +214,7 @@ class SheerkaExecute(BaseService):
 
         # cache for preregistered parsers
         # Same construction than the evaluators
-        # Except 1 : the key does not have a step component. It is simple the list of parsers' names
+        # Except 1 : the key is a tuple (concept_hint or DEFAULT, names of the parsers)
         # Except 2 : we store the type of the parser, not its instance
         self.grouped_parsers_cache = {}
 
@@ -229,8 +234,13 @@ class SheerkaExecute(BaseService):
         from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
         self.rules_eval_service = self.sheerka.services[SheerkaEvaluateRules.NAME]
 
+        self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.on_concepts_modified)
+        self.sheerka.subscribe(EVENT_CONCEPT_MODIFIED, self.on_concepts_modified)
+        self.sheerka.subscribe(EVENT_CONCEPT_DELETED, self.on_concepts_modified)
+
     def reset_state(self):
         self.pi_cache.clear()
+        self.parsers_cache.clear()
 
     def reset_registered_evaluators(self):
         # instantiate evaluators, once for all, only keep when it's enabled
@@ -240,7 +250,7 @@ class SheerkaExecute(BaseService):
 
         # get default evaluators by process step
         for process_step in ALL_STEPS:
-            self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped(
+            self.grouped_evaluators_cache[(process_step, DEFAULT)] = self.get_grouped(
                 [e for e in self.instantiated_evaluators if process_step in e.steps])
 
     def reset_registered_parsers(self):
@@ -249,10 +259,20 @@ class SheerkaExecute(BaseService):
         :return:
         """
         self.instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()]
-        self.instantiated_parsers = [p for p in self.instantiated_parsers if p.enabled]
+        self.instantiated_parsers = [p for p in self.instantiated_parsers if p.enabled and p.name != STM_PARSER_NAME]
         self.parsers_by_name = {p.short_name: p for p in self.instantiated_parsers}
 
-        self.grouped_parsers_cache["__default"] = self.get_grouped(self.instantiated_parsers, use_classes=True)
+        default_parsers = [p for p in self.instantiated_parsers if p.hints is None]
+        self.grouped_parsers_cache[(DEFAULT, DEFAULT)] = self.get_grouped(default_parsers, use_classes=True)
+
+        # By default, we use the same parsers when it's a question
+        question_parsers = [p for p in self.instantiated_parsers if
+                            p.hints is not None and BuiltinConcepts.EVAL_QUESTION_REQUESTED in p.hints]
+        self.grouped_parsers_cache[(BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT)] = self.get_grouped(
+            # default_parsers,
+            question_parsers,
+            use_classes=True)
+        self.question_parsers = [p.name for p in question_parsers]
 
     @staticmethod
     def get_grouped(evaluators, use_classes=False):
@@ -302,12 +322,12 @@ class SheerkaExecute(BaseService):
         """
         # Normal case, the evaluators are the default one
         if not context.preprocess_evaluators and not context.preprocess:
-            return self.grouped_evaluators_cache[f"{process_step}|__default"]
+            return self.grouped_evaluators_cache[(process_step, DEFAULT)]
 
         # Other case, only use a subset of evaluators
         selected = context.preprocess_evaluators
         if selected and not context.preprocess:
-            key = str(process_step) + "|" + "|".join(selected)
+            key = (process_step, "|".join(selected))
             try:
                 return self.grouped_evaluators_cache[key]
             except KeyError:
@@ -357,6 +377,15 @@ class SheerkaExecute(BaseService):
         groups, sorted_priorities = self.get_grouped(parsers, use_classes=True)
         return key, *get_instances((groups, sorted_priorities))
 
+    def get_input_as_text(self, text):
+        if isinstance(text, str):
+            return text
+
+        if isinstance(text, ParserInput):
+            return text.as_text()
+
+        raise NotImplementedError()
+
     def get_parser_input(self, text, tokens=None):
         """
         Returns new or existing parser input
@@ -386,17 +415,44 @@ class SheerkaExecute(BaseService):
         """
         From the context.preprocess_parsers and context.preprocess,
         try to find a key to store the further results of the parsings
+        The key is a two values tuple
+            * The first part indicates the parsers to use (__default for the hardcoded default behaviour)
+            * The second part indicates some context hint, like for instance if it's question
         :param context:
         :return:
         """
-        if not context.preprocess_parsers and not context.preprocess:
-            return "__default"
+        # as of now, EVAL_QUESTION_REQUESTED is the only hint that can alter the parsing
+        if context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED):
+            in_context = BuiltinConcepts.EVAL_QUESTION_REQUESTED
+        else:
+            in_context = DEFAULT
 
-        if context.preprocess_parsers and not context.preprocess:
-            return "|".join(context.preprocess_parsers)
+        if context.preprocess:
+            from parsers.BaseParser import BaseParser
+            preprocess = [p for p in context.preprocess if p.preprocess_name.startswith(BaseParser.PREFIX)]
+        else:
+            preprocess = None
+
+        if not context.preprocess_parsers and not preprocess:
+            return in_context, DEFAULT
+
+        if context.preprocess_parsers and not preprocess:
+            return in_context, "|".join(context.preprocess_parsers)
 
         return None
 
+    def add_to_parser_cache(self, parsers_key, text, return_value):
+        if parsers_key is None:
+            return
+
+        key = (parsers_key, text)
+        if key in self.parsers_cache:
+            old = self.parsers_cache.get(key)
+            old.append((return_value.who, return_value.status, return_value.value))
+            self.parsers_cache.put(key, old)
+        else:
+            self.parsers_cache.put(key, [(return_value.who, return_value.status, return_value.value)])
+
     def call_parsers(self, context, return_values):
         """
         Call all the parsers, ordered by priority
@@ -414,7 +470,7 @@ class SheerkaExecute(BaseService):
         if not isinstance(return_values, list):
             return_values = [return_values]
 
-        # first make the distinguish between what is for the parsers and what is not
+        # 1. Make the distinguish between what is for the parsers and what is not
         result = []
         to_process = []
         for r in return_values:
@@ -431,58 +487,110 @@ class SheerkaExecute(BaseService):
 
         parsers_key, grouped_parsers, sorted_priorities = self.get_parsers(context)
 
-        stop_processing = False
-        for priority in sorted_priorities:
-            inputs_for_this_group = to_process[:]
+        # 2. Try the stm parser, as it depends on the context
+        from parsers.ShortTermMemoryParser import ShortTermMemoryParser
+        if parsers_key is None or parsers_key[1] == DEFAULT or ShortTermMemoryParser.NAME in parsers_key[1]:
+            try:
+                stm_parser = self.parsers_by_name[STM_PARSER_NAME]
+                if stm_parser.enabled:
+                    processed = []
+                    for return_value in to_process:
+                        to_parse = self.get_parser_input(return_value.body.body) \
+                            if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \
+                            else return_value.body
 
-            for parser in grouped_parsers[priority]:
-
-                for return_value in inputs_for_this_group:
-
-                    to_parse = self.get_parser_input(return_value.body.body) \
-                        if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \
-                        else return_value.body
-
-                    # if self.sheerka.log.isEnabledFor(logging.DEBUG):
-                    #     debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \
-                    #         else "'" + core.utils.get_text_from_tokens(to_parse) + "' as tokens"
-                    #     context.log(f"Parsing {debug_text}")
-
-                    with context.push(BuiltinConcepts.PARSING,
-                                      {"parser": parser.name},
-                                      desc=f"Parsing using {parser.name}") as sub_context:
-                        sub_context.add_inputs(to_parse=to_parse)
-                        res = parser.parse(sub_context, to_parse)
-                        if res is not None:
-                            if hasattr(res, "__iter__"):
-                                for r in res:
-                                    if r is None:
-                                        continue
-                                    r.parents = [return_value]
-                                    result.append(r)
-                                    if self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT):
-                                        # if a ParserResultConcept is returned, it will be used by the parsers
-                                        # of the following groups
-                                        to_process.append(r)
-                                    if r.status:
-                                        stop_processing = True
-
-                            else:
+                        with context.push(BuiltinConcepts.PARSING,
+                                          {"parser": stm_parser.name},
+                                          desc=f"Parsing using {stm_parser.name}") as sub_context:
+                            sub_context.add_inputs(to_parse=to_parse)
+                            res = stm_parser.parse(sub_context, to_parse)
+                            if res.status:
                                 res.parents = [return_value]
                                 result.append(res)
-                                if self.sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT):
-                                    # if a ParserResultConcept is returned, it will be used by the parsers
-                                    # of the following groups
-                                    to_process.append(res)
-                                if res.status:
-                                    stop_processing = True
-                        sub_context.add_values(return_values=res)
+                                processed.append(return_value)
+                            sub_context.add_values(return_values=res)
 
-            if stop_processing:
-                break  # Do not try the other priorities if a match is found
+                    to_process = core.utils.remove_list_from_list(to_process, processed)
+
+            except KeyError:
+                # stm_parser may not exist in some unit tests
+                pass
+
+        # 3. Try the cache
+        if to_process and parsers_key:
+            processed = []
+            for return_value in to_process:
+                to_parse_as_str = self.get_input_as_text(return_value.body.body) \
+                    if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \
+                    else return_value.body.source
+
+                key_to_use = (parsers_key, to_parse_as_str)
+                if key_to_use in self.parsers_cache:
+                    for who, status, value in self.parsers_cache.get(key_to_use):
+                        ret = self.sheerka.ret(who, status, value)
+                        ret.parents = [return_value]
+                        result.append(ret)
+                    processed.append(return_value)
+            to_process = core.utils.remove_list_from_list(to_process, processed)
+
+        # 4. Call the parsers
+        if to_process:
+            stop_processing = False
+            for priority in sorted_priorities:
+                inputs_for_this_group = to_process[:]
+
+                for parser in grouped_parsers[priority]:
+
+                    for return_value in inputs_for_this_group:
+
+                        if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT):
+                            to_parse_as_str = self.get_input_as_text(return_value.body.body)
+                            to_parse = self.get_parser_input(return_value.body.body)
+                        else:
+                            to_parse = return_value.body
+                            to_parse_as_str = return_value.body.source
+
+                        # if self.sheerka.log.isEnabledFor(logging.DEBUG):
+                        #     debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \
+                        #         else "'" + core.utils.get_text_from_tokens(to_parse) + "' as tokens"
+                        #     context.log(f"Parsing {debug_text}")
+
+                        with context.push(BuiltinConcepts.PARSING,
+                                          {"parser": parser.name},
+                                          desc=f"Parsing using {parser.name}") as sub_context:
+                            sub_context.add_inputs(to_parse=to_parse)
+                            res = parser.parse(sub_context, to_parse)
+                            if res is not None:
+                                if hasattr(res, "__iter__"):
+                                    for r in res:
+                                        if r is None:
+                                            continue
+                                        r.parents = [return_value]
+                                        result.append(r)
+                                        self.add_to_parser_cache(parsers_key, to_parse_as_str, r)
+                                        if self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT):
+                                            # if a ParserResultConcept is returned, it will be used by the parsers
+                                            # of the following groups
+                                            to_process.append(r)
+                                        if r.status:
+                                            stop_processing = True
+
+                                else:
+                                    res.parents = [return_value]
+                                    result.append(res)
+                                    self.add_to_parser_cache(parsers_key, to_parse_as_str, res)
+                                    if self.sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT):
+                                        # if a ParserResultConcept is returned, it will be used by the parsers
+                                        # of the following groups
+                                        to_process.append(res)
+                                    if res.status:
+                                        stop_processing = True
+                            sub_context.add_values(return_values=res)
+
+                if stop_processing:
+                    break  # Do not try the other priorities if a match is found
 
         result = core.utils.remove_list_from_list(result, user_inputs)
-
         return result
 
     def call_evaluators(self, context, return_values, process_step):
@@ -687,7 +795,12 @@ class SheerkaExecute(BaseService):
         else:
             return parser_or_evaluator_name == preprocessor_name
 
-    def parse_unrecognized(self, context, source, parsers, who=None, prop=None, filter_func=None):
+    def parse_unrecognized(self, context, source, parsers,
+                           who=None,
+                           prop=None,
+                           filter_func=None,
+                           is_question=False,
+                           possible_variables=None):
         """
         Try to recognize concepts or code from source using the given parsers
         :param context:
@@ -696,6 +809,8 @@ class SheerkaExecute(BaseService):
         :param who: who is asking the parsing ?
         :param prop: Extra info, when parsing a property
         :param filter_func: Once the result are found, call this function to filter them
+        :param is_question: Force EVAL_QUESTION_REQUESTED
+        :param possible_variables: concepts that must be considered as variables
         :return:
         """
         sheerka = context.sheerka
@@ -708,14 +823,18 @@ class SheerkaExecute(BaseService):
             desc = f"Parsing '{source}'"
 
         with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context:
+
+            if (prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN) or
+                is_question):
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+
             # disable all parsers but the requested ones
             if parsers != "all":
                 sub_context.preprocess_parsers = parsers
             else:
                 sub_context.preprocess_parsers = None
 
-            if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN):
-                sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+            sub_context.possible_variables = possible_variables
 
             sub_context.add_inputs(source=source)
             to_parse = sheerka.ret(context.who,
@@ -779,7 +898,7 @@ class SheerkaExecute(BaseService):
             python_parser = PythonParser()
             return python_parser.parse(sub_context, parser_input)
 
-    def parse_expression(self, context, source, desc=None):
+    def parse_expression(self, context, source, desc=None, auto_compile=False):
         """
         Helper function to parser expressions with AND, OR and NOT
         """
@@ -787,5 +906,8 @@ class SheerkaExecute(BaseService):
         with context.push(BuiltinConcepts.PARSE_CODE, source, desc) as sub_context:
             parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(source)
             from parsers.ExpressionParser import ExpressionParser
-            expr_parser = ExpressionParser()
+            expr_parser = ExpressionParser(auto_compile=auto_compile)
             return expr_parser.parse(sub_context, parser_input)
+
+    def on_concepts_modified(self, *args, **kwargs):
+        self.parsers_cache.clear()

src/core/sheerka/services/SheerkaIsAManager.py

@@ -1,12 +1,12 @@
 import core.builtin_helpers
 from cache.Cache import Cache
 from cache.SetCache import SetCache
-from core.ast_helpers import UnreferencedVariablesVisitor
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF
+from core.concept import Concept, DEFINITION_TYPE_BNF
 from core.global_symbols import NotFound
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.sheerka_service import BaseService
+from core.tokenizer import Tokenizer, TokenKind
 from core.utils import merge_sets
 
 
@@ -153,16 +153,7 @@ class SheerkaIsAManager(BaseService):
 
             # apply the where clause if any
             if sub_concept.get_metadata().where:
-                new_condition = self._validate_where_clause(context, sub_concept)
-                if not new_condition:
-                    return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=sub_concept)
-
-                # This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
-                # to do it properly now. It will be enhanced later
-                globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka}
-                locals_ = {}
-                exec(new_condition, globals_, locals_)
-                concepts = locals_["result"]
+                concepts = self._filter_results(context, sub_concept, concepts)
 
             return concepts
 
@@ -201,7 +192,7 @@ class SheerkaIsAManager(BaseService):
             return False
 
         for c in a.get_metadata().props[BuiltinConcepts.ISA]:
-            if c == b:
+            if c.id == b.id:
                 return True
             if self.isa(self.sheerka.get_by_id(c.id), b):
                 return True
@@ -237,28 +228,25 @@ class SheerkaIsAManager(BaseService):
 
         return self.isaset(context, concept.body)
 
-    def _validate_where_clause(self, context, concept):
-        python_parser_result = [r for r in concept.get_compiled()[ConceptParts.WHERE] if r.who == "parsers.Python"]
-        if not python_parser_result or not python_parser_result[0].status:
-            return None
+    def _filter_results(self, context, concept, results):
+        """
+        Filter the list of results, according to the specification of the concept
+        ex: def concept sub_concept as number where number < 4
+        We want to return the numbers that are < 4
+        :param context:
+        :param concept:
+        :param results:
+        :return:
+        """
+        # first get the pivot variable
+        possibles_variables = [t.value for t in Tokenizer(concept.get_metadata().body, yield_eof=False) if
+                               t.type in (TokenKind.IDENTIFIER, TokenKind.KEYWORD)]
+        if len(possibles_variables) != 1:
+            return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=concept)
 
-        ast_ = python_parser_result[0].body.body.ast_
-        visitor = UnreferencedVariablesVisitor(context)
-        names = list(visitor.get_names(ast_))
-        if len(names) != 1 or names[0] != concept.get_metadata().body:
-            return None
-
-        condition = concept.get_metadata().where.replace(concept.get_metadata().body, "sheerka.objvalue(x)")
-        expression = f"""
-result=[]
-for x in xx__concepts__xx:
-    try:
-        if {condition}:
-            result.append(x)
-    except Exception:
-        pass
-"""
-        return expression
+        predicate = concept.get_metadata().where.replace(possibles_variables[0], "sheerka.objvalue(self)")
+        res = self.sheerka.filter_objects(context, results, predicate)
+        return res
 
     def _get_concepts(self, context, ids, evaluate):
         """

src/core/sheerka/services/SheerkaMemory.py

@@ -222,12 +222,14 @@ class SheerkaMemory(BaseService):
         """
         name_to_use = name.name if isinstance(name, Concept) else name
         self.unregister_object(context, name_to_use)
+
+        # first try direct access
         obj = self.get_last_from_memory(context, name_to_use)
         if obj is not NotFound:
             return obj.obj
 
-        all_objects = self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)
-        all_objects_copy = []
+        all_objects = self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)  # not always a list of list
+        all_objects_copy = []  # to transform into list of list
         for obj in all_objects:
             if isinstance(obj, list):
                 all_objects_copy.append(obj.copy())
@@ -242,15 +244,15 @@ class SheerkaMemory(BaseService):
                 if len(obj) > 0:
                     temp.append(obj)
 
-            all_objects_copy = temp
+            all_objects_copy = temp  # list constructed with the last item of each item
             current_list = sorted(current_list, key=lambda o: o.timestamp, reverse=True)
             current_objects = [o.obj for o in current_list]
 
-            res = self.sheerka.filter_objects(context, current_objects, name)
+            res = self.sheerka.filter_objects(context, current_objects, name_to_use)
             if len(res) > 0:
                 return res[0]  # only the first, as it should have a better timestamp
 
-        return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name})
+        return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name_to_use})
 
     def mem(self):
         keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)])

src/core/sheerka/services/SheerkaQueryManager.py

@@ -19,7 +19,7 @@ class SheerkaQueryManager(BaseService):
     QUERY_PARAMETER_PREFIX = "__xxx__query_parameter__xx__"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka)
+        super().__init__(sheerka, order=16)
         self.queries = FastCache()
         self.conditions = FastCache()
         self.lexer = Lexer()

src/core/sheerka/services/SheerkaRuleManager.py

@@ -1,610 +1,38 @@
 import operator
-import re
 from dataclasses import dataclass
 from typing import Union, Set, List, Tuple
 
 from cache.Cache import Cache
 from cache.ListIfNeededCache import ListIfNeededCache
+from core.ast_helpers import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.builtin_helpers import ensure_evaluated, expect_one, evaluate_from_source, \
-    get_possible_variables_from_concept, is_a_question
+    get_possible_variables_from_concept, is_a_question, only_successful, is_only_successful, evaluate_return_values
 from core.concept import Concept
 from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound, ErrorObj, \
-    EVENT_RULE_CREATED, EVENT_RULE_DELETED, NotInit
+    EVENT_RULE_CREATED, EVENT_RULE_DELETED, NotInit, INIT_AST_PARSERS
 from core.rule import Rule, ACTION_TYPE_PRINT
 from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
-from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError
-from core.tokenizer import Keywords, TokenKind, Token, IterParser
-from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets, get_safe_str_value
+from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, UnknownVariableError
+from core.tokenizer import Keywords, TokenKind, Token
+from core.utils import merge_dictionaries, merge_sets, get_safe_str_value
 from evaluators.PythonEvaluator import PythonEvaluator
 from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode, \
     ComparisonType, NotNode, NameExprNode
 from parsers.BaseNodeParser import ConceptNode
+from parsers.FormatRuleActionParser import FormatRuleActionParser
 from parsers.LogicalOperatorParser import LogicalOperatorParser
 from sheerkapython.python_wrapper import Expando, sheerka_globals
 from sheerkarete.common import V
 from sheerkarete.conditions import AndConditions, Condition, NegatedCondition, NegatedConjunctiveConditions
 from sheerkarete.network import FACT_NAME, FACT_SELF
 
-CONCEPTS_ONLY_PARSERS = ["ExactConcept", "Bnf", "Sya", "Sequence"]
-CONDITIONS_VISITOR_EVALUATORS = ["Python", "Concept", "LexerNode"]
-
-identifier_regex = re.compile(r"[\w _.]+")
-
-
-@dataclass
-class FormatRuleError(ErrorObj):
-    pass
-
-
-@dataclass
-class BraceMismatch(FormatRuleError):
-    lbrace: Token
-
-
-@dataclass
-class UnexpectedEof(FormatRuleError):
-    message: str
-    token: Token = None
-
-    def __eq__(self, other):
-        if id(self) == id(other):
-            return True
-
-        if not isinstance(other, UnexpectedEof):
-            return False
-
-        return self.message == other.message and (other.token is None or other.token == self.token)
-
-    def __hash__(self):
-        return hash(self.message, self.token)
-
-
-@dataclass
-class FormatRuleSyntaxError(FormatRuleError):
-    message: str
-    token: Token
-
-
-@dataclass
-class FormatAstNode:
-    @staticmethod
-    def repr_value(items):
-        if items is None:
-            return ""
-
-        return ", ".join(repr(item) for item in items)
-
-    def clone(self, instance, props, **kwargs):
-        for prop_name in props:
-            setattr(instance, prop_name, getattr(self, prop_name))
-
-        for k, v in kwargs.items():
-            setattr(instance, k, v)
-
-        return instance
-
-
-@dataclass
-class FormatAstRawText(FormatAstNode):
-    text: str
-
-
-@dataclass
-class FormatAstVariable(FormatAstNode):
-    name: str
-    format: Union[str, None] = None
-    debug: bool = False
-    value: object = None
-    index: object = None
-
-    def clone(self, **kwargs):
-        return super().clone(FormatAstVariable(self.name),
-                             ("format", "debug", "value", "index"),
-                             **kwargs)
-
-
-@dataclass
-class FormatAstVariableNotFound(FormatAstNode):
-    name: str
-
-
-@dataclass
-class FormatAstGrid(FormatAstNode):
-    pass
-
-
-@dataclass
-class FormatAstList(FormatAstNode):
-    variable: str
-    items_prop: str = None  # where to search the list if variable does not resolve to an iterable
-    recurse_on: str = None
-    recursion_depth: int = 0
-    debug: bool = False
-    prefix: str = None
-    suffix: str = None
-    show_index: bool = False
-    index: object = None
-
-    items: object = None
-
-    def clone(self, **kwargs):
-        return super().clone(
-            FormatAstList(self.variable),
-            (
-                "items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "index",
-                "items"),
-            **kwargs)
-
-
-@dataclass
-class FormatAstDict(FormatAstNode):
-    variable: str
-    items_prop: str = None  # where to search the dict if variable does not resolve to an iterable
-    debug: bool = False
-    prefix: str = None
-    suffix: str = None
-
-    items: object = None
-
-    def clone(self, **kwargs):
-        return super().clone(
-            FormatAstDict(self.variable),
-            ("items_prop", "debug", "prefix", "suffix", "items"),
-            **kwargs)
-
-
-@dataclass
-class FormatAstColor(FormatAstNode):
-    color: str
-    format_ast: FormatAstNode
-
-    def __repr__(self):
-        return f"{self.color}({self.format_ast})"
-
-    def clone(self, **kwargs):
-        return super().clone(
-            FormatAstColor(self.color, self.format_ast),
-            (),
-            **kwargs)
-
-
-@dataclass
-class FormatAstFunction(FormatAstNode):
-    name: str
-    args: list = None
-    kwargs: dict = None
-
-
-@dataclass
-class FormatAstSequence(FormatAstNode):
-    items: list
-    debug: bool = False
-
-    def __repr__(self):
-        return "FormatAstSequence(" + self.repr_value(self.items) + ")"
-
-    def clone(self, **kwargs):
-        return super().clone(
-            FormatAstSequence(self.items),
-            ("debug",),
-            **kwargs)
-
-
-@dataclass
-class FormatAstMulti(FormatAstNode):
-    """
-    Used when there are multiple out to print, but they are not related
-    Just print them one by one
-    """
-    variable: str
-    items: list = None
-
-    def __repr__(self):
-        return f"FormatAstMulti({self.variable}, items={self.items})"
-
-    def clone(self, **kwargs):
-        return super().clone(
-            FormatAstMulti(self.variable),
-            ("items",),
-            **kwargs)
-
-
-class FormatRuleActionParser(IterParser):
-
-    @staticmethod
-    def to_text(list_or_dict_of_tokens):
-        """
-        Works on list of list of tokens
-        or dict of list of tokens
-        :param list_or_dict_of_tokens:
-        :return:
-        """
-        get_text = get_text_from_tokens
-        if isinstance(list_or_dict_of_tokens, list):
-            return [get_text(i) for i in list_or_dict_of_tokens]
-        if isinstance(list_or_dict_of_tokens, dict):
-            return {k: get_text(v) for k, v in list_or_dict_of_tokens.items()}
-        raise NotImplementedError("")
-
-    def to_value(self, tokens):
-        """
-        Works on list of tokens
-        return string or numeric value of the tokens
-        :return:
-        """
-
-        value = get_text_from_tokens(tokens)
-        if value[0] in ("'", '"'):
-            return value[1:-1]
-
-        if value in ("True", "False"):
-            return bool(value)
-
-        try:
-            return int(value)
-        except ValueError:
-            pass
-
-        try:
-            return float(value)
-        except ValueError:
-            self.error_sink = FormatRuleSyntaxError(f"'{value}' is not numeric", None)
-
-    def parse(self):
-        """
-        Parses the print part of the format rule
-        format ::= {variable'} | function(...) | rawtext
-        :return:
-        """
-
-        if self.source == "":
-            return FormatAstRawText("")
-
-        buffer = []
-        result = []
-        res = None
-        escaped = False
-
-        def _flush_buffer():
-            if len(buffer) > 0:
-                result.append(FormatAstRawText(get_text_from_tokens(buffer)))
-                buffer.clear()
-
-        while self.next_token(skip_whitespace=False):
-            if not escaped:
-                if self.token.type == TokenKind.IDENTIFIER and self.the_token_after().type == TokenKind.LPAR:
-                    _flush_buffer()
-                    res = self.parse_function(self.token)
-                elif self.token.type == TokenKind.LBRACE:
-                    _flush_buffer()
-                    res = self.parse_variable(self.token)
-                elif self.token.type == TokenKind.BACK_SLASH:
-                    escaped = True
-                else:
-                    buffer.append(self.token)
-            else:
-                escaped = False
-                buffer.append(self.token)
-
-            if self.error_sink:
-                break
-
-            if res:
-                result.append(res)
-                res = None
-
-        _flush_buffer()
-
-        return [] if len(result) == 0 else result[0] if len(result) == 1 else FormatAstSequence(result)
-
-    def parse_function(self, func_name):
-        self.next_token()
-        self.next_token()
-
-        if self.token.type == TokenKind.EOF:
-            self.error_sink = UnexpectedEof("while parsing function", func_name)
-            return None
-
-        param_buffer = []
-        args = []
-        kwargs = {}
-        get_text = get_text_from_tokens
-
-        def _process_parameters():
-            if len(param_buffer) == 0:
-                self.error_sink = FormatRuleSyntaxError("no parameter found", self.token)
-                return None
-            if (index := index_tokens(param_buffer, "=")) > 0:
-                kwargs[get_text(param_buffer[:index])] = param_buffer[index + 1:]
-            else:
-                args.append(param_buffer.copy())
-            param_buffer.clear()
-
-        while True:
-            if self.token.type == TokenKind.RPAR:
-                if len(param_buffer) > 0:
-                    _process_parameters()
-                break
-
-            elif self.token.type == TokenKind.COMMA:
-                _process_parameters()
-                if self.error_sink:
-                    break
-
-            else:
-                param_buffer.append(self.token)
-
-            if not self.next_token():
-                break
-
-        if self.error_sink:
-            return None
-
-        if self.token.type != TokenKind.RPAR:
-            self.error_sink = UnexpectedEof("while parsing function", func_name)
-            return None
-
-        if func_name.value in COLORS:
-            return self.return_color(func_name.value, args, kwargs)
-        elif func_name.value == "list":
-            return self.return_list(args, kwargs)
-        elif func_name.value == "dict":
-            return self.return_dict(args, kwargs)
-        elif func_name.value == "multi":
-            return self.return_multi(args, kwargs)
-
-        return FormatAstFunction(func_name.value, self.to_text(args), self.to_text(kwargs))
-
-    def parse_variable(self, lbrace):
-        self.next_token()
-
-        if self.token.type == TokenKind.EOF:
-            self.error_sink = UnexpectedEof("while parsing variable", lbrace)
-            return None
-
-        buffer = []
-        while True:
-            if self.token.type == TokenKind.RBRACE:
-                break
-            buffer.append(self.token)
-
-            if not self.next_token():
-                break
-
-        # if self.error_sink:
-        #     return None
-
-        if self.token.type != TokenKind.RBRACE:
-            self.error_sink = UnexpectedEof("while parsing variable", lbrace)
-            return None
-
-        if len(buffer) == 0:
-            self.error_sink = FormatRuleSyntaxError("variable name not found", None)
-            return None
-
-        variable = get_text_from_tokens(buffer)
-        try:
-            index = variable.index(":")
-            return FormatAstVariable(variable[:index], variable[index + 1:])
-        except ValueError:
-            return FormatAstVariable(variable)
-
-    def return_color(self, color, args, kwargs):
-        if len(kwargs) > 0:
-            self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None)
-            return None
-
-        if len(args) == 0:
-            return FormatAstColor(color, FormatAstRawText(""))
-
-        if len(args) > 1:
-            self.error_sink = FormatRuleSyntaxError("only one parameter supported", args[1][0])
-            return None
-
-        source = get_text_from_tokens(args[0])
-        if len(source) > 1 and source[0] in ("'", '"') and source[-1] in ("'", '"'):
-            source = source[1:-1]
-            parser = FormatRuleActionParser(source)
-            res = parser.parse()
-            self.error_sink = parser.error_sink
-            return FormatAstColor(color, res)
-        else:
-            try:
-                index = source.index(":")
-                variable, vformat = source[:index], source[index + 1:]
-            except ValueError:
-                variable, vformat = source, None
-
-            if not identifier_regex.fullmatch(variable):
-                self.error_sink = FormatRuleSyntaxError("Invalid identifier", None)
-                return None
-            return FormatAstColor(color, FormatAstVariable(variable, vformat))
-
-    def return_list(self, args, kwargs):
-        """
-        Looking for greeting_var, [recurse_on], [recursion_depth], [items_prop]
-        :param args:
-        :param kwargs:
-        :return:
-        """
-        len_args = len(args)
-        if len_args < 1:
-            self.error_sink = FormatRuleSyntaxError("variable name not found", None)
-            return None
-
-        if len_args > 4:
-            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[4][0])
-            return None
-
-        variable_name = get_text_from_tokens(args[0])
-        recurse_on, recursion_depth, items_prop = None, 0, None
-
-        if len_args == 2:
-            recursion_depth = self.to_value(args[1])
-        elif len_args == 3:
-            recursion_depth = self.to_value(args[1])
-            recurse_on = self.to_value(args[2])
-        elif len_args == 4:
-            recursion_depth = self.to_value(args[1])
-            recurse_on = self.to_value(args[2])
-            items_prop = self.to_value(args[3])
-
-        if "recurse_on" in kwargs:
-            recurse_on = self.to_value(kwargs["recurse_on"])
-
-        if "recursion_depth" in kwargs:
-            recursion_depth = self.to_value(kwargs["recursion_depth"])
-
-        if "items_prop" in kwargs:
-            items_prop = self.to_value(kwargs["items_prop"])
-
-        if self.error_sink:
-            return None
-
-        if not isinstance(recursion_depth, int):
-            self.error_sink = FormatRuleSyntaxError("'recursion_depth' must be an integer", None)
-            return None
-
-        return FormatAstList(variable_name, items_prop, recurse_on, recursion_depth)
-
-    def return_dict(self, args, kwargs):
-        len_args = len(args)
-        if len_args < 1:
-            self.error_sink = FormatRuleSyntaxError("variable name not found", None)
-            return None
-
-        if len_args > 1:
-            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0])
-            return None
-
-        variable_name = get_text_from_tokens(args[0])
-
-        kwargs_parameters = {}
-        for prop in ("items_prop", "prefix", "suffix", "debug"):
-            if prop in kwargs:
-                kwargs_parameters[prop] = self.to_value(kwargs[prop])
-
-        if "debug" in kwargs_parameters:
-            if "prefix" not in kwargs_parameters:
-                kwargs_parameters["prefix"] = "{"
-            if "suffix" not in kwargs_parameters:
-                kwargs_parameters["suffix"] = "}"
-
-        if self.error_sink:
-            return None
-
-        return FormatAstDict(variable_name, **kwargs_parameters)
-
-    def return_multi(self, args, kwargs):
-        if len(kwargs) > 0:
-            self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None)
-            return None
-
-        if len(args) > 1:
-            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0])
-            return None
-
-        return FormatAstMulti(get_text_from_tokens(args[0]))
-
 
 @dataclass
 class EmitPythonCodeException(Exception):
     error: object
 
 
-class PythonCodeEmitter:
-
-    def __init__(self, context, text=None):
-        self.context = context
-        self.text = text or ""
-        self.var_counter = 0
-        self.variables = []
-
-    def add(self, text):
-        self.text += f" and {text}" if self.text else text
-        return self
-
-    def recognize(self, obj, as_name, root=True):
-        if isinstance(obj, str):
-            return self.recognize_str(obj, as_name)
-        elif isinstance(obj, (int, float)):
-            return self.recognize_int(obj, as_name)
-        elif isinstance(obj, Concept):
-            return self.recognize_concept(obj, as_name, root)
-        elif isinstance(obj, Expando):
-            return self.recognize_expando(obj, as_name, root)
-        else:
-            raise NotImplementedError()
-
-    def recognize_str(self, text, as_name):
-        if self.text:
-            self.text += " and "
-
-        if "'" in text and '"' in text:
-            self.text += f"{as_name} == '{text}'"
-        elif "'" in text:
-            self.text += f'{as_name} == "{text}"'
-        else:
-            self.text += f"{as_name} == '{text}'"
-        return self
-
-    def recognize_int(self, value, as_name):
-        if self.text:
-            self.text += " and "
-
-        self.text += f"{as_name} == {value}"
-        return self
-
-    def recognize_expando(self, value, as_name, root=True):
-        if self.text:
-            self.text += " and "
-
-        if not root:
-            as_name = self.add_variable(as_name)
-
-        self.text += f"isinstance({as_name}, Expando) and {as_name}.get_name() == '{value.get_name()}'"
-        return self
-
-    def recognize_concept(self, concept, as_name, root=True):
-        if self.text:
-            self.text += " and "
-
-        if not root:
-            as_name = self.add_variable(as_name)
-
-        if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
-            self.text += f"isinstance({as_name}, Concept) and {as_name}.name == '{concept.name}'"
-        elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
-            self.text += f"isinstance({as_name}, Concept) and {as_name}.id == '{concept.id}'"
-        else:
-            self.text += f"isinstance({as_name}, Concept) and {as_name}.key == '{concept.key}'"
-            if len(concept.get_metadata().variables) > 0:
-                # add variables constraints
-                evaluated = ensure_evaluated(self.context, concept, eval_body=False, metadata=["variables"])
-
-                if not self.context.sheerka.is_success(evaluated) and evaluated.key != concept.key:
-                    raise EmitPythonCodeException(evaluated)
-
-                for k, v in concept.variables().items():
-                    self.recognize(v, f"{as_name}.get_value('{k}')", root=False)
-
-        return self
-
-    def add_variable(self, target):
-        var_name = f"__x_{self.var_counter:02}__"
-        self.var_counter += 1
-        self.variables.append((var_name, target))
-        return var_name
-
-    def get_text(self):
-        if self.variables:
-            variables_as_str = '\n'.join([f"{k} = {v}" for k, v in self.variables])
-            return variables_as_str + "\n" + self.text
-
-        return self.text
-
-
 class NoConditionFound(ErrorObj):
     def __eq__(self, other):
         return isinstance(other, NoConditionFound)
@@ -1149,7 +577,6 @@ class GetConditionExprVisitor(ExpressionVisitor):
     def evaluate_from_source(self, source, is_question=False, return_body=False):
         res = evaluate_from_source(self.context,
                                    source,
-                                   evaluators=CONDITIONS_VISITOR_EVALUATORS,
                                    desc=None,
                                    eval_body=not is_question,
                                    eval_where=False,
@@ -1160,6 +587,9 @@ class GetConditionExprVisitor(ExpressionVisitor):
 
         if return_body:
             if not res.status:
+                python_eval_error = self.context.sheerka.get_errors(self.context, res, __type="PythonEvalError")
+                if python_eval_error and isinstance(python_eval_error[0].error, NameError):
+                    raise UnknownVariableError(python_eval_error[0].source)
                 raise FailedToCompileError(res.body)
 
             return res.body
@@ -1268,7 +698,11 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
     def visit_ComparisonNode(self, expr_node: ComparisonNode):
         if isinstance(expr_node.left, VariableNode):
             conditions = []
-            value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True)
+            try:
+                value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True)
+            except UnknownVariableError:
+                value = expr_node.right.unpack()
+
             self.add_to_condition(expr_node.left.unpack(), value, conditions)
             return conditions
         else:
@@ -1334,7 +768,6 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
     def visit_NameExprNode(self, expr_node: NameExprNode):
         res = evaluate_from_source(self.context,
                                    expr_node.get_source(),
-                                   evaluators=CONDITIONS_VISITOR_EVALUATORS,
                                    desc=None,
                                    eval_body=True,
                                    eval_where=False,
@@ -1367,9 +800,9 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
         variable = self.init_or_get_variable_from_attr(variable_path, conditions)
         conditions.append(Condition(variable, "__is_concept__", True))
 
-        if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
+        if concept.get_hints().recognized_by == RECOGNIZED_BY_NAME:
             conditions.append(Condition(variable, "name", concept.name))
-        elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
+        elif concept.get_hints().recognized_by == RECOGNIZED_BY_ID:
             conditions.append(Condition(variable, "id", concept.id))
         else:
             conditions.append(Condition(variable, "key", concept.key))
@@ -1392,6 +825,9 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
         elif isinstance(value, Concept):
             res = self.recognize_concept(var_path, value, {})
             conditions.extend(res)
+        elif isinstance(value, list):
+            var_root, var_attr = self.init_or_get_variable_from_name(value, conditions)
+            conditions.append(Condition(left, attr, var_root))
         else:
             conditions.append(Condition(left, attr, value))
 
@@ -1483,12 +919,25 @@ class PythonConditionExprVisitorObj:
                                              node.variables,
                                              node.not_variables)
 
+    @staticmethod
+    def create_condition(text, op, left, right):
+        def get_source(a, b):
+            if op == ComparisonType.EQUALS and b == "sheerka":
+                return f"is_sheerka({a})"
+            else:
+                return ComparisonNode.rebuild_source(a, op, b)
+
+        return PythonConditionExprVisitorObj(text,
+                                             get_source(left.source, right.source),
+                                             merge_dictionaries(left.objects, right.objects),
+                                             merge_sets(left.variables, right.variables),
+                                             merge_sets(left.not_variables, right.not_variables))
+
 
 class PythonConditionExprVisitor(GetConditionExprVisitor):
 
     def __init__(self, context):
         super().__init__(context)
-        self.know_object_variables = {}
         self.check_variable_existence_only = True
         self.concepts_to_reset = set()
 
@@ -1537,7 +986,8 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
         return var_name
 
     def unpack_variable(self, variable_path: List[str], obj_variables):
-        obj_variables.add(variable_path[0])
+        if self.is_a_possible_variable(variable_path[0]):
+            obj_variables.add(variable_path[0])
         return self.inner_unpack_variable(variable_path)
 
     @staticmethod
@@ -1551,49 +1001,49 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
             # try to recognize a concept
             res = self.evaluate_from_source(expr_node.name, is_question=True)
             if res.status and isinstance(res.value, Concept):
+                if self.context.possible_variables and res.value.name in self.context.possible_variables:
+                    variable_name = expr_node.get_source()
+                    return PythonConditionExprVisitorObj(variable_name, variable_name, {}, {variable_name}, set())
+
+                # else / otherwise
                 self.check_variable_existence_only = False
-                if is_a_question(self.context, res.value):
-                    return self.evaluate_concept_as_question(expr_node.name, res.value)
-                else:
-                    return self.evaluate_concept(expr_node.name, res.value)
+                return self.manage_concept(expr_node.get_source(), res.value)
             else:
                 if self.context.sheerka.has_error(self.context, res, __type=BuiltinConcepts.TOO_MANY_SUCCESS):
                     raise FailedToCompileError([res])
 
                 variable_name = expr_node.get_source()
-                variables = {variable_name} if not res.status else set()
+                if res.status:
+                    variables = set()
+                    self.check_variable_existence_only = False
+                else:
+                    variables = {variable_name}
                 return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables, set())
 
-        variable_name = expr_node.get_source()
-        variables_detected = {variable_name} if self.is_a_possible_variable(variable_name) else set()
-        return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables_detected, set())
+        if self.check_variable_existence_only:
+            # special case where we want to check the existence of the whole string
+            variable_name = expr_node.get_source()
+            possible_variables = {variable_name} if self.is_a_possible_variable(variable_name) else set()
+        else:
+            # try to detect the variable
+            possible_variables = set()
+            var_root, var_attr = self.unpack_variable(expr_node.unpack(), possible_variables)
+            variable_name = self.construct_variable(var_root, var_attr)
+        return PythonConditionExprVisitorObj(variable_name, variable_name, {}, possible_variables, set())
 
     def visit_ComparisonNode(self, expr_node: ComparisonNode):
         self.check_variable_existence_only = False
-
-        if not isinstance(expr_node.left, VariableNode):
-            # KSI 2021-04-22. Not quite sure of the reason why I have this piece of code
-            left = self.visit(expr_node.left)
-            source = expr_node.get_source()
-            return PythonConditionExprVisitorObj(source, source, {}, left.variables, left.not_variables)
-
+        left = self.visit(expr_node.left)
         right = self.visit(expr_node.right)
-        if right.source in right.objects:
-            return self.create_comparison_condition(expr_node.left.unpack(),
-                                                    expr_node.comp,
-                                                    right.source,
-                                                    right.objects[right.source],
-                                                    right.objects,
-                                                    expr_node.get_source())
+
+        # special case when we call recognize concept when an equality with a concept is found
+        right_value = right.objects[right.source] if right.source in right.objects else None
+        if expr_node.comp == ComparisonType.EQUALS and isinstance(right_value, Concept):
+            res = self.recognize_concept(expr_node.left.unpack(), right_value, {}, expr_node.get_source())
         else:
-            value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True)
-            object_name, objects = self.get_object_name(value)
-            return self.create_comparison_condition(expr_node.left.unpack(),
-                                                    expr_node.comp,
-                                                    object_name,
-                                                    value,
-                                                    objects,
-                                                    expr_node.get_source())
+            res = PythonConditionExprVisitorObj.create_condition(expr_node.get_source(), expr_node.comp, left, right)
+
+        return res
 
     def visit_AndNode(self, expr_node: AndNode):
         current_visitor_obj = self.visit(expr_node.parts[0])
@@ -1636,13 +1086,43 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
     def visit_NameExprNode(self, expr_node: NameExprNode):
         self.check_variable_existence_only = False
         source = expr_node.get_source()
-        res = self.evaluate_from_source(source, is_question=False)
-        if res.status:
-            obj_name, objects = self.get_object_name(res.value)
-            return PythonConditionExprVisitorObj(source, obj_name, objects, set(), set())
-        else:
+        res = self.context.sheerka.parse_unrecognized(self.context,
+                                                      source,
+                                                      INIT_AST_PARSERS,
+                                                      filter_func=only_successful,
+                                                      is_question=True)
+        if not res.status:
             raise FailedToCompileError([expr_node])
 
+        return_values = res.body.body if is_only_successful(self.context.sheerka, res) else [res]
+        # if the result is a concept, create an object for it, otherwise just leave the source as it is
+        res = evaluate_return_values(self.context, source, return_values, is_question=True)
+        if res.status:
+            if isinstance(res.value, Concept):
+                return self.manage_concept(expr_node.get_source(), res.value)
+            else:
+                obj_name, objects = self.get_object_name(res.value)
+                return PythonConditionExprVisitorObj(source, obj_name, objects, set(), set())
+
+        else:
+            if len(return_values) == 1:
+                body = return_values[0].body.body
+                if hasattr(body, "get_python_node"):
+                    python_node = return_values[0].body.body.get_python_node()
+                    unreferenced_names_visitor = UnreferencedNamesVisitor(self.context)
+                    variables = unreferenced_names_visitor.get_names(python_node.ast_)
+                    variables = variables - set(python_node.objects.keys())
+                    return PythonConditionExprVisitorObj(python_node.original_source,
+                                                         python_node.source,
+                                                         python_node.objects,
+                                                         variables,
+                                                         set())
+                elif isinstance(body, Concept):
+                    return self.manage_concept(expr_node.get_source(), body)
+
+            else:
+                raise FailedToCompileError([expr_node])
+
     def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict, original_source=None):
         if not isinstance(concept_to_recognize, Concept):
             concept_as_str = concept_to_recognize.get_source()
@@ -1659,9 +1139,9 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
 
         source = f"isinstance({var_name}, Concept)"
 
-        if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
+        if concept.get_hints().recognized_by == RECOGNIZED_BY_NAME:
             source += f" and {var_name}.name == '{concept.name}'"
-        elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
+        elif concept.get_hints().recognized_by == RECOGNIZED_BY_ID:
             source += f" and {var_name}.id == '{concept.id}'"
         else:
             source += f" and {var_name}.key == '{concept.key}'"
@@ -1676,17 +1156,24 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
                                                                   ComparisonType.EQUALS,
                                                                   obj_name,
                                                                   var_value,
-                                                                  objects)
+                                                                  objects,
+                                                                  set())
             source += " and " + variable_condition.source
             text += " and " + variable_condition.text
 
         return PythonConditionExprVisitorObj(original_source or text, source, objects, obj_variables, set())
 
+    def manage_concept(self, source, concept):
+        if is_a_question(self.context, concept):
+            return self.evaluate_concept_as_question(source, concept)
+        else:
+            return self.evaluate_concept(source, concept)
+
     def evaluate_concept_as_question(self, original_text, concept):
         concept_var_name, objects = self.get_object_name(concept)
         source = f"evaluate_question({concept_var_name})"
         variables = get_possible_variables_from_concept(self.context, concept)
-        self.concepts_to_reset.add(concept)
+        self.concepts_to_reset.update(self.get_concepts_to_reset(concept))
         return PythonConditionExprVisitorObj(original_text, source, objects, variables, set())
 
     def evaluate_concept(self, original_text, concept):
@@ -1694,13 +1181,62 @@ class PythonConditionExprVisitor(GetConditionExprVisitor):
         source, objects = self.get_object_name(concept)
         return PythonConditionExprVisitorObj(original_text, source, objects, set(), set())
 
-    def create_comparison_condition(self, left_path, op, right_name, right_value, objects, original_source=None):
-        possible_variables = set()
+    def get_concepts_to_reset(self, concept):
+        """
+        Returns all the concept that might be reset before a second evaluation
+        The algo is empirical, there must be a theory to make sure that no concept is missed
+        :param concept:
+        :return:
+        """
+
+        res = set()
+
+        def _inner_get_concept_to_reset(_concept):
+            if _concept in res:  # prevent circular references
+                return
+
+            res.add(_concept)
+            assert not _concept.get_hints().use_copy
+            for part_name, asts in _concept.get_compiled().items():
+                if isinstance(asts, Concept):
+                    if is_a_question(self.context, asts):
+                        res.update(self.get_concepts_to_reset(asts))
+                else:
+                    for ret_val in asts:
+                        body_as_list = ret_val.body.body  # go through the ParserResult
+                        if not isinstance(body_as_list, list):
+                            body_as_list = [body_as_list]
+
+                        for body in body_as_list:
+
+                            if hasattr(body, "get_concept"):  # to manage Concept and ConceptNode
+                                c = body.get_concept()
+                                if is_a_question(self.context, c):
+                                    res.update(self.get_concepts_to_reset(c))
+
+                            elif hasattr(body, "get_python_node"):  # to manage PythonNode and SourceCodeNode like
+                                python_node = body.get_python_node()
+                                for obj in python_node.objects.values():
+                                    if isinstance(obj, Concept) and is_a_question(self.context, obj):
+                                        res.update(self.get_concepts_to_reset(obj))
+
+        _inner_get_concept_to_reset(concept)
+        return res
+
+    def create_comparison_condition(self,
+                                    left_path,
+                                    op,
+                                    right_name,
+                                    right_value,
+                                    objects,
+                                    variables,
+                                    original_source=None):
+        possible_variables = variables.copy()
         var_root, var_attr = self.unpack_variable(left_path, possible_variables)
         left = self.construct_variable(var_root, var_attr)
 
         if original_source is None:
-            right = get_safe_str_value(right_value)
+            right = get_safe_str_value(right_value or right_name)
             original_source = ComparisonNode.rebuild_source(left, op, right)
 
         if op == ComparisonType.EQUALS:

src/core/sheerka/services/sheerka_service.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 
+from core.concept import Concept
 from core.global_symbols import NotFound, ErrorObj
 from core.utils import sheerka_deepcopy
 
@@ -52,3 +53,13 @@ class BaseService:
 @dataclass()
 class FailedToCompileError(Exception, ErrorObj):
     cause: list
+
+
+@dataclass()
+class UnknownVariableError(Exception, ErrorObj):
+    variable: str
+
+
+@dataclass
+class ChickenAndEggException(Exception, ErrorObj):
+    error: Concept

src/evaluators/ConceptEvaluator.py

@@ -51,6 +51,10 @@ class ConceptEvaluator(OneReturnValueEvaluator):
 
             return sheerka.ret(self.name, True, value, parents=[return_value])
 
+        if concept.get_hints().use_copy:
+            concept = concept.copy()
+            concept.get_hints().use_copy = False
+
         evaluated = sheerka.evaluate_concept(context, concept)
 
         if not sheerka.is_success(evaluated) and evaluated.key != concept.key:
@@ -67,4 +71,3 @@ class ConceptEvaluator(OneReturnValueEvaluator):
             return sheerka.ret(self.name, True, evaluated.body, parents=[return_value])
         else:
             return sheerka.ret(self.name, True, evaluated, parents=[return_value])
-

src/evaluators/DefConceptEvaluator.py

@@ -8,6 +8,7 @@ from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import TokenKind, Tokenizer
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
+from parsers.BaseExpressionParser import ExprNode
 from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
 from parsers.DefConceptParser import DefConceptNode, NameNode
 from parsers.PythonParser import get_python_node
@@ -255,6 +256,15 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
             debugger.debug_var("names", visitor.variables, hint="from BNF")
             return visitor.variables
 
+        #
+        # case of ExprNode
+        #
+        if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, ExprNode):
+            variables = set()
+            for compiled in ret_value.value.value.compiled:
+                variables.update(compiled.variables)
+            return [PossibleVariable(v) for v in variables]
+
         #
         # Case of python code
         #

src/evaluators/ExpressionEvaluator.py

@@ -0,0 +1,56 @@
+from core.builtin_concepts import ParserResultConcept
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
+from evaluators.BaseEvaluator import OneReturnValueEvaluator
+from parsers.BaseExpressionParser import ExprNode
+from sheerkapython.python_wrapper import create_namespace
+
+
+class ExpressionEvaluator(OneReturnValueEvaluator):
+    NAME = "Expression"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
+
+    def matches(self, context, return_value):
+        if not return_value.status:
+            return False
+
+        if not isinstance(return_value.value, ParserResultConcept):
+            return False
+
+        return isinstance(return_value.value.body, ExprNode)
+
+    def eval(self, context, return_value):
+        sheerka = context.sheerka
+        conditions = return_value.value.value.compiled
+        rule_evaluator = sheerka.services[SheerkaEvaluateRules.NAME]
+
+        errors = []
+        success = False
+
+        with context.push(BuiltinConcepts.EXEC_CODE, return_value.value.value.source) as sub_context:
+            # sub condition is created only to add a namespace
+
+            requested_vars = set()
+            for c in conditions:
+                requested_vars.update(c.variables)
+            namespace = create_namespace(context, self.NAME, requested_vars, set(), {}, True, False)
+            # TODO: ADD NAMESPACE TO STM
+            missing_vars = set()
+            results = rule_evaluator.evaluate_conditions(sub_context, conditions, namespace, missing_vars)
+
+            if not results and missing_vars:
+                errors = [NameError(v) for v in missing_vars]
+
+            for res in results:
+                if not res.status:
+                    errors.append(res.body)
+                value = context.sheerka.objvalue(res.body)
+                if isinstance(value, bool) and value:
+                    success = True
+
+        if errors:
+            return sheerka.ret(self.name, False, sheerka.err(errors))
+
+        return sheerka.ret(self.name, True, success)

src/evaluators/MultipleErrorsEvaluator.py

@@ -52,4 +52,4 @@ class MultipleErrorsEvaluator(AllReturnValuesEvaluator):
             self.name,
             False,
             sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=self.return_values_in_error.copy()),
-            parents=self.eaten)
+            parents=self.eaten.copy())

src/evaluators/MultipleOutEvaluator.py

@@ -41,4 +41,4 @@ class MultipleOutEvaluator(AllReturnValuesEvaluator):
 
     def eval(self, context, return_values):
         to_multi = context.sheerka.new(BuiltinConcepts.TO_MULTI, body=[r.body for r in self.success])
-        return context.sheerka.ret(self.name, True, to_multi, parents=self.eaten)
+        return context.sheerka.ret(self.name, True, to_multi, parents=self.eaten.copy())

src/evaluators/MultipleSuccessEvaluator.py

@@ -81,7 +81,7 @@ class MultipleSuccessEvaluator(AllReturnValuesEvaluator):
             self.name,
             True,
             sheerka.new(BuiltinConcepts.MULTIPLE_SUCCESS, body=self.successful_return_values.copy()),
-            parents=self.eaten)
+            parents=self.eaten.copy())
 
     def already_seen(self, context, ret_val):
         for successful in self.successful_return_values:

src/evaluators/MutipleSameSuccessEvaluator.py

@@ -70,18 +70,18 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator):
         # give the priority to the ConceptEvaluator
         for s in self.success:
             if isinstance(s.value, Concept) and s.who == ConceptEvaluator().name:
-                return sheerka.ret(self.name, True, s.value, parents=self.eaten)
+                return sheerka.ret(self.name, True, s.value, parents=self.eaten.copy())
 
         # Then the PythonEvaluator
         for s in self.success:
             if isinstance(s.value, Concept) and s.who == PythonEvaluator().name:
-                return sheerka.ret(self.name, True, s.value, parents=self.eaten)
+                return sheerka.ret(self.name, True, s.value, parents=self.eaten.copy())
 
         # Then the first concept.
         # It's not predictable, so I guess that it's not a good implementation choice
         for s in self.success:
             if isinstance(s.value, Concept):
-                return sheerka.ret(self.name, True, s.value, parents=self.eaten)
+                return sheerka.ret(self.name, True, s.value, parents=self.eaten.copy())
 
-        return sheerka.ret(self.name, True, self.success[0].value, parents=self.eaten)
+        return sheerka.ret(self.name, True, self.success[0].value, parents=self.eaten.copy())
 

src/evaluators/OneErrorEvaluator.py

@@ -47,4 +47,4 @@ class OneErrorEvaluator(AllReturnValuesEvaluator):
         context.log(f"{self.return_value_in_error}", who=self)
 
         sheerka = context.sheerka
-        return sheerka.ret(self.name, False, self.return_value_in_error.value, parents=self.eaten)
+        return sheerka.ret(self.name, False, self.return_value_in_error.value, parents=self.eaten.copy())

src/evaluators/OneSuccessEvaluator.py

@@ -48,4 +48,4 @@ class OneSuccessEvaluator(AllReturnValuesEvaluator):
         context.log(f"{self.successful_return_value}", who=self)
 
         sheerka = context.sheerka
-        return sheerka.ret(self.name, True, self.value_to_return, parents=self.eaten)
+        return sheerka.ret(self.name, True, self.value_to_return, parents=self.eaten.copy())

src/evaluators/PythonEvaluator.py

@@ -79,7 +79,8 @@ class PythonEvaluator(OneReturnValueEvaluator):
         #   or if EVAL_QUESTION_REQUESTED is explicit
         # We need to disable the functions that may alter the state
         # It's a poor way to have source code security check
-        expression_only = context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        expression_only = context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED) or \
+                          context.in_context(BuiltinConcepts.VALIDATION_ONLY_REQUESTED)
 
         if not expression_only:
             attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE)
@@ -92,6 +93,11 @@ class PythonEvaluator(OneReturnValueEvaluator):
             my_globals = self.get_globals(context, node, expression_only)
             debugger.debug_var("globals", my_globals)
         except MethodAccessError as ex:
+            # Quick and dirty,
+            # When VALIDATION_ONLY_REQUESTED is enabled, it's normal to have some NameError exceptions
+            if context.in_context(BuiltinConcepts.VALIDATION_ONLY_REQUESTED):
+                return sheerka.ret(self.name, False, BuiltinConcepts.METHOD_ACCESS_ERROR, parents=[return_value])
+
             eval_error = PythonEvalError(ex,
                                          node.source,
                                          traceback.format_exc() if get_trace_back else None,

src/evaluators/RetEvaluator.py

@@ -25,7 +25,7 @@
 #         concept = return_value.value
 #         context.log(f"Processing ret value for concept {concept}.", self.name)
 #
-#         if not concept.get_metadata().is_evaluated:
+#         if not concept.get_hints().is_evaluated:
 #             evaluated = ensure_evaluated(context, concept)
 #             if evaluated.key != concept.key:
 #                 context.log(f"Failed to evaluate concept '{concept}'")

src/evaluators/TooManySuccessEvaluator.py

@@ -55,7 +55,7 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
         if not same_success:
             context.log(f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name)
             too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success.copy())
-            return sheerka.ret(self.name, False, too_many_success, parents=self.eaten)
+            return sheerka.ret(self.name, False, too_many_success, parents=self.eaten.copy())
 
         context.log(f"Values are the same. Nothing to do.", self.name)
         return None

src/evaluators/ValidateConceptEvaluator.py

@@ -0,0 +1,109 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from evaluators.BaseEvaluator import OneReturnValueEvaluator
+from parsers.BaseNodeParser import ConceptNode
+from parsers.BaseParser import BaseParser
+
+
+class ValidateConceptEvaluator(OneReturnValueEvaluator):
+    """
+    To recognize when the user input is a question
+    """
+
+    NAME = "ValidateConcept"
+
+    def __init__(self, **kwargs):
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_PARSING], 90)
+
+    def matches(self, context, return_value):
+        if not return_value.status or \
+            not context.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT) or \
+            not return_value.who.startswith(BaseParser.PREFIX):
+            return False
+
+        value = return_value.body.body
+        if isinstance(return_value.body.body, Concept):  # case of ExactConceptParser
+            return value.get_hints().need_validation and (
+                value.get_metadata().pre is not None and value.get_metadata().pre != "" or
+                value.get_metadata().where is not None and value.get_metadata().where != ""
+            )
+
+        # simple case first, cases with a longer list may be managed later if needed
+        if isinstance(value, list) and len(value) == 1 and isinstance(value[0], ConceptNode):
+            concept = value[0].concept
+            return concept.get_hints().need_validation and (
+                concept.get_metadata().pre is not None and concept.get_metadata().pre != "" or
+                concept.get_metadata().where is not None and concept.get_metadata().where != ""
+            )
+
+        return False
+
+    def eval(self, context, return_value):
+        """
+        This evaluator returns None is the concept validates its PRE and POST constraint or if the constraint cannot
+            be validated
+        If the constraint can be validated, but fails, an error is returned
+        :param context:
+        :param return_value:
+        :return:
+        """
+        sheerka = context.sheerka
+
+        if isinstance(return_value.body.body, list):
+            concept_node = return_value.body.body[0]
+            concept = concept_node.concept
+        else:
+            concept_node = None
+            concept = return_value.body.body
+
+        if context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_CONCEPT and
+                                          ec.action_context.id == concept.id):
+            # we are in an infinite loop trying to validate ourself
+            return None
+
+        if concept.get_hints().use_copy:
+            use_copy = True
+            concept = concept.copy()
+            concept.get_hints().use_copy = False
+        else:
+            use_copy = False
+
+        res = sheerka.evaluate_concept(context,
+                                       concept,
+                                       eval_body=False,
+                                       validation_only=True,
+                                       metadata=["variables", ConceptParts.PRE, ConceptParts.WHERE])
+
+        # either the 'pre' or the 'where' condition is not fulfilled
+        if sheerka.isinstance(res, BuiltinConcepts.CONDITION_FAILED):
+            filtered = sheerka.new(BuiltinConcepts.FILTERED, filtered=return_value.body.body, reason=res)
+            failed = sheerka.ret(self.name, False, filtered)
+            return failed
+
+        # CAUTION
+        # Make sure that the return value and the parser result are not modified
+        # As they might be cached
+        if concept_node:
+            node = concept_node.clone()
+            node.concept = concept
+            return self.new_ret_val(context, return_value, [node])
+
+        if use_copy:
+            return self.new_ret_val(context, return_value, concept)
+
+        else:
+            return None  # tells the Execute() engine to use the old return value
+
+    def new_ret_val(self, context, old_ret_value, new_value):
+        parser_result_copy = context.sheerka.new(
+            BuiltinConcepts.PARSER_RESULT,
+            parser=old_ret_value.body.parser,
+            source=old_ret_value.body.source,
+            body=new_value,
+            try_parsed=new_value)
+
+        return context.sheerka.ret(
+            self.name,
+            True,
+            parser_result_copy,
+            parents=[old_ret_value])

src/out/AsStrVisitor.py

@@ -1,6 +1,6 @@
 import re
 
-from core.sheerka.services.SheerkaRuleManager import FormatAstNode
+from parsers.FormatRuleActionParser import FormatAstNode
 from core.utils import CONSOLE_COLORS_MAP as CCM, no_color_str, CONSOLE_COLUMNS
 from out.OutVisitor import OutVisitor
 

src/out/DeveloperVisitor.py

@@ -1,5 +1,5 @@
 from core.builtin_helpers import evaluate_expression
-from core.sheerka.services.SheerkaRuleManager import FormatAstVariable, FormatAstVariableNotFound, FormatAstColor, \
+from parsers.FormatRuleActionParser import FormatAstVariable, FormatAstVariableNotFound, FormatAstColor, \
     FormatAstList, FormatAstRawText, FormatAstDict
 from core.utils import as_bag
 

src/parsers/BaseExpressionParser.py

@@ -44,6 +44,7 @@ class ExprNode(Node):
         self.end = end
         self.tokens = tokens
         self.source = None
+        self.compiled = None
 
     def eval(self, obj):
         return True
@@ -426,6 +427,8 @@ class BaseExpressionParser(BaseParser):
                 False,
                 context.sheerka.new(BuiltinConcepts.ERROR, body=error_sink.sink))
 
+        # Do not compile the node here, as it merely be useless
+        # The node is compiled in ExpressionParser.parse() or FunctionParser.parse(), depending of the requirement
         node = self.parse_input(context, parser_input, error_sink)
 
         token = parser_input.token
@@ -582,7 +585,9 @@ class TrueifyVisitor(ExpressionVisitor):
 
 
 is_question_tokens = list(Tokenizer("is_question()"))
-eval_question_requested_in_context = list(Tokenizer("context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)"))
+eval_question_requested_in_context_tokens = list(
+    Tokenizer("context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)"))
+in_context_tokens = list(Tokenizer("in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)"))
 
 
 class IsAQuestionVisitor(ExpressionVisitor):
@@ -593,13 +598,13 @@ class IsAQuestionVisitor(ExpressionVisitor):
 
     def visit_NameExprNode(self, expr_node):
         if tokens_are_matching(expr_node.tokens, is_question_tokens) or \
-            tokens_are_matching(expr_node.tokens, eval_question_requested_in_context):
+            tokens_are_matching(expr_node.tokens, eval_question_requested_in_context_tokens):
             return True
         return None
 
     def visit_FunctionNode(self, expr_node: FunctionNode):
         if tokens_are_matching(expr_node.tokens, is_question_tokens) or \
-            tokens_are_matching(expr_node.tokens, eval_question_requested_in_context):
+            tokens_are_matching(expr_node.tokens, in_context_tokens):
             return True
         return None
 

src/parsers/BaseNodeParser.py

@@ -223,6 +223,8 @@ class ConceptNode(LexerNode):
     def to_short_str(self):
         return f'CN({self.concept})'
 
+    def get_concept(self):
+        return self.concept
 
 class SourceCodeNode(LexerNode):
     """
@@ -388,7 +390,8 @@ class SourceCodeWithConceptNode(LexerNode):
         return self._all_nodes
 
     def clone(self):
-        clone = SourceCodeWithConceptNode(self.first, self.last, self.nodes.copy(), self.has_unrecognized)
+        nodes = [n.clone() for n in self.nodes]
+        clone = SourceCodeWithConceptNode(self.first.clone(), self.last.clone(), nodes, self.has_unrecognized)
         clone.python_node = self.python_node
         clone.return_value = self.return_value
         return clone

src/parsers/BaseParser.py

@@ -71,7 +71,15 @@ class UnexpectedEofParsingError(ParsingError):
 class BaseParser:
     PREFIX = "parsers."
 
-    def __init__(self, name, priority: int, enabled=True, yield_eof=False):
+    def __init__(self, name, priority: int, enabled=True, yield_eof=False, hints=None):
+        """
+
+        :param name:
+        :param priority:
+        :param enabled:
+        :param yield_eof:
+        :param hints: Dictionary context_hint: priority. When not null, priority is taken from it
+        """
         # self.log = get_logger("parsers." + self.__class__.__name__)
         # self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__)
         # self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__)
@@ -81,6 +89,7 @@ class BaseParser:
         self.priority = priority
         self.enabled = enabled
         self.yield_eof = yield_eof
+        self.hints = hints
 
     def __eq__(self, other):
         if not isinstance(other, self.__class__):

src/parsers/BnfNodeParser.py

@@ -1227,6 +1227,18 @@ class BnfNodeConceptExpressionVisitor(ParsingExpressionVisitor):
         self.references.append(pe.concept)
 
 
+class HasAChoiceExpressionVisitor(ParsingExpressionVisitor):
+    def __init__(self):
+        super().__init__()
+        self.result = False
+
+    def visit_OrderedChoice(self, parsing_expression):
+        self.result = True
+
+    def visit_UnOrderedChoice(self, parsing_expression):
+        self.result = True
+
+
 class BnfConceptParserHelper:
     def __init__(self, parser, debugger):
         self.parser = parser
@@ -1480,6 +1492,7 @@ class BnfConceptParserHelper:
         key = (template.key, template.id) if template.id else template.key
         concept = sheerka.new(key)
         concept = self.finalize_concept(sheerka, concept, underlying)
+        concept.get_hints().use_copy = True
         concept_node = ConceptNode(concept,
                                    underlying.start,
                                    underlying.end,
@@ -1555,7 +1568,7 @@ class BnfConceptParserHelper:
                     _underlying.parsing_expression.rule_name not in _concept.get_compiled()):
                     var_value = _get_underlying_value(_underlying)
                     _add_compiled(_concept, _underlying.parsing_expression.rule_name, var_value)
-                    _concept.get_metadata().need_validation = True
+                    _concept.get_hints().need_validation = True
 
             elif isinstance(_underlying, NonTerminalNode):
                 for child in _underlying.children:
@@ -1567,7 +1580,7 @@ class BnfConceptParserHelper:
             concept.get_compiled()[ConceptParts.BODY] = value
             if underlying.parsing_expression.rule_name:
                 _add_compiled(concept, underlying.parsing_expression.rule_name, value)
-                # KSI : Why don't we set concept.get_metadata().need_validation to True ?
+                # KSI : Why don't we set concept.get_hints().need_validation to True ?
 
         # then recursively browse children to update concept variables
         if isinstance(underlying, NonTerminalNode) and not isinstance(underlying.parsing_expression, ConceptExpression):

src/parsers/DefConceptParser.py

@@ -3,9 +3,11 @@ from dataclasses import dataclass
 import core.builtin_helpers
 import core.utils
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
-from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
+from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, ConceptParts
 from core.global_symbols import NotInit
+from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept, ChickenAndEggException
 from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
+from core.sheerka.services.sheerka_service import FailedToCompileError
 from core.tokenizer import TokenKind, Keywords
 from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, SyntaxErrorNode, NameNode, CustomGrammarParserNode
 from parsers.BaseParser import ParsingError, UnexpectedTokenParsingError
@@ -54,7 +56,7 @@ class DefConceptParser(BaseCustomGrammarParser):
     """
     Parse sheerka specific grammar (like def concept)
     """
-
+    NAME = "DefConcept"
     KEYWORDS = [Keywords.CONCEPT,
                 Keywords.FROM,
                 Keywords.AS,
@@ -67,7 +69,7 @@ class DefConceptParser(BaseCustomGrammarParser):
     KEYWORDS_VALUES = [k.value for k in KEYWORDS]
 
     def __init__(self, **kwargs):
-        BaseCustomGrammarParser.__init__(self, "DefConcept", 60)
+        BaseCustomGrammarParser.__init__(self, DefConceptParser.NAME, 60)
 
     def parse(self, context, parser_input: ParserInput):
         # default parser can only manage string text
@@ -140,11 +142,11 @@ class DefConceptParser(BaseCustomGrammarParser):
         node.definition_type, node.definition = self.get_concept_definition(node, parts)
 
         # get the bodies
-        node.body = self.get_ast(Keywords.AS, parts)
-        node.where = self.get_ast(Keywords.WHERE, parts)
-        node.pre = self.get_ast(Keywords.PRE, parts)
-        node.post = self.get_ast(Keywords.POST, parts)
-        node.ret = self.get_ast(Keywords.RET, parts)
+        node.body = self.get_ast(Keywords.AS, ConceptParts.BODY, parts)
+        node.where = self.get_ast(Keywords.WHERE, ConceptParts.WHERE, parts)
+        node.pre = self.get_ast(Keywords.PRE, ConceptParts.PRE, parts)
+        node.post = self.get_ast(Keywords.POST, ConceptParts.POST, parts)
+        node.ret = self.get_ast(Keywords.RET, ConceptParts.RET, parts)
 
         # other information
         node.auto_eval = self.get_concept_auto_eval(parts)
@@ -263,7 +265,7 @@ class DefConceptParser(BaseCustomGrammarParser):
 
         return DEFINITION_TYPE_DEF, NameNode(tokens)
 
-    def get_ast(self, keyword, parts):
+    def get_ast(self, keyword, concept_part, parts):
         if keyword not in parts:
             return NotInit
 
@@ -273,12 +275,26 @@ class DefConceptParser(BaseCustomGrammarParser):
             return None
 
         source = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens[1:])
-        parsed = self.sheerka.parse_unrecognized(self.context,
-                                                 source,
-                                                 parsers="all",
-                                                 who=self.name,
-                                                 prop=keyword,
-                                                 filter_func=core.builtin_helpers.expect_one)
+
+        try:
+            parsed = self.sheerka.services[SheerkaEvaluateConcept.NAME].get_asts(self.context,
+                                                                                 self.NAME,
+                                                                                 source,
+                                                                                 None,
+                                                                                 concept_part,
+                                                                                 None)
+
+            parsed = core.builtin_helpers.expect_one(self.context, parsed)
+            if not parsed.status:
+                self.add_error(parsed.value)
+                return None
+
+        except ChickenAndEggException as ex:
+            self.add_error(ex.error)
+            return None
+        except FailedToCompileError as ex:
+            self.add_error(ex.cause[0] if len(ex.cause) == 1 else ex.cause)
+            return None
 
         if not parsed.status:
             self.add_error(parsed.value)

src/parsers/DefRuleParser.py

@@ -6,11 +6,12 @@ from core.builtin_concepts import ReturnValueConcept
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatAstNode, CompiledCondition
+from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition
 from core.sheerka.services.sheerka_service import FailedToCompileError
 from core.tokenizer import Keywords, TokenKind
 from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, NameNode, KeywordNotFound, SyntaxErrorNode
 from parsers.BaseParser import Node, UnexpectedEofParsingError
+from parsers.FormatRuleActionParser import FormatAstNode
 from sheerkarete.conditions import AndConditions
 
 

src/parsers/ExactConceptParser.py

@@ -4,18 +4,18 @@ from core.concept import VARIABLE_PREFIX
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import TokenKind
 from core.utils import str_concept
-from parsers.BaseParser import BaseParser, BaseParserInputParser
+from parsers.BaseParser import BaseParserInputParser
 
 
 class ExactConceptParser(BaseParserInputParser):
     """
     Tries to recognize a single concept
     """
-
+    NAME = "ExactConcept"
     MAX_WORDS_SIZE = 6
 
     def __init__(self, max_word_size=None, **kwargs):
-        BaseParserInputParser.__init__(self, "ExactConcept", 80)
+        BaseParserInputParser.__init__(self, ExactConceptParser.NAME, 80)
         self.max_word_size = max_word_size
 
     def parse(self, context, parser_input: ParserInput):
@@ -47,7 +47,7 @@ class ExactConceptParser(BaseParserInputParser):
         for combination in self.combinations(words):
 
             concept_key = " ".join(combination)
-            result = sheerka.get_by_key(concept_key)  # use new(), not get() because we need a new instance
+            result = sheerka.get_by_key(concept_key)
 
             if sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT):
                 continue
@@ -74,13 +74,15 @@ class ExactConceptParser(BaseParserInputParser):
                         index = int(token[len(VARIABLE_PREFIX):])
                         value = words[i]
                         concept.def_var_by_index(index, str_concept(value) if isinstance(value, tuple) else value)
-                        concept.get_metadata().need_validation = True
+                        concept.get_hints().need_validation = True
 
                 already_recognized.append(concept)
 
         by_name = sheerka.fast_resolve(parser_input.as_text())
         core.builtin_helpers.set_is_evaluated(by_name)
         recognized = self.merge_concepts(already_recognized, by_name)
+        for c in recognized:
+            c.get_hints().use_copy = True
 
         if len(recognized) == 0:
             ret = sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT,

src/parsers/ExpressionParser.py

@@ -1,3 +1,7 @@
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.sheerka.services.SheerkaRuleManager import PythonConditionExprVisitor
+from core.sheerka.services.sheerka_service import FailedToCompileError
 from parsers.BaseExpressionParser import BaseExpressionParser
 from parsers.FunctionParser import FunctionParser
 from parsers.LogicalOperatorParser import LogicalOperatorParser
@@ -12,15 +16,45 @@ class ExpressionParser(BaseExpressionParser):
 
     NAME = "Expression"
 
-    def __init__(self, **kwargs):
-        super().__init__(ExpressionParser.NAME, 60, False, yield_eof=False)
+    def __init__(self, auto_compile=True, **kwargs):
+        super().__init__(ExpressionParser.NAME,
+                         0,
+                         True,
+                         yield_eof=False,
+                         hints={BuiltinConcepts.EVAL_QUESTION_REQUESTED: 60})
         self.variable_parser = VariableOrNamesParser()
         self.function_parser = FunctionParser(expr_parser=self, tokens_parser=self.variable_parser)
         self.relational_parser = RelationalOperatorParser(expr_parser=self.function_parser)
         self.logical_parser = LogicalOperatorParser(expr_parser=self.relational_parser)
+        self.auto_compile = auto_compile
 
     def parse_input(self, context, parser_input, error_sink):
         return self.logical_parser.parse_input(context, parser_input, error_sink)
 
     def parse_tokens_stop_condition(self, token, parser_input):
         pass
+
+    def parse(self, context, parser_input: ParserInput):
+        ret = super().parse(context, parser_input)
+
+        if not self.auto_compile:
+            return ret
+
+        if ret is None:
+            return None
+
+        if not ret.status:
+            return ret
+
+        # The parsing is successful
+        # let's validate it
+        try:
+            python_visitor = PythonConditionExprVisitor(context)
+            python_conditions = python_visitor.get_conditions(ret.body.body)
+            ret.body.body.compiled = python_conditions
+            return ret
+        except FailedToCompileError as ex:
+            return context.sheerka.ret(
+                self.name,
+                False,
+                context.sheerka.err(ex.cause))

src/parsers/FormatRuleActionParser.py

@@ -0,0 +1,485 @@
+import re
+from dataclasses import dataclass
+from typing import Union
+
+from core.global_symbols import ErrorObj
+from core.tokenizer import IterParser, TokenKind, Token
+from core.utils import get_text_from_tokens, COLORS, index_tokens
+
+identifier_regex = re.compile(r"[\w _.]+")
+
+
+@dataclass
+class FormatRuleError(ErrorObj):
+    pass
+
+
+@dataclass
+class BraceMismatch(FormatRuleError):
+    lbrace: Token
+
+
+@dataclass
+class UnexpectedEof(FormatRuleError):
+    message: str
+    token: Token = None
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, UnexpectedEof):
+            return False
+
+        return self.message == other.message and (other.token is None or other.token == self.token)
+
+    def __hash__(self):
+        return hash(self.message, self.token)
+
+
+@dataclass
+class FormatRuleSyntaxError(FormatRuleError):
+    message: str
+    token: Token
+
+
+@dataclass
+class FormatAstNode:
+    @staticmethod
+    def repr_value(items):
+        if items is None:
+            return ""
+
+        return ", ".join(repr(item) for item in items)
+
+    def clone(self, instance, props, **kwargs):
+        for prop_name in props:
+            setattr(instance, prop_name, getattr(self, prop_name))
+
+        for k, v in kwargs.items():
+            setattr(instance, k, v)
+
+        return instance
+
+
+@dataclass
+class FormatAstRawText(FormatAstNode):
+    text: str
+
+
+@dataclass
+class FormatAstVariable(FormatAstNode):
+    name: str
+    format: Union[str, None] = None
+    debug: bool = False
+    value: object = None
+    index: object = None
+
+    def clone(self, **kwargs):
+        return super().clone(FormatAstVariable(self.name),
+                             ("format", "debug", "value", "index"),
+                             **kwargs)
+
+
+@dataclass
+class FormatAstVariableNotFound(FormatAstNode):
+    name: str
+
+
+@dataclass
+class FormatAstGrid(FormatAstNode):
+    pass
+
+
+@dataclass
+class FormatAstList(FormatAstNode):
+    variable: str
+    items_prop: str = None  # where to search the list if variable does not resolve to an iterable
+    recurse_on: str = None
+    recursion_depth: int = 0
+    debug: bool = False
+    prefix: str = None
+    suffix: str = None
+    show_index: bool = False
+    index: object = None
+
+    items: object = None
+
+    def clone(self, **kwargs):
+        return super().clone(
+            FormatAstList(self.variable),
+            (
+                "items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "index",
+                "items"),
+            **kwargs)
+
+
+@dataclass
+class FormatAstDict(FormatAstNode):
+    variable: str
+    items_prop: str = None  # where to search the dict if variable does not resolve to an iterable
+    debug: bool = False
+    prefix: str = None
+    suffix: str = None
+
+    items: object = None
+
+    def clone(self, **kwargs):
+        return super().clone(
+            FormatAstDict(self.variable),
+            ("items_prop", "debug", "prefix", "suffix", "items"),
+            **kwargs)
+
+
+@dataclass
+class FormatAstColor(FormatAstNode):
+    color: str
+    format_ast: FormatAstNode
+
+    def __repr__(self):
+        return f"{self.color}({self.format_ast})"
+
+    def clone(self, **kwargs):
+        return super().clone(
+            FormatAstColor(self.color, self.format_ast),
+            (),
+            **kwargs)
+
+
+@dataclass
+class FormatAstFunction(FormatAstNode):
+    name: str
+    args: list = None
+    kwargs: dict = None
+
+
+@dataclass
+class FormatAstSequence(FormatAstNode):
+    items: list
+    debug: bool = False
+
+    def __repr__(self):
+        return "FormatAstSequence(" + self.repr_value(self.items) + ")"
+
+    def clone(self, **kwargs):
+        return super().clone(
+            FormatAstSequence(self.items),
+            ("debug",),
+            **kwargs)
+
+
+@dataclass
+class FormatAstMulti(FormatAstNode):
+    """
+    Used when there are multiple out to print, but they are not related
+    Just print them one by one
+    """
+    variable: str
+    items: list = None
+
+    def __repr__(self):
+        return f"FormatAstMulti({self.variable}, items={self.items})"
+
+    def clone(self, **kwargs):
+        return super().clone(
+            FormatAstMulti(self.variable),
+            ("items",),
+            **kwargs)
+
+
+class FormatRuleActionParser(IterParser):
+
+    @staticmethod
+    def to_text(list_or_dict_of_tokens):
+        """
+        Works on list of list of tokens
+        or dict of list of tokens
+        :param list_or_dict_of_tokens:
+        :return:
+        """
+        get_text = get_text_from_tokens
+        if isinstance(list_or_dict_of_tokens, list):
+            return [get_text(i) for i in list_or_dict_of_tokens]
+        if isinstance(list_or_dict_of_tokens, dict):
+            return {k: get_text(v) for k, v in list_or_dict_of_tokens.items()}
+        raise NotImplementedError("")
+
+    def to_value(self, tokens):
+        """
+        Works on list of tokens
+        return string or numeric value of the tokens
+        :return:
+        """
+
+        value = get_text_from_tokens(tokens)
+        if value[0] in ("'", '"'):
+            return value[1:-1]
+
+        if value in ("True", "False"):
+            return bool(value)
+
+        try:
+            return int(value)
+        except ValueError:
+            pass
+
+        try:
+            return float(value)
+        except ValueError:
+            self.error_sink = FormatRuleSyntaxError(f"'{value}' is not numeric", None)
+
+    def parse(self):
+        """
+        Parses the print part of the format rule
+        format ::= {variable'} | function(...) | rawtext
+        :return:
+        """
+
+        if self.source == "":
+            return FormatAstRawText("")
+
+        buffer = []
+        result = []
+        res = None
+        escaped = False
+
+        def _flush_buffer():
+            if len(buffer) > 0:
+                result.append(FormatAstRawText(get_text_from_tokens(buffer)))
+                buffer.clear()
+
+        while self.next_token(skip_whitespace=False):
+            if not escaped:
+                if self.token.type == TokenKind.IDENTIFIER and self.the_token_after().type == TokenKind.LPAR:
+                    _flush_buffer()
+                    res = self.parse_function(self.token)
+                elif self.token.type == TokenKind.LBRACE:
+                    _flush_buffer()
+                    res = self.parse_variable(self.token)
+                elif self.token.type == TokenKind.BACK_SLASH:
+                    escaped = True
+                else:
+                    buffer.append(self.token)
+            else:
+                escaped = False
+                buffer.append(self.token)
+
+            if self.error_sink:
+                break
+
+            if res:
+                result.append(res)
+                res = None
+
+        _flush_buffer()
+
+        return [] if len(result) == 0 else result[0] if len(result) == 1 else FormatAstSequence(result)
+
+    def parse_function(self, func_name):
+        self.next_token()
+        self.next_token()
+
+        if self.token.type == TokenKind.EOF:
+            self.error_sink = UnexpectedEof("while parsing function", func_name)
+            return None
+
+        param_buffer = []
+        args = []
+        kwargs = {}
+        get_text = get_text_from_tokens
+
+        def _process_parameters():
+            if len(param_buffer) == 0:
+                self.error_sink = FormatRuleSyntaxError("no parameter found", self.token)
+                return None
+            if (index := index_tokens(param_buffer, "=")) > 0:
+                kwargs[get_text(param_buffer[:index])] = param_buffer[index + 1:]
+            else:
+                args.append(param_buffer.copy())
+            param_buffer.clear()
+
+        while True:
+            if self.token.type == TokenKind.RPAR:
+                if len(param_buffer) > 0:
+                    _process_parameters()
+                break
+
+            elif self.token.type == TokenKind.COMMA:
+                _process_parameters()
+                if self.error_sink:
+                    break
+
+            else:
+                param_buffer.append(self.token)
+
+            if not self.next_token():
+                break
+
+        if self.error_sink:
+            return None
+
+        if self.token.type != TokenKind.RPAR:
+            self.error_sink = UnexpectedEof("while parsing function", func_name)
+            return None
+
+        if func_name.value in COLORS:
+            return self.return_color(func_name.value, args, kwargs)
+        elif func_name.value == "list":
+            return self.return_list(args, kwargs)
+        elif func_name.value == "dict":
+            return self.return_dict(args, kwargs)
+        elif func_name.value == "multi":
+            return self.return_multi(args, kwargs)
+
+        return FormatAstFunction(func_name.value, self.to_text(args), self.to_text(kwargs))
+
+    def parse_variable(self, lbrace):
+        self.next_token()
+
+        if self.token.type == TokenKind.EOF:
+            self.error_sink = UnexpectedEof("while parsing variable", lbrace)
+            return None
+
+        buffer = []
+        while True:
+            if self.token.type == TokenKind.RBRACE:
+                break
+            buffer.append(self.token)
+
+            if not self.next_token():
+                break
+
+        # if self.error_sink:
+        #     return None
+
+        if self.token.type != TokenKind.RBRACE:
+            self.error_sink = UnexpectedEof("while parsing variable", lbrace)
+            return None
+
+        if len(buffer) == 0:
+            self.error_sink = FormatRuleSyntaxError("variable name not found", None)
+            return None
+
+        variable = get_text_from_tokens(buffer)
+        try:
+            index = variable.index(":")
+            return FormatAstVariable(variable[:index], variable[index + 1:])
+        except ValueError:
+            return FormatAstVariable(variable)
+
+    def return_color(self, color, args, kwargs):
+        if len(kwargs) > 0:
+            self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None)
+            return None
+
+        if len(args) == 0:
+            return FormatAstColor(color, FormatAstRawText(""))
+
+        if len(args) > 1:
+            self.error_sink = FormatRuleSyntaxError("only one parameter supported", args[1][0])
+            return None
+
+        source = get_text_from_tokens(args[0])
+        if len(source) > 1 and source[0] in ("'", '"') and source[-1] in ("'", '"'):
+            source = source[1:-1]
+            parser = FormatRuleActionParser(source)
+            res = parser.parse()
+            self.error_sink = parser.error_sink
+            return FormatAstColor(color, res)
+        else:
+            try:
+                index = source.index(":")
+                variable, vformat = source[:index], source[index + 1:]
+            except ValueError:
+                variable, vformat = source, None
+
+            if not identifier_regex.fullmatch(variable):
+                self.error_sink = FormatRuleSyntaxError("Invalid identifier", None)
+                return None
+            return FormatAstColor(color, FormatAstVariable(variable, vformat))
+
+    def return_list(self, args, kwargs):
+        """
+        Looking for greeting_var, [recurse_on], [recursion_depth], [items_prop]
+        :param args:
+        :param kwargs:
+        :return:
+        """
+        len_args = len(args)
+        if len_args < 1:
+            self.error_sink = FormatRuleSyntaxError("variable name not found", None)
+            return None
+
+        if len_args > 4:
+            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[4][0])
+            return None
+
+        variable_name = get_text_from_tokens(args[0])
+        recurse_on, recursion_depth, items_prop = None, 0, None
+
+        if len_args == 2:
+            recursion_depth = self.to_value(args[1])
+        elif len_args == 3:
+            recursion_depth = self.to_value(args[1])
+            recurse_on = self.to_value(args[2])
+        elif len_args == 4:
+            recursion_depth = self.to_value(args[1])
+            recurse_on = self.to_value(args[2])
+            items_prop = self.to_value(args[3])
+
+        if "recurse_on" in kwargs:
+            recurse_on = self.to_value(kwargs["recurse_on"])
+
+        if "recursion_depth" in kwargs:
+            recursion_depth = self.to_value(kwargs["recursion_depth"])
+
+        if "items_prop" in kwargs:
+            items_prop = self.to_value(kwargs["items_prop"])
+
+        if self.error_sink:
+            return None
+
+        if not isinstance(recursion_depth, int):
+            self.error_sink = FormatRuleSyntaxError("'recursion_depth' must be an integer", None)
+            return None
+
+        return FormatAstList(variable_name, items_prop, recurse_on, recursion_depth)
+
+    def return_dict(self, args, kwargs):
+        len_args = len(args)
+        if len_args < 1:
+            self.error_sink = FormatRuleSyntaxError("variable name not found", None)
+            return None
+
+        if len_args > 1:
+            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0])
+            return None
+
+        variable_name = get_text_from_tokens(args[0])
+
+        kwargs_parameters = {}
+        for prop in ("items_prop", "prefix", "suffix", "debug"):
+            if prop in kwargs:
+                kwargs_parameters[prop] = self.to_value(kwargs[prop])
+
+        if "debug" in kwargs_parameters:
+            if "prefix" not in kwargs_parameters:
+                kwargs_parameters["prefix"] = "{"
+            if "suffix" not in kwargs_parameters:
+                kwargs_parameters["suffix"] = "}"
+
+        if self.error_sink:
+            return None
+
+        return FormatAstDict(variable_name, **kwargs_parameters)
+
+    def return_multi(self, args, kwargs):
+        if len(kwargs) > 0:
+            self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None)
+            return None
+
+        if len(args) > 1:
+            self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0])
+            return None
+
+        return FormatAstMulti(get_text_from_tokens(args[0]))

src/parsers/FunctionParser.py

@@ -62,6 +62,10 @@ class FunctionParser(BaseExpressionParser):
         if not ret.status:
             return ret
 
+        # FunctionParser returns LexerNodes, rather than an ExprNode
+        # I know that is is not very logical, but at the beginning, the FunctionParser was
+        #   uses exclusively by the SyaNodeParser.
+        # It has been refactored to fit in ExpressionParser. So it has two main usages
         node = ret.body.body
         source_code_nodes = self.to_source_code_node(context, node)
         res = []

src/parsers/LogicalOperatorParser.py

@@ -1,15 +1,10 @@
-from itertools import product
-
-from core.builtin_helpers import only_successful, get_inner_body, get_lexer_nodes_using_positions
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import FailedToCompileError
-from core.tokenizer import TokenKind, Tokenizer, Keywords
+from core.tokenizer import TokenKind, Tokenizer
 from core.utils import get_text_from_tokens
-from parsers.BaseExpressionParser import ParenthesisNode, OrNode, AndNode, NotNode, ExprNode, VariableNode, \
+from parsers.BaseExpressionParser import ParenthesisNode, OrNode, AndNode, NotNode, VariableNode, \
     ComparisonNode, BaseExpressionParser
-from parsers.BaseNodeParser import UnrecognizedTokensNode
 from parsers.BaseParser import UnexpectedEofParsingError, ErrorSink
-from parsers.PythonWithConceptsParser import PythonWithConceptsParser
 from sheerkarete.common import V
 from sheerkarete.conditions import Condition, AndConditions
 
@@ -154,96 +149,96 @@ class LogicalOperatorParser(BaseExpressionParser):
         return token.type == TokenKind.IDENTIFIER and token.value in ("and", "or") or \
                token.value == "not" and parser_input.the_token_after(True).value != "in"
 
-    def compile_conjunctions(self, context, conjunctions, who):
-        """
-        Transform a list of conjunctions (AND and OR) into one or multiple CompiledExpr
-        :param context:
-        :param conjunctions: list of ExprNode
-        :param who: service that calls the method
-        :returns: List Of CompiledExpr
-        May throw FailedToRecognized if a conjunction cannot be parsed
-        """
-        recognized = []
-        for conjunction in conjunctions:
-            # try to recognize conjunction, one by one
-            # negative conjunction can be a concept starting with 'not'
-            parsed_ret = context.sheerka.parse_unrecognized(
-                context,
-                conjunction.get_value(),  # we remove the 'NOT' part when needed to ease the recognition
-                parsers="all",
-                who=who,
-                prop=Keywords.WHEN,
-                filter_func=only_successful)
-
-            if parsed_ret.status:
-                recognized.append(get_inner_body(context, parsed_ret.body))
-            else:
-                raise FailedToCompileError(parsed_ret.body)
-
-        # for each conjunction, we have a list of recognized concepts (or python node)
-        # we need a cartesian product of the results
-        # Explanation for later
-        # conjunction[0] : 'x is a y' that can be resolved with two concepts c:|1001: and c:|1002:
-        # conjunction[1] : 'y is an z' that can also be resolved with two concepts (c:|1003: and c:|1004)
-        # so to understand the full question 'x is a y and y is an z'
-        # we can have c:|1001: then c:|1003:
-        # or          c:|1001: then c:|1004:
-        # or          c:|1002: then c:|1003:
-        # or          c:|1002: then c:|1004:
-        # if one of this combination works, it means that the question 'x is a y and y is an z' was matched
-        # hence the cartesian product
-        product_of_recognized = list(product(*recognized))
-
-        return_values = []
-        for recognized_conjunctions in product_of_recognized:
-            if len(recognized_conjunctions) == 1 and not isinstance(conjunctions[0], NotNode):
-                return_values.append(recognized_conjunctions[0])
-            elif len(recognized_conjunctions) == 1 and recognized_conjunctions[0].who == "parsers.Python":
-                # it is a negated python Node. Need to parse again
-                ret = context.sheerka.parse_python(context, source=str(conjunctions[0]))
-                if ret.status:
-                    return_values.append(ret)
-                else:
-                    # find a way to track the failure
-                    pass
-            else:
-                # complex result. Use PythonWithNode
-                lexer_nodes = get_lexer_nodes_using_positions(recognized_conjunctions,
-                                                              self._get_positions(conjunctions))
-
-                # put back the 'and' / 'not' node
-                for i in range(len(lexer_nodes) - 1, 0, -1):
-                    end = lexer_nodes[i].start - 1
-                    start = lexer_nodes[i - 1].end + 1
-                    if isinstance(conjunctions[i], NotNode):
-                        lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_not_tokens))
-                    else:
-                        lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_tokens))
-
-                # add the starting 'not' if needed
-                # and reindex the following positions
-                if isinstance(conjunctions[0], NotNode):
-                    lexer_nodes[0].start = 2
-                    lexer_nodes.insert(0, UnrecognizedTokensNode(0, 1, self.not_tokens))
-
-                python_with_concept_node_ret = PythonWithConceptsParser().parse_nodes(context, lexer_nodes)
-                if not python_with_concept_node_ret.status:
-                    # find a way to track the failure
-                    pass
-                return_values.append(python_with_concept_node_ret)
-
-        rete_cond_emitter = ReteConditionsEmitter(context)
-        rete_disjunctions = rete_cond_emitter.get_conditions(conjunctions)
-
-        return return_values, rete_disjunctions
-
-    @staticmethod
-    def _get_positions(expr_nodes):
-        """
-        simply manage NotNodes to address the fact that the 'not' part in removed
-        """
-        for expr in expr_nodes:
-            if isinstance(expr, NotNode):
-                yield ExprNode(expr.start + 2, expr.end, expr.tokens[2:])
-            else:
-                yield expr
+    # def compile_conjunctions(self, context, conjunctions, who):
+    #     """
+    #     Transform a list of conjunctions (AND and OR) into one or multiple CompiledExpr
+    #     :param context:
+    #     :param conjunctions: list of ExprNode
+    #     :param who: service that calls the method
+    #     :returns: List Of CompiledExpr
+    #     May throw FailedToRecognized if a conjunction cannot be parsed
+    #     """
+    #     recognized = []
+    #     for conjunction in conjunctions:
+    #         # try to recognize conjunction, one by one
+    #         # negative conjunction can be a concept starting with 'not'
+    #         parsed_ret = context.sheerka.parse_unrecognized(
+    #             context,
+    #             conjunction.get_value(),  # we remove the 'NOT' part when needed to ease the recognition
+    #             parsers="all",
+    #             who=who,
+    #             prop=Keywords.WHEN,
+    #             filter_func=only_successful)
+    #
+    #         if parsed_ret.status:
+    #             recognized.append(get_inner_body(context, parsed_ret.body))
+    #         else:
+    #             raise FailedToCompileError(parsed_ret.body)
+    #
+    #     # for each conjunction, we have a list of recognized concepts (or python node)
+    #     # we need a cartesian product of the results
+    #     # Explanation for later
+    #     # conjunction[0] : 'x is a y' that can be resolved with two concepts c:|1001: and c:|1002:
+    #     # conjunction[1] : 'y is an z' that can also be resolved with two concepts (c:|1003: and c:|1004)
+    #     # so to understand the full question 'x is a y and y is an z'
+    #     # we can have c:|1001: then c:|1003:
+    #     # or          c:|1001: then c:|1004:
+    #     # or          c:|1002: then c:|1003:
+    #     # or          c:|1002: then c:|1004:
+    #     # if one of this combination works, it means that the question 'x is a y and y is an z' was matched
+    #     # hence the cartesian product
+    #     product_of_recognized = list(product(*recognized))
+    #
+    #     return_values = []
+    #     for recognized_conjunctions in product_of_recognized:
+    #         if len(recognized_conjunctions) == 1 and not isinstance(conjunctions[0], NotNode):
+    #             return_values.append(recognized_conjunctions[0])
+    #         elif len(recognized_conjunctions) == 1 and recognized_conjunctions[0].who == "parsers.Python":
+    #             # it is a negated python Node. Need to parse again
+    #             ret = context.sheerka.parse_python(context, source=str(conjunctions[0]))
+    #             if ret.status:
+    #                 return_values.append(ret)
+    #             else:
+    #                 # find a way to track the failure
+    #                 pass
+    #         else:
+    #             # complex result. Use PythonWithNode
+    #             lexer_nodes = get_lexer_nodes_using_positions(recognized_conjunctions,
+    #                                                           self._get_positions(conjunctions))
+    #
+    #             # put back the 'and' / 'not' node
+    #             for i in range(len(lexer_nodes) - 1, 0, -1):
+    #                 end = lexer_nodes[i].start - 1
+    #                 start = lexer_nodes[i - 1].end + 1
+    #                 if isinstance(conjunctions[i], NotNode):
+    #                     lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_not_tokens))
+    #                 else:
+    #                     lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_tokens))
+    #
+    #             # add the starting 'not' if needed
+    #             # and reindex the following positions
+    #             if isinstance(conjunctions[0], NotNode):
+    #                 lexer_nodes[0].start = 2
+    #                 lexer_nodes.insert(0, UnrecognizedTokensNode(0, 1, self.not_tokens))
+    #
+    #             python_with_concept_node_ret = PythonWithConceptsParser().parse_nodes(context, lexer_nodes)
+    #             if not python_with_concept_node_ret.status:
+    #                 # find a way to track the failure
+    #                 pass
+    #             return_values.append(python_with_concept_node_ret)
+    #
+    #     rete_cond_emitter = ReteConditionsEmitter(context)
+    #     rete_disjunctions = rete_cond_emitter.get_conditions(conjunctions)
+    #
+    #     return return_values, rete_disjunctions
+    #
+    # @staticmethod
+    # def _get_positions(expr_nodes):
+    #     """
+    #     simply manage NotNodes to address the fact that the 'not' part in removed
+    #     """
+    #     for expr in expr_nodes:
+    #         if isinstance(expr, NotNode):
+    #             yield ExprNode(expr.start + 2, expr.end, expr.tokens[2:])
+    #         else:
+    #             yield expr

src/parsers/PythonParser.py

@@ -134,7 +134,7 @@ class PythonParser(BaseParserInputParser):
 
         if self.reset_parser(context, parser_input):
             source_code = parser_input.as_text(python_switcher, tracker)
-            source_code = source_code.strip()
+            source_code = source_code.lstrip()  # right side spaces must be kept
 
             #  first, try to parse an expression
             res, tree, error = self.try_parse_expression(source_code)

src/parsers/PythonWithConceptsParser.py

@@ -2,7 +2,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import CreateObjectIdentifiers
 from parsers.BaseNodeParser import ConceptNode, RuleNode, VariableNode
 from parsers.BaseNodeParser import SourceCodeWithConceptNode
-from parsers.BaseParser import BaseParser, BaseParserInputParser
+from parsers.BaseParser import BaseParserInputParser
 from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
 
 unrecognized_nodes_parser = UnrecognizedNodeParser()
@@ -76,7 +76,6 @@ class PythonWithConceptsParser(BaseParserInputParser):
                 python_ids_mappings[python_id] = var_ref
                 last_token_index = node.end
 
-
             else:
                 source += node.source
                 to_parse += node.get_source_to_parse()

src/parsers/SequenceNodeParser.py

@@ -380,9 +380,11 @@ class SequenceNodeParser(BaseNodeParser):
             for node in parser_helper.sequence:
                 # if isinstance(node, ConceptNode):
                 #     if len(node.concept.get_metadata().variables) > 0:
-                #         node.concept.get_metadata().is_evaluated = True  # Do not try to evaluate those concepts
+                #         node.concept.get_hints().is_evaluated = True  # Do not try to evaluate those concepts
                 node.tokens = self.parser_input.tokens[node.start:node.end + 1]
                 node.fix_source()
+                if isinstance(node, ConceptNode):
+                    node.concept.get_hints().use_copy = True
 
             parser_helper_hash_code = compute_hash_code(parser_helper)
             if parser_helper_hash_code in already_seen:
@@ -426,7 +428,7 @@ class SequenceNodeParser(BaseNodeParser):
                         self.sheerka.new(
                             BuiltinConcepts.PARSER_RESULT,
                             parser=self,
-                            source=parser_input,
+                            source=parser_input.as_text(),
                             body=parser_helper.sequence,
                             try_parsed=parser_helper.sequence)))
 

src/parsers/ShortTermMemoryParser.py

@@ -8,9 +8,10 @@ class ShortTermMemoryParser(BaseParser):
     """
     This parser is used to recognize concept that are already instantiated
     """
+    NAME = "ShortTermMemory"
 
     def __init__(self, **kwargs):
-        super().__init__("ShortTermMemory", 85)
+        super().__init__(ShortTermMemoryParser.NAME, 85)  # priority is irrelevant for ShortTermMemory parser
 
     def parse(self, context, parser_input):
         """

src/parsers/SyaNodeParser.py

@@ -5,6 +5,7 @@ from typing import List
 
 from core import builtin_helpers
 from core.builtin_concepts import BuiltinConcepts
+from core.builtin_helpers import update_compiled
 from core.concept import Concept, DEFINITION_TYPE_BNF
 from core.global_symbols import CONCEPT_COMPARISON_CONTEXT, SyaAssociativity
 from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
@@ -1270,8 +1271,10 @@ class SyaNodeParser(BaseNodeParser):
             while len(item.nodes) > 0:
                 res = self.postfix_to_item(sheerka, item.nodes)
                 if isinstance(res, PostFixToItem):
-                    items.append(
-                        ConceptNode(res.concept, res.start, res.end, self.parser_input.tokens[res.start: res.end + 1]))
+                    items.append(ConceptNode(res.concept,
+                                             res.start,
+                                             res.end,
+                                             self.parser_input.tokens[res.start: res.end + 1]))
                 else:
                     items.append(res)
                 item.has_unrecognized |= hasattr(res, "has_unrecognized") and res.has_unrecognized or \
@@ -1314,6 +1317,8 @@ class SyaNodeParser(BaseNodeParser):
             assert meta_orig[0] == meta_new[0]
         # ---- Sanity check. To remove at some point
         concept.get_metadata().variables = concept_metadata
+        concept.get_hints().use_copy = True
+        concept.get_hints().need_validation = True
 
         source = get_text_from_tokens(self.parser_input.tokens[start:end + 1])
         return PostFixToItem(concept, start, end, has_unrecognized, source)
@@ -1354,6 +1359,7 @@ class SyaNodeParser(BaseNodeParser):
         for infix_to_postfix in valid_infix_to_postfixs:
             sequence = []
             has_unrecognized = False
+            errors = []
             while len(infix_to_postfix.out) > 0:
                 item = self.postfix_to_item(context.sheerka, infix_to_postfix.out)
                 has_unrecognized |= hasattr(item, "has_unrecognized") and item.has_unrecognized or \
@@ -1363,10 +1369,30 @@ class SyaNodeParser(BaseNodeParser):
                                             item.start,
                                             item.end,
                                             self.parser_input.tokens[item.start: item.end + 1])
+
+                    # validate the concept
+                    update_compiled(context, item.concept, errors)
+                    if errors:
+                        break
                 else:
                     to_insert = item
                 sequence.insert(0, to_insert)
 
+            if errors:
+                if len(errors) == 1:
+                    ret.append(
+                        self.sheerka.ret(
+                            self.name,
+                            False,
+                            errors[0]))
+                else:
+                    ret.append(
+                        self.sheerka.ret(
+                            self.name,
+                            False,
+                            self.sheerka.err([e.body for e in errors])))
+                continue
+
             if has_unrecognized:
                 # Manage some sick cases where missing parenthesis mess the order or the sequence
                 # example "foo bar(one plus two"
@@ -1380,7 +1406,7 @@ class SyaNodeParser(BaseNodeParser):
                     self.sheerka.new(
                         BuiltinConcepts.PARSER_RESULT,
                         parser=self,
-                        source=parser_input,
+                        source=parser_input.as_text(),
                         body=sequence,
                         try_parsed=sequence)))
 

src/parsers/UnrecognizedNodeParser.py

@@ -6,6 +6,7 @@ from core.builtin_helpers import only_successful, get_lexer_nodes, update_compil
 from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode, SourceCodeWithConceptNode
 from parsers.BaseParser import BaseParser, ParsingError, BaseParserInputParser
 from parsers.BnfNodeParser import BnfNodeParser
+from parsers.PythonParser import PythonParser
 from parsers.SequenceNodeParser import SequenceNodeParser
 from parsers.SyaNodeParser import SyaNodeParser
 
@@ -14,7 +15,7 @@ PARSERS = ["EmptyString",
            SequenceNodeParser.NAME,
            BnfNodeParser.NAME,
            SyaNodeParser.NAME,
-           "Python"]
+           PythonParser.NAME]
 
 
 @dataclass()

src/sheerkaql/symbols.py

@@ -258,6 +258,7 @@ def comparison_value(value1, op, value2):
         return op(value1(namespace), value2(namespace))
 
     object.__setattr__(where, '__objquery__', True)
+    object.__setattr__(where, '__objcond__', True)
     return where
 
 
@@ -297,6 +298,7 @@ def set_expression_value(val, op, s):
         return op(val(namespace), s(namespace))
 
     object.__setattr__(where, '__objquery__', True)
+    object.__setattr__(where, '__objcond__', True)
     return where
 
 
@@ -309,6 +311,7 @@ def boolean_expression_value(value1, op, value2):
         return op(value1, value2, namespace)
 
     object.__setattr__(where, '__objquery__', True)
+    object.__setattr__(where, '__objcond__', True)
     return where
 
 
@@ -321,6 +324,7 @@ def unary_expression_value(op, val):
         return op(val(namespace))
 
     object.__setattr__(where, '__objquery__', True)
+    object.__setattr__(where, '__objcond__', True)
     return where
 
 
@@ -345,6 +349,8 @@ def where_value(val):
         return val(namespace)
 
     object.__setattr__(where, '__objquery__', True)
+    if hasattr(val, "__objcond__"):
+        object.__setattr__(where, '__objcond__', True)
     return where
 
 
@@ -414,12 +420,11 @@ def query_value(q):
                 index = object.__getattribute__(current_namespace, '_objquery__i')
                 attr_name, where = attrs_path[index]
 
-                # if sheerka_hasattr(current_namespace, attr_name):  # the current object has the attr
                 attr_value = sheerka_getattr(current_namespace, attr_name)
                 # it is iterable
                 if not isinstance(attr_value, str) and hasattr(attr_value, '__iter__'):
                     # # try to use where clause as an indexer
-                    if where is not None:
+                    if where is not None and not hasattr(where, "__objcond__"):
                         try:
                             namespace_copy = dict(namespace)
                             res = where(namespace_copy)

src/sheerkarete/network.py

@@ -262,7 +262,7 @@ class ReteNetwork:
 
                 # Manage list of requested attributes when bounding a new variable
                 if (cond.identifier in vars_ids_mappings and
-                    isinstance(cond.attribute, str) and
+                    (isinstance(cond.attribute, str) and cond.attribute != FACT_SELF) and
                     isinstance(cond.value, V)):
                     vars_ids_mappings[cond.value] = f"{vars_ids_mappings[cond.identifier]}.{cond.attribute}"
 
@@ -272,6 +272,11 @@ class ReteNetwork:
                 if identifier:
                     attr = "*" if isinstance(cond.attribute, V) else cond.attribute
                     self.attributes_by_id.setdefault(identifier, []).append(attr)
+
+                    # to manage conditions like V(x) == V(y)
+                    if cond.attribute == FACT_SELF and cond.value in vars_ids_mappings:
+                        self.attributes_by_id.setdefault(vars_ids_mappings[cond.value], []).append(FACT_SELF)
+
                 elif not isinstance(cond.attribute, V):
                     self.default_attributes.add(cond.attribute)
 

tests/BaseTest.py

@@ -45,7 +45,7 @@ class InitTestHelper:
                         c.get_metadata().definition_type = DEFINITION_TYPE_BNF
                     else:
                         raise Exception(f"Error in bnf definition '{c.get_metadata().definition}'",
-                                        self.sheerka.get_errors(res))
+                                        self.sheerka.get_errors(self.context, res))
 
             if create_new:
                 self.sheerka.create_new_concept(self.context, c)
@@ -264,3 +264,7 @@ class BaseTest:
             assert res[0].status, f"Error while executing '{expression}'"
 
         return sheerka
+
+    @staticmethod
+    def successful_return_values(return_values):
+        return [ret_val for ret_val in return_values if ret_val.status]

tests/core/test_ExecutionContext.py

@@ -47,6 +47,7 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka):
         a.preprocess_evaluators = ["list of evaluators"]
         a.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
         a.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        a.possible_variables = {"a", "b", "c"}
 
         b = a.push(BuiltinConcepts.EVALUATION, "sub action context", desc="sub description")
 
@@ -65,8 +66,9 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka):
         assert b.preprocess_evaluators == a.preprocess_evaluators
         assert b.protected_hints == {BuiltinConcepts.EVAL_BODY_REQUESTED}
         assert b.global_hints == a.global_hints
+        assert b.possible_variables == a.possible_variables
 
-    def test_children_i_created_when_i_push(self):
+    def test_children_are_created_when_i_push(self):
         sheerka = self.get_sheerka()
 
         e = ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None)
@@ -75,19 +77,9 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka):
         e.push(BuiltinConcepts.NOP, None, who="c", desc="I do something else")
 
         assert len(e._children) == 3
-        assert e._children[0].who, e._children[0].who == ("a", "I do something")
-        assert e._children[1].who, e._children[1].who == ("b", "oups! I did a again")
-        assert e._children[2].who, e._children[2].who == ("c", "I do something else")
-
-    # def test_i_can_add_variable_when_i_push(self):
-    #     sheerka = self.get_sheerka()
-    #
-    #     e = ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None)
-    #     sub_e = e.push(BuiltinConcepts.NOP, None, who="a", my_new_var="new var value")
-    #
-    #     assert sub_e.my_new_var == "new var value"
-    #     with pytest.raises(AttributeError):
-    #         assert e.my_new_var == ""  # my_new_var does not exist in parent
+        assert (e._children[0].who, e._children[0].desc) == ("a", "I do something")
+        assert (e._children[1].who, e._children[1].desc) == ("b", "oups! I did a again")
+        assert (e._children[2].who, e._children[2].desc) == ("c", "I do something else")
 
     def test_local_hints_are_local_and_global_hints_are_global(self):
         sheerka = self.get_sheerka()

tests/core/test_SheerkaConceptManager.py

@@ -5,7 +5,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import ensure_bnf
 from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs, \
     DEFINITION_TYPE_BNF
-from core.global_symbols import NotInit, NotFound, SyaAssociativity
+from core.global_symbols import NotInit, NotFound, SyaAssociativity, CONCEPT_COMPARISON_CONTEXT
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \
     UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced, NoFirstTokenError
 from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore, \
@@ -293,8 +293,6 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         "definition",
         "definition_type",
         "desc",
-        "is_evaluated",
-        "need_validation",
         "full_serialization",
     ])
     def test_i_can_modify_a_metadata_attribute(self, attr):
@@ -1422,6 +1420,33 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         res = sheerka.smart_get_attr(table_instance, size)
         assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
 
+    def test_i_can_set_concept_precedence(self):
+        sheerka, context, one, two, three = self.init_concepts("one", "two", "three")
+
+        res = sheerka.set_precedence(context, one, two, three)
+        assert sheerka.isinstance(res, BuiltinConcepts.SUCCESS)
+
+        weights = sheerka.get_weights(BuiltinConcepts.PRECEDENCE, comparison_context=CONCEPT_COMPARISON_CONTEXT)
+        assert weights == {'c:one|1001:': 3, 'c:two|1002:': 2, 'c:three|1003:': 1}
+
+    def test_i_cannot_set_precedence_when_too_few_argument(self):
+        sheerka, context, one = self.init_concepts("one")
+
+        res = sheerka.set_precedence(context)
+        assert res == sheerka.err("Not enough elements")
+
+        res = sheerka.set_precedence(context, one)
+        assert res == sheerka.err("Not enough elements")
+
+    def test_i_cannot_set_precedence_when_error(self):
+        sheerka, context, one, two = self.init_concepts("one", "two")
+
+        ret = sheerka.set_precedence(context, one, two, one)
+        assert not ret.status
+
+        weights = sheerka.get_weights(BuiltinConcepts.PRECEDENCE, comparison_context=CONCEPT_COMPARISON_CONTEXT)
+        assert weights == {'c:one|1001:': 2, 'c:two|1002:': 1}
+
 
 class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
     def test_i_can_add_several_concepts(self):

tests/core/test_SheerkaDebugManager.py

@@ -672,16 +672,15 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         res = sheerka.inspect(context, return_values[0], values=True)
         concept_debug_obj = ConceptDebugObj(return_values[0].body)
-        assert res.body == {
-            'body': concept_debug_obj,
-            '#type#': 'ReturnValueConcept',
-            'id': f'{self.return_value_id}',
-            'key': '__RETURN_VALUE',
-            'name': '__RETURN_VALUE',
-            'parents': [concept_debug_obj],
-            'status': True,
-            'value': concept_debug_obj,
-            'who': 'evaluators.OneSuccess'}
+        assert res.body['body'] == concept_debug_obj
+        assert res.body['#type#'] == 'ReturnValueConcept'
+        assert res.body['id'] == f'{self.return_value_id}'
+        assert res.body['key'] == '__RETURN_VALUE'
+        assert res.body['name'] == '__RETURN_VALUE'
+        assert isinstance(res.body['parents'], list)
+        assert res.body['status'] == True
+        assert res.body['value'] == concept_debug_obj
+        assert res.body['who'] == 'evaluators.OneSuccess'
 
         # I also can print it using bag
         res = sheerka.inspect(context, return_values[0], '#type#', "who", "status", "value", values=True, as_bag=True)

tests/core/test_SheerkaEvaluateConcept.py

@@ -5,9 +5,13 @@ from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionR
     DEFINITION_TYPE_DEF
 from core.global_symbols import NotInit, NotFound
 from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
+from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from parsers.BaseParser import BaseParser
+from parsers.ExactConceptParser import ExactConceptParser
+from parsers.ExpressionParser import ExpressionParser
 from parsers.PythonParser import PythonNode, PythonParser
+from parsers.SyaNodeParser import SyaNodeParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, python_ret_val
 from tests.parsers.parsers_utils import CB, compare_with_test_object
@@ -37,7 +41,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.get_metadata().post is None
         assert evaluated.get_metadata().where is None
         assert evaluated.variables() == {}
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
         assert len(evaluated.values()) == 0 if body is None else 1
 
         assert "foo" in sheerka.services[SheerkaMemory.NAME].registration
@@ -70,7 +74,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.get_metadata().where is None
         assert evaluated.get_value(ConceptParts.POST) == expected
         assert evaluated.variables() == {}
-        assert not evaluated.get_metadata().is_evaluated
+        assert not evaluated.get_hints().is_evaluated
         assert len(evaluated.values) == 0 if expr is None else 1
 
     @pytest.mark.parametrize("expr, expected", [
@@ -94,7 +98,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.get_metadata().post is None
         assert evaluated.get_metadata().where is None
         assert evaluated.variables() == {"a": expected}
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
 
     def test_i_can_evaluate_when_the_body_is_the_name_of_the_concept(self):
         # to prove that I can distinguish from a string
@@ -114,7 +118,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept)
 
         assert evaluated.body == "do not resolve"
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
 
     def test_i_can_evaluate_variable_using_do_not_resolve(self):
         sheerka, context, concept = self.init_concepts(Concept("foo").def_var("a"), eval_body=True)
@@ -123,7 +127,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept)
 
         assert evaluated.get_value("a") == "do not resolve"
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
 
     def test_original_value_is_overridden_when_using_do_no_resolve(self):
         concept = Concept("foo", body="original value").def_var("a", "original value")
@@ -135,7 +139,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         assert evaluated.body == "do not resolve"
         assert evaluated.get_value("a") == "do not resolve"
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
 
     def test_variables_are_evaluated_before_body(self):
         sheerka, context, concept = self.init_concepts(Concept("foo", body="a+1").def_var("a", "10"), eval_body=True)
@@ -153,8 +157,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept)
 
         compare_with_test_object(evaluated, CB("foo", CB("a", NotInit)))
-        assert evaluated.get_metadata().is_evaluated
-        assert evaluated.body.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
+        assert evaluated.body.get_hints().is_evaluated
 
     def test_i_can_evaluate_when_the_referenced_concept_has_a_body(self):
         sheerka, context, concept_a, concept = self.init_concepts(
@@ -166,8 +170,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         assert evaluated.key == concept.key
         compare_with_test_object(evaluated.body, CB("a", 1))
-        assert not concept_a.get_metadata().is_evaluated
-        assert evaluated.get_metadata().is_evaluated
+        assert not concept_a.get_hints().is_evaluated
+        assert evaluated.get_hints().is_evaluated
 
     def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(self):
         sheerka, context, concept_a, concept_b, concept_c, concept_d = self.init_concepts(
@@ -183,7 +187,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         expected = CB("c", CB("b", CB("a", "a")))
         compare_with_test_object(evaluated.body, expected)
         assert sheerka.objvalue(evaluated) == 'a'
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
 
     def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(self):
         sheerka, context, concept_a, concept_b, concept_c, concept_d = self.init_concepts(
@@ -199,7 +203,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         expected = CB("c", CB("b", CB("a", NotInit)))
         compare_with_test_object(evaluated.body, expected)
         compare_with_test_object(sheerka.objvalue(evaluated), CB("a", NotInit))
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
 
     def test_i_can_evaluate_concept_when_variables_reference_others_concepts_1(self):
         """
@@ -349,7 +353,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, add_instance)
 
         assert evaluated.key == add_instance.key
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
         assert sheerka.objvalue(evaluated) == 3
 
     def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_and_different_names(self):
@@ -363,7 +367,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, add_instance)
 
         assert evaluated.key == add_instance.key
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
         assert sheerka.objvalue(evaluated) == 3
 
     def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_multiple_levels(self):
@@ -378,7 +382,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, inc_instance)
 
         assert evaluated.key == inc_instance.key
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
         assert sheerka.objvalue(evaluated) == 2
 
     def test_i_can_evaluate_a_concept_that_references_another_concept_twice(self):
@@ -556,7 +560,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         sheerka, context, one_1, one_str, is_an_int, plus = self.init_concepts(
             Concept("one", body="1"),
             Concept("one", body="'one'"),
-            Concept("x is an int", body="isinstance(x, int)").def_var("x"),
+            Concept("x is an int", body="isinstance(x, int)", pre="is_question()").def_var("x"),
             Concept("a plus b", body="a + b", where="a is an int").def_var("a", "one").def_var("b", "2"),
             eval_body=True,
             eval_where=True,
@@ -570,8 +574,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         sheerka, context, one_1, one_str, is_an_int, is_an_integer, plus = self.init_concepts(
             Concept("one", body="1"),
             Concept("one", body="'one'"),
-            Concept("x is an int", body="isinstance(x, int)").def_var("x"),
-            Concept("y is an integer", body="y is an int").def_var("y"),
+            Concept("x is an int", body="isinstance(x, int)", pre="is_question()").def_var("x"),
+            Concept("y is an integer", body="y is an int", pre="is_question()").def_var("y"),
             Concept("a plus b", body="a + b", where="a is an integer").def_var("a", "one").def_var("b", "2"),
             eval_body=True,
             eval_where=True,
@@ -600,14 +604,15 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == plus.key
         assert evaluated.body == 3
 
-    @pytest.mark.skip("Not ready for that")
     def test_i_can_apply_intermediate_where_condition_when_multiple_variables(self):
+        # The test does not work because the and condition is not correctly supported
+        # We need the ExpressionParser
         sheerka, context, one_1, one_str, two_2, two_str, is_an_int, plus = self.init_concepts(
             Concept("one", body="1"),
             Concept("one", body="'one'"),
             Concept("two", body="2"),
             Concept("two", body="'two'"),
-            Concept("x is an int", body="isinstance(x, int)").def_var("x"),
+            Concept("x is an int", body="isinstance(x, int)", pre="is_question()").def_var("x"),
             Concept("a plus b",
                     body="a + b",
                     where="a is an int and isinstance(b, int)").def_var("a", "one").def_var("b", "two"),
@@ -622,7 +627,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
     def test_i_cannot_evaluate_when_intermediate_where_condition_fails(self):
         sheerka, context, one_1, is_an_int, plus = self.init_concepts(
             Concept("one", body="'one'"),
-            Concept("x is an int", body="isinstance(x, int)").def_var("x"),
+            Concept("x is an int", body="isinstance(x, int)", pre="is_question()").def_var("x"),
             Concept("a plus b", body="a + b", where="a is an int").def_var("a", "one").def_var("b", "2"),
             eval_body=True,
             eval_where=True,
@@ -685,10 +690,10 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
             eval_body=True
         )
 
-        for concept in (c1, c2, c3, c4):
+        for concept, expected in ((c1, 3), (c2, 1), (c3, 2), (c4, 3)):
             evaluated = sheerka.evaluate_concept(context, concept)
             assert evaluated.key == concept.key
-            assert evaluated.body == InfiniteRecursionResolved(3)
+            assert evaluated.body == InfiniteRecursionResolved(expected)
 
     def test_i_can_detect_infinite_recursion_when_no_constant(self):
         sheerka, context, foo, bar, baz, qux = self.init_concepts(
@@ -877,26 +882,31 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept, eval_body=True)
 
         assert evaluated.key == concept.key
-        assert concept.get_metadata().is_evaluated == expected
+        assert concept.get_hints().is_evaluated == expected
 
     def test_i_only_compute_the_requested_metadata(self):
         sheerka, context, concept = self.init_concepts(
-            Concept("foo", pre="'pre'", post="'post'", ret="'ret'", where="'where'", body="'body'").def_var("a", "'a'")
+            Concept("foo", pre="True", post="'post'", ret="'ret'", where="True", body="'body'").def_var("a", "'a'")
         )
         context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)  # to prove that we do not care
         context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)  # to prove that we do not care
 
         evaluated = sheerka.evaluate_concept(context, concept, metadata=[ConceptParts.PRE])
-        assert evaluated.values() == {"a": NotInit, ConceptParts.PRE: 'pre'}
+        assert evaluated.values() == {"a": NotInit, ConceptParts.PRE: True}
 
     def test_i_can_manage_ret(self):
         sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="foo"))
 
         res = sheerka.evaluate_concept(context, bar)
-        assert res.id == bar.id
+        assert sheerka.isinstance(res, "bar")
 
         res = sheerka.evaluate_concept(context, bar, eval_body=True)
-        assert res.id == foo.id
+        assert sheerka.isinstance(res, "foo")
+
+        # And the result is still the same after a second call
+        assert bar.get_hints().is_evaluated
+        res = sheerka.evaluate_concept(context, bar, eval_body=True)
+        assert sheerka.isinstance(res, "foo")
 
     def test_ret_is_evaluated_only_is_body_is_requested(self):
         sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="__NOT_FOUND"))
@@ -930,7 +940,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         # 'def concept foo as foo'
         return_values = [pr_ret_val(foo, parser="ExactConcept"), python_ret_val("foo")]
 
-        res = evaluator.get_recursive_definitions(foo, return_values)
+        res = evaluator.get_recursive_definitions(context, foo, return_values)
 
         assert list(res) == [BaseParser.get_name("ExactConcept")]
 
@@ -941,7 +951,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         # 'def concept foo as bar'
         return_values = [pr_ret_val(bar, parser="ExactConcept"), python_ret_val("foo")]
 
-        res = evaluator.get_recursive_definitions(foo, return_values)
+        res = evaluator.get_recursive_definitions(context, foo, return_values)
 
         assert list(res) == []
 
@@ -953,18 +963,120 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         # i dunno how to construct the return value
         return_values = [pr_ret_val(q, parser="ExactConcept")]
 
-        res = evaluator.get_recursive_definitions(q, return_values)
+        res = evaluator.get_recursive_definitions(context, q, return_values)
 
         assert list(res) == []
 
-    # I cannot implement value cache for now
-    # def test_values_when_no_variables_are_computed_only_once(self):
-    #     sheerka, context, foo = self.init_concepts(Concept("foo", body="10"))
-    #
-    #     evaluated = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True)
-    #     assert evaluated.body == 10
-    #     assert len(evaluated.get_compiled()) > 0
-    #
-    #     evaluated_2 = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True)
-    #     assert evaluated_2.body == 10
-    #     assert len(evaluated_2.get_compiled()) == 0
+    def test_i_do_not_mess_up_use_copy_when_exact_concept(self):
+        sheerka, context, one, number, isa = self.init_concepts(
+            "one",
+            "number",
+            Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y"))
+
+        evaluator = SheerkaEvaluateConcept(sheerka)
+
+        parsed_return_value = ExactConceptParser().parse(context, ParserInput("one is a number"))
+        concept = parsed_return_value[0].body.body
+
+        # just get the compiled
+        context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        evaluated = evaluator.evaluate_concept(context, concept)
+        assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy
+        assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy
+
+        # get the body
+        evaluated = evaluator.evaluate_concept(context, concept, eval_body=True)
+        assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy
+        assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy
+        assert not evaluated.get_value("x").get_hints().use_copy
+        assert not evaluated.get_value("y").get_hints().use_copy
+
+    def test_i_do_not_mess_up_use_copy_when_expression_parser(self):
+        sheerka, context, one, number, isa = self.init_concepts(
+            "one",
+            "number",
+            Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y"))
+
+        evaluator = SheerkaEvaluateConcept(sheerka)
+
+        parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number"))
+        concept = next(iter(parsed_return_value.body.body.compiled[0].return_value.body.body.objects.values()))
+        assert concept.get_compiled()["x"][0].body.body.get_hints().use_copy
+        assert concept.get_compiled()["y"][0].body.body.get_hints().use_copy
+
+        # get the body
+        context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        evaluated = evaluator.evaluate_concept(context, concept, eval_body=True)
+        assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy
+        assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy
+        assert not evaluated.get_value("x").get_hints().use_copy
+        assert not evaluated.get_value("y").get_hints().use_copy
+
+    def test_i_do_not_evaluate_the_body_when_validation_only_is_set(self):
+        sheerka, context, red, shirt, a_x, red_x = self.init_concepts(
+            "red",
+            Concept("shirt", body="set_attr(self, 'body_shirt_is_evaluated', True)"),
+            Concept("a x", body="set_attr(x, 'body_ax_is_evaluated', True)", ret="x").def_var("x"),
+            Concept("red x", body="set_attr(x, 'color', 'red')", ret="x").def_var("x"),
+            create_new=True)
+
+        parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt"))
+
+        # Sanity check for normal behaviour
+        to_evaluate1 = parsed_ret_val.body.body[0].concept.copy()
+        evaluated1 = sheerka.evaluate_concept(context, to_evaluate1, eval_body=True, validation_only=False)
+
+        assert sheerka.isinstance(evaluated1, shirt)
+        assert evaluated1.get_value("body_ax_is_evaluated") == True
+        assert evaluated1.get_value("body_shirt_is_evaluated") == True
+        assert evaluated1.get_value("color") == "red"
+        assert evaluated1.body == sheerka.new(BuiltinConcepts.SUCCESS)
+
+        # check validation_only behaviour
+        to_evaluate2 = parsed_ret_val.body.body[0].concept.copy()
+        evaluated2 = sheerka.evaluate_concept(context, to_evaluate2, eval_body=True, validation_only=True)
+
+        assert sheerka.isinstance(evaluated2, shirt)
+        assert evaluated2.get_value("body_ax_is_evaluated") == NotInit
+        assert evaluated2.get_value("body_shirt_is_evaluated") == NotInit
+        assert evaluated2.get_value("color") == NotInit
+        assert evaluated2.body == NotInit
+
+    def test_methods_with_side_effect_are_not_called_when_eval_body_is_false(self):
+        sheerka, context, red, shirt, a_x, red_x = self.init_concepts(
+            "red",
+            Concept("shirt", body="set_attr(self, 'body_shirt_is_evaluated', True)"),
+            Concept("a x", body="set_attr(x, 'body_ax_is_evaluated', True)", ret="x").def_var("x"),
+            Concept("red x", body="set_attr(x, 'color', 'red')", ret="x").def_var("x"),
+            create_new=True,
+        )
+
+        parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt"))
+        to_evaluate = parsed_ret_val.body.body[0].concept
+
+        evaluated = sheerka.evaluate_concept(context, to_evaluate, eval_body=False)
+
+        assert sheerka.isinstance(evaluated, a_x)
+        assert "x" in evaluated.get_compiled()
+        assert ConceptParts.BODY in evaluated.get_compiled()
+        assert ConceptParts.RET in evaluated.get_compiled()
+        assert sheerka.isinstance(evaluated.get_compiled()["x"], red_x)
+        assert evaluated.get_compiled()["x"].get_compiled()["x"] == shirt  # so, it's not evaluated
+
+        # sanity check
+        parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt"))
+        to_evaluate = parsed_ret_val.body.body[0].concept
+        evaluated = sheerka.evaluate_concept(context, to_evaluate, eval_body=True)
+
+        assert sheerka.isinstance(evaluated, shirt)
+        assert evaluated.get_value("body_ax_is_evaluated") == True
+        assert evaluated.get_value("body_shirt_is_evaluated") == True
+        assert evaluated.get_value("color") == "red"
+
+    def test_concept_is_not_evaluated_when_method_access_error(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo", body="set_attr(self, 'prop_name', 'prop_value')"))
+
+        evaluated = sheerka.evaluate_concept(context, foo, eval_body=True, validation_only=True)
+
+        assert sheerka.isinstance(evaluated, foo)
+        assert not foo.get_hints().is_evaluated

tests/core/test_SheerkaEvaluateRules.py

@@ -2,14 +2,14 @@ import operator
 
 import pytest
 
-from core.builtin_concepts_ids import BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_DEF
 from core.rule import Rule, ACTION_TYPE_EXEC
 from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME
 from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules, LOW_PRIORITY_RULES, DISABLED_RULES
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition
+from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition, PythonConditionExprVisitor
 from evaluators.PythonEvaluator import PythonEvaluator
+from parsers.ExpressionParser import ExpressionParser
 from parsers.PythonParser import PythonParser
 from sheerkapython.python_wrapper import Expando
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -40,7 +40,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
             DISABLED_RULES: [r7]
         }
 
-    @pytest.mark.skip("Not ready for that")
+    @pytest.mark.skip("Rete is not ready for that")
     def test_i_can_evaluate_question_concept_rules(self):
         sheerka, context, concept, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_test().with_concepts(
             Concept("x equals y", body="x == y", pre="is_question()").def_var("x").def_var("y"),
@@ -152,7 +152,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
 
         there_instance = sheerka.new_from_template(there, there.key)
         if recognized_by:
-            there_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, recognized_by)
+            there_instance.get_hints().recognized_by = recognized_by
         ret = sheerka.ret("evaluator", True, sheerka.new(greetings, a=there_instance))
         res = service.evaluate_rules(context, [rule], {"__ret": ret}, set())
         assert res == {True: [rule]}
@@ -175,7 +175,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
 
         my_friend_instance = sheerka.new_from_template(my_friend, my_friend.key)
         if recognized_by:
-            my_friend_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, recognized_by)
+            my_friend_instance.get_hints().recognized_by = recognized_by
         ret = sheerka.ret("evaluator", True, sheerka.new(greetings, a=my_friend_instance))
         res = service.evaluate_rules(context, [rule], {"__ret": ret}, set())
         assert res == {True: [rule]}
@@ -211,7 +211,8 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
     def test_i_can_evaluate_concept_rule_with_the_same_name_when_the_second_concept_is_declared_after(self):
         sheerka, context, g1, rule, g2 = self.init_test().with_concepts(
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
-            create_new=True).with_format_rules(Rule(predicate="recognize(__ret.body, greetings)", action="")).with_concepts(
+            create_new=True).with_format_rules(
+            Rule(predicate="recognize(__ret.body, greetings)", action="")).with_concepts(
             Concept("greetings", definition="hi a", definition_type=DEFINITION_TYPE_DEF).def_var("a")).unpack()
 
         service = sheerka.services[SheerkaEvaluateRules.NAME]
@@ -249,3 +250,22 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
 
         assert rule in evaluate_rule_service.network.rules
         assert rule.rete_net == evaluate_rule_service.network
+
+    def test_i_can_get_missing_variables_when_evaluate_conditions(self):
+        sheerka, context = self.init_test().unpack()
+
+        expression = "isinstance(a, int)"
+        parser = ExpressionParser()
+        ret_val = parser.parse(context, ParserInput(expression))
+        parsed = ret_val.body.body
+
+        visitor = PythonConditionExprVisitor(context)
+        conditions = visitor.get_conditions(parsed)
+
+        missing_vars = set()
+        service = sheerka.services[SheerkaEvaluateRules.NAME]
+        res = service.evaluate_conditions(context, conditions, {}, missing_vars)
+
+        assert res == []
+        assert missing_vars == {"a"}
+

tests/core/test_SheerkaIsAManager.py

@@ -84,7 +84,19 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(error, BuiltinConcepts.NOT_A_SET)
         assert error.body == one
 
-    def test_isa_and_isa_group(self):
+    def test_isa(self):
+        sheerka, context, blue, color = self.init_concepts(Concept("blue"), Concept("color"))
+
+        assert not sheerka.isa(blue, color)
+
+        sheerka.set_isa(context, blue, color)
+        assert sheerka.isa(blue, color)
+
+        # isa tests the id of a concept, not it's content
+        another_color_instance_but_with_a_body = sheerka.new(color, body="a body")
+        assert sheerka.isa(blue, another_color_instance_but_with_a_body)
+
+    def test_isaset(self):
         sheerka, context, group, foo = self.init_concepts(Concept("group"), Concept("foo"))
 
         assert not sheerka.isaset(context, group)

tests/core/test_SheerkaRuleManager.py

@@ -4,28 +4,18 @@ from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept, DEFINITION_TYPE_DEF
 from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound, EVENT_RULE_DELETED
 from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
-from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleActionParser, \
-    FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \
-    FormatRuleSyntaxError, FormatAstList, UnexpectedEof, FormatAstColor, FormatAstDict, \
-    FormatAstMulti, \
-    PythonCodeEmitter, FormatAstNode, ReteConditionExprVisitor
+from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, ReteConditionExprVisitor
 from core.tokenizer import Token, TokenKind
 from parsers.BaseParser import ErrorSink
 from parsers.ExpressionParser import ExpressionParser
+from parsers.FormatRuleActionParser import FormatAstNode
 from sheerkarete.conditions import FilterCondition
 from sheerkarete.network import ReteNetwork
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND, NCCOND
 
-seq = FormatAstSequence
-raw = FormatAstRawText
-var = FormatAstVariable
-func = FormatAstFunction
-lst = FormatAstList
-
 PYTHON_EVALUATOR_NAME = "Python"
 CONCEPT_EVALUATOR_NAME = "Concept"
 
@@ -175,7 +165,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         sheerka, context = self.init_test(cache_only=False).unpack()
         service = sheerka.services[SheerkaRuleManager.NAME]
 
-        rule = Rule(ACTION_TYPE_EXEC, "name", "cannot build = False", "'Hello back at you !'")
+        rule = Rule(action_type, "name", "cannot build = False", action)
         rule.metadata.is_enabled = True  # it should be disabled
         rule = service.init_rule(context, rule)
 
@@ -209,71 +199,6 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         assert not rule.metadata.is_enabled
         assert rule.compiled_action is None
 
-    @pytest.mark.parametrize("text, expected", [
-        ("", FormatAstRawText("")),
-        (" ", FormatAstRawText(" ")),
-        ("  raw text  ", FormatAstRawText("  raw text  ")),
-        ("{variable}", FormatAstVariable("variable")),
-        ("{ variable }", FormatAstVariable("variable")),
-        (" xy {v} z", seq([raw(" xy "), var("v"), raw(" z")])),
-        (r"\{variable}", FormatAstRawText("{variable}")),
-        (r"\\{variable}", seq([raw("\\"), var("variable")])),
-        (r"\\\{variable}", FormatAstRawText(r"\{variable}")),
-        (r"{var1}{var2}", seq([var("var1"), var("var2")])),
-        ("func()", FormatAstFunction("func", [], {})),
-        ("func(a, 'string value', c)", FormatAstFunction("func", ["a", "'string value'", "c"], {})),
-        ("func(a=10, b='string value')", FormatAstFunction("func", [], {"a": "10", "b": "'string value'"})),
-        ("func('string value'='another string value')", func("func", [], {"'string value'": "'another string value'"})),
-        ("red(' xy {v}')", FormatAstColor("red", seq([raw(" xy "), var("v")]))),
-        ('blue(" xy {v}")', FormatAstColor("blue", seq([raw(" xy "), var("v")]))),
-        ('green(  xy )', FormatAstColor("green", var("xy"))),
-        ('green()', FormatAstColor("green", raw(""))),
-        ('green("")', FormatAstColor("green", raw(""))),
-        ("list(var_name, 2, 'children')", FormatAstList("var_name", recurse_on="children", recursion_depth=2)),
-        ("list(var_name, recursion_depth=2, recurse_on='children')", FormatAstList("var_name",
-                                                                                   recurse_on="children",
-                                                                                   recursion_depth=2)),
-        ("list(var_name, recursion_depth=2, 'children')", FormatAstList("var_name", recursion_depth=2)),
-        ("list(var_name, 'children', recursion_depth=2)", FormatAstList("var_name", recursion_depth=2)),
-        ("list(var_name)", FormatAstList("var_name")),
-        ("{obj.prop1.prop2[0].prop3['value']}", FormatAstVariable("obj.prop1.prop2[0].prop3['value']")),
-        ("[{id}]", seq([raw("["), var("id"), raw("]")])),
-        ("{variable:format}", FormatAstVariable("variable", "format")),
-        ("{variable:3}", FormatAstVariable("variable", "3")),
-        (r"\not_a_function(a={var})", seq([raw("not_a_function(a="), var("var"), raw(")")])),
-        ("dict(var_name)", FormatAstDict("var_name")),
-        ("dict(var_name, items_prop='props')", FormatAstDict("var_name", items_prop='props')),
-        ("dict(var_name, debug=True)", FormatAstDict("var_name", debug=True, prefix="{", suffix="}")),
-        ("multi(var_name)", FormatAstMulti("var_name")),
-    ])
-    def test_i_can_parse_format_rule(self, text, expected):
-        assert FormatRuleActionParser(text).parse() == expected
-
-    @pytest.mark.parametrize("text, expected_error", [
-        ("{", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
-        ("{var_name", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
-        ("{}", FormatRuleSyntaxError("variable name not found", None)),
-        ("func(", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
-        ("func(a,b,c", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
-        ("func(a,,c", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
-        ("func(a,,c)", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
-        ("red(a,b)", FormatRuleSyntaxError("only one parameter supported", Token(TokenKind.IDENTIFIER, "b", 6, 1, 7))),
-        ("red(a=b)", FormatRuleSyntaxError("keyword arguments are not supported", None)),
-        ("red(xy {v})", FormatRuleSyntaxError("Invalid identifier", None)),
-        ("list()", FormatRuleSyntaxError("variable name not found", None)),
-        ("list(recursion_depth=2)", FormatRuleSyntaxError("variable name not found", None)),
-        ("list(a,b,c,d,e)", FormatRuleSyntaxError("too many positional arguments",
-                                                  Token(TokenKind.IDENTIFIER, "e", 13, 1, 14))),
-        ("list(a, recursion_depth=hello)", FormatRuleSyntaxError("'hello' is not numeric", None)),
-        ("list(a, recursion_depth='hello')", FormatRuleSyntaxError("'recursion_depth' must be an integer", None)),
-        ("dict()", FormatRuleSyntaxError("variable name not found", None)),
-    ])
-    def test_i_cannot_parse_invalid_format(self, text, expected_error):
-        parser = FormatRuleActionParser(text)
-        parser.parse()
-
-        assert parser.error_sink == expected_error
-
     def test_i_can_get_rule_priorities(self):
         sheerka, context, rule_true, rule_false = self.init_test().with_format_rules(("True", "True"),
                                                                                      ("False", "False")).unpack()
@@ -337,106 +262,6 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         unresolved.metadata.id_is_unresolved = True
         assert sheerka.resolve_rule(context, unresolved) == rule
 
-    @pytest.mark.parametrize("obj, expected", [
-        ("text value", "var == 'text value'"),
-        ("text 'value'", '''var == "text 'value'"'''),
-        ('text "value"', """var == 'text "value"'"""),
-        (10, "var == 10"),
-        (10.01, "var == 10.01"),
-    ])
-    def test_i_can_test_python_code_emitter_for_basic_types(self, obj, expected):
-        sheerka, context = self.init_test().unpack()
-
-        assert PythonCodeEmitter(context).recognize(obj, "var").get_text() == expected
-        assert PythonCodeEmitter(context, "status").recognize(obj, "var").get_text() == "status and " + expected
-
-    @pytest.mark.parametrize("recognized_by, expected", [
-        (RECOGNIZED_BY_ID, "isinstance(var, Concept) and var.id == '1001'"),
-        (RECOGNIZED_BY_NAME, "isinstance(var, Concept) and var.name == 'greetings'"),
-        (None, "isinstance(var, Concept) and var.key == 'hello'"),
-    ])
-    def test_i_can_test_python_code_emitter_for_concepts(self, recognized_by, expected):
-        sheerka, context, foo = self.init_concepts(
-            Concept("greetings", definition="hello", definition_type=DEFINITION_TYPE_DEF))
-
-        instance = sheerka.new_from_template(foo, foo.key)
-        if recognized_by:
-            instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, recognized_by)
-
-        assert PythonCodeEmitter(context).recognize(instance, "var").get_text() == expected
-        assert PythonCodeEmitter(context, "status").recognize(instance, "var").get_text() == "status and " + expected
-
-    def test_i_can_test_python_code_emitter_for_concepts_with_variable(self):
-        sheerka, context, greetings, little, foo, bar, and_concept = self.init_concepts(
-            Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
-            Concept("little x").def_var("x"),
-            "foo",
-            "bar",
-            Concept("a and b").def_var("a").def_var("b")
-        )
-
-        # variable is a string
-        greetings_instance = sheerka.new_from_template(greetings, greetings.key, a='sheerka')
-        expected = "isinstance(var, Concept) and var.key == 'hello __var__0' and var.get_value('a') == 'sheerka'"
-        text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
-        assert text == expected
-
-        # variable is a concept recognized by id
-        foo_instance = sheerka.new_from_template(foo, foo.key)
-        foo_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_ID)
-        greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
-        expected = """__x_00__ = var.get_value('a')
-isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.id == '1003'"""
-        text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
-        assert text == expected
-
-        # variable is a concept recognized by name
-        foo_instance = sheerka.new_from_template(foo, foo.key)
-        foo_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_NAME)
-        greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
-        expected = """__x_00__ = var.get_value('a')
-isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.name == 'foo'"""
-        text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
-        assert text == expected
-
-        # variable is a concept recognized by value
-        foo_instance = sheerka.new_from_template(foo, foo.key)
-        greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
-        expected = """__x_00__ = var.get_value('a')
-isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.key == 'foo'"""
-        text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
-        assert text == expected
-
-        # variable is a concept witch has itself some variable
-        foo_instance = sheerka.new_from_template(foo, foo.key)
-        little_instance = sheerka.new_from_template(little, little.key, x=foo_instance)
-        greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=little_instance)
-        expected = """__x_00__ = var.get_value('a')
-__x_01__ = __x_00__.get_value('x')
-isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
-                   " and isinstance(__x_00__, Concept) and __x_00__.key == 'little __var__0'" + \
-                   " and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'"""
-        text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
-        assert text == expected
-
-        # concept with multiple variables (which are themselves concepts)
-        foo_instance = sheerka.new_from_template(foo, foo.key)
-        bar_instance = sheerka.new_from_template(bar, bar.key)
-        little_instance = sheerka.new_from_template(little, little.key, x=foo_instance)
-        and_instance = sheerka.new_from_template(and_concept, and_concept.key, a=bar_instance, b=little_instance)
-        greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=and_instance)
-        expected = """__x_00__ = var.get_value('a')
-__x_01__ = __x_00__.get_value('a')
-__x_02__ = __x_00__.get_value('b')
-__x_03__ = __x_02__.get_value('x')
-isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
-                   " and isinstance(__x_00__, Concept) and __x_00__.key == '__var__0 and __var__1'" + \
-                   " and isinstance(__x_01__, Concept) and __x_01__.key == 'bar'" + \
-                   " and isinstance(__x_02__, Concept) and __x_02__.key == 'little __var__0'" + \
-                   " and isinstance(__x_03__, Concept) and __x_03__.key == 'foo'"
-        text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
-        assert text == expected
-
     def test_i_can_get_format_rules(self):
         sheerka, context = self.init_test().unpack()
         service = sheerka.services[SheerkaRuleManager.NAME]
@@ -674,3 +499,103 @@ class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
             assert rule.compiled_conditions == expected.compiled_conditions
             assert rule.priority is not None
             assert rule.priority == expected.priority
+
+#     @pytest.mark.parametrize("obj, expected", [
+#         ("text value", "var == 'text value'"),
+#         ("text 'value'", '''var == "text 'value'"'''),
+#         ('text "value"', """var == 'text "value"'"""),
+#         (10, "var == 10"),
+#         (10.01, "var == 10.01"),
+#     ])
+#     def test_i_can_test_python_code_emitter_for_basic_types(self, obj, expected):
+#         sheerka, context = self.init_test().unpack()
+#
+#         assert PythonCodeEmitter(context).recognize(obj, "var").get_text() == expected
+#         assert PythonCodeEmitter(context, "status").recognize(obj, "var").get_text() == "status and " + expected
+#
+#     @pytest.mark.parametrize("recognized_by, expected", [
+#         (RECOGNIZED_BY_ID, "isinstance(var, Concept) and var.id == '1001'"),
+#         (RECOGNIZED_BY_NAME, "isinstance(var, Concept) and var.name == 'greetings'"),
+#         (None, "isinstance(var, Concept) and var.key == 'hello'"),
+#     ])
+#     def test_i_can_test_python_code_emitter_for_concepts(self, recognized_by, expected):
+#         sheerka, context, foo = self.init_concepts(
+#             Concept("greetings", definition="hello", definition_type=DEFINITION_TYPE_DEF))
+#
+#         instance = sheerka.new_from_template(foo, foo.key)
+#         if recognized_by:
+#             instance.get_hints().recognized_by = recognized_by
+#
+#         assert PythonCodeEmitter(context).recognize(instance, "var").get_text() == expected
+#         assert PythonCodeEmitter(context, "status").recognize(instance, "var").get_text() == "status and " + expected
+#
+#     def test_i_can_test_python_code_emitter_for_concepts_with_variable(self):
+#         sheerka, context, greetings, little, foo, bar, and_concept = self.init_concepts(
+#             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
+#             Concept("little x").def_var("x"),
+#             "foo",
+#             "bar",
+#             Concept("a and b").def_var("a").def_var("b")
+#         )
+#
+#         # variable is a string
+#         greetings_instance = sheerka.new_from_template(greetings, greetings.key, a='sheerka')
+#         expected = "isinstance(var, Concept) and var.key == 'hello __var__0' and var.get_value('a') == 'sheerka'"
+#         text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
+#         assert text == expected
+#
+#         # variable is a concept recognized by id
+#         foo_instance = sheerka.new_from_template(foo, foo.key)
+#         foo_instance.get_hints().recognized_by = RECOGNIZED_BY_ID
+#         greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
+#         expected = """__x_00__ = var.get_value('a')
+# isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.id == '1003'"""
+#         text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
+#         assert text == expected
+#
+#         # variable is a concept recognized by name
+#         foo_instance = sheerka.new_from_template(foo, foo.key)
+#         foo_instance.get_hints().recognized_by = RECOGNIZED_BY_NAME
+#         greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
+#         expected = """__x_00__ = var.get_value('a')
+# isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.name == 'foo'"""
+#         text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
+#         assert text == expected
+#
+#         # variable is a concept recognized by value
+#         foo_instance = sheerka.new_from_template(foo, foo.key)
+#         greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance)
+#         expected = """__x_00__ = var.get_value('a')
+# isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.key == 'foo'"""
+#         text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
+#         assert text == expected
+#
+#         # variable is a concept witch has itself some variable
+#         foo_instance = sheerka.new_from_template(foo, foo.key)
+#         little_instance = sheerka.new_from_template(little, little.key, x=foo_instance)
+#         greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=little_instance)
+#         expected = """__x_00__ = var.get_value('a')
+# __x_01__ = __x_00__.get_value('x')
+# isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
+#                    " and isinstance(__x_00__, Concept) and __x_00__.key == 'little __var__0'" + \
+#                    " and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'"""
+#         text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
+#         assert text == expected
+#
+#         # concept with multiple variables (which are themselves concepts)
+#         foo_instance = sheerka.new_from_template(foo, foo.key)
+#         bar_instance = sheerka.new_from_template(bar, bar.key)
+#         little_instance = sheerka.new_from_template(little, little.key, x=foo_instance)
+#         and_instance = sheerka.new_from_template(and_concept, and_concept.key, a=bar_instance, b=little_instance)
+#         greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=and_instance)
+#         expected = """__x_00__ = var.get_value('a')
+# __x_01__ = __x_00__.get_value('a')
+# __x_02__ = __x_00__.get_value('b')
+# __x_03__ = __x_02__.get_value('x')
+# isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
+#                    " and isinstance(__x_00__, Concept) and __x_00__.key == '__var__0 and __var__1'" + \
+#                    " and isinstance(__x_01__, Concept) and __x_01__.key == 'bar'" + \
+#                    " and isinstance(__x_02__, Concept) and __x_02__.key == 'little __var__0'" + \
+#                    " and isinstance(__x_03__, Concept) and __x_03__.key == 'foo'"
+#         text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text()
+#         assert text == expected

tests/core/test_SheerkaRuleManagerRulesCompilation.py

@@ -4,6 +4,7 @@ import pytest
 
 from core.builtin_concepts import ReturnValueConcept
 from core.builtin_concepts_ids import BuiltinConcepts
+from core.builtin_helpers import ensure_evaluated
 from core.concept import Concept, DEFINITION_TYPE_DEF
 from core.rule import Rule
 from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
@@ -23,6 +24,19 @@ from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND
 
 
 class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka):
+
+    @staticmethod
+    def get_conditions(context, expression):
+        parser = ExpressionParser()
+        error_sink = ErrorSink()
+        parser_input = ParserInput(expression)
+        parser.reset_parser_input(parser_input, error_sink)
+        parsed = parser.parse_input(context, parser_input, error_sink)
+
+        visitor = PythonConditionExprVisitor(context)
+        conditions = visitor.get_conditions(parsed)
+        return conditions
+
     @staticmethod
     def check_against_rete(rule_expression, rule_conditions, objects):
         """
@@ -72,6 +86,9 @@ class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka):
             sub_context.sheerka.add_many_to_short_term_memory(sub_context, objects)
 
             evaluator = PythonEvaluator()
+            for c in condition.concepts_to_reset:
+                c.get_hints().is_evaluated = False
+
             return evaluator.eval(sub_context, condition.return_value)
 
     @classmethod
@@ -130,15 +147,7 @@ class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka):
                              expected_not_variables,
                              expected_objects):
         sheerka = context.sheerka
-
-        parser = ExpressionParser()
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = PythonConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
+        conditions = BaseTestSheerkaRuleManagerRulesCompilation.get_conditions(context, expression)
 
         assert len(conditions) == 1
         assert isinstance(conditions[0], CompiledCondition)
@@ -312,6 +321,132 @@ class TestSheerkaRuleManagerRulesCompilationNotExists(BaseTestSheerkaRuleManager
         self.check_against_python(context, expression, conditions, objects)
 
 
+class TestSheerkaRuleManagerRulesCompilationSimplePython(BaseTestSheerkaRuleManagerRulesCompilation):
+    """
+    Testing Python
+        True
+        False
+        10 + 5
+        'hello world'
+        a + self
+        a + 10
+        a + " world !"
+        a + foo
+        a + twenty one
+        a + my friend
+    """
+
+    @pytest.mark.parametrize("expression, e_compiled, e_text, e_variables, e_objects,  e_result", [
+        (
+            "True",
+            "True",
+            "True",
+            set(),
+            set(),
+            True
+        ),
+        (
+            "False",
+            "False",
+            "False",
+            set(),
+            set(),
+            False
+        ),
+        (
+            "10 + 5",
+            "__o_00__",
+            "10 + 5",
+            set(),
+            {("__o_00__", 15)},
+            15
+        ),
+        (
+            "'hello world'",
+            "__o_00__",
+            "'hello world'",
+            set(),
+            {("__o_00__", 'hello world')},
+            'hello world'
+        ),
+        (
+            "a + self",
+            "a + self",
+            "a + self",
+            {("a", 10), ("self", "foo")},
+            set(),
+            15
+        ),
+        (
+            "a + 10",
+            "a + 10",
+            "a + 10",
+            {("a", 10)},
+            set(),
+            20
+        ),
+        (
+            "a + 'world !'",
+            "a + 'world !'",
+            "a + 'world !'",
+            {("a", "hello ")},
+            set(),
+            "hello world !"
+        ),
+        (
+            "a + foo",
+            "a + foo",
+            "a + foo",
+            {("a", 10), ("foo", "foo")},
+            set(),
+            15
+        ),
+        (
+            "a + twenty one",
+            "a + __C__twenties__1004__C__",
+            "a + twenty one",
+            {("a", 10)},
+            {"__C__twenties__1004__C__"},
+            31
+        ),
+        (
+            "a + my friend",
+            "a + __C__my0friend__1005__C__",
+            "a + my friend",
+            {("a", "hello ")},
+            {'__C__my0friend__1005__C__'},
+            "hello my friend"
+        ),
+    ])
+    def test_python(self, expression, e_compiled, e_text, e_variables, e_objects, e_result):
+        sheerka, context, foo, one, two, twenties, my_friend = self.init_concepts(
+            Concept("foo", body="5"),
+            Concept("one", body="1"),
+            Concept("two", body="2"),
+            Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"),
+            Concept("my friend", body="'my friend'"),
+            create_new=True
+        )
+        ensure_evaluated(context, foo, eval_body=True)
+        ensure_evaluated(context, my_friend, eval_body=True)
+        conditions = self.validate_python_test(context,
+                                               expression,
+                                               e_compiled,
+                                               e_text,
+                                               e_variables,
+                                               set(),
+                                               e_objects)
+
+        # check against SheerkaEvaluateRules
+        variables_mapping = {
+            "foo": foo,
+        }
+        namespace = self.get_testing_objects(context, e_variables, variables_mapping)
+        res = self.evaluate_condition(context, expression, conditions[0], namespace)
+        assert res.status
+        assert sheerka.objvalue(res) == e_result
+
+
 class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerRulesCompilation):
     """
     Testing simple equality:
@@ -321,6 +456,7 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR
         self == sheerka
         self == BuiltinConcepts.TO_DICT
         self == hello 'my friend'
+        a == b
     """
 
     @pytest.mark.parametrize("expression, expected_as_list_of_str, expected_variables", [
@@ -339,7 +475,12 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR
           "#__x_00__|key|'hello __var__0'",
           "#__x_00__|a|'my friend'"],
          {("self", "hello_my_friend")}
-         )
+         ),
+        # ("a == b",
+        #  ["#__x_00__|__name__|'a'",
+        #   "#__x_01__|__name__|'b'",
+        #   "#__x_00__|__self__|#__x_01__"],
+        #  {("a", 10), ("b", 10)}),
     ])
     def test_rete(self, expression, expected_as_list_of_str, expected_variables):
         sheerka, context, greetings = self.init_test().with_concepts(
@@ -365,23 +506,27 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR
         objects = self.get_testing_objects(context, expected_variables, objects_mappings)
         self.check_against_rete(expression, conditions, objects)
 
+        # KSI: 2021-05-06 The last test done not produce any match because the WME (b, __self__, 10)
+        # is not added to memory.
+
     @pytest.mark.parametrize("expression, expected_compiled, expected_variables, expected_objects", [
         ("a == 10", "a == __o_00__", {("a", 10)}, {("__o_00__", 10)}),
-        ("__ret.status == True", "__ret.status == __o_00__", {"__ret"}, {("__o_00__", True)}),
+        ("__ret.status == True", "__ret.status == True", {"__ret"}, set()),
         ("self == 'a'", "self == __o_00__", {("self", 'a')}, {("__o_00__", 'a')}),
         ("self == sheerka", "is_sheerka(self)", {("self", "sheerka")}, {}),
         (
             "self == BuiltinConcepts.TO_DICT",
-            "self == __o_00__",
+            "self == BuiltinConcepts.TO_DICT",
             {("self", BuiltinConcepts.TO_DICT)},
-            {("__o_00__", BuiltinConcepts.TO_DICT)}
+            set()
         ),
         (
             "self == hello 'my friend'",
             """isinstance(self, Concept) and self.key == 'hello __var__0' and self.a == __o_01__""",
             {("self", "hello_my_friend")},
             {("__o_01__", "my friend")}
-        )
+        ),
+        ("a == b", "a == b", {("a", 10), ("b", 10)}, {}),
     ])
     def test_python(self, expression, expected_compiled, expected_variables, expected_objects):
         sheerka, context, greetings = self.init_test().with_concepts(
@@ -403,6 +548,176 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR
         self.check_against_python(context, expression, conditions, objects)
 
 
+class TestSheerkaRuleManagerRulesCompilationOtherConditions(BaseTestSheerkaRuleManagerRulesCompilation):
+    """
+    Testing other conditions than equality
+        a > 10
+        a >= 10
+        a < 10
+        a <= 10
+        a != 10
+        a > 10 and b <= 5
+
+        __ret.value > 10
+        10 > __ret.value
+
+        a + self > 10
+        a + 10 > 10
+        a + " world !" == "hello world !"
+        a + foo > 10
+        a + twenty one > 21
+        a + my friend == 'hello my friend'
+
+        10 < a + self
+        10 < a + 10
+        'hello world !' == a + ' world !'
+        10 < a + foo
+        10 > a + twenty one
+        'hello my friend' == a + my friend
+    """
+
+    @pytest.mark.parametrize("expression, e_compiled, e_variables, e_objects, e_result", [
+        ("a > 10", "a > __o_00__", {("a", 10)}, {("__o_00__", 10)}, False),
+        ("a >= 10", "a >= __o_00__", {("a", 10)}, {("__o_00__", 10)}, True),
+        ("a < 10", "a < __o_00__", {("a", 10)}, {("__o_00__", 10)}, False),
+        ("a <= 10", "a <= __o_00__", {("a", 10)}, {("__o_00__", 10)}, True),
+        ("a != 10", "a != __o_00__", {("a", 10)}, {("__o_00__", 10)}, False),
+        (
+            "a > 10 and b <= 5",
+            "a > __o_00__ and b <= __o_01__",
+            {("a", 11), ("b", 4)},
+            {("__o_00__", 10), ("__o_01__", 5)},
+            True
+        ),
+        (
+            "__ret.value > 10",
+            "__ret.value > __o_00__",
+            {("__ret", 15)},
+            {("__o_00__", 10)},
+            True
+        ),
+        (
+            "10 > __ret.value",
+            "__o_00__ > __ret.value",
+            {("__ret", 15)},
+            {("__o_00__", 10)},
+            False
+        ),
+        (
+            "a + self > 10",
+            "a + self > __o_00__",
+            {("a", 6), ("self", "foo")},
+            {("__o_00__", 10)},
+            True
+        ),
+        (
+            "a + 10 > 10",
+            "a + 10 > __o_00__",
+            {("a", 5)},
+            {("__o_00__", 10)},
+            True
+        ),
+        (
+            "a + 'world !' == 'hello world !'",
+            "a + 'world !' == __o_00__",
+            {("a", "hello ")},
+            {("__o_00__", 'hello world !')},
+            True
+        ),
+        (
+            "a + foo > 10",
+            "a + foo > __o_00__",
+            {("a", 6), ("foo", "foo")},
+            {("__o_00__", 10)},
+            True
+        ),
+        (
+            "a + twenty one > 21",
+            "a + __C__twenties__1004__C__ > __o_00__",
+            {("a", 5)},
+            {"__C__twenties__1004__C__", ("__o_00__", 21)},
+            True
+        ),
+        (
+            "a + my friend == 'hello my friend'",
+            "a + __C__my0friend__1005__C__ == __o_00__",
+            {("a", "hello ")},
+            {"__C__my0friend__1005__C__", ("__o_00__", 'hello my friend')},
+            True
+        ),
+
+        (
+            "10 < a + self",
+            "__o_00__ < a + self",
+            {("a", 6), ("self", "foo")},
+            {("__o_00__", 10)},
+            True
+        ),
+        (
+            "10 > a + 10",
+            "__o_00__ > a + 10",
+            {("a", 5)},
+            {("__o_00__", 10)},
+            False
+        ),
+        (
+            "'hello world !' != a + 'world !'",
+            "__o_00__ != a + 'world !'",
+            {("a", "hello ")},
+            {("__o_00__", 'hello world !')},
+            False
+        ),
+        (
+            "10 < a + foo",
+            "__o_00__ < a + foo",
+            {("a", 6), ("foo", "foo")},
+            {("__o_00__", 10)},
+            True
+        ),
+        (
+            "21 > a + twenty one",
+            "__o_00__ > a + __C__twenties__1004__C__",
+            {("a", 5)},
+            {"__C__twenties__1004__C__", ("__o_00__", 21)},
+            False
+        ),
+        (
+            "'hello my friend' == a + my friend",
+            "__o_00__ == a + __C__my0friend__1005__C__",
+            {("a", "hello ")},
+            {"__C__my0friend__1005__C__", ("__o_00__", 'hello my friend')},
+            True
+        ),
+
+    ])
+    def test_python(self, expression, e_compiled, e_variables, e_objects, e_result):
+        sheerka, context, foo, one, two, twenties, my_friend = self.init_concepts(
+            Concept("foo", body="5"),
+            Concept("one", body="1"),
+            Concept("two", body="2"),
+            Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"),
+            Concept("my friend", body="'my friend'"),
+            create_new=True
+        )
+        ensure_evaluated(context, foo, eval_body=True)
+        ensure_evaluated(context, my_friend, eval_body=True)
+
+        conditions = self.validate_python_test(context,
+                                               expression,
+                                               e_compiled,
+                                               expression,
+                                               e_variables,
+                                               set(),
+                                               e_objects)
+
+        # check against SheerkaEvaluateRules
+        variables_mapping = {
+            "foo": foo,
+        }
+        objects = self.get_testing_objects(context, e_variables, variables_mapping)
+        self.check_against_python(context, expression, conditions, objects, expected_result=e_result)
+
+
 class TestSheerkaRuleManagerRulesCompilationFunctionsCall(BaseTestSheerkaRuleManagerRulesCompilation):
     """
     Testing functions
@@ -795,6 +1110,8 @@ class TestSheerkaRuleManagerRulesCompilationEvalQuestionConcept(BaseTestSheerkaR
         with long concept   : the little boy is a human being
         with long concept + variable  : the little boy is a self
         with long concept + variable  : self is a human being
+        with a special symbol : self is a 'human'
+        with a special symbol : the little boy is a 'human'
     """
 
     def test_rete(self):
@@ -826,6 +1143,16 @@ class TestSheerkaRuleManagerRulesCompilationEvalQuestionConcept(BaseTestSheerkaR
             "evaluate_question(__o_00__)",
             {"self"},
         ),
+        (
+            "self is a 'human'",
+            "evaluate_question(__o_00__)",
+            {"self"},
+        ),
+        (
+            "the little boy is a 'human'",
+            "evaluate_question(__o_00__)",
+            set(),
+        ),
     ])
     def test_python(self, expression, expected_compiled, expected_variables):
         sheerka, context, girl, human, little_boy, human_being, isa = self.init_test().with_concepts(
@@ -980,11 +1307,11 @@ class TestSheerkaRuleManagerRulesCompilationEvalConceptMixedWithOther(BaseTestSh
                                                expected_objects)
 
         # check against SheerkaEvaluateRules
-        variable_mapping = {
+        variables_mapping = {
             "girl": girl,
             "human being": human_being
         }
-        testing_objects = self.get_testing_objects(context, expected_variables, variable_mapping)
+        testing_objects = self.get_testing_objects(context, expected_variables, variables_mapping)
         self.check_against_python(context,
                                   expression,
                                   conditions,
@@ -1005,6 +1332,8 @@ class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheer
         with function: func_identity(twenty two + twenty one)
         with function: func_identity(twenty two plus one)
         with function: func_identity(twenty two plus twenty one)
+        with special char : 'one' plus 'two'
+        with special char : twenty two plus 2
     """
 
     def test_rete(self):
@@ -1081,6 +1410,20 @@ class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheer
             {"__o_00__"},
             43
         ),
+        (
+            "'one' plus 'two'",
+            "__o_00__",
+            "'one' plus 'two'",
+            {"__o_00__"},
+            'onetwo'
+        ),
+        (
+            "twenty two plus 2",
+            "__o_00__",
+            "twenty two plus 2",
+            {"__o_00__"},
+            24
+        ),
     ])
     def test_python(self, expression, e_compiled, e_text, e_objects, e_result):
         sheerka, context, one, two, twenties, plus = self.init_test().with_concepts(
@@ -1109,7 +1452,7 @@ class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheer
 class TestSheerkaRuleManagerRulesCompilationMultipleSameConcept(BaseTestSheerkaRuleManagerRulesCompilation):
     """
     Test when a concept returns multiple results
-    The compilation should fail : No need to execute a condition if we are not sure of the meaning ?
+    The compilation should fail : No need to execute a condition if we are not sure of the meaning
         self is a bar
     """
 
@@ -1156,6 +1499,99 @@ class TestSheerkaRuleManagerRulesCompilationNot(BaseTestSheerkaRuleManagerRulesC
     Testing not
         not __ret.status == True
         not recognize(__ret.body, hello sheerka)
+        not a cat is a pet and not bird is an animal  # where x is a y is a concept
+        not a cat is a pet and not x > 5 # concept mixed with python
 
     """
     pass
+
+
+class TestCompiledCondition(BaseTestSheerkaRuleManagerRulesCompilation):
+    @pytest.mark.parametrize("expression, expected", [
+        ("self is a 'foo'", {"x is a y"}),
+        ("set self is a 'foo'", set()),
+    ])
+    def test_i_can_get_concept_to_reset(self, expression, expected):
+        """
+        When compiled conditions, sometimes there are concepts to reset between two usages
+        :param expression:
+        :param expected:
+        :return:
+        """
+        sheerka, context, question, not_a_question = self.init_concepts(
+            Concept("x is a y", pre="is_question()").def_var("x").def_var("y"),
+            Concept("set x is a y").def_var("x").def_var("y"),
+        )
+
+        conditions = self.get_conditions(context, expression)
+
+        assert len(conditions) == 1
+        assert set(c.name for c in conditions[0].concepts_to_reset) == expected
+
+    def test_i_can_reset_concepts_when_multiple_levels(self):
+        """
+        When compiled conditions, sometimes there are concepts to reset between two usages
+        :return:
+        """
+        sheerka, context, is_instance, is_int, is_integer = self.init_concepts(
+            Concept("x is an instance of y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+            Concept("x is a int", pre="is_question()", body="x is an instance of int").def_var("x"),
+            Concept("x is an integer", pre="is_question()", body="x is a int").def_var("x"),
+        )
+
+        expression = "self is an integer"
+        conditions = conditions = self.get_conditions(context, expression)
+
+        assert len(conditions) == 1
+        assert set(c.name for c in conditions[0].concepts_to_reset) == {"x is an instance of y",
+                                                                        "x is a int",
+                                                                        "x is an integer"}
+
+        # So I can evaluate multiple times
+        res = self.evaluate_condition(context, expression, conditions[0], {'self': 10})
+        assert res.status
+        assert sheerka.objvalue(res.body)
+
+        res = self.evaluate_condition(context, expression, conditions[0], {'self': "string"})
+        assert res.status
+        assert not sheerka.objvalue(res.body)
+
+    def test_i_can_reset_concepts_when_multiple_levels_and_concept_node(self):
+        """
+        When compiled conditions, sometimes there are concepts to reset between two usages
+        :return:
+        """
+        # in this example, x + 2 is an int won't be parsed as an ExactNodeConcept, but as a ConceptNode
+        sheerka, context, is_int, is_integer = self.init_concepts(
+            Concept("x is a int", pre="is_question()", body="isinstance(x, int)").def_var("x"),
+            Concept("x is an integer", pre="is_question()", body="x + 2 is a int").def_var("x"),
+            create_new=True
+        )
+
+        expression = "self is an integer"
+        conditions = self.get_conditions(context, expression)
+
+        assert len(conditions) == 1
+        assert set(c.name for c in conditions[0].concepts_to_reset) == {"x is a int",
+                                                                        "x is an integer"}
+
+        # So I can evaluate multiple times
+        res = self.evaluate_condition(context, expression, conditions[0], {'self': 10})
+        assert res.status
+        assert sheerka.objvalue(res.body)
+
+        res = self.evaluate_condition(context, expression, conditions[0], {'self': "string"})
+        assert not res.status
+
+    def test_long_name_concept_set_are_not_considered_as_variables(self):
+        sheerka, context, one, number = self.init_concepts(
+            "one",
+            "all numbers",
+        )
+        sheerka.set_isa(context, one, number)
+
+        expression = "all numbers < 5"
+        conditions = self.get_conditions(context, expression)
+
+        assert len(conditions) == 1
+        assert conditions[0].return_value.body.body.source == '__o_00__ < __o_01__'

tests/core/test_builtin_helpers.py

@@ -4,6 +4,13 @@ import core.builtin_helpers
 from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
 from core.concept import Concept
 from core.global_symbols import NotInit
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import Tokenizer
+from evaluators.BaseEvaluator import BaseEvaluator
+from evaluators.ValidateConceptEvaluator import ValidateConceptEvaluator
+from parsers.BaseNodeParser import ConceptNode
+from parsers.BaseParser import BaseParser
+from parsers.SyaNodeParser import SyaNodeParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -162,6 +169,7 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
         (" is_question ( ) ", True),
         ("context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", True),
         (" context . in_context ( BuiltinConcepts . EVAL_QUESTION_REQUESTED ) ", True),
+        ("in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", True),
         (None, False),
         ("", False),
         (NotInit, False),
@@ -193,6 +201,58 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
         evaluated = [r for r in res if r.status][0].body
         assert evaluated.body is NotInit
 
+    def test_i_can_evaluate_from_source_with_specific_evaluators(self):
+        sheerka, context, one = self.init_concepts(Concept("foo", body="'hello world'"))
+
+        res = core.builtin_helpers.evaluate_from_source(context, "foo", eval_body=True, evaluators=["Python"])
+        res = self.successful_return_values(res)
+        assert len(res) == 1
+        assert res[0].who.startswith(BaseParser.PREFIX)  # Cannot evaluate concept with PythonEvaluator
+
+        res = core.builtin_helpers.evaluate_from_source(context, "foo", eval_body=True, evaluators=["Concept"])
+        res = self.successful_return_values(res)
+        assert len(res) == 1
+        assert res[0].who == BaseEvaluator.PREFIX + "Concept"
+        assert sheerka.isinstance(res[0].body.body,
+                                  BuiltinConcepts.PARSER_RESULT)  # cannot eval 'hello world' without PythonEvaluator
+
+        res = core.builtin_helpers.evaluate_from_source(context, "foo", eval_body=True,
+                                                        evaluators=["Concept", "Python"])
+        res = self.successful_return_values(res)
+        assert len(res) == 1
+        assert res[0].who == BaseEvaluator.PREFIX + "Concept"
+        assert res[0].body.body == "hello world"
+
+    def test_i_can_get_lexer_nodes_after_parsing_validation(self):
+        sheerka, context, the, foo = self.init_concepts(
+            Concept("the x", ret="x", where="isinstance(x, Concept)").def_var("x"),
+            "foo",
+            create_new=True)
+
+        parsed_ret_val = SyaNodeParser().parse(context, ParserInput("the foo"))
+        validated_ret_val = ValidateConceptEvaluator().eval(context, parsed_ret_val)
+
+        res = core.builtin_helpers.get_lexer_nodes([validated_ret_val], 0, list(Tokenizer("the foo", yield_eof=False)))
+
+        assert isinstance(res, list)
+        assert isinstance(res[0][0], ConceptNode)
+
+    def test_ensure_evaluated_returns_the_ret_value(self):
+        """
+        When a concept has a RET defined, make sure to return it
+        :return:
+        """
+
+        sheerka, context, foo, bar = self.init_concepts(
+            "foo",
+            Concept("bar", ret="foo")
+        )
+
+        assert core.builtin_helpers.ensure_evaluated(context, bar) == foo
+
+        # a second time, now that bar is already evaluated
+        assert core.builtin_helpers.ensure_evaluated(context, bar) == foo
+
     # @pytest.mark.parametrize("return_values", [
     #     None,
     #     []

tests/core/test_sheerka.py

@@ -75,7 +75,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert loaded is not None
         assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
         assert loaded.body == {"key": "key_that_does_not_exist"}
-        assert loaded.get_metadata().is_evaluated
+        assert loaded.get_hints().is_evaluated
 
     def test_i_cannot_get_when_id_is_not_found(self):
         sheerka = self.get_sheerka()
@@ -85,7 +85,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert loaded is not None
         assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
         assert loaded.body == {"id": "id_that_does_not_exist"}
-        assert loaded.get_metadata().is_evaluated
+        assert loaded.get_hints().is_evaluated
 
     def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self):
         sheerka = self.get_sheerka()
@@ -192,15 +192,15 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
                                                                                   create_new=True).unpack()
 
         sheerka.evaluate_concept(context, sheerka.get_by_id(template.id))
-        assert template.get_metadata().is_evaluated
+        assert template.get_hints().is_evaluated
         assert template.body == "foo body"
 
         new = sheerka.new(template.key)
-        assert not new.get_metadata().is_evaluated
+        assert not new.get_hints().is_evaluated
         assert new.body == NotInit
 
         new = sheerka.new((None, template.id))
-        assert not new.get_metadata().is_evaluated
+        assert not new.get_hints().is_evaluated
         assert new.body == NotInit
 
     def test_i_cannot_instantiate_an_unknown_concept(self):

tests/core/test_sheerka_call_evaluators.py

@@ -287,7 +287,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
         assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]}
         assert sorted_priorities == [20, 15]
 
-        key = BuiltinConcepts.EVALUATION + "|" + "|".join(evaluators_names)
+        key = (BuiltinConcepts.EVALUATION,  "|".join(evaluators_names))
         assert key in service.grouped_evaluators_cache
         groups, sorted_priorities = service.grouped_evaluators_cache[key]
         assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]}
@@ -343,7 +343,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
         assert groups == {99: [EvaluatorAllWithPriority15(99)]}
         assert sorted_priorities == [99]
 
-        key = BuiltinConcepts.EVALUATION + "|" + "|".join(evaluators_names)
+        key = (BuiltinConcepts.EVALUATION, "|".join(evaluators_names))
         assert key not in service.grouped_evaluators_cache
 
     def test_i_can_revert_back_evaluators_alterations(self):
@@ -570,7 +570,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
 
         # grouped evaluator is in cache
         service = sheerka.services[SheerkaExecute.NAME]
-        assert "__EVALUATION|all_priority10" in service.grouped_evaluators_cache
+        assert ("__EVALUATION", "all_priority10") in service.grouped_evaluators_cache
 
     def test_evaluators_priorities_can_be_tweaked_by_the_context(self):
         sheerka, context = self.init_concepts()

tests/core/test_sheerka_call_parsers.py

@@ -1,14 +1,38 @@
-from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept
-from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
-from parsers.BaseParser import BaseParser
+import pytest
 
+from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept
+from core.concept import Concept
+from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute, DEFAULT, PARSE_STEPS
+from parsers.BaseExpressionParser import ExprNode
+from parsers.BaseParser import BaseParser
+from parsers.BnfNodeParser import BnfNodeParser
+from parsers.SequenceNodeParser import SequenceNodeParser
+from parsers.SyaNodeParser import SyaNodeParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
-def get_ret_val(text, who="who"):
+def get_user_input(text, who="who"):
     return ReturnValueConcept(who, True, UserInputConcept(text, "user_name"))
 
 
+def check_same_results(result1, result2, user_input):
+    assert len(result1) == len(result2)
+    for previous, current in zip(result1, result2):
+        assert current.parents == user_input
+        assert current.who == previous.who
+        assert current.status == previous.status
+        assert id(current.body) == id(previous.body)
+
+
+def check_same_values(sheerka, context, ret_val1, ret_val2):
+    previous = sheerka.execute(context, ret_val1, [BuiltinConcepts.EVALUATION])
+    current = sheerka.execute(context, ret_val2, [BuiltinConcepts.EVALUATION])
+
+    assert len(previous) == len(current)
+    for p, c in zip(previous, current):
+        assert p == c
+
+
 class BaseTestParser(BaseParser):
     debug_out = []
 
@@ -136,12 +160,23 @@ class ListOfNoneParser(BaseTestParser):
 
 
 class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
+    default_parsers = None
+
+    @classmethod
+    def setup_class(cls):
+        cls.default_parsers = cls().get_sheerka().parsers
+
     @classmethod
     def teardown_class(cls):
         # At the end of the tests, sheerka singleton instance will be corrupted
         # Ask for a new one
         TestUsingMemoryBasedSheerka.sheerka = None
 
+    def reset_parsers(self, sheerka):
+        sheerka.parsers = self.default_parsers
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_registered_parsers()
+
     def test_i_can_get_parser_when_context_is_not_altered(self):
         sheerka, context = self.init_concepts()
         sheerka.parsers = {
@@ -152,7 +187,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service.reset_registered_parsers()
 
         parsers_key, groups, sorted_priorities = service.get_parsers(context)
-        assert parsers_key == "__default"
+        assert parsers_key == ("__default", "__default")
         assert groups == {80: [Enabled80FalseParser()], 90: [Enabled90FalseParser()]}
         assert sorted_priorities == [90, 80]
 
@@ -171,13 +206,13 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         context.preprocess_parsers = parsers_names
 
         parsers_key, groups, sorted_priorities = service.get_parsers(context)
-        assert parsers_key == "Enabled50True|Enabled70False|Disabled"
+        assert parsers_key == (DEFAULT, "Enabled50True|Enabled70False|Disabled")
         assert groups == {50: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]}
         assert sorted_priorities == [70, 50]  # Disabled parser does not appear
 
         key = "|".join(parsers_names)
-        assert key in service.grouped_parsers_cache
-        groups, sorted_priorities = service.grouped_parsers_cache[key]
+        assert (DEFAULT, key) in service.grouped_parsers_cache
+        groups, sorted_priorities = service.grouped_parsers_cache[(DEFAULT, key)]
         assert groups == {50: [Enabled50TrueParser], 70: [Enabled70FalseParser]}
         assert sorted_priorities == [70, 50]
 
@@ -202,8 +237,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         assert groups == {80: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]}
         assert sorted_priorities == [80, 70]  # Disabled parsers does not appear
 
-        key = "|".join(parsers_names)
-        assert key not in service.grouped_parsers_cache # not saved in cache
+        key = (DEFAULT, "|".join(parsers_names))
+        assert key not in service.grouped_parsers_cache  # not saved in cache
 
     def test_disabled_parsers_are_not_executed(self):
         sheerka = self.get_sheerka()
@@ -214,7 +249,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
         BaseTestParser.debug_out = []
         sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
 
@@ -230,7 +265,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
         BaseTestParser.debug_out = []
         sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
 
@@ -254,7 +289,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
         BaseTestParser.debug_out = []
         sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
 
@@ -279,7 +314,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
         BaseTestParser.debug_out = []
         sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
 
@@ -310,7 +345,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
         BaseTestParser.debug_out = []
         sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
 
@@ -342,7 +377,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
 
         res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
         res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
@@ -365,7 +400,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
 
         BaseTestParser.debug_out = []
         res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
@@ -391,7 +426,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
         BaseTestParser.debug_out = []
         sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
 
@@ -412,7 +447,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
         BaseTestParser.debug_out = []
         res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
 
@@ -432,7 +467,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
             ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'),
         ]
 
-    def test_i_can_manage_parser_with_multiple_results_and_a_sucess(self):
+    def test_i_can_manage_parser_with_multiple_results_and_a_success(self):
         sheerka = self.get_sheerka()
         sheerka.parsers = {
             "Enabled80MultipleTrue": Enabled80MultipleTrueParser,
@@ -441,7 +476,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaExecute.NAME]
         service.reset_registered_parsers()
 
-        user_input = [get_ret_val("hello world")]
+        user_input = [get_user_input("hello world")]
         BaseTestParser.debug_out = []
         res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
 
@@ -454,3 +489,390 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
             ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'),
             ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'),
         ]
+
+    def test_parser_calls_are_put_in_cache(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaExecute.NAME]
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("1")]
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        assert len(res) > 0
+        assert len(service.parsers_cache.cache) == 1
+        assert (('__default', '__default'), '1') in service.parsers_cache
+
+    def test_parser_calls_are_put_in_cache_when_question(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaExecute.NAME]
+        self.reset_parsers(sheerka)
+        context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+
+        user_input = [get_user_input("1")]
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        assert len(res) > 0
+        assert len(service.parsers_cache.cache) == 2  # one for EVAL_QUESTION, one to compile the Python
+        assert ((BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT), '1') in service.parsers_cache
+
+    def test_parser_calls_are_not_put_in_cache_when_preprocess_parsers(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaExecute.NAME]
+        self.reset_parsers(sheerka)
+        context.preprocess_parsers = [SequenceNodeParser.NAME, BnfNodeParser.NAME, SyaNodeParser.NAME]
+
+        user_input = [get_user_input("1")]
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        expected_key = (DEFAULT, "|".join([SequenceNodeParser.NAME, BnfNodeParser.NAME, SyaNodeParser.NAME]))
+
+        assert len(res) == 3
+        assert len(service.parsers_cache.cache) == 1
+        assert (expected_key, '1') in service.parsers_cache
+
+    def test_parser_calls_are_not_put_in_cache_when_preprocess(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaExecute.NAME]
+        self.reset_parsers(sheerka)
+        context.add_preprocess(BaseParser.PREFIX + "parser_name", some_attribute='some_value')
+
+        user_input = [get_user_input("1")]
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        assert len(res) > 0
+        assert len(service.parsers_cache.cache) == 0
+
+    def test_i_can_use_parser_memoization_on_python(self):
+        sheerka, context = self.init_test().unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("1 + 1")]
+        first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        user_input_2 = [get_user_input("1 + 1")]
+        second_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        check_same_results(first_request, second_request, user_input_2)
+        check_same_values(sheerka, context, first_request, second_request)
+
+    @pytest.mark.parametrize("concept", {
+        Concept("foo"),
+        Concept("foo", body="1"),
+    })
+    def test_i_can_use_parser_memoization_when_exact_concept(self, concept):
+        sheerka, context, foo = self.init_test(eval_body=True, eval_where=True).with_concepts(concept).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("foo")]
+        first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        successful = [r.who for r in first_request if r.status]
+        assert successful == ['parsers.ExactConcept']
+
+        user_input_2 = [get_user_input("foo")]
+        second_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        check_same_results(first_request, second_request, user_input_2)
+        check_same_values(sheerka, context, first_request, second_request)
+
+    def test_i_can_use_parser_memoization_when_multiple_results(self):
+        sheerka, context, *foo = self.init_test(eval_body=True, eval_where=True).with_concepts(
+            Concept("foo", body="1"),
+            Concept("foo", body="2")).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("foo")]
+        first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        successful = [r.who for r in first_request if r.status]
+        assert successful == ['parsers.ExactConcept', 'parsers.ExactConcept']
+
+        user_input_2 = [get_user_input("foo")]
+        second_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        check_same_results(first_request, second_request, user_input_2)
+        check_same_values(sheerka, context, first_request, second_request)
+
+    def test_i_can_use_parser_memoization_when_bnf_node(self):
+        sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts(
+            Concept("one", body="1"),
+            Concept("two", body="2"),
+            Concept("twenties", definition="'twenty' (one|two)=unit", body="20 + unit").def_var("unit"),
+            create_new=True).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("twenty one")]
+        first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        successful = [r.who for r in first_request if r.status]
+        assert successful == ['parsers.Bnf']
+
+        user_input_2 = [get_user_input("twenty one")]
+        second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING])
+
+        check_same_results(first_request, second_request, user_input_2)
+        check_same_values(sheerka, context, first_request, second_request)
+
+    def test_i_can_use_parser_memoization_when_sequence_node(self):
+        sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts(
+            Concept("one", body="1"),
+            Concept("two", body="2"),
+            create_new=True).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("one two")]
+        first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        successful = [r.who for r in first_request if r.status]
+        assert successful == ['parsers.Sequence']
+
+        user_input_2 = [get_user_input("one two")]
+        second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING])
+
+        check_same_results(first_request, second_request, user_input_2)
+        check_same_values(sheerka, context, first_request, second_request)
+
+    def test_i_can_use_parser_memoization_when_sya_node(self):
+        sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts(
+            Concept("one", body="1"),
+            Concept("twenty two", body="22"),
+            Concept("a plus b", body="a + b").def_var("a").def_var("b"),
+            create_new=True).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("one plus twenty two")]
+        first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        successful = [r.who for r in first_request if r.status]
+        assert successful == ['parsers.Sya']
+
+        user_input_2 = [get_user_input("one plus twenty two")]
+        second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING])
+
+        check_same_results(first_request, second_request, user_input_2)
+        check_same_values(sheerka, context, first_request, second_request)
+
+    def test_i_can_use_parser_memoization_when_not_parser_input_from_sequence_node(self):
+        # In the test, 'one two' will be partially parsed by the SequenceNodeParser
+        # We test that the results of the UnrecognizedNodeParser and the PythonWithConceptParser
+        #  which both do not used ParsingInput as an input (but a ParserResult)
+        #  are added to the list of results
+
+        sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts(
+            Concept("one", body="1")).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("one two")]
+        first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        successful = [r.who for r in first_request if r.status]
+        assert successful == []
+
+        user_input_2 = [get_user_input("one two")]
+        second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING])
+
+        check_same_results(first_request, second_request, user_input_2)
+        check_same_values(sheerka, context, first_request, second_request)
+
+    def test_i_can_use_parser_memoization_when_not_parser_input_from_bnf_node(self):
+        # check the comment from test_i_can_use_parser_memoization_when_not_parser_input_from_sequence_node
+        # for more information on this test
+
+        sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts(
+            Concept("one", body="1"),
+            Concept("two", body="2"),
+            Concept("twenties", definition="'twenty' (one|two)=unit", body="20 + unit"),
+            create_new=True).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("twenty two foo")]
+        first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        successful = [r.who for r in first_request if r.status]
+        assert successful == []
+
+        user_input_2 = [get_user_input("twenty two foo")]
+        second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING])
+
+        check_same_results(first_request, second_request, user_input_2)
+        check_same_values(sheerka, context, first_request, second_request)
+
+    def test_i_can_use_parser_memoization_when_not_parser_input_from_sy_node(self):
+        # check the comment from test_i_can_use_parser_memoization_when_not_parser_input_from_sequence_node
+        # for more information on this test
+
+        sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts(
+            Concept("one", body="1"),
+            Concept("twenty two", body="22"),
+            Concept("a plus b", body="a + b").def_var("a").def_var("b"),
+            create_new=True).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("twenty two plus one foo")]
+        first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+
+        successful = [r.who for r in first_request if r.status]
+        assert successful == []
+
+        user_input_2 = [get_user_input("twenty two plus one foo")]
+        second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING])
+
+        check_same_results(first_request, second_request, user_input_2)
+        check_same_values(sheerka, context, first_request, second_request)
+
+    def test_cache_is_reset_on_concept_creation(self):
+        sheerka, context, *concepts = self.init_test().with_concepts(
+            Concept("foo", body="1"), create_new=True).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("foo")]
+
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        successful = [r for r in res if r.status]
+        assert len(successful) == 1
+
+        sheerka.create_new_concept(context, Concept("foo", body="2"))
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        successful = [r for r in res if r.status]
+        assert len(successful) == 2
+
+    def test_cache_is_reset_on_concept_deletion(self):
+        sheerka, context, foo1, foo2 = self.init_test().with_concepts(
+            Concept("foo", body="1"),
+            Concept("foo", body="2"),
+            create_new=True).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("foo")]
+
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        successful = [r for r in res if r.status]
+        assert len(successful) == 2
+
+        sheerka.remove_concept(context, foo2)
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        successful = [r for r in res if r.status]
+        assert len(successful) == 1
+
+    def test_cache_is_reset_on_concept_modification(self):
+        sheerka, context, foo1, foo2 = self.init_test().with_concepts(
+            Concept("foo", body="1"),
+            Concept("foo", body="2"),
+            create_new=True).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("foo")]
+
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        successful = [r for r in res if r.status]
+        assert len(successful) == 2
+
+        sheerka.modify_concept(context, foo2, to_add={'meta': {"name": "bar"}})
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        successful = [r for r in res if r.status]
+        assert len(successful) == 1
+
+    def test_i_do_not_use_short_term_memory_when_not_requested(self):
+        sheerka, context, foo = self.init_test().with_concepts(
+            Concept("foo"),
+            create_new=True).unpack()
+        self.reset_parsers(sheerka)
+
+        user_input = [get_user_input("foo")]
+
+        # put something is STM
+        context.add_to_short_term_memory("foo", "value")
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        assert len(res) == 1
+        assert res[0].body == "value"
+
+        # specify the parsers to use
+        context.preprocess_parsers = ["ExactConcept"]
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        assert len(res) == 1
+        assert res[0].body.body == foo
+
+        # I can force STM
+        context.preprocess_parsers = ["ExactConcept", "ShortTermMemory"]  # order in not relevant for STM parser
+        res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING])
+        assert len(res) == 1
+        assert res[0].body == "value"
+
+    def test_i_can_compute_parsers_key(self):
+        sheerka = self.get_sheerka()
+
+        context = self.get_context(sheerka)
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key == (DEFAULT, DEFAULT)
+
+        context = self.get_context(sheerka)
+        context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT)
+
+        context = self.get_context(sheerka)
+        context.preprocess_parsers = ["foo", "bar", "baz"]
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key == (DEFAULT, 'foo|bar|baz')
+
+        context = self.get_context(sheerka)
+        context.preprocess_parsers = ["foo", "bar", "baz"]
+        context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, 'foo|bar|baz')
+
+        context = self.get_context(sheerka)
+        context.add_preprocess(BaseParser.PREFIX + "foo", key="some_value")
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key is None
+
+        context = self.get_context(sheerka)
+        context.add_preprocess(BaseParser.PREFIX + "foo", key="some_value")
+        context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key is None
+
+        context = self.get_context(sheerka)
+        context.add_preprocess("foo", key="some_value")
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key == (DEFAULT, DEFAULT)
+
+        context = self.get_context(sheerka)
+        context.add_preprocess("foo", key="some_value")
+        context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT)
+
+        context = self.get_context(sheerka)
+        context.preprocess_parsers = ["foo", "bar", "baz"]
+        context.add_preprocess("foo", key="some_value")
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key == (DEFAULT, 'foo|bar|baz')
+
+        context = self.get_context(sheerka)
+        context.preprocess_parsers = ["foo", "bar", "baz"]
+        context.add_preprocess("foo", key="some_value")
+        context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        key = SheerkaExecute.get_parsers_key(context)
+        assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, 'foo|bar|baz')
+
+    def test_i_use_expression_parser_when_needed(self):
+        sheerka, context, one, number, isa_q, isa = self.init_concepts(
+            "one",
+            "number",
+            Concept("x is a y ", pre="is_question()").def_var("x").def_var("y"),
+            Concept("x is a y ").def_var("x").def_var("y"),
+        )
+        user_input = get_user_input("one is a number")
+
+        res = sheerka.execute(context, [user_input], PARSE_STEPS)
+        res = self.successful_return_values(res)
+        assert len(res) == 1
+        assert res[0].status
+        assert isinstance(res[0].body.body, Concept)
+
+        context = self.get_context(sheerka)
+        context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        res = sheerka.execute(context, [user_input], PARSE_STEPS)
+        assert len(res) == 1
+        assert res[0].status
+        assert isinstance(res[0].body.body, ExprNode)

tests/evaluators/EvaluatorTestsUtils.py

@@ -11,53 +11,112 @@ reduced_requested = ReturnValueConcept("Sheerka", True, Concept(name=BuiltinConc
 
 
 def ret_val(value="value", who="who", status=True):
+    """
+    ReturnValueConcept
+    :param value:
+    :param who:
+    :param status:
+    :return:
+    """
     return ReturnValueConcept(who, status, value)
 
 
 def p_ret_val(value="value", parser="parser", status=True):
+    """
+    ReturnValueConcept from parser
+    :param value:
+    :param parser:
+    :param status:
+    :return:
+    """
     return ReturnValueConcept(BaseParser.get_name(parser), status, value)
 
 
 def e_ret_val(value="value", evaluator="evaluator", status=True):
+    """
+    ReturnValueConcept from evaluator
+    :param value:
+    :param evaluator:
+    :param status:
+    :return:
+    """
     return ReturnValueConcept(BaseEvaluator.PREFIX + evaluator, status, value)
 
 
 def p_ret_val_false(value="value", parser="parser"):
+    """
+    Failed ReturnValueConcept from parser
+    :param value:
+    :param parser:
+    :return:
+    """
     return p_ret_val(value, parser, status=False)
 
 
 def p_ret_val_true(value="value", parser="parser"):
+    """
+    Successful ReturnValueConcept from parser
+    :param value:
+    :param parser:
+    :return:
+    """
     return p_ret_val(value, parser, status=True)
 
 
 def e_ret_val_false(value="value", parser="parser"):
+    """
+    Failed ReturnValueConcept from evaluator
+    :param value:
+    :param parser:
+    :return:
+    """
     return e_ret_val(value, parser, status=False)
 
 
 def e_ret_val_true(value="value", parser="parser"):
+    """
+    Successful ReturnValueConcept from evaluator
+    :param value:
+    :param parser:
+    :return:
+    """
     return e_ret_val(value, parser, status=True)
 
 
 def e_ret_val_new(key, evaluator="evaluator", status=True, **kwargs):
+    """
+    Successful ReturnValueConcept from evaluator that returns a concept
+    :param key:
+    :param evaluator:
+    :param status:
+    :param kwargs:
+    :return:
+    """
     body = new_concept(key, **kwargs)
     return e_ret_val(body, evaluator, status)
 
 
-def pr_ret_val(value, parser="parser", source=None):
+def pr_ret_val(value, parser="parser", source=None, status=True):
     """
     ParserResult ReturnValue
     eg: ReturnValue with a ParserResult
     :param value:
     :param parser:
     :param source:
+    :param status:
     :return:
     """
     source = source or (value.name if isinstance(value, Concept) else "source")
     parser_result = ParserResultConcept(BaseParser.get_name(parser), source=source, value=value)
-    return p_ret_val(parser_result, parser)
+    return p_ret_val(value=parser_result, parser=parser, status=status)
 
 
 def python_ret_val(source):
+    """
+    ReturnValueConcept with a PythonNode
+    :param source:
+    :return:
+    """
     python_node = PythonNode(source.strip(), ast.parse(source.strip(), f"<source>", 'eval'))
     return pr_ret_val(python_node, parser="Python", source=source)
 
@@ -68,5 +127,5 @@ def new_concept(key, **kwargs):
         to_use = "#" + k + "#" if k in ("body", "pre", "post", "ret") else k
         res.set_value(to_use, v)
 
-    res.get_metadata().is_evaluated = True
+    res.get_hints().is_evaluated = True
     return res

tests/evaluators/test_ConceptEvaluator.py

@@ -47,7 +47,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         concept = Concept(name="foo",
                           body="'I have a value'",
                           where="True",
-                          pre="2",
+                          pre="True",
                           post="3").set_value("a", "4").set_value("b", "5")
 
         evaluator = ConceptEvaluator(return_body=True)
@@ -64,7 +64,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         concept = Concept(name="foo",
                           body="'I have a value'",
                           where="True",
-                          pre="2",
+                          pre="True",
                           post="3").set_value("a", "4").set_value("b", "5")
 
         evaluator = ConceptEvaluator(return_body=False)  # which is the default behaviour

tests/evaluators/test_DefConceptEvaluator.py

@@ -204,7 +204,9 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert DefConceptEvaluator.get_variables(context, concept_definition, []) == expected
 
     def test_i_can_recognize_variables_when_referencing_other_concepts(self):
-        sheerka, context, isa_concept = self.init_concepts(Concept("x is an y").def_var("x").def_var("y"))
+        sheerka, context, isa_concept = self.init_concepts(
+            Concept("x is an y", pre="is_question()").def_var("x").def_var("y")
+        )
 
         text = "def concept what x is y pre is_question() where x is an adjective as get_attr(x, y)"
         def_ret_val = DefConceptParser().parse(context, ParserInput(text))

tests/evaluators/test_ExpressionEvaluator.py

@@ -0,0 +1,83 @@
+import pytest
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.SheerkaExecute import ParserInput
+from evaluators.BaseEvaluator import BaseEvaluator
+from evaluators.ExpressionEvaluator import ExpressionEvaluator
+from parsers.BaseExpressionParser import AndNode
+from parsers.ExpressionParser import ExpressionParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, e_ret_val_true
+
+
+class TestExpressionEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("return_value, expected", [
+        (pr_ret_val(AndNode(0, 0, [])), True),
+        (pr_ret_val(AndNode(0, 0, []), status=False), False),
+        (pr_ret_val("not a ExprNode", status=True), False),
+        (e_ret_val_true("no parser result"), False),
+    ])
+    def test_i_can_match(self, return_value, expected):
+        sheerka, context = self.init_concepts()
+        evaluator = ExpressionEvaluator()
+
+        assert evaluator.matches(context, return_value) == expected
+
+    def test_i_can_eval(self):
+        sheerka, context, one, number, isa = self.init_concepts(
+            "one",
+            "number",
+            Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y"))
+        evaluator = ExpressionEvaluator()
+
+        context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+
+        parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number"))
+
+        # first time, it returns False
+        res = evaluator.eval(context, parsed_return_value)
+        assert res.status
+        assert res.who == BaseEvaluator.PREFIX + ExpressionEvaluator.NAME
+        assert not res.body
+
+        # second time
+        sheerka.set_isa(context, one, number)
+        parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number"))
+        res = evaluator.eval(context, parsed_return_value)
+        assert res.status
+        assert res.body
+
+    def test_i_do_not_mess_up_use_copy(self):
+        sheerka, context, one, number, isa = self.init_concepts(
+            "one",
+            "number",
+            Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y"))
+        evaluator = ExpressionEvaluator()
+
+        parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number"))
+        concept = next(iter(parsed_return_value.body.body.compiled[0].return_value.body.body.objects.values()))
+        assert concept.get_compiled()["x"][0].body.body.get_hints().use_copy
+        assert concept.get_compiled()["y"][0].body.body.get_hints().use_copy
+
+        evaluator.eval(context, parsed_return_value)
+        concept = next(iter(parsed_return_value.body.body.compiled[0].return_value.body.body.objects.values()))
+        assert concept.get_compiled()["x"][0].body.body.get_hints().use_copy
+        assert concept.get_compiled()["y"][0].body.body.get_hints().use_copy
+
+    def test_i_cannot_eval_when_variables_are_missing(self):
+        sheerka, context, one, number, isa = self.init_concepts(
+            "one",
+            "number",
+            Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y"))
+        evaluator = ExpressionEvaluator()
+
+        parsed_return_value = ExpressionParser().parse(context, ParserInput("self is a number"))
+        res = evaluator.eval(context, parsed_return_value)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert len(res.body.body) == 1
+        assert isinstance(res.body.body[0], NameError)
+        assert res.body.body[0].args == ("self",)

tests/evaluators/test_ValidateConceptEvaluator.py

@@ -0,0 +1,198 @@
+import pytest
+
+from core.builtin_concepts import ParserResultConcept
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.SheerkaExecute import ParserInput
+from evaluators.BaseEvaluator import BaseEvaluator
+from evaluators.ValidateConceptEvaluator import ValidateConceptEvaluator
+from parsers.BaseNodeParser import ConceptNode
+from parsers.BaseParser import BaseParser
+from parsers.BnfNodeParser import BnfNodeParser
+from parsers.SyaNodeParser import SyaNodeParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.evaluators.EvaluatorTestsUtils import p_ret_val, pr_ret_val, ret_val
+
+
+class TestValidateConceptEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("return_value, need_validation, expected", [
+        (pr_ret_val(Concept("foo", where="something"), status=True), True, True),
+        (pr_ret_val(Concept("foo", pre="something"), status=True), True, True),
+        (pr_ret_val(Concept("foo", where="something"), status=True), False, False),
+        (pr_ret_val(Concept("foo", pre="something"), status=True), False, False),
+        (pr_ret_val(Concept("foo"), status=True), True, False),
+        (pr_ret_val(Concept("foo"), status=True), True, False),
+        (pr_ret_val(Concept("foo", where=None), status=True), True, False),
+        (pr_ret_val(Concept("foo", pre=None), status=True), True, False),
+        (pr_ret_val(Concept("foo", where=""), status=True), True, False),
+        (pr_ret_val(Concept("foo", pre=""), status=True), True, False),
+        (pr_ret_val(Concept("foo", where="something"), status=False), True, False),
+        (pr_ret_val(Concept("foo", pre="something"), status=False), True, False),
+        (pr_ret_val("Not a concept", status=True), True, False),
+        (pr_ret_val("Not a concept", status=False), True, False),
+        (p_ret_val("Not a parser result", status=True), True, False),
+        (p_ret_val("Not a parser result", status=False), True, False),
+    ])
+    def test_i_can_match(self, return_value, need_validation, expected):
+        sheerka, context = self.init_concepts()
+        evaluator = ValidateConceptEvaluator()
+        if (sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT) and
+            isinstance(return_value.body.body, Concept)):
+            return_value.body.body.get_hints().need_validation = need_validation
+
+        assert evaluator.matches(context, return_value) == expected
+
+    def test_i_cannot_match_when_the_return_value_is_not_a_direct_parsing_result(self):
+        # I match only if the return_value comes from a parser (not from an after_parsing evaluator)
+        sheerka, context = self.init_concepts()
+
+        concept = Concept("foo", pre="something")
+        parser_result = ParserResultConcept(BaseParser.get_name("parser"), source=concept.name, value=concept)
+        return_value = ret_val(value=parser_result, who="evaluators.something", status=True)
+        return_value.body.body.get_hints().need_validation = True
+
+        evaluator = ValidateConceptEvaluator()
+        assert not evaluator.matches(context, return_value)
+
+    @pytest.mark.parametrize("concept", [
+        Concept("foo", pre="False"),
+        Concept("foo", where="False"),
+    ])
+    def test_i_can_eval_when_constraint_is_false(self, concept):
+        sheerka, context, foo = self.init_concepts(concept)
+
+        ret_val = pr_ret_val(foo, status=True)
+        res = ValidateConceptEvaluator().eval(context, ret_val)
+
+        assert not res.status
+        assert res.who == BaseEvaluator.PREFIX + ValidateConceptEvaluator.NAME
+        assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED)
+        assert res.body.body == foo
+
+    @pytest.mark.parametrize("concept", [
+        Concept("foo"),
+        Concept("foo", pre="True"),
+        Concept("foo", where="True"),
+    ])
+    def test_i_can_eval_when_constraint_is_true_or_when_no_constrain(self, concept):
+        sheerka, context, foo = self.init_concepts(concept)
+
+        ret_val = pr_ret_val(foo, status=True)
+        res = ValidateConceptEvaluator().eval(context, ret_val)
+
+        assert res is None
+
+    @pytest.mark.parametrize("concept", [
+        Concept("foo"),
+        Concept("foo", pre="True"),
+        Concept("foo", where="True"),
+    ])
+    def test_i_can_eval_when_constraint_is_true_or_when_no_constrain_use_copy(self, concept):
+        sheerka, context, foo = self.init_concepts(concept)
+
+        foo.get_hints().use_copy = True
+        ret_val = pr_ret_val(foo, status=True)
+        res = ValidateConceptEvaluator().eval(context, ret_val)
+
+        assert res.status
+        assert res.who == BaseEvaluator.PREFIX + ValidateConceptEvaluator.NAME
+        assert res.body.body.id == foo.id
+        assert not res.body.body.get_hints().use_copy
+
+    def test_i_can_eval_when_is_question(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo", pre="is_question()"))
+
+        ret_val = pr_ret_val(foo, status=True)
+
+        res = ValidateConceptEvaluator().eval(context, ret_val)
+        assert not res.status
+
+        context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        res = ValidateConceptEvaluator().eval(context, ret_val)
+        assert res is None
+
+    def test_i_can_eval_when_unknown_variables_but_no_constraint(self):
+        sheerka, context, foo = self.init_concepts(Concept("a plus b").def_var("a").def_var("b"))
+
+        ret_val = pr_ret_val(foo, status=True)
+
+        res = ValidateConceptEvaluator().eval(context, ret_val)
+        assert res is None
+
+    def test_i_can_eval_when_unknown_variables(self):
+        # ValidateConceptEvaluator must filter only if all the variables are known
+        # In this example, 'b' is not set, so the return_value must not be filtered
+        sheerka, context, foo = self.init_concepts(
+            Concept("a plus b", where="isinstance(b, int)").def_var("a", "10").def_var("b"))
+
+        ret_val = pr_ret_val(foo, status=True)
+
+        res = ValidateConceptEvaluator().eval(context, ret_val)
+        assert res is None
+
+    def test_i_can_eval_when_constraint_on_variable_fails(self):
+        sheerka, context, foo = self.init_concepts(
+            Concept("a plus b", where="isinstance(b, int)").def_var("a").def_var("b", "'a string'"))
+
+        ret_val = pr_ret_val(foo, status=True)
+
+        res = ValidateConceptEvaluator().eval(context, ret_val)
+        assert not res.status
+
+    def test_i_can_eval_bnf_concepts(self):
+        sheerka, context, quantify_x = self.init_concepts(
+            Concept("quantify x", definition="('one'|'two') x", where="x != 'one'").def_var("x"),
+            create_new=True)
+        evaluator = ValidateConceptEvaluator()
+
+        # success
+        ret_val = BnfNodeParser().parse(context, ParserInput("one 'two'"))
+        assert evaluator.matches(context, ret_val)
+        res = evaluator.eval(context, ret_val)
+        assert isinstance(res.body.body, list)
+        assert len(res.body.body) == 1
+        assert isinstance(res.body.body[0], ConceptNode)
+        assert res.body.body[0].concept.id == ret_val.body.body[0].concept.id
+
+        # failure
+        ret_val = BnfNodeParser().parse(context, ParserInput("one 'one'"))
+        assert evaluator.matches(context, ret_val)
+        res = evaluator.eval(context, ret_val)
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED)
+        assert res.body.body == ret_val.body.body
+
+    def test_i_can_eval_sya_concepts(self):
+        sheerka, context, quantify_x = self.init_concepts(
+            Concept("a plus b", where="a < 10").def_var("a").def_var("b"),
+            create_new=True)
+        evaluator = ValidateConceptEvaluator()
+
+        # success
+        ret_val = SyaNodeParser().parse(context, ParserInput("5 plus 3"))
+        assert evaluator.matches(context, ret_val)
+        res = evaluator.eval(context, ret_val)
+        assert isinstance(res.body.body, list)
+        assert len(res.body.body) == 1
+        assert isinstance(res.body.body[0], ConceptNode)
+        assert res.body.body[0].concept.id == ret_val.body.body[0].concept.id
+
+        # failure
+        ret_val = SyaNodeParser().parse(context, ParserInput("15 plus 3"))
+        assert evaluator.matches(context, ret_val)
+        res = evaluator.eval(context, ret_val)
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED)
+        assert res.body.body == ret_val.body.body
+
+    def test_i_can_manage_infinite_recursion(self):
+        sheerka, context, a_and_b = self.init_concepts(
+            Concept("a and b", where="is_question()", body="a and b").def_var("a").def_var("b"),
+            create_new=True)
+        evaluator = ValidateConceptEvaluator()
+
+        ret_val = pr_ret_val(a_and_b)
+        res = evaluator.eval(context, ret_val)
+
+        assert res is None  # infinite recursion detected, res is None to drop the validator

tests/non_reg/test_sheerka_non_reg.py

@@ -7,7 +7,6 @@ from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
 from evaluators.PythonEvaluator import PythonEvalError
-from sheerkapython.python_wrapper import MethodAccessError
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, CB
 
@@ -211,9 +210,9 @@ as:
         # sanity check
         evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value)
         assert evaluated.body == "hello foo"
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
         compare_with_test_object(evaluated.get_value("a"), CB("foo", "foo"))
-        assert evaluated.get_value("a").get_metadata().is_evaluated
+        assert evaluated.get_value("a").get_hints().is_evaluated
 
     def test_i_can_recognize_duplicate_concepts_with_same_value(self):
         # when multiple result, choose the one that is the more specific (that has the less variables)
@@ -296,9 +295,9 @@ as:
         # sanity check
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka=sheerka, eval_body=True), return_value)
         assert evaluated.body == "one three"
-        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.get_hints().is_evaluated
         assert evaluated.get_value("a") == sheerka.new(concept_a.key, body="one").init_key()
-        assert evaluated.get_value("a").get_metadata().is_evaluated
+        assert evaluated.get_value("a").get_hints().is_evaluated
 
     @pytest.mark.parametrize("user_input", [
         "def concept greetings from def hello a where a",
@@ -314,7 +313,7 @@ as:
         concept_found = res[0].value
         assert sheerka.isinstance(concept_found, greetings)
         assert concept_found.get_value("a") == "foo"
-        assert concept_found.get_metadata().need_validation
+        assert concept_found.get_hints().need_validation
 
         res = sheerka.evaluate_user_input("greetings")
         assert len(res) == 1
@@ -322,7 +321,7 @@ as:
         concept_found = res[0].value
         assert sheerka.isinstance(concept_found, greetings)
         assert concept_found.get_value("a") == NotInit
-        assert not concept_found.get_metadata().need_validation
+        assert not concept_found.get_hints().need_validation
 
     @pytest.mark.parametrize("desc, definitions", [
         ("Simple form", [
@@ -681,14 +680,9 @@ as:
         assert res[0].body == 22
 
         res = sheerka.evaluate_user_input("twenty three")
-        assert len(res) == 1
-        assert not res[0].status
-        assert sheerka.isinstance(res[0].body, BuiltinConcepts.MULTIPLE_ERRORS)
         assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED)
 
         res = sheerka.evaluate_user_input("eval twenty three")
-        assert len(res) == 1
-        assert not res[0].status
         assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED)
 
     def test_i_can_manage_some_type_of_infinite_recursion(self):
@@ -725,20 +719,20 @@ as:
         assert res[0].body == "hello world"
 
         res = sheerka.evaluate_user_input("foo baz")
-        assert len(res) == 1
-        assert not res[0].status
-        assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
+        assert sheerka.has_error(self.get_context(sheerka), res, __type=BuiltinConcepts.CONDITION_FAILED)
 
         res = sheerka.evaluate_user_input("eval foo baz")
-        assert len(res) == 1
-        assert not res[0].status
-        assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
+        assert sheerka.has_error(self.get_context(sheerka), res, __type=BuiltinConcepts.CONDITION_FAILED)
 
-        res = sheerka.evaluate_user_input("foobar")
+        res = sheerka.evaluate_user_input("c:foobar:")  # where clause must not be evaluated
         assert len(res) == 1
         assert res[0].status
 
-        res = sheerka.evaluate_user_input("eval foobar")
+        res = sheerka.evaluate_user_input("foobar")  # where clause must not be evaluated
+        assert len(res) == 1
+        assert res[0].status
+
+        res = sheerka.evaluate_user_input("eval foobar")  # where clause is forced
         assert len(res) == 1  # error
         assert not res[0].status
         assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
@@ -835,7 +829,6 @@ as:
     def test_concepts_parsed_by_atom_parser_must_not_be_evaluated(self):
         definitions = [
             "def concept mult from a mult b as a * b",
-            "def concept a mult b as a * b",
         ]
 
         sheerka = self.init_scenario(definitions)
@@ -844,9 +837,17 @@ as:
         assert res[0].status
         assert isinstance(res[0].body, Concept)
 
-        # res = sheerka.evaluate_user_input("eval a mult b")
-        # assert res[0].status
-        # assert isinstance(res[0].body, Concept)
+    @pytest.mark.skip("Need to be fixed")
+    def test_concepts_parsed_by_atom_parser_must_not_be_evaluated_2(self):
+        definitions = [
+            "def concept a mult b as a * b",
+        ]
+
+        sheerka = self.init_scenario(definitions)
+
+        res = sheerka.evaluate_user_input("eval a mult b")
+        assert res[0].status
+        assert isinstance(res[0].body, Concept)
 
     def test_i_can_express_comparison(self):
         definitions = [
@@ -1059,8 +1060,7 @@ as:
             "def concept foo",
             "def concept number",
             "set_isa(one, number)",
-            "def concept q from q ? as question(q)",
-            "set_auto_eval(q)",
+            "def concept q from q ? as question(q) auto_eval True",
             "def concept is_a from x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
             "set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:, 'Sya')",
         ]
@@ -1069,18 +1069,16 @@ as:
         res = sheerka.evaluate_user_input("one is a number ?")  # automatically evaluated
         assert len(res) == 1
         assert res[0].status
-        assert res[0].body == True  # the body MUST be a boolean
+        assert isinstance(res[0].body, bool) and res[0].body  # the body MUST be a boolean
 
         res = sheerka.evaluate_user_input("foo is a number ?")  # automatically evaluated
         assert len(res) == 1
         assert res[0].status
-        assert res[0].body == False  # the body MUST be a boolean
+        assert isinstance(res[0].body, bool) and not res[0].body  # the body MUST be a boolean
 
         # x is a y is supposed to be a question. It cannot be used if not in a context of a question
         res = sheerka.evaluate_user_input("one is a number")
-        assert len(res) == 1
-        assert not res[0].status
-        assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
+        assert sheerka.has_error(self.get_context(sheerka), res, __type=BuiltinConcepts.CONDITION_FAILED)
 
     def test_i_can_evaluate_source_code_with_concept(self):
         init = [
@@ -1299,7 +1297,7 @@ as:
         assert sheerka.objvalue(res[0].body.get_value("qty")) == 2
 
     def test_i_can_implement_the_concept_and(self):
-        # Normally, redefining and leads to a circular ref between the concept and the python
+        # Normally, redefining 'and' leads to a circular ref between the concept and the python
         init = [
             "def concept x and y as x and y",
             "set_is_lesser(__PRECEDENCE, c:x and y:, 'Sya')",
@@ -1362,3 +1360,47 @@ as:
 
         res = sheerka.evaluate_user_input("eval foo")
         assert sheerka.has_error(context, res, __type="MethodAccessError")
+
+    def test_i_can_use_function_parser_and_complex_concept(self):
+        init = [
+            "def concept the x ret x",
+            "def concept foo",
+            "def concept bar",
+        ]
+        sheerka = self.init_scenario(init)
+
+        res = sheerka.evaluate_user_input("smart_get_attr(the foo, bar)")
+
+        assert len(res) == 1
+        assert res[0].status
+
+    def test_i_can_get_smart_get_attr_for_complex_concepts(self):
+        init = [
+            "def concept q from q ? as question(q) auto_eval True",
+            "set_is_lesser(__PRECEDENCE, q, 'Sya')",
+            "def concept a x ret x where isinstance(x, Concept)",
+            "def concept the x ret memory(x)",
+            "def concept short",
+            "def concept color",
+            "def concept red",
+            "def concept blue",
+            "def concept adjective",
+            "set_isa(red, color)",
+            "set_isa(blue, color)",
+            "set_isa(color, adjective)",
+            "def concept what is the x of y pre is_question() as smart_get_attr(y, x)",
+            "def concept qualify x from bnf adjective x as set_attr(x, c:adjective:, adjective) ret x",
+            "eval a red short",
+        ]
+        sheerka = self.init_scenario(init)
+
+        res = sheerka.evaluate_user_input("what is the color of the short ?")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == "red"
+
+        res = sheerka.evaluate_user_input("eval a blue short")
+        res = sheerka.evaluate_user_input("what is the color of the short ?")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == "blue"

tests/non_reg/test_sheerka_non_reg2.py

@@ -0,0 +1,18 @@
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaNonRegMemory2(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_select_the_correct_concept_when_ambiguity(self):
+        init = [
+            "def concept foo",
+            "def concept x is a y pre is_question() as isa(x,y)",
+            "def concept x is a foo pre is_question() as isinstance(x, foo)",
+        ]
+        sheerka = self.init_scenario(init)
+
+        res = sheerka.evaluate_user_input("question(foo is a foo)")
+
+        # assert len(res) == 1
+        # assert res[0].status
+        # assert res[0].value

tests/out/test_AsStrVisitor.py

@@ -1,5 +1,5 @@
 import pytest
-from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, FormatAstVariable, FormatAstVariableNotFound, \
+from parsers.FormatRuleActionParser import FormatAstRawText, FormatAstVariable, FormatAstVariableNotFound, \
     FormatAstSequence, FormatAstList, FormatAstDict
 from out.AsStrVisitor import AsStrVisitor
 

tests/out/test_ConsoleVisitor.py

@@ -1,5 +1,5 @@
 import pytest
-from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, FormatAstVariable, FormatAstVariableNotFound, \
+from parsers.FormatRuleActionParser import FormatAstRawText, FormatAstVariable, FormatAstVariableNotFound, \
     FormatAstSequence, FormatAstList, FormatAstDict
 from out.ConsoleVisistor import ConsoleVisitor
 

tests/out/test_DeveloperVisitor.py

@@ -1,7 +1,7 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.services.SheerkaDebugManager import NullDebugLogger
 from core.sheerka.services.SheerkaOut import SheerkaOut
-from core.sheerka.services.SheerkaRuleManager import FormatAstList, FormatAstVariable, FormatAstDict, FormatAstMulti
+from parsers.FormatRuleActionParser import FormatAstList, FormatAstVariable, FormatAstDict, FormatAstMulti
 from out.DeveloperVisitor import DeveloperVisitor
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka

tests/out/test_SheerkaOut.py

@@ -8,8 +8,9 @@ from core.sheerka.Sheerka import Sheerka
 from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager
 from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
 from core.sheerka.services.SheerkaOut import SheerkaOut
-from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, FormatAstVariable, FormatAstSequence, \
-    FormatAstColor, FormatAstVariableNotFound, FormatAstList, FormatAstDict, SheerkaRuleManager
+from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
+from parsers.FormatRuleActionParser import FormatAstRawText, FormatAstVariable, FormatAstSequence, \
+    FormatAstColor, FormatAstVariableNotFound, FormatAstList, FormatAstDict
 from core.utils import flatten_all_children
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka

tests/parsers/parsers_utils.py

@@ -230,7 +230,7 @@ class CC:
                       self.exclude_body,
                       **compiled)
 
-        raise NotImplementedError(f"CC, {other=}")
+        raise Exception(f"Expecting Concept but received {other=}")
 
 
 class CB:
@@ -269,7 +269,7 @@ class CB:
                 body = other.body
             return CB(concept, body)
 
-        raise NotImplementedError(f"CB, {other=}")
+        raise Exception(f"Expecting Concept but received {other=}")
 
 
 class CV:
@@ -310,7 +310,7 @@ class CV:
             values = get_test_obj_delegate(other.values(), self.values, get_test_obj_delegate)
             return CV(concept, **values)
 
-        raise NotImplementedError(f"CV, {other=}")
+        raise Exception(f"Expecting Concept but received {other=}")
 
 
 class CMV:
@@ -361,7 +361,7 @@ class CMV:
             variables = {name: value for name, value in other.get_metadata().variables}
             return CMV(concept, **variables)
 
-        raise NotImplementedError(f"CMV, {other=}")
+        raise Exception(f"Expecting Concept but received {other=}")
 
 
 class CIO:
@@ -408,7 +408,7 @@ class CIO:
         if isinstance(other, Concept):
             return CIO(other)
 
-        raise NotImplementedError(f"CIO, {other=}")
+        raise Exception(f"Expecting Concept but received {other=}")
 
 
 class HelperWithPos:
@@ -496,7 +496,7 @@ class SCN(HelperWithPos):
                        other.start if self.start is not None else None,
                        other.end if self.end is not None else None)
 
-        raise NotImplementedError(f"SCN, {other=}")
+        raise Exception(f"Expecting SourceCodeNode but received {other=}")
 
 
 class SCWC(HelperWithPos):
@@ -572,7 +572,7 @@ class SCWC(HelperWithPos):
             res.end = other.end
             return res
 
-        raise NotImplementedError(f"SCWC, {other=}")
+        raise Exception(f"Expecting SourceCodeWithConceptNode but received {other=}")
 
     @property
     def source(self):
@@ -663,7 +663,7 @@ class CN(HelperWithPos):
                       other.start if self.start is not None else None,
                       other.end if self.end is not None else None)
 
-        raise NotImplementedError(f"CN, {other=}")
+        raise Exception(f"Expecting ConceptNode but received {other=}")
 
 
 class CNC(CN):
@@ -737,8 +737,7 @@ class CNC(CN):
                        other.end if self.end is not None else None,
                        self.exclude_body,
                        **compiled)
-
-        raise NotImplementedError(f"CNC, {other=}")
+        raise Exception(f"Expecting ConceptNode but received {other=}")
 
 
 class UTN(HelperWithPos):
@@ -799,7 +798,7 @@ class UTN(HelperWithPos):
                        other.start,
                        other.end)
 
-        raise NotImplementedError(f"UTN, {other=}")
+        raise Exception(f"Expecting UnrecognizedTokensNode but received {other=}")
 
 
 class RN(HelperWithPos):
@@ -863,7 +862,7 @@ class RN(HelperWithPos):
                       other.start if self.start is not None else None,
                       other.end if self.end is not None else None)
 
-        raise NotImplementedError(f"RN, {other=}")
+        raise Exception(f"Expecting RuleNode but received {other=}")
 
 
 class FN:
@@ -930,7 +929,7 @@ class FN:
 
             return FN(other.first.value, other.last.value, params)
 
-        raise NotImplementedError(f"FN, {other=}")
+        raise Exception(f"Expecting FunctionNode but received {other=}")
 
 
 @dataclass()

tests/parsers/test_BnfNodeParser.py

@@ -1927,6 +1927,27 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert res.status
         compare_with_test_object(res.value.value, expected_array)
 
+    def test_i_cannot_parse_regex_concept_mixed_with_unrecognized_sya(self):
+        my_map = {
+            "hex": self.bnf_concept("hex", RegExMatch("[a-f0-9]{8}")),
+            "isa": Concept("x is an y", body="isinstance(x, y)", pre="is_question()").def_var("x").def_var("y"),
+            "isafoo": Concept("x is an foo", body="False", pre="is_question()").def_var("x"),
+            "q": Concept("q ?", body="question(a)").def_var("q")
+        }
+        
+        # I need the concept isafoo to fool SyaNodeParser when parsing the sub text 'is an hex ?'"
+        # The parser will try to recognize 'is an foo', will fail and will revert the result to UTN()
+        # It's this UTN that need to be properly handled
+
+        sheerka, context, parser = self.init_parser(my_map, init_from_sheerka=True, create_new=True)
+        sheerka.set_precedence(context, my_map["isa"], my_map["q"])
+        sheerka.set_precedence(context, my_map["isafoo"], my_map["q"])
+
+        text = "01234567 is an hexadecimal ?"
+        res = parser.parse(context, ParserInput(text))
+        assert not res.status
+
+
     # @pytest.mark.parametrize("parser_input, expected", [
     #     ("one", [
     #         (True, [CNC("bnf_one", source="one", one="one", body="one")]),

tests/parsers/test_DefConceptParser.py

@@ -8,16 +8,18 @@ from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept
 from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Keywords, Tokenizer, LexerError
+from parsers.BaseExpressionParser import VariableNode, ExprNode
 from parsers.BaseParser import UnexpectedEofParsingError
 from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence, RegExMatch, OneOrMore, \
     VariableExpression
 from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode, CannotHandleParsingError
 from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode
+from parsers.ExpressionParser import ExpressionParser
 from parsers.FunctionParser import FunctionParser
 from parsers.PythonParser import PythonParser, PythonNode
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import compute_expected_array, SCWC, CV, compare_with_test_object
+from tests.parsers.parsers_utils import compute_expected_array, SCWC, compare_with_test_object, CIO
 
 
 def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None, ret=None):
@@ -26,9 +28,9 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition
     if body:
         def_concept.body = get_concept_part(body)
     if where:
-        def_concept.where = get_concept_part(where)
+        def_concept.where = get_concept_part(where, use_expression=True)
     if pre:
-        def_concept.pre = get_concept_part(pre)
+        def_concept.pre = get_concept_part(pre, use_expression=True)
     if post:
         def_concept.post = get_concept_part(post)
     if ret:
@@ -46,11 +48,21 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition
     return def_concept
 
 
-def get_concept_part(part):
-    if isinstance(part, str):
-        node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval"))
+def get_concept_part(part, use_expression=False):
+    if use_expression:
+        node = VariableNode(0, 0, [], part)
         return ReturnValueConcept(
-            who="parsers.DefConcept",
+            who="parsers.Expression",
+            status=True,
+            value=ParserResultConcept(
+                source=part,
+                parser=ExpressionParser(),
+                value=node))
+
+    if isinstance(part, str):
+        node = PythonNode(part.lstrip(), ast.parse(part.lstrip(), mode="eval"))
+        return ReturnValueConcept(
+            who="parsers.Python",
             status=True,
             value=ParserResultConcept(
                 source=part,
@@ -61,7 +73,7 @@ def get_concept_part(part):
         # node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval"))
         nodes = compute_expected_array({}, part.source, [SCWC(part.first, part.last, *part.content)])
         return ReturnValueConcept(
-            who="parsers.DefConcept",
+            who="parsers.Python",
             status=True,
             value=ParserResultConcept(
                 source=part.source,
@@ -70,9 +82,9 @@ def get_concept_part(part):
                 try_parsed=nodes[0]))
 
     if isinstance(part, PN):
-        node = PythonNode(part.source.strip(), ast.parse(part.source.strip(), mode=part.mode))
+        node = PythonNode(part.source.lstrip(), ast.parse(part.source.lstrip(), mode=part.mode))
         return ReturnValueConcept(
-            who="parsers.DefConcept",
+            who="parsers.Python",
             status=True,
             value=ParserResultConcept(
                 source=part.source,
@@ -81,7 +93,7 @@ def get_concept_part(part):
 
     if isinstance(part, PythonNode):
         return ReturnValueConcept(
-            who="parsers.DefConcept",
+            who="parsers.Python",
             status=True,
             value=ParserResultConcept(
                 source=part.source,
@@ -248,7 +260,7 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka):
         assert isinstance(res.value, ParserResultConcept)
 
         part_mapping = "body" if part == "as" else part
-        args = {part_mapping: get_concept_part("True")}
+        args = {part_mapping: "True"}
         expected = get_def_concept("foo", **args)
         assert node == expected
 
@@ -260,28 +272,6 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
 
-    def test_i_can_parse_complex_def_concept_statement(self):
-        text = """def concept a mult b
-where a,b
-pre isinstance(a, int) and isinstance(b, int)
-as res = a * b
-ret a if isinstance(a, Concept) else self
-"""
-        sheerka, context, parser, *concepts = self.init_parser()
-        res = parser.parse(context, ParserInput(text))
-        return_value = res.value
-        expected_concept = get_def_concept(
-            name="a mult b",
-            where="a,b\n",
-            pre="isinstance(a, int) and isinstance(b, int)\n",
-            body=PN("res = a * b\n", "exec"),
-            ret="a if isinstance(a, Concept) else self\n"
-        )
-
-        assert res.status
-        assert isinstance(return_value, ParserResultConcept)
-        assert return_value.value == expected_concept
-
     def test_i_can_parse_mutilines_declarations(self):
         text = """
 def concept add one to a as
@@ -547,7 +537,10 @@ from give me the date !
         assert isinstance(res.value, ParserResultConcept)
         assert isinstance(node, DefConceptNode)
         assert sheerka.isinstance(node.where, BuiltinConcepts.RETURN_VALUE)
-        compare_with_test_object(node.where.body.body, CV(concepts[0], pre=True))
+        where_condition = node.where.body.body
+        assert isinstance(where_condition, ExprNode)
+        concept_found = where_condition.compiled[0].objects["__o_00__"]
+        compare_with_test_object(concept_found, CIO(concepts[0]))
 
         text = "def concept foo x y pre x is a y"
         res = parser.parse(context, ParserInput(text))
@@ -559,7 +552,10 @@ from give me the date !
         assert isinstance(res.value, ParserResultConcept)
         assert isinstance(node, DefConceptNode)
         assert sheerka.isinstance(node.pre, BuiltinConcepts.RETURN_VALUE)
-        compare_with_test_object(node.pre.body.body, CV(concepts[0], pre=True))
+        pre_condition = node.pre.body.body
+        assert isinstance(pre_condition, ExprNode)
+        concept_found = pre_condition.compiled[0].objects["__o_00__"]
+        compare_with_test_object(concept_found, CIO(concepts[0]))
 
     def test_i_can_parse_bnf_concept_with_regex(self):
         sheerka, context, parser, number = self.init_parser("number")

tests/parsers/test_DefRuleParser.py

@@ -3,12 +3,13 @@ import pytest
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka.services.SheerkaRuleManager import FormatAstNode, CompiledCondition
+from core.sheerka.services.SheerkaRuleManager import CompiledCondition
 from core.tokenizer import Tokenizer, Keywords
 from core.utils import tokens_are_matching
 from parsers.BaseCustomGrammarParser import KeywordNotFound, NameNode, SyntaxErrorNode
 from parsers.BaseParser import UnexpectedEofParsingError
 from parsers.DefRuleParser import DefRuleParser, DefExecRuleNode, DefFormatRuleNode
+from parsers.FormatRuleActionParser import FormatAstNode
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 cmap = {

tests/parsers/test_ExactConceptParser.py

@@ -62,8 +62,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         assert concept_found == concept
-        assert not concept_found.get_metadata().need_validation
-        assert not concept_found.get_metadata().is_evaluated
+        assert not concept_found.get_hints().need_validation
+        assert not concept_found.get_hints().is_evaluated
 
     def test_i_can_parse_concepts_defined_several_times(self):
         sheerka = self.get_sheerka(singleton=True)
@@ -80,11 +80,11 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert results[0].status
         assert results[0].value.value.name == "hello a"
         assert variable_def(results[0].value.value, "a") == "world"
-        assert results[0].value.value.get_metadata().need_validation
+        assert results[0].value.value.get_hints().need_validation
 
         assert results[1].status
         assert results[1].value.value.name == "hello world"
-        assert not results[1].value.value.get_metadata().need_validation
+        assert not results[1].value.value.get_hints().need_validation
 
     def test_i_can_parse_a_concept_with_variables(self):
         sheerka = self.get_sheerka(singleton=True)
@@ -99,8 +99,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
 
         concept_found = results[0].value.value
         compare_with_test_object(concept_found, CMV(concept, a="10", b="5"))
-        assert concept_found.get_metadata().need_validation
-        assert not concept_found.get_metadata().is_evaluated
+        assert concept_found.get_hints().need_validation
+        assert not concept_found.get_hints().is_evaluated
 
     def test_i_can_parse_a_concept_with_duplicate_variables(self):
         sheerka = self.get_sheerka(singleton=True)
@@ -115,7 +115,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
 
         concept_found = results[0].value.value
         compare_with_test_object(concept_found, CMV(concept, a="10", b="5"))
-        assert concept_found.get_metadata().need_validation
+        assert concept_found.get_hints().need_validation
 
     def test_i_can_parse_concept_when_defined_using_from_def(self):
         sheerka, context, plus = self.init_concepts(
@@ -129,8 +129,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         compare_with_test_object(concept_found, CMV(plus, a="10", b="5"))
-        assert concept_found.get_metadata().need_validation
-        assert not concept_found.get_metadata().is_evaluated
+        assert concept_found.get_hints().need_validation
+        assert not concept_found.get_hints().is_evaluated
 
     def test_i_can_parse_concept_token(self):
         sheerka, context, foo = self.init_concepts("foo")
@@ -142,8 +142,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         assert concept_found == foo
-        assert not concept_found.get_metadata().need_validation
-        assert concept_found.get_metadata().is_evaluated
+        assert not concept_found.get_hints().need_validation
+        assert concept_found.get_hints().is_evaluated
 
     def test_i_can_parse_concept_with_concept_tokens(self):
         sheerka, context, one, two, plus = self.init_concepts(
@@ -159,8 +159,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         compare_with_test_object(concept_found, CMV(plus, a="c:one:", b="c:two:"))
-        assert concept_found.get_metadata().need_validation
-        assert not concept_found.get_metadata().is_evaluated
+        assert concept_found.get_hints().need_validation
+        assert not concept_found.get_hints().is_evaluated
 
     def test_i_can_parse_when_expression_contains_keyword(self):
         sheerka, context, isa, def_concept = self.init_concepts(
@@ -175,8 +175,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         compare_with_test_object(concept_found, CMV(isa, c="z"))
-        assert concept_found.get_metadata().need_validation
-        assert not concept_found.get_metadata().is_evaluated
+        assert concept_found.get_hints().need_validation
+        assert not concept_found.get_hints().is_evaluated
 
         source = "def concept z"
         results = ExactConceptParser().parse(context, ParserInput(source))
@@ -185,8 +185,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         compare_with_test_object(concept_found, CMV(def_concept, a="z"))
-        assert concept_found.get_metadata().need_validation
-        assert not concept_found.get_metadata().is_evaluated
+        assert concept_found.get_hints().need_validation
+        assert not concept_found.get_hints().is_evaluated
 
     def test_i_can_manage_unknown_concept(self):
         context = self.get_context(self.get_sheerka(singleton=True))
@@ -219,4 +219,4 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
     #     assert len(results) == 1
     #     assert results[0].status
     #     assert results[0].value.value == concept
-    #     assert not results[0].value.value.get_metadata().need_validation
+    #     assert not results[0].value.value.get_hints().need_validation

tests/parsers/test_ExpressionParser.py

@@ -3,8 +3,8 @@ import pytest
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
-from parsers.BaseExpressionParser import VariableNode, ComparisonNode
-from parsers.BaseParser import ErrorSink
+from parsers.BaseExpressionParser import VariableNode, ComparisonNode, ExprNode
+from parsers.BaseParser import ErrorSink, BaseParser
 from parsers.ExpressionParser import ExpressionParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, FN, AND, NOT, OR, GT, GTE, LT, LTE, EQ, \
@@ -71,7 +71,7 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         "var.attr1.attr2",
         "var . attr1 . attr2",
     ])
-    def test_i_can_parse_variable(self, expression):
+    def test_i_can_parse_input_variable(self, expression):
         sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
         parsed = parser.parse_input(context, parser_input, error_sink)
 
@@ -80,7 +80,7 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         assert parsed.name == "var"
         assert parsed.attributes == ["attr1", "attr2"]
 
-    def test_i_can_parse_sub_tokens(self):
+    def test_i_can_parse_input_sub_tokens(self):
         sheerka, context, parser = self.init_parser()
 
         expression = "do not care var1 + var2 do not care either"
@@ -105,3 +105,23 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
 
         new_source = ComparisonNode.rebuild_source("new_var", parsed.comp, parsed.right.get_source())
         assert new_source == expected
+
+    def test_i_cannot_parse_empty_string(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(""))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
+
+    def test_i_can_compile(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = ParserInput("a > b and c < d")
+        res = parser.parse(context, text)
+
+        assert res.who == BaseParser.PREFIX + ExpressionParser.NAME
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(res.body.body, ExprNode)
+        assert res.body.body.compiled is not None

tests/parsers/test_FormatRuleActionParser.py

@@ -0,0 +1,81 @@
+import pytest
+
+from core.tokenizer import Token, TokenKind
+from parsers.FormatRuleActionParser import FormatAstSequence, FormatAstRawText, FormatAstVariable, FormatAstFunction, \
+    FormatAstList, FormatAstColor, FormatAstDict, FormatAstMulti, FormatRuleActionParser, UnexpectedEof, \
+    FormatRuleSyntaxError
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+seq = FormatAstSequence
+raw = FormatAstRawText
+var = FormatAstVariable
+func = FormatAstFunction
+lst = FormatAstList
+
+
+class TestFormatRuleActionParser(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("text, expected", [
+        ("", FormatAstRawText("")),
+        (" ", FormatAstRawText(" ")),
+        ("  raw text  ", FormatAstRawText("  raw text  ")),
+        ("{variable}", FormatAstVariable("variable")),
+        ("{ variable }", FormatAstVariable("variable")),
+        (" xy {v} z", seq([raw(" xy "), var("v"), raw(" z")])),
+        (r"\{variable}", FormatAstRawText("{variable}")),
+        (r"\\{variable}", seq([raw("\\"), var("variable")])),
+        (r"\\\{variable}", FormatAstRawText(r"\{variable}")),
+        (r"{var1}{var2}", seq([var("var1"), var("var2")])),
+        ("func()", FormatAstFunction("func", [], {})),
+        ("func(a, 'string value', c)", FormatAstFunction("func", ["a", "'string value'", "c"], {})),
+        ("func(a=10, b='string value')", FormatAstFunction("func", [], {"a": "10", "b": "'string value'"})),
+        ("func('string value'='another string value')", func("func", [], {"'string value'": "'another string value'"})),
+        ("red(' xy {v}')", FormatAstColor("red", seq([raw(" xy "), var("v")]))),
+        ('blue(" xy {v}")', FormatAstColor("blue", seq([raw(" xy "), var("v")]))),
+        ('green(  xy )', FormatAstColor("green", var("xy"))),
+        ('green()', FormatAstColor("green", raw(""))),
+        ('green("")', FormatAstColor("green", raw(""))),
+        ("list(var_name, 2, 'children')", FormatAstList("var_name", recurse_on="children", recursion_depth=2)),
+        ("list(var_name, recursion_depth=2, recurse_on='children')", FormatAstList("var_name",
+                                                                                   recurse_on="children",
+                                                                                   recursion_depth=2)),
+        ("list(var_name, recursion_depth=2, 'children')", FormatAstList("var_name", recursion_depth=2)),
+        ("list(var_name, 'children', recursion_depth=2)", FormatAstList("var_name", recursion_depth=2)),
+        ("list(var_name)", FormatAstList("var_name")),
+        ("{obj.prop1.prop2[0].prop3['value']}", FormatAstVariable("obj.prop1.prop2[0].prop3['value']")),
+        ("[{id}]", seq([raw("["), var("id"), raw("]")])),
+        ("{variable:format}", FormatAstVariable("variable", "format")),
+        ("{variable:3}", FormatAstVariable("variable", "3")),
+        (r"\not_a_function(a={var})", seq([raw("not_a_function(a="), var("var"), raw(")")])),
+        ("dict(var_name)", FormatAstDict("var_name")),
+        ("dict(var_name, items_prop='props')", FormatAstDict("var_name", items_prop='props')),
+        ("dict(var_name, debug=True)", FormatAstDict("var_name", debug=True, prefix="{", suffix="}")),
+        ("multi(var_name)", FormatAstMulti("var_name")),
+    ])
+    def test_i_can_parse_format_rule(self, text, expected):
+        assert FormatRuleActionParser(text).parse() == expected
+
+    @pytest.mark.parametrize("text, expected_error", [
+        ("{", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
+        ("{var_name", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
+        ("{}", FormatRuleSyntaxError("variable name not found", None)),
+        ("func(", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
+        ("func(a,b,c", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
+        ("func(a,,c", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
+        ("func(a,,c)", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
+        ("red(a,b)", FormatRuleSyntaxError("only one parameter supported", Token(TokenKind.IDENTIFIER, "b", 6, 1, 7))),
+        ("red(a=b)", FormatRuleSyntaxError("keyword arguments are not supported", None)),
+        ("red(xy {v})", FormatRuleSyntaxError("Invalid identifier", None)),
+        ("list()", FormatRuleSyntaxError("variable name not found", None)),
+        ("list(recursion_depth=2)", FormatRuleSyntaxError("variable name not found", None)),
+        ("list(a,b,c,d,e)", FormatRuleSyntaxError("too many positional arguments",
+                                                  Token(TokenKind.IDENTIFIER, "e", 13, 1, 14))),
+        ("list(a, recursion_depth=hello)", FormatRuleSyntaxError("'hello' is not numeric", None)),
+        ("list(a, recursion_depth='hello')", FormatRuleSyntaxError("'recursion_depth' must be an integer", None)),
+        ("dict()", FormatRuleSyntaxError("variable name not found", None)),
+    ])
+    def test_i_cannot_parse_invalid_format(self, text, expected_error):
+        parser = FormatRuleActionParser(text)
+        parser.parse()
+
+        assert parser.error_sink == expected_error

tests/parsers/test_LogicalOperatorParser.py

@@ -1,21 +1,15 @@
-import ast
-
 import pytest
 
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
-from core.concept import Concept, DoNotResolve
-from core.rule import Rule
+from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import TokenKind
-from parsers.BaseExpressionParser import TrueifyVisitor, IsAQuestionVisitor, AndNode, LeftPartNotFoundError, \
+from parsers.BaseExpressionParser import TrueifyVisitor, IsAQuestionVisitor, LeftPartNotFoundError, \
     ParenthesisMismatchError
 from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
 from parsers.LogicalOperatorParser import LogicalOperatorParser
-from parsers.PythonParser import PythonNode
-from sheerkarete.network import ReteNetwork
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import compute_expected_array, resolve_test_concept, EXPR, OR, AND, NOT, \
-    get_expr_node_from_test_node, get_rete_conditions, CMV, CNC, CC, compare_with_test_object
+from tests.parsers.parsers_utils import EXPR, OR, AND, NOT, \
+    get_expr_node_from_test_node
 
 
 class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka):
@@ -181,281 +175,281 @@ class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka):
 
         assert IsAQuestionVisitor().visit(expr_node) == expected
 
-    @pytest.mark.parametrize("expression, expected", [
-        ("foo", "foo"),
-        ("one two", "one two"),
-        ("foo is a bar", CMV("is a", x='foo', y='bar')),
-        ("one two is a bar", [CNC("is a", "one two is a bar", x="one two", y="bar")]),
-        ("foo is an foo bar",
-         [CNC("is an", "foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]),
-    ])
-    def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected):
-        concepts_map = {
-            "foo": Concept("foo"),
-            "bar": Concept("bar"),
-            "one two": Concept("one two"),
-            "is a": Concept("x is a y").def_var("x").def_var("y"),
-            "is an": Concept("x is an y", definition="('foo'|'bar')=x 'is an' 'foo bar'").def_var("x"),
-        }
-        sheerka, context, *concepts = self.init_test().with_concepts(*concepts_map.values(), create_new=True).unpack()
+    # @pytest.mark.parametrize("expression, expected", [
+    #     ("foo", "foo"),
+    #     ("one two", "one two"),
+    #     ("foo is a bar", CMV("is a", x='foo', y='bar')),
+    #     ("one two is a bar", [CNC("is a", "one two is a bar", x="one two", y="bar")]),
+    #     ("foo is an foo bar",
+    #      [CNC("is an", "foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]),
+    # ])
+    # def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected):
+    #     concepts_map = {
+    #         "foo": Concept("foo"),
+    #         "bar": Concept("bar"),
+    #         "one two": Concept("one two"),
+    #         "is a": Concept("x is a y").def_var("x").def_var("y"),
+    #         "is an": Concept("x is an y", definition="('foo'|'bar')=x 'is an' 'foo bar'").def_var("x"),
+    #     }
+    #     sheerka, context, *concepts = self.init_test().with_concepts(*concepts_map.values(), create_new=True).unpack()
+    #
+    #     parser = LogicalOperatorParser()
+    #     expr_node = parser.parse(context, ParserInput(expression)).body.body
+    #     return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
+    #
+    #     assert len(return_values) == 1
+    #     ret = return_values[0]
+    #
+    #     if isinstance(expected, list):
+    #         expected_nodes = compute_expected_array(concepts_map, expression, expected)
+    #         compare_with_test_object(ret.body.body, expected_nodes)
+    #     else:
+    #         expected_concept = resolve_test_concept(concepts_map, expected)
+    #         compare_with_test_object(ret.body.body, expected_concept)
 
-        parser = LogicalOperatorParser()
-        expr_node = parser.parse(context, ParserInput(expression)).body.body
-        return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
-
-        assert len(return_values) == 1
-        ret = return_values[0]
-
-        if isinstance(expected, list):
-            expected_nodes = compute_expected_array(concepts_map, expression, expected)
-            compare_with_test_object(ret.body.body, expected_nodes)
-        else:
-            expected_concept = resolve_test_concept(concepts_map, expected)
-            compare_with_test_object(ret.body.body, expected_concept)
-
-    @pytest.mark.parametrize("expression", [
-        "a == 5",
-        "foo > 5",
-        "func() == 5",
-        "not a == 5",
-        "not foo > 5",
-        "not func() == 5",
-        "isinstance(a, int)",
-        "func()",
-        "not isinstance(a, int)",
-        "not func()"
-    ])
-    def test_i_can_get_compiled_expr_from_simple_python_expressions(self, expression):
-        sheerka, context, = self.init_test().unpack()
-
-        parser = LogicalOperatorParser()
-        expr_node = parser.parse(context, ParserInput(expression)).body.body
-        return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
-
-        assert len(return_values) == 1
-        ret = return_values[0]
-
-        assert ret.status
-        python_node = ret.body.body.get_python_node()
-        _ast = ast.parse(expression, mode="eval")
-        expected_python_node = PythonNode(expression, _ast)
-        assert python_node == expected_python_node
-
-    @pytest.mark.parametrize("expression", [
-        "a and not b",
-        "not b and a",
-        "__ret and not __ret.status",
-    ])
-    def test_i_can_compile_negative_conjunctions_when_pure_python(self, expression):
-        sheerka, context, *concepts = self.init_concepts("foo")
-
-        parser = LogicalOperatorParser()
-        expr_node = parser.parse(context, ParserInput(expression)).body.body
-        return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
-
-        ast_ = ast.parse(expression, "<source>", 'eval')
-        expected_python_node = PythonNode(expression, ast_)
-
-        assert len(return_values) == 1
-        ret = return_values[0]
-
-        assert sheerka.objvalue(ret) == expected_python_node
-
-    @pytest.mark.parametrize("expression, text_to_compile", [
-        ("foo bar == 5", "__C__foo0bar__1001__C__ == 5"),
-        ("not foo bar == 5", "not __C__foo0bar__1001__C__ == 5"),
-    ])
-    def test_i_can_get_compiled_expr_from_python_and_concept(self, expression, text_to_compile):
-        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack()
-
-        parser = LogicalOperatorParser()
-        expr_node = parser.parse(context, ParserInput(expression)).body.body
-        return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
-
-        assert len(return_values) == 1
-        ret = return_values[0]
-
-        assert ret.status
-        python_node = ret.body.body.get_python_node()
-        _ast = ast.parse(text_to_compile, mode="eval")
-        expected_python_node = PythonNode(text_to_compile, _ast, expression)
-        assert python_node == expected_python_node
-
-    def test_i_can_get_compiled_expr_from__mix_of_concepts_and_python(self):
-        sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
-            Concept("animal"),
-            Concept("a cat"),
-            Concept("dog"),
-            Concept("pet"),
-            Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
-            Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
-            create_new=True
-        ).unpack()
-
-        parser = LogicalOperatorParser()
-        expression = "not a cat is a pet and not bird is an animal and not x > 5 and not dog is a pet"
-        expr_node = parser.parse(context, ParserInput(expression)).body.body
-        return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
-
-        to_compile = 'not __C__00var0000is0a000var001__1005__C__'
-        to_compile += ' and not __C__00var0000is0an0y__1006__C__'
-        to_compile += ' and not x > 5'
-        to_compile += ' and not __C__00var0000is0a000var001__1005_1__C__'
-        ast_ = ast.parse(to_compile, "<source>", 'eval')
-        expected_python_node = PythonNode(to_compile, ast_, expression)
-
-        assert len(return_values) == 1
-        ret = return_values[0]
-        python_node = ret.body.body
-        assert python_node == expected_python_node
-        compare_with_test_object(python_node.objects, {
-            "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
-            "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
-            "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
-        })
-
-    def test_i_can_get_compiled_expr_from_mix(self):
-        sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
-            Concept("animal"),
-            Concept("a cat"),
-            Concept("dog"),
-            Concept("pet"),
-            Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
-            Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
-            create_new=True
-        ).unpack()
-
-        expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
-        parser = LogicalOperatorParser()
-        expr_node = parser.parse(context, ParserInput(expression)).body.body
-        return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
-
-        assert len(return_values) == 1
-        ret = return_values[0]
-
-        to_compile = '__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1006__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__'
-        ast_ = ast.parse(to_compile, "<source>", 'eval')
-        expected_python_node = PythonNode(to_compile, ast_, expression)
-
-        python_node = ret.body.body
-        assert python_node == expected_python_node
-        compare_with_test_object(python_node.objects, {
-            "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
-            "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
-            "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
-        })
-
-    def test_i_can_get_compiled_expr_when_multiple_choices(self):
-        sheerka, context, *concepts = self.init_test().with_concepts(
-            Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
-            Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
-            create_new=True
-        ).unpack()
-
-        parser = LogicalOperatorParser()
-        expression = "a is a b"
-        expr_node = parser.parse(context, ParserInput(expression)).body.body
-        return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
-
-        assert len(return_values) == 2
-
-        ret = return_values[0]
-        compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[0], x="a", y="b"))
-
-        ret = return_values[1]
-        compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[1], x="a", y="b"))
-
-    def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self):
-        sheerka, context, *concepts = self.init_test().with_concepts(
-            Concept("animal"),
-            Concept("a cat"),
-            Concept("dog"),
-            Concept("pet"),
-            Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
-            Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
-            Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
-            create_new=True
-        ).unpack()
-
-        expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
-        parser = LogicalOperatorParser()
-        expr_node = parser.parse(context, ParserInput(expression)).body.body
-        return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
-
-        assert len(return_values) == 4
-        trimmed_source = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
-
-        current_ret = return_values[0]
-        python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__"
-        ast_ = ast.parse(python_source, "<source>", 'eval')
-        resolved_expected = PythonNode(python_source, ast_, trimmed_source)
-        assert sheerka.objvalue(current_ret) == resolved_expected
-
-        current_ret = return_values[1]
-        assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
-        python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006__C__"
-        ast_ = ast.parse(python_source, "<source>", 'eval')
-        resolved_expected = PythonNode(python_source, ast_, trimmed_source)
-        assert sheerka.objvalue(current_ret) == resolved_expected
-
-        current_ret = return_values[2]
-        assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
-        python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005__C__"
-        ast_ = ast.parse(python_source, "<source>", 'eval')
-        resolved_expected = PythonNode(python_source, ast_, trimmed_source)
-        assert sheerka.objvalue(current_ret) == resolved_expected
-
-        current_ret = return_values[3]
-        python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006_1__C__"
-        ast_ = ast.parse(python_source, "<source>", 'eval')
-        resolved_expected = PythonNode(python_source, ast_, trimmed_source)
-        assert sheerka.objvalue(current_ret) == resolved_expected
-
-    @pytest.mark.skip
-    @pytest.mark.parametrize("expression, expected_conditions, test_obj", [
-        (
-            "__ret",
-            ["#__x_00__|__name__|'__ret'"],
-            ReturnValueConcept("Test", True, None)
-        ),
-        (
-            "__ret.status == True",
-            ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
-            ReturnValueConcept("Test", True, None)
-        ),
-        (
-            "__ret.status",
-            ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
-            ReturnValueConcept("Test", True, None)
-        ),
-        (
-            "__ret and __ret.status",
-            ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
-            ReturnValueConcept("Test", True, None)
-        ),
-    ])
-    def test_i_can_get_rete_condition_from_python(self, expression, expected_conditions, test_obj):
-        sheerka, context, = self.init_test().unpack()
-        expected_full_condition = get_rete_conditions(*expected_conditions)
-
-        parser = LogicalOperatorParser()
-        expr_node = parser.parse(context, ParserInput(expression)).body.body
-
-        nodes = expr_node.parts if isinstance(expr_node, AndNode) else [expr_node]
-        _, rete_disjunctions = parser.compile_conjunctions(context, nodes, "test")
-
-        assert len(rete_disjunctions) == 1
-        assert rete_disjunctions == [expected_full_condition]
-
-        # check against a Rete network
-        network = ReteNetwork()
-        rule = Rule("test", expression, None)
-        rule.metadata.id = 9999
-        rule.metadata.is_compiled = True
-        rule.metadata.is_enabled = True
-        rule.rete_disjunctions = rete_disjunctions
-        network.add_rule(rule)
-
-        network.add_obj("__ret", test_obj)
-        matches = list(network.matches)
-        assert len(matches) > 0
+    # @pytest.mark.parametrize("expression", [
+    #     "a == 5",
+    #     "foo > 5",
+    #     "func() == 5",
+    #     "not a == 5",
+    #     "not foo > 5",
+    #     "not func() == 5",
+    #     "isinstance(a, int)",
+    #     "func()",
+    #     "not isinstance(a, int)",
+    #     "not func()"
+    # ])
+    # def test_i_can_get_compiled_expr_from_simple_python_expressions(self, expression):
+    #     sheerka, context, = self.init_test().unpack()
+    #
+    #     parser = LogicalOperatorParser()
+    #     expr_node = parser.parse(context, ParserInput(expression)).body.body
+    #     return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
+    #
+    #     assert len(return_values) == 1
+    #     ret = return_values[0]
+    #
+    #     assert ret.status
+    #     python_node = ret.body.body.get_python_node()
+    #     _ast = ast.parse(expression, mode="eval")
+    #     expected_python_node = PythonNode(expression, _ast)
+    #     assert python_node == expected_python_node
+    #
+    # @pytest.mark.parametrize("expression", [
+    #     "a and not b",
+    #     "not b and a",
+    #     "__ret and not __ret.status",
+    # ])
+    # def test_i_can_compile_negative_conjunctions_when_pure_python(self, expression):
+    #     sheerka, context, *concepts = self.init_concepts("foo")
+    #
+    #     parser = LogicalOperatorParser()
+    #     expr_node = parser.parse(context, ParserInput(expression)).body.body
+    #     return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
+    #
+    #     ast_ = ast.parse(expression, "<source>", 'eval')
+    #     expected_python_node = PythonNode(expression, ast_)
+    #
+    #     assert len(return_values) == 1
+    #     ret = return_values[0]
+    #
+    #     assert sheerka.objvalue(ret) == expected_python_node
+    #
+    # @pytest.mark.parametrize("expression, text_to_compile", [
+    #     ("foo bar == 5", "__C__foo0bar__1001__C__ == 5"),
+    #     ("not foo bar == 5", "not __C__foo0bar__1001__C__ == 5"),
+    # ])
+    # def test_i_can_get_compiled_expr_from_python_and_concept(self, expression, text_to_compile):
+    #     sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack()
+    #
+    #     parser = LogicalOperatorParser()
+    #     expr_node = parser.parse(context, ParserInput(expression)).body.body
+    #     return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
+    #
+    #     assert len(return_values) == 1
+    #     ret = return_values[0]
+    #
+    #     assert ret.status
+    #     python_node = ret.body.body.get_python_node()
+    #     _ast = ast.parse(text_to_compile, mode="eval")
+    #     expected_python_node = PythonNode(text_to_compile, _ast, expression)
+    #     assert python_node == expected_python_node
+    #
+    # def test_i_can_get_compiled_expr_from__mix_of_concepts_and_python(self):
+    #     sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
+    #         Concept("animal"),
+    #         Concept("a cat"),
+    #         Concept("dog"),
+    #         Concept("pet"),
+    #         Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+    #         Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
+    #         create_new=True
+    #     ).unpack()
+    #
+    #     parser = LogicalOperatorParser()
+    #     expression = "not a cat is a pet and not bird is an animal and not x > 5 and not dog is a pet"
+    #     expr_node = parser.parse(context, ParserInput(expression)).body.body
+    #     return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
+    #
+    #     to_compile = 'not __C__00var0000is0a000var001__1005__C__'
+    #     to_compile += ' and not __C__00var0000is0an0y__1006__C__'
+    #     to_compile += ' and not x > 5'
+    #     to_compile += ' and not __C__00var0000is0a000var001__1005_1__C__'
+    #     ast_ = ast.parse(to_compile, "<source>", 'eval')
+    #     expected_python_node = PythonNode(to_compile, ast_, expression)
+    #
+    #     assert len(return_values) == 1
+    #     ret = return_values[0]
+    #     python_node = ret.body.body
+    #     assert python_node == expected_python_node
+    #     compare_with_test_object(python_node.objects, {
+    #         "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
+    #         "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
+    #         "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
+    #     })
+    #
+    # def test_i_can_get_compiled_expr_from_mix(self):
+    #     sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
+    #         Concept("animal"),
+    #         Concept("a cat"),
+    #         Concept("dog"),
+    #         Concept("pet"),
+    #         Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+    #         Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
+    #         create_new=True
+    #     ).unpack()
+    #
+    #     expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
+    #     parser = LogicalOperatorParser()
+    #     expr_node = parser.parse(context, ParserInput(expression)).body.body
+    #     return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
+    #
+    #     assert len(return_values) == 1
+    #     ret = return_values[0]
+    #
+    #     to_compile = '__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1006__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__'
+    #     ast_ = ast.parse(to_compile, "<source>", 'eval')
+    #     expected_python_node = PythonNode(to_compile, ast_, expression)
+    #
+    #     python_node = ret.body.body
+    #     assert python_node == expected_python_node
+    #     compare_with_test_object(python_node.objects, {
+    #         "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
+    #         "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
+    #         "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
+    #     })
+    #
+    # def test_i_can_get_compiled_expr_when_multiple_choices(self):
+    #     sheerka, context, *concepts = self.init_test().with_concepts(
+    #         Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+    #         Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
+    #         create_new=True
+    #     ).unpack()
+    #
+    #     parser = LogicalOperatorParser()
+    #     expression = "a is a b"
+    #     expr_node = parser.parse(context, ParserInput(expression)).body.body
+    #     return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
+    #
+    #     assert len(return_values) == 2
+    #
+    #     ret = return_values[0]
+    #     compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[0], x="a", y="b"))
+    #
+    #     ret = return_values[1]
+    #     compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[1], x="a", y="b"))
+    #
+    # def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self):
+    #     sheerka, context, *concepts = self.init_test().with_concepts(
+    #         Concept("animal"),
+    #         Concept("a cat"),
+    #         Concept("dog"),
+    #         Concept("pet"),
+    #         Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+    #         Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
+    #         Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
+    #         create_new=True
+    #     ).unpack()
+    #
+    #     expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
+    #     parser = LogicalOperatorParser()
+    #     expr_node = parser.parse(context, ParserInput(expression)).body.body
+    #     return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
+    #
+    #     assert len(return_values) == 4
+    #     trimmed_source = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
+    #
+    #     current_ret = return_values[0]
+    #     python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__"
+    #     ast_ = ast.parse(python_source, "<source>", 'eval')
+    #     resolved_expected = PythonNode(python_source, ast_, trimmed_source)
+    #     assert sheerka.objvalue(current_ret) == resolved_expected
+    #
+    #     current_ret = return_values[1]
+    #     assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
+    #     python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006__C__"
+    #     ast_ = ast.parse(python_source, "<source>", 'eval')
+    #     resolved_expected = PythonNode(python_source, ast_, trimmed_source)
+    #     assert sheerka.objvalue(current_ret) == resolved_expected
+    #
+    #     current_ret = return_values[2]
+    #     assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
+    #     python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005__C__"
+    #     ast_ = ast.parse(python_source, "<source>", 'eval')
+    #     resolved_expected = PythonNode(python_source, ast_, trimmed_source)
+    #     assert sheerka.objvalue(current_ret) == resolved_expected
+    #
+    #     current_ret = return_values[3]
+    #     python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006_1__C__"
+    #     ast_ = ast.parse(python_source, "<source>", 'eval')
+    #     resolved_expected = PythonNode(python_source, ast_, trimmed_source)
+    #     assert sheerka.objvalue(current_ret) == resolved_expected
+    #
+    # @pytest.mark.skip
+    # @pytest.mark.parametrize("expression, expected_conditions, test_obj", [
+    #     (
+    #         "__ret",
+    #         ["#__x_00__|__name__|'__ret'"],
+    #         ReturnValueConcept("Test", True, None)
+    #     ),
+    #     (
+    #         "__ret.status == True",
+    #         ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
+    #         ReturnValueConcept("Test", True, None)
+    #     ),
+    #     (
+    #         "__ret.status",
+    #         ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
+    #         ReturnValueConcept("Test", True, None)
+    #     ),
+    #     (
+    #         "__ret and __ret.status",
+    #         ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
+    #         ReturnValueConcept("Test", True, None)
+    #     ),
+    # ])
+    # def test_i_can_get_rete_condition_from_python(self, expression, expected_conditions, test_obj):
+    #     sheerka, context, = self.init_test().unpack()
+    #     expected_full_condition = get_rete_conditions(*expected_conditions)
+    #
+    #     parser = LogicalOperatorParser()
+    #     expr_node = parser.parse(context, ParserInput(expression)).body.body
+    #
+    #     nodes = expr_node.parts if isinstance(expr_node, AndNode) else [expr_node]
+    #     _, rete_disjunctions = parser.compile_conjunctions(context, nodes, "test")
+    #
+    #     assert len(rete_disjunctions) == 1
+    #     assert rete_disjunctions == [expected_full_condition]
+    #
+    #     # check against a Rete network
+    #     network = ReteNetwork()
+    #     rule = Rule("test", expression, None)
+    #     rule.metadata.id = 9999
+    #     rule.metadata.is_compiled = True
+    #     rule.metadata.is_enabled = True
+    #     rule.rete_disjunctions = rete_disjunctions
+    #     network.add_rule(rule)
+    #
+    #     network.add_obj("__ret", test_obj)
+    #     matches = list(network.matches)
+    #     assert len(matches) > 0

tests/parsers/test_PythonWithConceptsParser.py

@@ -9,8 +9,10 @@ from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Token, TokenKind, Tokenizer
 from core.var_ref import VariableRef
 from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode, VariableNode
+from parsers.BnfNodeParser import BnfNodeParser
 from parsers.PythonParser import PythonNode
 from parsers.PythonWithConceptsParser import PythonWithConceptsParser
+from parsers.SequenceNodeParser import SequenceNodeParser
 from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import get_source_code_node
@@ -226,3 +228,39 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         assert result_python_node.ast_str == PythonNode.get_dump(expected_ast)
         assert result_python_node.original_source == "not foo == 1 and bar < 1"
         assert result_python_node.objects == {"__C__foo__C__": foo, "__C__bar__C__": bar}
+
+    def test_can_parse_after_unrecognized_bnf(self):
+        sheerka, context, one, two, twenties = self.init_concepts(
+            Concept("one", body="1"),
+            Concept("two", body="2"),
+            Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"),
+            create_new=True
+        )
+
+        bnf_parser_ret_val = BnfNodeParser().parse(context, ParserInput("a + twenty one"))
+        unrec_node_ret_val = UnrecognizedNodeParser().parse(context, bnf_parser_ret_val.body)
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, unrec_node_ret_val.body)
+
+        assert result.status
+        python_node = result.body.body
+        assert isinstance(python_node, PythonNode)
+        assert python_node.source == 'a + __C__twenties__1003__C__'
+        assert "__C__twenties__1003__C__" in python_node.objects
+
+    def test_i_cannot_parse_unrecognized_sequence(self):
+        sheerka, context, one, two, twenties = self.init_concepts(
+            Concept("one", body="1"),
+            Concept("two", body="2"),
+            Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"),
+            create_new=True
+        )
+
+        sequence_parser_ret_val = SequenceNodeParser().parse(context, ParserInput("a + twenty one"))
+        unrec_node_ret_val = UnrecognizedNodeParser().parse(context, sequence_parser_ret_val.body)
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, unrec_node_ret_val.body)
+
+        assert not result.status

tests/parsers/test_SequenceNodeParser.py

@@ -9,7 +9,7 @@ from tests.parsers.parsers_utils import compute_expected_array, CN, CNC, SCN, ge
     UTN
 
 
-class TestAtomsParser(TestUsingMemoryBasedSheerka):
+class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
     def init_parser(self, my_map, create_new=False, singleton=True, use_sheerka=False):
         sheerka, context, *updated_concepts = self.init_test().with_concepts(
             *my_map.values(),
@@ -283,8 +283,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
     @pytest.mark.parametrize("text, expected", [
         ("hello foo bar",
          [
-             (True, [CNC("hello1", "hello foo ", a="foo "), "bar"]),
-             (True, [CNC("hello2", "hello foo ", b="foo "), "bar"]),
+             ("a", [CN("hello1", "hello foo "), "bar"]),
+             ("b", [CN("hello2", "hello foo "), "bar"]),
          ]),
     ])
     def test_i_can_parse_when_unrecognized_yield_multiple_values(self, text, expected):
@@ -303,11 +303,12 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
             wrapper = res.body
             lexer_nodes = res.body.body
 
-            assert res.status == expected[0]
+            assert res.status
             assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
             expected_array = compute_expected_array(concepts_map, text, expected[1])
             transformed_nodes = get_test_obj(lexer_nodes, expected_array)
             assert transformed_nodes == expected_array
+            assert lexer_nodes[0].concept.get_metadata().variables == [(expected[0], "foo ")]
 
     @pytest.mark.parametrize("text, expected", [
         ("1 + twenty one", [SCN("1 + twenty "), "one"]),
@@ -352,7 +353,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         lexer_nodes = res.body.body
 
         assert res.status
-        assert lexer_nodes[0].concept.get_metadata().is_evaluated == expected_is_evaluated
+        assert lexer_nodes[0].concept.get_hints().is_evaluated == expected_is_evaluated
 
     def test_the_parser_always_return_a_new_instance_of_the_concept(self):
         concepts_map = {
@@ -383,3 +384,24 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+
+    @pytest.mark.parametrize("text", [
+        "foo",
+        "foo bar",
+        "long concept",
+        "unrecognized foo",
+        "bar unrecognized",
+    ])
+    def test_i_correctly_set_up_use_copy(self, text):
+        concepts_map = {
+            "foo": Concept("foo"),
+            "bar": Concept("bar"),
+            "long concept": Concept("long concept"),
+        }
+
+        sheerka, context, parser = self.init_parser(concepts_map)
+        res = parser.parse(context, ParserInput(text))
+
+        for node in res.body.body:
+            if hasattr(node, "concept"):
+                assert node.concept.get_hints().use_copy

tests/parsers/test_SyaNodeParser.py

@@ -7,7 +7,6 @@ from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
 from core.utils import NextIdManager
-from parsers.BaseNodeParser import UnrecognizedTokensNode
 from parsers.PythonParser import PythonNode
 from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
     NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError
@@ -1081,29 +1080,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         assert isinstance(concept_plus_b[0].body.body, PythonNode)
         assert concept_suffixed_a == cmap["two"]
 
-    @pytest.mark.parametrize("text, expected_status, expected_result", [
-        ("f1(one prefixed) plus f2(suffixed two)", False, [
-            CNC("plus",
-                a=SCWC("f1(", ")", CNC("prefixed", a="one")),
-                b=SCWC("f2(", (")", 1), CNC("suffixed", a="two")))
-        ]),
-        ("one is a concept", True, [CNC("is a concept", c="one")]),
-        ("a is a concept", False, [CNC("is a concept", c=UTN("a"))]),
-    ])
-    def test_i_can_parse_when_one_result(self, text, expected_status, expected_result):
-        sheerka, context, parser = self.init_parser()
-
-        res = parser.parse(context, ParserInput(text))
-        wrapper = res.body
-        lexer_nodes = res.body.body
-
-        expected_array = compute_expected_array(cmap, text, expected_result)
-        assert res.status == expected_status
-        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-
-        transformed_nodes = get_test_obj(lexer_nodes, expected_array)
-        assert transformed_nodes == expected_array
-
     @pytest.mark.parametrize("text", [
         "function(suffixed one)",
         "function(one plus two mult three)",
@@ -1144,8 +1120,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         ("one plus two a long other b", [CNC("plus", a="one", b="two"), UTN(" a long other b")]),
         ("one plus two a long infixed", [CNC("plus", a="one", b="two"), UTN(" a long infixed")]),
         ("one plus two a long", [CNC("plus", a="one", b="two"), UTN(" a long")]),
-        ("one ? a long infixed : two", [CNC("?", a="one", b=UTN("a long infixed"), c="two")]),
-        ("one ? a long infix : two", [CNC("?", a="one", b=UTN("a long infix"), c="two")]),
     ])
     def test_i_can_almost_parse_when_one_part_is_recognized_but_not_the_rest(self, text, expected_result):
         """
@@ -1195,31 +1169,25 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         assert transformed_nodes == expected_array
         # assert lexer_nodes == expected_array
 
-    @pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [
-        ("x$!# prefixed", "prefixed", ["a"]),
-        ("suffixed x$!#", "suffixed", ["a"]),
-        ("one infix x$!#", "infix", ["b"]),
-        ("x$!# infix one", "infix", ["a"]),
-        ("x$!# infix z$!#", "infix", ["a", "b"]),
+    @pytest.mark.parametrize("text, expected_error", [
+        ("x$!# prefixed", "Cannot parse 'x$!#'"),
+        ("suffixed x$!#", "Cannot parse 'x$!#'"),
+        ("one infix x$!#", "Cannot parse 'x$!#'"),
+        ("x$!# infix one", "Cannot parse 'x$!#'"),
+        ("x$!# infix z$!#", ["Cannot parse 'z$!#'", "Cannot parse 'x$!#'"]),
+        ("suffixed alpha beta", "Cannot parse 'alpha beta'"),
+        ("alpha beta prefixed", "Cannot parse 'alpha beta'"),
+        ("one plus alpha beta", "Cannot parse 'alpha beta'"),
     ])
-    def test_i_cannot_parse_when_unrecognized(self, text, expected_concept, expected_unrecognized):
+    def test_i_cannot_parse_when_unrecognized(self, text, expected_error):
         sheerka, context, parser = self.init_parser()
 
         res = parser.parse(context, ParserInput(text))
         wrapper = res.body
-        lexer_nodes = res.body.body
-        expected_end = len(list(Tokenizer(text))) - 2
 
         assert not res.status
-        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-        expected_array = [CN(cmap[expected_concept], text, 0, expected_end)]
-        transformed_nodes = get_test_obj(lexer_nodes, expected_array)
-        assert transformed_nodes == expected_array
-        # assert lexer_nodes == [CN(cmap[expected_concept], text, 0, expected_end)]
-
-        concept_found = lexer_nodes[0].concept
-        for unrecognized in expected_unrecognized:
-            assert isinstance(concept_found.get_compiled()[unrecognized], UnrecognizedTokensNode)
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.ERROR)
+        assert wrapper.body == expected_error
 
     @pytest.mark.parametrize("text, expected", [
         ("x$!# suffixed one", [UTN("x$!# ", 0, 4), CN("suffixed __var__0", "suffixed one", 5, 7)]),
@@ -1364,6 +1332,26 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
         compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)])
 
+    def test_i_correctly_set_up_use_copy(self):
+        my_map = {
+            "shirt": Concept("shirt"),
+            "a x": Concept("a x", ret="x").def_var("x"),
+            "red x": Concept("red x", ret="x").def_var("x"),
+        }
+
+        sheerka, context, parser = self.init_parser(my_map)
+
+        res = parser.parse(context, ParserInput("a red shirt"))
+        concept_found = res.body.body[0].concept
+
+        assert concept_found.get_hints().use_copy
+
+        concept_found_x = concept_found.get_compiled()["x"]
+        assert concept_found_x.get_hints().use_copy
+
+        concept_found_x_x = concept_found_x.get_compiled()["x"]
+        assert concept_found_x_x.get_hints().use_copy
+
 
 class TestFileBaseSyaNodeParser(TestUsingFileBasedSheerka):
     def test_i_can_parse_after_restart(self):

tests/parsers/test_UnrecognizedNodeParser.py

@@ -363,7 +363,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         assert parser_result.source == expression
         assert len(actual_nodes) == 1
         assert actual_nodes[0].nodes[
-            0].concept.get_metadata().is_evaluated  # 'a plus b' is recognized as concept definition
+            0].concept.get_hints().is_evaluated  # 'a plus b' is recognized as concept definition
 
     def test_i_can_parse_unrecognized_source_code_with_concept_node_when_var_in_short_term_memory(self):
         sheerka, context, parser = self.init_parser()
@@ -373,7 +373,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         nodes = get_input_nodes_from(concepts_map, expression, source_code_concepts)
         parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
 
-        context.add_to_short_term_memory("a", 1)
+        context.add_to_short_term_memory("a", 1)  # -> a plus b is now an instance of the concept
         res = parser.parse(context, parser_input)
         parser_result = res.body
         actual_nodes = res.body.body
@@ -382,7 +382,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert parser_result.source == expression
         assert len(actual_nodes) == 1
-        assert not actual_nodes[0].nodes[0].concept.get_metadata().is_evaluated  # 'a plus b' need to be evaluated
+        assert not actual_nodes[0].nodes[0].concept.get_hints().is_evaluated  # 'a plus b' need to be evaluated
 
     def test_i_can_parse_unrecognized_sya_concept_that_references_source_code(self):
         sheerka, context, parser = self.init_parser()

tests/sheerkarete/test_conditions.py

@@ -1,3 +1,5 @@
+import pytest
+
 from sheerkarete.common import WME, V
 from sheerkarete.conditions import NotEqualsCondition, AndConditions, Condition, NegatedCondition, \
     NegatedConjunctiveConditions, FilterCondition, BindCondition
@@ -27,6 +29,31 @@ class TestReteConditions(TestUsingMemoryBasedSheerka):
         network.remove_wme(wme)
         assert len(list(network.matches)) == 0
 
+    @pytest.mark.skip("Comparison between variables need to be implemented.")
+    def test_i_can_test_condition_between_variables(self):
+        network = ReteNetwork()
+        conditions = [Condition(V("x"), "__name__", "x"),
+                      Condition(V("y"), "__name__", "y"),
+                      Condition(V("x"), "__self__", V("y"))],
+        rule = RuleForTestingRete(AndConditions(conditions))
+        network.add_rule(rule)
+
+        assert len(list(network.matches)) == 0
+
+        wme_x = WME("x", "__self__", "10")
+        wme_y = WME("y", "__self__", "10")
+        network.add_wme(wme_x)
+        network.add_wme(wme_y)
+
+        matches = list(network.matches)
+        assert len(matches) == 1
+        assert matches[0].pnode.rules == [rule]
+        assert matches[0].token.wmes == [wme_x, wme_y]
+
+        # remove wme
+        network.remove_wme(wme_x)
+        assert len(list(network.matches)) == 0
+
     def test_i_can_manage_not_equals_condition(self):
         network = ReteNetwork()
 

tests/sheerkarete/test_network.py

@@ -88,6 +88,20 @@ class TestReteNetwork(TestUsingMemoryBasedSheerka):
             "fact_name.value.body": ["sub_value"],
         }
 
+    def test_i_can_update_conditions_attributes_when_value_is_a_variable(self):
+        network = ReteNetwork()
+        conditions = [Condition(V("x"), "__name__", "x"),
+                      Condition(V("y"), "__name__", "y"),
+                      Condition(V("x"), "__self__", V("y"))]
+
+        rule = RuleForTestingRete(AndConditions(conditions))
+        network.add_rule(rule)
+
+        assert network.attributes_by_id == {
+            "x": ["__name__", "__self__"],
+            "y": ["__name__", "__self__"],
+        }
+
     def test_adding_obj_when_no_rule_has_no_effect(self):
         network = ReteNetwork()
         ret = ReturnValueConcept("test", True, "value")