Added first implementation of concepts ambiguity resolution + Jenkins file test

Jenkinsfile

@@ -0,0 +1,16 @@
+pipeline {
+    agent none
+    stages {
+        stage('Build') {
+            agent {
+                docker {
+                    image 'python:2-alpine'
+                }
+            }
+            steps {
+                sh 'make clean'
+                // stash(name: 'compiled-results', includes: 'sources/*.py*')
+            }
+        }
+    }
+}

_concepts_lite.txt

@@ -2,4 +2,10 @@ def concept one as 1
 def concept two as 2
 def concept plus from a plus b as a + b
 def concept explain as get_results() | filter("id == 0") | recurse(2)
+set_isa(c:explain:, __COMMAND)
 def concept explain last as get_last_results() | filter("id == 0") | recurse(2)
+set_isa(c:explain last:, __COMMAND)
+def concept precedence a > precedence b as set_is_greater_than(BuiltinConcepts.PRECEDENCE, a, b)
+set_isa(c:precedence a > precedence b:, __COMMAND)
+def concept x is a command as set_isa(x, __COMMAND)
+set_isa(c:x is a command:, __COMMAND)

src/core/builtin_concepts.py

@@ -15,7 +15,15 @@ class BuiltinConcepts(Enum):
     """
     SHEERKA = "sheerka"
 
-    # Execution context actions
+    # processing instructions during sheerka.execute()
+    EVAL_BODY_REQUESTED = "eval body"  # to evaluate the body
+    EVAL_WHERE_REQUESTED = "eval where"  # to evaluate the where clause
+    RETURN_VALUE_REQUESTED = "return value"  # returns the body of the concept instead of the concept itself
+    REDUCE_REQUESTED = "reduce"  # remove meaningless error when possible
+    EVAL_UNTIL_SUCCESS_REQUESTED = "eval until success"  # PythonEvaluator tries combination until True is found
+    QUESTION_REQUESTED = "question"  # a question is asked
+
+    # possible actions during sheerka.execute()
     INIT_SHEERKA = "init sheerka"  #
     PROCESS_INPUT = "process input"  # Processing user input or other input
     PROCESSING = "processing input"  # Processing user input or other input
@@ -30,22 +38,28 @@ class BuiltinConcepts(Enum):
     AFTER_RENDERING = "after rendering"  # rendering the response from sheerka
     EVALUATE_CONCEPT = "evaluate concept"  # a concept will be evaluated
     EVALUATING_CONCEPT = "evaluating concept"  # a concept will be evaluated
+    EVALUATING_ATTRIBUTE = "evaluating concept attribute"  #
     VALIDATE_CONCEPT = "validate concept"
     VALIDATING_CONCEPT = "validating concept"
     INIT_COMPILED = "initializing concept compiled"
-    INIT_BNF = "ensure bnf"
+    INIT_BNF = "initialize bnf"
     MANAGE_INFINITE_RECURSION = "manage infinite recursion"
     PARSE_CODE = "execute source code"
     EXEC_CODE = "execute source code"
     TESTING = "testing"
 
-    USER_INPUT = "user input"  # represent an input from an user
-    SUCCESS = "success"
-    ERROR = "error"
+    # builtin attributes
+    ISA = "is a"  # when a concept is an instance of another one
+    COMMAND = "command"  # when the concept must be auto evaluated
+
+    # object
+    USER_INPUT = "user input concept"  # represent an input from an user
+    SUCCESS = "success concept"
+    ERROR = "error concept"
     UNKNOWN_CONCEPT = "unknown concept"  # the request concept is not recognized
     CANNOT_RESOLVE_CONCEPT = "cannot resolve concept"  # when too many concepts with the same name
-    RETURN_VALUE = "return value"  # a value is returned
-    CONCEPT_TOO_LONG = "concept too long"  # concept cannot be processed by exactConcept parser
+    RETURN_VALUE = "return value concept"  # a value is returned
+    CONCEPT_TOO_LONG = "concept too long concept"  # concept cannot be processed by exactConcept parser
     NEW_CONCEPT = "new concept"  # when a new concept is added
     UNKNOWN_PROPERTY = "unknown property"  # when requesting for a unknown property
     PARSER_RESULT = "parser result"
@@ -64,15 +78,9 @@ class BuiltinConcepts(Enum):
     FILTERED = "filtered"  # represents the result of a filtering
     CONCEPT_ALREADY_IN_SET = "concept already in set"
     EVALUATOR_PRE_PROCESS = "evaluator pre process"  # used modify / tweak behaviour of evaluators
-    EVAL_BODY_REQUESTED = "eval body requested"  # to evaluate the body
-    EVAL_WHERE_REQUESTED = "eval where requested"  # to evaluate the where clause
-    CONCEPT_VALUE_REQUESTED = "concept value requested"  # returns the body of the concept instead of the concept itself
-    REDUCE_REQUESTED = "reduce requested"  # remove meaningless error when possible
-    EVAL_SUCCESS_REQUESTED = "Try to find a successful evaluation"  # PyhtonEvaluator tries combination until True is found
     NOT_A_SET = "not a set"  # the concept has no entry in sets
-    WHERE_CLAUSE_FAILED = "where clause failed"  # failed to validate where clause during evaluation
+    CONDITION_FAILED = "where clause failed"  # failed to validate where clause during evaluation
     CHICKEN_AND_EGG = "chicken and egg"  # infinite recursion when declaring concept
-    ISA = "is a"  # builtin concept to express that a concept is an instance of another one
     EXPLANATION = "explanation"
     PRECEDENCE = "precedence"  # use to set priority among concepts when parsing
     ASSOCIATIVITY = "associativity"  # use to set priority among concepts when parsing
@@ -80,6 +88,7 @@ class BuiltinConcepts(Enum):
     NOT_FOUND = "not found"  # when the wanted resource is not found
     FORMAT_INSTRUCTIONS = "format instructions"  # to express how to print the concept
     NOT_IMPLEMENTED = "not implemented"  # instead of raise an error
+    PYTHON_SECURITY_ERROR = "security error"  # when trying to execute statement when only expression is allowed
 
     NODE = "node"
     GENERIC_NODE = "generic node"
@@ -108,6 +117,16 @@ class BuiltinConcepts(Enum):
 
 
 BuiltinUnique = [
+    BuiltinConcepts.EVAL_BODY_REQUESTED,
+    BuiltinConcepts.EVAL_WHERE_REQUESTED,
+    BuiltinConcepts.RETURN_VALUE_REQUESTED,
+    BuiltinConcepts.REDUCE_REQUESTED,
+    BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED,
+    BuiltinConcepts.QUESTION_REQUESTED,
+
+    BuiltinConcepts.INIT_SHEERKA,
+    BuiltinConcepts.PROCESS_INPUT,
+    BuiltinConcepts.PROCESSING,
     BuiltinConcepts.BEFORE_PARSING,
     BuiltinConcepts.PARSING,
     BuiltinConcepts.AFTER_PARSING,
@@ -117,10 +136,20 @@ BuiltinUnique = [
     BuiltinConcepts.BEFORE_RENDERING,
     BuiltinConcepts.RENDERING,
     BuiltinConcepts.AFTER_RENDERING,
-    BuiltinConcepts.SUCCESS,
-    BuiltinConcepts.NOP,
-    BuiltinConcepts.EVAL_BODY_REQUESTED,
-    BuiltinConcepts.REDUCE_REQUESTED,
+    BuiltinConcepts.EVALUATE_CONCEPT,
+    BuiltinConcepts.EVALUATING_CONCEPT,
+    BuiltinConcepts.EVALUATING_ATTRIBUTE,
+    BuiltinConcepts.VALIDATE_CONCEPT,
+    BuiltinConcepts.VALIDATING_CONCEPT,
+    BuiltinConcepts.INIT_COMPILED,
+    BuiltinConcepts.INIT_BNF,
+    BuiltinConcepts.MANAGE_INFINITE_RECURSION,
+    BuiltinConcepts.PARSE_CODE,
+    BuiltinConcepts.EXEC_CODE,
+    BuiltinConcepts.TESTING,
+
+    BuiltinConcepts.ISA,
+    BuiltinConcepts.COMMAND,
 ]
 
 BuiltinErrors = [str(e) for e in {
@@ -137,7 +166,7 @@ BuiltinErrors = [str(e) for e in {
     BuiltinConcepts.CONCEPT_EVAL_ERROR,
     BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
     BuiltinConcepts.NOT_A_SET,
-    BuiltinConcepts.WHERE_CLAUSE_FAILED,
+    BuiltinConcepts.CONDITION_FAILED,
     BuiltinConcepts.CHICKEN_AND_EGG,
     BuiltinConcepts.NOT_INITIALIZED,
     BuiltinConcepts.NOT_FOUND
@@ -194,7 +223,7 @@ class ReturnValueConcept(Concept):
     It's the main input for the evaluators
     """
 
-    def __init__(self, who=None, status=None, value=None, message=None, parents=None):
+    def __init__(self, who=None, status=None, value=None, message=None, parents=None, concept_id=None):
         super().__init__(BuiltinConcepts.RETURN_VALUE, True, False, BuiltinConcepts.RETURN_VALUE)
         self.set_value(ConceptParts.BODY, value)
         self.set_value("who", who)
@@ -202,6 +231,7 @@ class ReturnValueConcept(Concept):
         self.set_value("message", message)
         self.set_value("parents", parents)
         self.metadata.is_evaluated = True
+        self.metadata.id = concept_id
 
     @property
     def who(self):
@@ -429,21 +459,19 @@ class ConceptAlreadyInSet(Concept):
         return self.get_value("concept_set")
 
 
-class WhereClauseFailed(Concept):
-    def __init__(self, concept=None):
-        super().__init__(BuiltinConcepts.WHERE_CLAUSE_FAILED,
+class ConditionFailed(Concept):
+    def __init__(self, condition=None, concept=None, prop=None):
+        super().__init__(BuiltinConcepts.CONDITION_FAILED,
                          True,
                          False,
-                         BuiltinConcepts.WHERE_CLAUSE_FAILED)
-        self.set_value(ConceptParts.BODY, concept)
+                         BuiltinConcepts.CONDITION_FAILED)
+        self.set_value(ConceptParts.BODY, condition)
+        self.set_value("concept", concept)
+        self.set_value("prop", prop)
         self.metadata.is_evaluated = True
 
     def __repr__(self):
-        return f"WhereClauseFailed(concept={self.concept})"
-
-    @property
-    def concept(self):
-        return self.body
+        return f"ConditionFailed(condition='{self.body}', concept='{self.concept}', prop='{self.prop}')"
 
 
 class NotForMeConcept(Concept):
@@ -472,3 +500,18 @@ class ExplanationConcept(Concept):
         self.set_value("instructions", instructions)  # instructions for SheerkaPrint
         self.set_value(ConceptParts.BODY, execution_result)  # list of results
         self.metadata.is_evaluated = True
+
+
+class PythonSecurityError(Concept):
+    def __init__(self, prop=None, source_code=None, source=None, line=None, column=None):
+        super().__init__(BuiltinConcepts.PYTHON_SECURITY_ERROR,
+                         True,
+                         False,
+                         BuiltinConcepts.PYTHON_SECURITY_ERROR)
+
+        self.set_value("prop", prop)  # property or variable that was evaluated
+        self.set_value("source", source)  # origin of the source code (eg. file name)
+        self.set_value("line", line)  # line number
+        self.set_value("column", column)  # column number
+        self.set_value(ConceptParts.BODY, source_code)  # code being executed
+        self.metadata.is_evaluated = True

src/core/builtin_helpers.py

@@ -5,7 +5,7 @@ import core.ast.nodes
 from core.ast.nodes import CallNodeConcept
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, NotInit
+from core.concept import Concept, NotInit, ConceptParts
 from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode
 from parsers.BaseParser import BaseParser, ErrorNode
 
@@ -25,13 +25,10 @@ def is_same_success(context, return_values):
 
         if isinstance(ret_val.body, Concept):
             if not ret_val.body.metadata.is_evaluated:
-                with context.push(BuiltinConcepts.EVALUATE_CONCEPT,
-                                  ret_val.body,
-                                  desc=f"Evaluating concept '{ret_val.body}'") as sub_context:
-                    sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-                    evaluated = context.sheerka.evaluate_concept(sub_context, ret_val.body)
-                    if evaluated.key != ret_val.body.key:
-                        raise Exception("Failed to evaluate evaluate")
+                evaluated = context.sheerka.evaluate_concept(context, ret_val.body, eval_body=True)
+                if not context.sheerka.is_success(evaluated):
+                    raise Exception("Failed to evaluate evaluate")
+
                 return context.sheerka.objvalue(evaluated)
             else:
                 return context.sheerka.objvalue(ret_val.body)
@@ -173,6 +170,129 @@ def only_successful(context, return_values):
         sheerka.new(BuiltinConcepts.ONLY_SUCCESSFUL, body=successful_results),
         parents=return_values)
 
+#
+# def remove_ambiguity(context, return_values):
+#     """
+#     When multiple concepts are matching, try to find the one(s) which is/are the more accurate
+#     :param context:
+#     :param return_values:
+#     :return:
+#     """
+#     # example :
+#     # Concept("x is a y", PRE=QUESTION)
+#     # Concept("x is a y")
+#     # Concept("x is a command")
+#     # with the source 'foo is a command'
+#     #   The first one do not respect the pre condition (assuming that QUESTION is not in context)
+#     #   The second one is Ok, but the third is more specific (as it's a concept that explicitly asks for 'command')
+#     #   The third one will remain
+#     if not return_values:
+#         return return_values
+#
+#     sheerka = context.sheerka
+#     by_number_of_vars = {}  # sorted by number of variables
+#     to_keep = []  # return values values that are not eligible (ex non parser result)
+#     passed_validation = []
+#
+#     # sort by number of variables
+#     # The less variables there are, the more accurate is the concept
+#     for r in return_values:
+#         if (r.status and
+#             sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT) and
+#             isinstance(r.body.body, Concept)):
+#             by_number_of_vars.setdefault(len(r.body.body.metadata.variables), []).append(r)
+#         else:
+#             to_keep.append(r)
+#
+#         # Check the concepts, starting with the ones that have the less variables
+#     # Once a concept with a given number of variables is found, we do not process the concepts with an higher
+#     #   number of variables
+#     for nb_vars in sorted(by_number_of_vars.keys()):
+#         for r in by_number_of_vars[nb_vars]:
+#             concept = r.body.body
+#             if concept.metadata.pre is None or concept.metadata.pre.strip() == "":
+#                 passed_validation.append(r)
+#             else:
+#                 evaluated = context.sheerka.evaluate_concept(context, concept, metadata=["pre"])
+#                 if evaluated.key == concept.key and evaluated.get_value(ConceptParts.PRE):
+#                     passed_validation.append(r)
+#         if len(passed_validation) > 0:
+#             break
+#
+#     # Nothing was filtered, return the original return values
+#     if len(passed_validation) + len(to_keep) == len(return_values):
+#         return return_values  # nothing to filter
+#
+#     # All return_values fail, return empty list
+#     if len(passed_validation) == 0:
+#         return sheerka.ret(
+#             context.who,
+#             True,
+#             sheerka.new(BuiltinConcepts.FILTERED,
+#                         body=to_keep,
+#                         iterable=return_values,
+#                         predicate="remove_ambiguity(context, iterable)"),
+#             parents=return_values)
+#
+#     # Final check, we consider that the concepts that match a PRE condition are better than concepts with no condition
+#     by_condition_complexity = {}
+#     for r in passed_validation:
+#         by_condition_complexity.setdefault(get_condition_complexity(r.body.body, "pre"), []).append(r)
+#
+#     return sheerka.ret(
+#         context.who,
+#         True,
+#         sheerka.new(BuiltinConcepts.FILTERED,
+#                     body=to_keep + by_condition_complexity[max(by_condition_complexity.keys())],
+#                     iterable=return_values,
+#                     predicate="remove_ambiguity(context, iterable)"),
+#         parents=return_values)
+
+
+def resolve_ambiguity(context, concepts):
+    """
+    From the list of concept, elect the one(s) that best suit(s) the context
+    :param context:
+    :param concepts:
+    :return:
+    """
+
+    # we first sort by condition complexity. The more complex is the PRE condition, the more likely
+    # the concept matches the context
+    by_complexity = {}
+    for c in concepts:
+        by_complexity.setdefault(get_condition_complexity(c, "pre"), []).append(c)
+
+    remaining_concepts = []
+    for complexity in sorted(by_complexity.keys(), reverse=True):
+        for c in by_complexity[complexity]:
+            evaluated = context.sheerka.evaluate_concept(context, c, metadata=["pre"])
+            if evaluated.key == c.key:
+                remaining_concepts.append(c)
+
+        if len(remaining_concepts) > 0:
+            break  # no need to check concept with lower complexity
+
+    if len(remaining_concepts) in (0, 1):
+        return remaining_concepts  # they all failed the pre conditions or one champ is found
+
+    # for concepts with the same condition complexity, we choose the one that has the less number of variables
+    # We consider that Concept("hello world") is more specific than Concept("hello a").def_var("a")
+    #   when the input is "hello world"
+    by_number_of_vars = {}
+    for c in remaining_concepts:
+        by_number_of_vars.setdefault(len(c.metadata.variables), []).append(c)
+
+    return by_number_of_vars[min(by_number_of_vars.keys())]
+
+
+def get_condition_complexity(concept, concept_part_str):
+    concept_part_value = getattr(concept.metadata, concept_part_str)
+    if concept_part_value is None or concept_part_value.strip() == 0:
+        return 0
+
+    return 1  # no real computing as of now
+
 
 def only_parsers_results(context, return_values):
     """
@@ -197,7 +317,8 @@ def only_parsers_results(context, return_values):
             sheerka.new(BuiltinConcepts.IS_EMPTY, body=return_values),
             parents=return_values)
 
-    return_values_ok = [item for item in return_values if sheerka.isinstance(item.body, BuiltinConcepts.PARSER_RESULT)]
+    return_values_ok = [item for item in return_values if
+                        sheerka.isinstance(item.body, BuiltinConcepts.PARSER_RESULT)]
 
     # hack because some parsers don't follow the NOT_FOR_ME rule
     temp_ret_val = []
@@ -291,7 +412,8 @@ def get_lexer_nodes(return_values, start, tokens):
                 continue
 
             end = start + len(tokens) - 1
-            lexer_nodes.append([SourceCodeNode(ret_val.body.body, start, end, tokens, ret_val.body.source, ret_val)])
+            lexer_nodes.append(
+                [SourceCodeNode(ret_val.body.body, start, end, tokens, ret_val.body.source, ret_val)])
 
         elif ret_val.who == "parsers.ExactConcept":
             concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
@@ -333,12 +455,7 @@ def ensure_evaluated(context, concept, eval_body=True):
             (var[0] not in concept.values or concept.get_value(var[0]) == NotInit):
             return concept
 
-    with context.push(BuiltinConcepts.EVALUATE_CONCEPT, concept, desc=f"Evaluating concept {concept}") as sub_context:
-        if eval_body:
-            sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-        evaluated = context.sheerka.evaluate_concept(sub_context, concept)
-        sub_context.add_values(return_values=evaluated)
-
+    evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=eval_body)
     return evaluated
 
 

src/core/sheerka/Sheerka.py

@@ -1,5 +1,6 @@
 import inspect
 import logging
+from dataclasses import dataclass
 
 import core.builtin_helpers
 import core.utils
@@ -22,6 +23,15 @@ BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
 EXIT_COMMANDS = ("quit", "exit", "bye")
 
 
+@dataclass
+class SheerkaMethod:
+    """
+    Wrapper to sheerka method, to indicate if it's safe to call
+    """
+    method: object
+    has_side_effect: bool
+
+
 class Sheerka(Concept):
     """
     Main controller for the project
@@ -55,6 +65,7 @@ class Sheerka(Concept):
         self.log.debug("Starting Sheerka.")
 
         self.bnp = None  # reference to the BaseNodeParser class (to compute first keyword token)
+        self.return_value_concept_id = None
 
         # a concept can be instantiated
         # ex: File is a concept, but File('foo.txt') is an instance
@@ -84,10 +95,10 @@ class Sheerka(Concept):
 
         self.save_execution_context = True
 
-        self.methods_with_context = {"test_using_context"}
+        self.methods_with_context = {"test_using_context"}  # only the names, the method is defined in sheerka_methods
         self.sheerka_methods = {
-            "test": self.test,
-            "test_using_context": self.test_using_context
+            "test": SheerkaMethod(self.test, False),
+            "test_using_context": SheerkaMethod(self.test_using_context, False)
         }
         self.sheerka_pipeables = {}
 
@@ -119,10 +130,11 @@ class Sheerka(Concept):
     def chicken_and_eggs(self):
         return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
 
-    def bind_service_method(self, bound_method, as_name=None):
+    def bind_service_method(self, bound_method, has_side_effect, as_name=None):
         """
         Bind service method to sheerka instance for ease of use ?
         :param bound_method:
+        :param has_side_effect: False if the method is safe
         :param as_name:
         :return:
         """
@@ -132,18 +144,19 @@ class Sheerka(Concept):
         signature = inspect.signature(bound_method)
         if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
             self.methods_with_context.add(as_name)
-        self.sheerka_methods[as_name] = bound_method
+        self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect)
 
         setattr(self, as_name, bound_method)
 
-    def add_pipeable(self, func_name, function):
+    def add_pipeable(self, func_name, function, has_side_effect):
         """
         Adds a function that can bu used with pipe '|'
         :param func_name:
         :param function:
+        :param has_side_effect:
         :return:
         """
-        self.sheerka_pipeables[func_name] = function
+        self.sheerka_pipeables[func_name] = SheerkaMethod(function, has_side_effect)
 
     def initialize(self, root_folder: str = None, save_execution_context=True):
         """
@@ -292,6 +305,10 @@ class Sheerka(Concept):
                 self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
                 self.set_id_if_needed(concept, True)
                 self.cache_manager.add_concept(concept)
+
+                if key == BuiltinConcepts.RETURN_VALUE:
+                    self.return_value_concept_id = concept.id
+
             else:
                 self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
                 concept.update_from(from_db)
@@ -595,6 +612,8 @@ class Sheerka(Concept):
         :param concept_id:
         :return:
         """
+        if concept_id is None:
+            return False
         return self.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id)
 
     def has_key(self, concept_key):
@@ -652,6 +671,7 @@ class Sheerka(Concept):
             return self.new_from_template(template, concept_key, **kwargs)
 
     def new_from_template(self, template, key, **kwargs):
+        # core.utils.my_debug(f"Created {template}, {key=}, {kwargs=}")
         # manage singleton
         if template.metadata.is_unique:
             return template
@@ -677,7 +697,7 @@ class Sheerka(Concept):
                 return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept)
 
         # TODO : add the concept to the list of known concepts (self.instances)
-        concept.metadata.is_evaluated = True # because we have manually set the variables
+        concept.metadata.is_evaluated = True  # because we have manually set the variables
         return concept
 
     def ret(self, who: str, status: bool, value, message=None, parents=None):
@@ -690,13 +710,23 @@ class Sheerka(Concept):
         :param parents:
         :return:
         """
-        return self.new(
-            BuiltinConcepts.RETURN_VALUE,
+
+        # 1 second saved every twenty seconds in unit tests
+        return ReturnValueConcept(
             who=who,
             status=status,
             value=value,
             message=message,
-            parents=parents)
+            parents=parents,
+            concept_id=self.return_value_concept_id
+        )
+        # return self.new(
+        #     BuiltinConcepts.RETURN_VALUE,
+        #     who=who,
+        #     status=status,
+        #     value=value,
+        #     message=message,
+        #     parents=parents)
 
     def objvalue(self, obj, reduce_simple_list=False):
         if obj is None:
@@ -812,6 +842,10 @@ class Sheerka(Concept):
         if isinstance(obj, ReturnValueConcept):
             return obj.status
 
+        # other cases ?
+        # ...
+
+        # manage internal errors
         if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
             return False
 

src/core/sheerka/services/SheerkaAdmin.py

@@ -14,11 +14,11 @@ class SheerkaAdmin(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.caches_names)
-        self.sheerka.bind_service_method(self.cache)
-        self.sheerka.bind_service_method(self.restore)
-        self.sheerka.bind_service_method(self.concepts)
-        self.sheerka.bind_service_method(self.last_created_concept)
+        self.sheerka.bind_service_method(self.caches_names, False)
+        self.sheerka.bind_service_method(self.cache, False)
+        self.sheerka.bind_service_method(self.restore, True)
+        self.sheerka.bind_service_method(self.concepts, False)
+        self.sheerka.bind_service_method(self.last_created_concept, False)
 
     def caches_names(self):
         """

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -90,10 +90,10 @@ class SheerkaComparisonManager(BaseService):
         cache = Cache()
         self.sheerka.cache_manager.register_cache(self.RESOLVED_COMPARISON_ENTRY, cache, persist=False)
 
-        self.sheerka.bind_service_method(self.set_is_greater_than)
-        self.sheerka.bind_service_method(self.set_is_less_than)
-        self.sheerka.bind_service_method(self.get_partition)
-        self.sheerka.bind_service_method(self.get_concepts_weights)
+        self.sheerka.bind_service_method(self.set_is_greater_than, True)
+        self.sheerka.bind_service_method(self.set_is_less_than, True)
+        self.sheerka.bind_service_method(self.get_partition, False)
+        self.sheerka.bind_service_method(self.get_concepts_weights, False)
 
     def set_is_greater_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
         """

src/core/sheerka/services/SheerkaCreateNewConcept.py

@@ -20,7 +20,7 @@ class SheerkaCreateNewConcept(BaseService):
         self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS)  # BaseNodeParser
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.create_new_concept)
+        self.sheerka.bind_service_method(self.create_new_concept, True)
 
     def create_new_concept(self, context, concept: Concept):
         """

src/core/sheerka/services/SheerkaDump.py

@@ -21,8 +21,8 @@ class SheerkaDump(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.dump_desc, "desc")
-        self.sheerka.bind_service_method(self.dump_sdp, "dump_sdp")
+        self.sheerka.bind_service_method(self.dump_desc, True, "desc") # because concept is evaluated
+        self.sheerka.bind_service_method(self.dump_sdp, False, "dump_sdp")
 
     def dump_desc(self, *concept_names, eval=False):
         first = True
@@ -42,7 +42,7 @@ class SheerkaDump(BaseService):
 
             for c in concepts:
                 if eval:
-                    evaluated = self.sheerka.evaluate_concept(context, c)
+                    evaluated = self.sheerka.evaluate_concept(context, c, eval_body=eval)
                     value = evaluated.body if evaluated.key == c.key else evaluated
 
                 if not first:

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -2,6 +2,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import expect_one, only_successful
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
 from core.sheerka.services.sheerka_service import BaseService
+from core.tokenizer import Tokenizer
 from core.utils import unstr_concept
 
 CONCEPT_EVALUATION_STEPS = [
@@ -17,16 +18,23 @@ class SheerkaEvaluateConcept(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.evaluate_concept)
+        self.sheerka.bind_service_method(self.evaluate_concept, True)
 
     @staticmethod
     def infinite_recursion_detected(context, concept):
+        """
+        Browse the parents, looking for another evaluation of the same concept
+        :param context:
+        :param concept:
+        :return:
+        """
         if concept is None:
             return False
 
         parent = context.get_parent()
         while parent is not None:
-            if parent.who == context.who and parent.obj == concept and parent.obj.compiled == concept.compiled:
+            if parent.who == context.who and parent.action == BuiltinConcepts.EVALUATING_CONCEPT and \
+                parent.obj == concept and parent.obj.compiled == concept.compiled:
                 return True
 
             parent = parent.get_parent()
@@ -64,7 +72,7 @@ class SheerkaEvaluateConcept(BaseService):
         parent = context
         concepts_found = set()
         while parent and parent.obj:
-            if parent.who == context.who and parent.desc == context.desc:
+            if parent.who == context.who and parent.action == BuiltinConcepts.EVALUATING_CONCEPT:
                 body = parent.obj.metadata.body
                 try:
                     return self.sheerka.ret(self.NAME, True, InfiniteRecursionResolved(eval(body)))
@@ -180,8 +188,7 @@ class SheerkaEvaluateConcept(BaseService):
 
         path = get_path(context, current_prop)
         desc = f"Evaluating {path} (concept={current_concept})"
-        context.log(desc, self.NAME)
-        with context.push(BuiltinConcepts.EVALUATING_CONCEPT,
+        with context.push(BuiltinConcepts.EVALUATING_ATTRIBUTE,
                           current_prop,
                           desc=desc,
                           obj=current_concept,
@@ -191,14 +198,14 @@ class SheerkaEvaluateConcept(BaseService):
                 sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             if expect_success:
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_SUCCESS_REQUESTED)
+                sub_context.local_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
 
             # when it's a concept, evaluate it
             if isinstance(to_resolve, Concept) and \
                 not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
                 evaluated = self.evaluate_concept(sub_context, to_resolve)
                 sub_context.add_values(return_values=evaluated)
-                if evaluated.key == to_resolve.key:
+                if evaluated.key == to_resolve.key:  # quicker (and dirtier) than sheerka.is_success()
                     return self.apply_ret(evaluated)
                 else:
                     error = evaluated
@@ -253,51 +260,78 @@ class SheerkaEvaluateConcept(BaseService):
 
         return res
 
-    def evaluate_concept(self, context, concept: Concept):
+    def evaluate_concept(self, context, concept: Concept, eval_body=False, metadata=None):
         """
         Evaluation a concept
         It means that if the where clause is True, will evaluate the body
         :param context:
         :param concept:
+        :param eval_body:
+        :param metadata: list of metadata to evaluate ('pre', 'post'...)
         :return: value of the evaluation or error
         """
 
         if concept.metadata.is_evaluated:
             return concept
 
-        self.initialize_concept_asts(context, concept)
+        # I cannot use cache because of concept like 'number'.
+        # They don't have variables, but their values change every time they are instanciated
+        # TODO: Need to find a way to cache despite of them
+        # need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        # if need_body and len(concept.metadata.variables) == 0 and context.sheerka.has_id(concept.id):
+        #     from_cache = context.sheerka.get_by_id(concept.id)
+        #     if from_cache.metadata.is_evaluated:
+        #         concept.set_value(ConceptParts.BODY, from_cache.body)
+        #         concept.metadata.is_evaluated = True
+        #         return concept
 
-        # to make sure of the order, it don't use ConceptParts.get_parts()
-        # variables must be evaluated first, body must be evaluated before where
-        all_metadata_to_eval = self.choose_metadata_to_eval(context, concept)
+        desc = f"Evaluating concept {concept}"
+        with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc, eval_body=eval_body) as sub_context:
 
-        for metadata_to_eval in all_metadata_to_eval:
-            if metadata_to_eval == "variables":
-                for var_name in (v for v in concept.variables() if v in concept.compiled):
-                    prop_ast = concept.compiled[var_name]
+            if eval_body:
+                # ask for body evaluation
+                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
-                    if isinstance(prop_ast, list):
-                        # Do not send the current concept for the properties
-                        resolved = self.resolve_list(context, prop_ast, var_name, None, True, False)
-                    else:
-                        # Do not send the current concept for the properties
-                        resolved = self.resolve(context, prop_ast, var_name, None, True, False)
+            # auto evaluate commands
+            if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)):
+                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
-                    if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
-                        resolved.set_value("concept", concept)  # since current concept was not sent
-                        return resolved
-                    else:
-                        concept.set_value(var_name, resolved)
-            else:
-                part_key = ConceptParts(metadata_to_eval)
+            self.initialize_concept_asts(sub_context, concept)
 
-                # do not evaluate where when the body is a set
-                # Indeed, the way that the where clause is expressed is not a valid python or concept code
-                if part_key == ConceptParts.WHERE and self.sheerka.isaset(context, concept.body):
-                    continue
+            # to make sure of the order, it don't use ConceptParts.get_parts()
+            # variables must be evaluated first, body must be evaluated before where
+            all_metadata_to_eval = metadata or self.compute_metadata_to_eval(sub_context, concept)
+
+            for metadata_to_eval in all_metadata_to_eval:
+                if metadata_to_eval == "variables":
+                    for var_name in (v for v in concept.variables() if v in concept.compiled):
+                        prop_ast = concept.compiled[var_name]
+
+                        if isinstance(prop_ast, list):
+                            # Do not send the current concept for the properties
+                            resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False)
+                        else:
+                            # Do not send the current concept for the properties
+                            resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False)
+
+                        if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
+                            resolved.set_value("concept", concept)  # since current concept was not sent
+                            return resolved
+                        else:
+                            concept.set_value(var_name, resolved)
+                else:
+                    part_key = ConceptParts(metadata_to_eval)
+
+                    # do not evaluate where when the body is a set
+                    # Indeed, the way that the where clause is expressed is not a valid python or concept code
+                    if part_key == ConceptParts.WHERE and self.sheerka.isaset(sub_context, concept.body):
+                        continue
+
+                    if part_key not in concept.compiled or concept.compiled[part_key] is None:
+                        continue
 
-                if part_key in concept.compiled and concept.compiled[part_key] is not None:
                     metadata_ast = concept.compiled[part_key]
+
                     # if part_key is PRE, POST or WHERE, the concept need to be evaluated
                     # if we want the predicates to be resolved => so force_eval = True
                     # otherwise no need to force
@@ -306,91 +340,111 @@ class SheerkaEvaluateConcept(BaseService):
                     # when resolving predicate (where or pre), we need to make sure that PythonEvaluator
                     # will try every possibilities before returning False
                     expect_success = part_key in (ConceptParts.WHERE, ConceptParts.PRE)
-                    resolved = self.resolve(context,
+
+                    # resolve
+                    resolved = self.resolve(sub_context,
                                             metadata_ast,
                                             part_key,
                                             concept,
                                             force_concept_eval,
                                             expect_success)
-                    if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
+
+                    # 'FATAL' error is detected, let's stop
+                    if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
                         return resolved
-                    else:
-                        concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved)
 
-        #
-        # TODO : Validate the PRE condition
-        #
+                    concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved)
 
-        # validate where clause
-        if ConceptParts.WHERE in concept.values:
-            where_value = concept.get_value(ConceptParts.WHERE)
-            if not (where_value is None or self.sheerka.objvalue(where_value)):
-                return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
+                    # validate PRE and WHERE condition
+                    if part_key in (ConceptParts.PRE, ConceptParts.WHERE) and not self.sheerka.objvalue(resolved):
+                        return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED,
+                                                body=getattr(concept.metadata, metadata_to_eval),
+                                                concept=concept,
+                                                prop=part_key)
 
-        #
-        # TODO : Validate the POST condition
-        #
+            #
+            # TODO : Validate the POST condition
+            #
 
-        concept.init_key()  # only does it if needed
-        concept.metadata.is_evaluated = "body" in all_metadata_to_eval
+            concept.init_key()  # Necessary for old unit tests. To remove someday
 
-        # update the cache for concepts with no variable
-        if len(concept.metadata.variables) == 0:
-            self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
+            if "body" in all_metadata_to_eval:
+                concept.metadata.is_evaluated = True
 
-        return concept
+                # # update the cache for concepts with no variables
+                # Cannot use cache. See the comment at the beginning of this method
+                # if len(concept.metadata.variables) == 0:
+                #     self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
 
-    def choose_metadata_to_eval(self, context, concept):
-        if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
-            return ["pre", "post", "variables", "body", "where", "ret"]
+            return concept
+
+    def compute_metadata_to_eval(self, context, concept):
+        to_eval = []
+
+        needed, variables, body = self.get_needed_metadata(concept, ConceptParts.PRE, True, True)
+        to_eval.extend(needed)
 
-        metadata = ["pre", "post", "ret"]
         if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.metadata.need_validation:
-            needed = self.needed_metadata(concept, ConceptParts.WHERE)
-            for e in needed:
-                if e not in metadata:
-                    metadata.append(e)
-            if "where" not in metadata:
-                metadata.append("where")
+            # What are the cases where we do not need a validation ?
+            # see test_sheerka_non_reg::test_i_can_evaluate_bnf_concept_with_where_clause()
+            # res = sheerka.evaluate_user_input("foobar")
+            needed, v, b = self.get_needed_metadata(concept, ConceptParts.WHERE, not variables, not body)
+            variables |= v
+            body |= b
+            to_eval.extend(needed)
 
-        return metadata
+        needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body)
+        variables |= v
+        body |= b
+        to_eval.extend(needed)
 
-    def needed_metadata(self, concept, metadata):
+        needed, v, b = self.get_needed_metadata(concept, ConceptParts.POST, not variables, not body)
+        variables |= v
+        body |= b
+        to_eval.extend(needed)
+
+        if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED):
+            if not variables:
+                to_eval.append('variables')
+
+            if not body:
+                to_eval.append("body")
+
+        return to_eval
+
+    @staticmethod
+    def get_needed_metadata(concept, concept_part, check_vars, check_body):
         """
-        Tries to find out if the evaluation of the body is necessary
-        It's a very basic approach that will need to be improved
+        Check if the concept_part has to be evaluated
+        It also checks if the variables and the body need to be evaluated prior to it
         :param concept:
-        :param metadata:
+        :param concept_part:
+        :param check_vars:
+        :param check_body:
         :return:
         """
+        ret = []
+        vars_needed = False
+        body_needed = False
 
-        if metadata not in concept.compiled:
-            return []
+        if concept_part in concept.compiled and concept.compiled[concept_part] is not None:
+            concept_part_source = getattr(concept.metadata, concept_part.value)
 
-        return_values = concept.compiled[metadata]
-        if not isinstance(return_values, list):
-            return []
+            assert concept_part_source is not None
 
-        needed = []
-        for return_value in return_values:
-            if not self.sheerka.isinstance(return_value, BuiltinConcepts.RETURN_VALUE):
-                continue
+            tokens = [t.str_value for t in Tokenizer(concept_part_source)]
 
-            if not return_value.status:
-                continue
+            if check_vars:
+                for var_name in (v[0] for v in concept.metadata.variables):
+                    if var_name in tokens:
+                        vars_needed = True
+                        ret.append("variables")
+                        break
 
-            if not self.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT):
-                continue
+            if check_body and "self" in tokens:
+                body_needed = True
+                ret.append("body")
 
-            if not isinstance(return_value.body.source, str):
-                continue
+            ret.append(concept_part.value)
 
-            for var_name in (p[0] for p in concept.metadata.variables):
-                if var_name in return_value.body.source:
-                    needed.append("variables")
-                    break
-
-            if "self" in return_value.body.source:
-                needed.append("body")
-
-        return needed
+        return ret, vars_needed, body_needed

src/core/sheerka/services/SheerkaExecute.py

@@ -144,7 +144,7 @@ class SheerkaExecute(BaseService):
         self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.execute)
+        self.sheerka.bind_service_method(self.execute, True)
 
         self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False)
 
@@ -362,7 +362,8 @@ class SheerkaExecute(BaseService):
                                     if not isinstance(results, list):
                                         results = [results]
                                     for result in results:
-                                        evaluated_items.append(result)
+                                        if result.body:
+                                            evaluated_items.append(result)
                                         to_delete.extend(result.parents)
                                     sub_context.add_values(return_values=results)
                                 else:

src/core/sheerka/services/SheerkaFilter.py

@@ -96,9 +96,9 @@ class SheerkaFilter(BaseService):
         for k, v in SheerkaFilter.__dict__.items():
             if k.startswith("pipe_"):
                 if isinstance(v, staticmethod):
-                    self.sheerka.add_pipeable(k[5:], v.__func__)
+                    self.sheerka.add_pipeable(k[5:], v.__func__, True)
                 else:
-                    self.sheerka.add_pipeable(k[5:], v.__get__(self, self.__class__))
+                    self.sheerka.add_pipeable(k[5:], v.__get__(self, self.__class__), True)
 
         self.sheerka.cache_manager.register_cache(self.PREDICATES_ENTRY, self.cache, False, False)
 

src/core/sheerka/services/SheerkaHistoryManager.py

@@ -57,7 +57,7 @@ class SheerkaHistoryManager(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.history)
+        self.sheerka.bind_service_method(self.history, False)
 
     def history(self, depth=10, start=0):
         """

src/core/sheerka/services/SheerkaModifyConcept.py

@@ -10,7 +10,7 @@ class SheerkaModifyConcept(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.modify_concept)
+        self.sheerka.bind_service_method(self.modify_concept, True)
 
     def modify_concept(self, context, concept):
         old_version = self.sheerka.get_by_id(concept.id)

src/core/sheerka/services/SheerkaResultManager.py

@@ -10,10 +10,10 @@ class SheerkaResultConcept(BaseService):
         self.page_size = page_size
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.get_results_by_digest)
-        self.sheerka.bind_service_method(self.get_results_by_command)
-        self.sheerka.bind_service_method(self.get_last_results)
-        self.sheerka.bind_service_method(self.get_results)
+        self.sheerka.bind_service_method(self.get_results_by_digest, True)  # digest is recorded
+        self.sheerka.bind_service_method(self.get_results_by_command, True)  # digest is recorded
+        self.sheerka.bind_service_method(self.get_last_results, True)  # digest is recorded
+        self.sheerka.bind_service_method(self.get_results, False)
 
     def get_results_by_digest(self, context, digest, record_digest=True):
         """

src/core/sheerka/services/SheerkaSetsManager.py

@@ -13,7 +13,7 @@ GROUP_PREFIX = 'All_'
 class SheerkaSetsManager(BaseService):
     NAME = "SetsManager"
     CONCEPTS_GROUPS_ENTRY = "SetsManager:Concepts_Groups"
-    CONCEPTS_IN_GROUPS_ENTRY = "SetsManager:Concepts_In_Groups" # cache for get_set_elements()
+    CONCEPTS_IN_GROUPS_ENTRY = "SetsManager:Concepts_In_Groups"  # cache for get_set_elements()
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
@@ -21,12 +21,12 @@ class SheerkaSetsManager(BaseService):
         self.concepts_in_set = Cache()
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.set_isa)
-        self.sheerka.bind_service_method(self.get_set_elements)
-        self.sheerka.bind_service_method(self.add_concept_to_set)
-        self.sheerka.bind_service_method(self.isinset)
-        self.sheerka.bind_service_method(self.isa)
-        self.sheerka.bind_service_method(self.isaset)
+        self.sheerka.bind_service_method(self.set_isa, True)
+        self.sheerka.bind_service_method(self.get_set_elements, True)  # concepts are evaluated
+        self.sheerka.bind_service_method(self.add_concept_to_set, True)
+        self.sheerka.bind_service_method(self.isinset, False)
+        self.sheerka.bind_service_method(self.isa, False)
+        self.sheerka.bind_service_method(self.isaset, True)  # concept is evaluated, need to change the code
 
         self.sheerka.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, self.sets)
         self.sheerka.cache_manager.register_cache(self.CONCEPTS_IN_GROUPS_ENTRY, self.concepts_in_set, persist=False)
@@ -49,6 +49,8 @@ class SheerkaSetsManager(BaseService):
                 False,
                 self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
 
+        # KSI 20200709 add the concept, not its 'id' or 'key'
+        # It will allow conditions handling if concept set has its WHERE or PRE set to something
         concept.add_prop(BuiltinConcepts.ISA, concept_set)
 
         res = self.sheerka.modify_concept(context, concept)
@@ -141,7 +143,7 @@ class SheerkaSetsManager(BaseService):
             if sub_concept.metadata.where:
                 new_condition = self._validate_where_clause(sub_concept)
                 if not new_condition:
-                    return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=sub_concept)
+                    return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=sub_concept)
 
                 # This methods sucks, but I don't have enough tools (like proper AST manipulation functions)
                 # to do it properly now. It will be enhanced later
@@ -206,7 +208,7 @@ class SheerkaSetsManager(BaseService):
         if not (isinstance(concept, Concept) and concept.id):
             return False
 
-        # KSI 29062020
+        # KSI 20200629
         # To resolve infinite recursion between group concepts and BNF concepts
         if concept.metadata.definition_type == DEFINITION_TYPE_BNF:
             return False

src/core/sheerka/services/SheerkaVariableManager.py

@@ -27,11 +27,11 @@ class SheerkaVariableManager(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.record)
-        self.sheerka.bind_service_method(self.load)
-        self.sheerka.bind_service_method(self.delete)
-        self.sheerka.bind_service_method(self.set)
-        self.sheerka.bind_service_method(self.get)
+        self.sheerka.bind_service_method(self.record, True)
+        self.sheerka.bind_service_method(self.load, False)
+        self.sheerka.bind_service_method(self.delete, True)
+        self.sheerka.bind_service_method(self.set, True)
+        self.sheerka.bind_service_method(self.get, False)
 
         cache = Cache(default=lambda k: self.sheerka.sdp.get(self.VARIABLES_ENTRY, k))
         self.sheerka.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True)

src/core/tokenizer.py

@@ -487,6 +487,11 @@ class Tokenizer:
         return result, lines_count, column_index
 
     def eat_word(self, start):
+        """
+        Word is an alphanum (no space)
+        :param start:
+        :return:
+        """
         result = self.text[start]
         i = start + 1
         while i < self.text_len:

src/evaluators/AddConceptEvaluator.py

@@ -2,7 +2,7 @@ from core.ast.nodes import python_to_concept
 from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
 from core.builtin_helpers import get_names
 from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
-from core.tokenizer import TokenKind
+from core.tokenizer import TokenKind, Tokenizer
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.BaseParser import NotInitializedNode
 from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
@@ -53,7 +53,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         sheerka = context.sheerka
 
         # validate the node
-        props_found = set()
+        variables_found = set()
 
         concept = Concept(def_concept_node.name)
         concept.metadata.definition_type = def_concept_node.definition_type
@@ -80,16 +80,16 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
 
             # try to find what can be a property
             for p in self.get_variables(sheerka, part_ret_val, name_to_use):
-                props_found.add(p)
+                variables_found.add(p)
 
         # add variables by order of appearance when possible
         for name_part in name_to_use:
-            if name_part in props_found:
+            if name_part in variables_found:
                 concept.def_var(name_part, None)
 
         # add the remaining properties
         # They mainly come from BNF definition
-        for p in props_found:
+        for p in variables_found:
             if p not in concept.values:
                 concept.def_var(p, None)
 
@@ -130,24 +130,31 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
             names = [str(t.value) for t in ret_value.tokens if t.type in (
                 TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)]
             variables = filter(lambda x: x in concept_name, names)
-            return list(variables)
+            return set(variables)
 
         #
         # Case of python code
         #
         if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode):
             if len(concept_name) > 1:
-                python_node = ret_value.value.value
-                as_concept_node = python_to_concept(python_node.ast_)
-                names = get_names(sheerka, as_concept_node)
-                variables = filter(lambda x: x in concept_name, names)
-                return list(variables)
+                # tokens from ParserResult or source from python node
+                variables = set()
+                tokens = ret_value.value.tokens or list(Tokenizer(ret_value.value.value.source))
+                tokens = [t.str_value for t in tokens]
+                for identifier in [i for i in concept_name if str(i).isalnum()]:
+                    if identifier in tokens:
+                        variables.add(identifier)
+                # python_node = ret_value.value.value
+                # as_concept_node = python_to_concept(python_node.ast_)
+                # names = get_names(sheerka, as_concept_node)
+                # variables = filter(lambda x: x in concept_name, names)
+                return variables
 
         #
         # case of concept
         #
         if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, Concept):
-            return list(ret_value.value.value.values.keys())
+            return set(ret_value.value.value.values.keys())
 
         #
         # case of BNF
@@ -155,6 +162,6 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, ParsingExpression):
             visitor = ConceptOrRuleNameVisitor()
             visitor.visit(ret_value.value.value)
-            return sorted(list(visitor.names))
+            return set(visitor.names)
 
         return []

src/evaluators/ConceptEvaluator.py

@@ -25,7 +25,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
     #         self.evaluate_body = True
     #
     #     for r in return_values:
-    #         if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.CONCEPT_VALUE_REQUESTED):
+    #         if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.RETURN_VALUE_REQUESTED):
     #             self.evaluate_body = True
     #             break
     #

src/evaluators/EvalEvaluator.py

@@ -16,7 +16,7 @@ class EvalEvaluator(AllReturnValuesEvaluator):
     def matches(self, context, return_values):
         evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING)
         is_root = len(evaluation_parents) <= 1
-        return context.in_context(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) and is_root
+        return context.in_context(BuiltinConcepts.RETURN_VALUE_REQUESTED) and is_root
 
     def eval(self, context, return_values):
         sheerka = context.sheerka

src/evaluators/PostExecutionEvaluator.py

@@ -0,0 +1,34 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from evaluators.BaseEvaluator import OneReturnValueEvaluator
+
+
+class PostExecutionEvaluator(OneReturnValueEvaluator):
+    """
+    Last chance to alter the return_value
+    This evaluator is supposed to be a generic evaluator for all rules that must be executed just before
+        the aggregations
+    """
+
+    NAME = "PostExecution"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 90)
+
+    def matches(self, context, return_value):
+        evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING)
+        if len(evaluation_parents) > 1:
+            return False  # It must be executed only when the top level context
+
+        # only support the rule for the COMMANDS
+        value = return_value.body
+        return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.COMMAND))
+
+    def eval(self, context, return_value):
+        # only support the rule for the COMMANDS
+        body = return_value.body.body
+        return context.sheerka.ret(
+            self.name,
+            True,
+            body if body != BuiltinConcepts.NOT_INITIALIZED else return_value.body,
+            parents=[return_value])

src/evaluators/PrepareEvalEvaluator.py

@@ -34,6 +34,7 @@ class PrepareEvalEvaluator(OneReturnValueEvaluator):
             True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.text[5:], user_name=context.event.user_id))
 
         context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-        context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED)
+        context.global_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+        context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED)
 
         return new_text_to_parse

src/evaluators/PythonEvaluator.py

@@ -7,7 +7,7 @@ import core.ast.nodes
 import core.utils
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
-from core.concept import ConceptParts, Concept
+from core.concept import ConceptParts, Concept, NotInit
 from core.sheerka.services.SheerkaFilter import Pipe
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.PythonParser import PythonNode
@@ -73,15 +73,29 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
                 return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
 
+        attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE)
+        if attr_under_eval:
+            attr_under_eval = attr_under_eval[0]
+            expression_only = attr_under_eval.action_context != ConceptParts.BODY
+
+            if expression_only and isinstance(node.ast_, ast.Module):
+                # Module execution is forbidden in where, pre, post and ret concept parts
+                security_error = sheerka.new(BuiltinConcepts.PYTHON_SECURITY_ERROR,
+                                             prop=attr_under_eval.action_context,
+                                             body=node.source)
+                return sheerka.ret(self.name, False, security_error, parents=[return_value])
+        else:
+            expression_only = False
+
         # get globals
-        my_globals = self.get_globals(context, node)
+        my_globals = self.get_globals(context, node, expression_only)
         context.log(f"globals={my_globals}", self.name)
 
         all_possible_globals = self.get_all_possible_globals(context, my_globals)
         concepts_entries = None
         evaluated = BuiltinConcepts.NOT_INITIALIZED
         errors = []
-        expect_success = BuiltinConcepts.EVAL_SUCCESS_REQUESTED in context.local_hints
+        expect_success = BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED in context.local_hints
         for globals_ in all_possible_globals:
             try:
                 # eval
@@ -117,14 +131,27 @@ class PythonEvaluator(OneReturnValueEvaluator):
         context.log(f"{evaluated=}", self.name)
         return sheerka.ret(self.name, True, evaluated, parents=[return_value])
 
-    def get_globals(self, context, node):
+    def get_globals(self, context, node, expression_only):
+        """
+        Creates the global variables for python source code evaluation
+        :param context:
+        :param node:
+        :param expression_only: most of the commands are refused
+        :return:
+        """
         my_globals = {
             "Concept": core.concept.Concept,
             "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts,
+            "in_context": context.in_context
         }
 
+        if expression_only:
+            # disable builtin
+            my_globals["__builtins__"] = None
+
         # has to be the first, to allow override
-        method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context)
+        method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context, expression_only)
+
         self.update_globals_with_context(my_globals, context)
         already_know = set(my_globals.keys())
         self.update_globals_with_node(my_globals, context, node, already_know)
@@ -136,32 +163,54 @@ class PythonEvaluator(OneReturnValueEvaluator):
         return my_globals
 
     @staticmethod
-    def update_globals_with_sheerka_methods(my_locals, context):
+    def update_globals_with_sheerka_methods(my_locals, context, expression_only):
+        # methods_from_sheerka = {}
+        #
+        # # Make sure that methods that need the concept are correctly wrapped
+        # for method_name, method in context.sheerka.sheerka_methods.items():
+        #     if expression_only and method.has_side_effect:
+        #         continue
+        #
+        #     if method_name in context.sheerka.methods_with_context:
+        #         methods_from_sheerka[method_name] = inject_context(context)(method.method)
+        #     else:
+        #         methods_from_sheerka[method_name] = method.method
         methods_from_sheerka = {}
 
-        # Make sure that methods that need the concept are correctly wrapped
-        for method_name, method in context.sheerka.sheerka_methods.items():
-            if method_name in context.sheerka.methods_with_context:
-                methods_from_sheerka[method_name] = inject_context(context)(method)
-            else:
-                methods_from_sheerka[method_name] = method
-
         # Add all the methods as a direct access
-        for method_name, method in methods_from_sheerka.items():
-            my_locals[method_name] = method
+        for method_name, method in context.sheerka.sheerka_methods.items():
+            if expression_only and method.has_side_effect:
+                continue
+
+            if method_name in context.sheerka.methods_with_context:
+                my_locals[method_name] = inject_context(context)(method.method)
+            else:
+                my_locals[method_name] = method.method
+            methods_from_sheerka[method_name] = my_locals[method_name]
 
         # Add pipeable functions
         for func_name, function in context.sheerka.sheerka_pipeables.items():
-            my_locals[func_name] = Pipe(function, context)
+            if expression_only and function.has_side_effect:
+                continue
+
+            my_locals[func_name] = Pipe(function.method, context)
 
         return methods_from_sheerka  # to allow access using prefix "sheerka."
 
     def update_globals_with_context(self, my_globals, context):
+        """
+        Update globals with the current object being evaluated (and its variables)
+        :param my_globals:
+        :param context:
+        :return:
+        """
         if context.obj:
             context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name)
 
             for prop_name in context.obj.variables():
-                my_globals[prop_name] = context.obj.get_value(prop_name)
+                value = context.obj.get_value(prop_name)
+                if value != NotInit:
+                    my_globals[prop_name] = value
             my_globals["self"] = context.obj
 
     def update_globals_with_node(self, my_globals, context, node, already_known):
@@ -202,19 +251,11 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 context.log(f"Concept {name} is already evaluated.", self.name)
             else:
                 context.log(f"Evaluating '{concept}'", self.name)
-                with context.push(BuiltinConcepts.EVALUATE_CONCEPT,
-                                  concept,
-                                  who=self.name,
-                                  desc=f"Evaluating '{concept}'",
-                                  obj=concept) as sub_context:
-                    sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-                    evaluated = context.sheerka.evaluate_concept(sub_context, concept)
-                    sub_context.add_values(return_values=evaluated)
-
-                    if evaluated.key != concept.key:
-                        context.log(f"Error while evaluating '{name}'. Skipping.", self.name)
-                        continue
-                    concept = evaluated
+                evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True)
+                if evaluated.key != concept.key:
+                    context.log(f"Error while evaluating '{name}'. Skipping.", self.name)
+                    continue
+                concept = evaluated
 
             my_globals[name] = concept
 
@@ -293,7 +334,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
         last_ast = copy.deepcopy(code_ast)
         last_ast.body = code_ast.body[-1:]
 
-        exec(compile(init_ast, "<ast>", "exec"), {}, my_locals)
+        exec(compile(init_ast, "<ast>", "exec"), my_globals, my_locals)
         if type(last_ast.body[0]) == ast.Expr:
             return eval(compile(self.expr_to_expression(last_ast.body[0]), "<ast>", "eval"), my_globals, my_locals)
         else:

src/evaluators/ResolveAmbiguityEvaluator.py

@@ -0,0 +1,60 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.builtin_helpers import resolve_ambiguity
+from core.concept import Concept
+from evaluators.BaseEvaluator import AllReturnValuesEvaluator
+
+
+class ResolveAmbiguityEvaluator(AllReturnValuesEvaluator):
+    """
+    Use when multiple concepts match the same input source
+    """
+
+    NAME = "ResolveAmbiguity"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_PARSING], 50)
+        self.sources = None
+
+    def matches(self, context, return_values):
+        # first, arrange return_values by sources.
+        # If they share the same source, that means that there are multiple results for one ParserInput
+        self.sources = {}
+        success = False
+        for ret in [ret for ret in return_values if ret.status]:
+            source = self.get_source(context, ret)
+
+            if source:
+                self.sources.setdefault(source, []).append(ret)
+
+                if len(self.sources[source]) > 1:
+                    success = True  # at least one source are more than one entry -> let's resolve it
+
+        return success
+
+    def eval(self, context, return_values):
+        ret = []
+
+        for ret_vals in self.sources.values():
+            parser_results = {id(r.body.body): r.body for r in ret_vals}
+            resolved = resolve_ambiguity(context, [r.body.body for r in ret_vals])
+            if len(resolved) == 0:
+                ret.append(context.sheerka.ret(self.name, True, [], parents=ret_vals))
+            else:
+                for c in resolved:
+                    ret.append(context.sheerka.ret(self.name, True, parser_results[id(c)], parents=ret_vals))
+
+        return ret
+
+    @staticmethod
+    def get_source(context, return_value):
+        """
+        Try to get the source (the ParserInput) of the return_value
+        We only consider parser result of concepts
+        :param context:
+        :param return_value:
+        :return:
+        """
+        if context.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT) and \
+            isinstance(return_value.body.body, Concept):
+            return return_value.body.source
+        return None

src/evaluators/RetEvaluator.py

@@ -6,7 +6,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator
 
 class RetEvaluator(OneReturnValueEvaluator):
     """
-    The evaluator transform the a concept, using the ret value
+    The evaluator transforms the concept, using the RET metadata value
     """
 
     NAME = "Ret"

src/parsers/ExactConceptParser.py

@@ -2,7 +2,7 @@ import logging
 
 import core.builtin_helpers
 from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
-from core.concept import VARIABLE_PREFIX
+from core.concept import VARIABLE_PREFIX, ConceptParts
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Keywords, TokenKind, LexerError
 from core.utils import str_concept

src/repl/SheerkaPromptCompleter.py

@@ -36,18 +36,16 @@ class SheerkaPromptCompleter(Completer):
         self.params_history_service = self.sheerka.services[SheerkaFunctionsParametersHistory.NAME]
         self.builtins = []
         for name, bound_method in sheerka.sheerka_methods.items():
-            self.builtins.append(self.get_completion_desc(name, bound_method, "builtin", ["context"]))
+            self.builtins.append(self.get_completion_desc(name, bound_method.method, "builtin", ["context"]))
 
         self.pipeable_builtins = []
         for name, pipeable in self.sheerka.sheerka_pipeables.items():
-            self.pipeable_builtins.append(self.get_completion_desc(name, pipeable, "builtin", ["context", "iterable"]))
+            self.pipeable_builtins.append(
+                self.get_completion_desc(name, pipeable.method, "builtin", ["context", "iterable"]))
 
         self.exit_commands = [CompletionDesc(c, c, "command") for c in EXIT_COMMANDS]
-        self.globals = self.sheerka.sheerka_methods.copy()
-        self.globals.update(self.sheerka.sheerka_pipeables)
-
-    def get_locals(self):
-        return self.sheerka.sheerka_methods
+        self.globals = {k: v.method for k, v in self.sheerka.sheerka_methods.items()}
+        self.globals.update({k: v.method for k, v in self.sheerka.sheerka_pipeables.items()})
 
     def get_completions(self, document, complete_event):
 

tests/BaseTest.py

@@ -81,7 +81,7 @@ class BaseTest:
     @staticmethod
     def retval(obj, who="who", status=True):
         """ret_val"""
-        return ReturnValueConcept.ret(who, status, obj)
+        return ReturnValueConcept(who, status, obj)
 
     @staticmethod
     def tretval(sheerka, obj, who="who"):
@@ -93,11 +93,11 @@ class BaseTest:
         return sheerka.ret(who, True, obj)
 
     @staticmethod
-    def pretval(concept, source=None, parser="parsers.name", who="some_name"):
+    def pretval(concept, source=None, parser="parsers.name", who="some_name", status=True):
         """ParserResult ret_val (p stands for ParserResult)"""
         return ReturnValueConcept(
             who,
-            True,
+            status,
             ParserResultConcept(parser=parser,
                                 source=source or concept.name,
                                 value=concept,

tests/core/test_SheerkaEvaluateConcept.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
 from core.concept import Concept, DoNotResolve, ConceptParts, Property, InfiniteRecursionResolved, CB, NotInit
+from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
 from parsers.PythonParser import PythonNode
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -50,16 +51,16 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         :return:
         """
 
-        sheerka, context, concept = self.init_concepts(Concept("foo", pre=expr))
+        sheerka, context, concept = self.init_concepts(Concept("foo", post=expr))
 
         evaluated = sheerka.evaluate_concept(context, concept)
 
         assert evaluated.key == concept.key
         assert evaluated.metadata.body is None
-        assert evaluated.metadata.pre == expr
-        assert evaluated.metadata.post is None
+        assert evaluated.metadata.post == expr
+        assert evaluated.metadata.pre is None
         assert evaluated.metadata.where is None
-        assert evaluated.get_value(ConceptParts.PRE) == expected
+        assert evaluated.get_value(ConceptParts.POST) == expected
         assert evaluated.variables() == {}
         assert not evaluated.metadata.is_evaluated
         assert len(evaluated.values) == 0 if expr is None else 1
@@ -353,27 +354,34 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         assert evaluated.key == concept.init_key().key
 
-    @pytest.mark.parametrize("where_clause, expected", [
-        ("True", True),
-        ("False", False),
-        ("self < 10", False),
-        ("self < 11", True),
-        ("a < 20", False),
-        ("a > 19", True),
-        ("a + self > 20", True),
+    @pytest.mark.parametrize("where_clause, expected, expected_body", [
+        ("True", True, BuiltinConcepts.NOT_INITIALIZED),
+        ("False", False, BuiltinConcepts.NOT_INITIALIZED),
+        ("self < 10", False, 10),
+        ("self < 11", True, 10),
+        ("a < 20", False, BuiltinConcepts.NOT_INITIALIZED),
+        ("a > 19", True, BuiltinConcepts.NOT_INITIALIZED),
+        ("a + self > 20", True, 10),
     ])
-    def test_i_can_evaluate_simple_where(self, where_clause, expected):
+    def test_i_can_evaluate_simple_where(self, where_clause, expected, expected_body):
+        # We check that the WHERE condition is correctly evaluated
+        # We also check that the body is evaluated only when it's mandatory
+
         sheerka, context, concept = self.init_concepts(
             Concept("foo", body="10", where=where_clause).def_var("a", "20"),
         )
 
-        evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept, eval_body=False)
 
         if expected:
             assert evaluated.key == concept.key
+            assert concept.body == expected_body
         else:
-            assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED)
-            assert evaluated.body == concept
+            assert sheerka.isinstance(evaluated, BuiltinConcepts.CONDITION_FAILED)
+            assert evaluated.body == where_clause
+            assert evaluated.concept == concept
+            assert evaluated.prop == ConceptParts.WHERE
+            assert concept.body == expected_body
 
     def test_i_can_evaluate_where_when_using_other_concept(self):
         sheerka, context, foo_true, foo_false = self.init_concepts(
@@ -381,20 +389,20 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
             Concept("foo_false", body="False"),
         )
 
-        concept = Concept("foo", where="foo_true").init_key()
+        concept = Concept("foo", where="foo_true")
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept)
         assert evaluated.key == concept.key
 
         concept = Concept("foo", where="foo_false")
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept)
-        assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED)
-        assert evaluated.body == concept
+        assert sheerka.isinstance(evaluated, BuiltinConcepts.CONDITION_FAILED)
+        assert evaluated.body == "foo_false"
 
-        concept = Concept("foo", where="foo_false").init_key()
+        concept = Concept("foo", where="foo_false")
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, False), concept)
         assert evaluated.key == concept.key
 
-    def test_i_can_evaluate_disable_where_clause_evaluation(self):
+    def test_i_can_enable_disable_where_clause_evaluation(self):
         sheerka, context, concept = self.init_concepts(
             Concept("foo", body="10", where="a > 10").def_var("a", None),
         )
@@ -405,7 +413,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept)
         assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
 
-    def test_i_can_detect_infinite_recursion_with_numeric_constant(self):
+    def test_i_can_detect_and_resolve_infinite_recursion_with_numeric_constant(self):
         sheerka, context, one_str, one_digit = self.init_concepts(
             Concept("one", body="1"),
             Concept("1", body="one"),
@@ -423,7 +431,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == one_str.key
         assert evaluated.body == InfiniteRecursionResolved(1)
 
-    def test_i_can_detect_infinite_recursion_with_boolean_constant(self):
+    def test_i_can_detect_and_resolve_infinite_recursion_with_boolean_constant(self):
         sheerka, context, true_str, true_bool = self.init_concepts(
             Concept("true", body="True"),
             Concept("True", body="true"),
@@ -523,3 +531,147 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, forty_one_thousand)
 
         assert evaluated.body == 41000
+
+    def test_i_can_evaluate_command(self):
+        sheerka, context, command = self.init_concepts(Concept("command", body="a = 10"))
+
+        evaluated = sheerka.evaluate_concept(context, command)
+        assert evaluated.key == command.key
+        assert "a" not in sheerka.locals
+
+        sheerka.set_isa(context, command, sheerka.new(BuiltinConcepts.COMMAND))
+        evaluated = sheerka.evaluate_concept(context, sheerka.new("command"))
+        assert evaluated.key == command.key
+        assert "a" in sheerka.locals
+
+    @pytest.mark.parametrize("metadata", [
+        "where",
+        "pre",
+        "post",
+        "ret"
+    ])
+    def test_i_cannot_evaluate_python_statement_in_where_pre_post_ret(self, metadata, capsys):
+        sheerka, context, foo = self.init_concepts("foo")
+        setattr(foo.metadata, metadata, "a=10; print('10')")
+        foo.metadata.need_validation = True
+
+        evaluated = sheerka.evaluate_concept(context, foo)
+        captured = capsys.readouterr()
+
+        assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        error = evaluated.body
+        assert sheerka.isinstance(error, BuiltinConcepts.PYTHON_SECURITY_ERROR)
+        assert error.prop.value == metadata
+        assert error.body == "a=10; print('10')"
+        assert captured.out == ""
+
+    def test_python_builtin_function_are_forbidden_in_where_pre_post_ret(self, capsys):
+        # I do the test only for PRE, as it will be the same for the other CounceptPart
+        sheerka, context, foo, bar = self.init_concepts(
+            Concept("foo", body="print('10')"),  # print will be executed
+            Concept("bar", pre="print('10')"),  # print won't be executed
+        )
+
+        evaluated = sheerka.evaluate_concept(context, foo, eval_body=True)
+        captured = capsys.readouterr()
+        assert evaluated.key == foo.key
+        assert captured.out == "10\n"
+
+        evaluated = sheerka.evaluate_concept(context, bar)
+        captured = capsys.readouterr()
+        assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        error = evaluated.body
+        assert sheerka.isinstance(error, BuiltinConcepts.ERROR)
+        assert captured.out == ""
+
+    def test_i_can_failed_a_concept_when_pre_clause_is_not_validated(self, capsys):
+        sheerka, context, concept = self.init_concepts(
+            Concept("foo", pre="in_context('foo')", body="print('10')"),
+        )
+
+        evaluated = sheerka.evaluate_concept(context, concept, eval_body=True)
+        assert sheerka.isinstance(evaluated, BuiltinConcepts.CONDITION_FAILED)
+        assert evaluated.body == "in_context('foo')"
+        assert evaluated.concept == concept
+        assert evaluated.prop == ConceptParts.PRE
+
+        captured = capsys.readouterr()
+        assert captured.out == ""
+
+    @pytest.mark.parametrize("concept, expected", [
+        (Concept("foo"), []),
+        (Concept("foo", pre="pre", post="post", ret="ret", where="where"), ["pre", "ret", "post"]),
+        (Concept("foo", pre="a").def_var("a"), ["variables", "pre"]),
+        (Concept("foo", pre="self"), ["body", "pre"]),
+        (Concept("foo", pre="self + a").def_var("a"), ["variables", "body", "pre"]),
+        (Concept("foo", pre="self + a", ret="ret").def_var("a"), ["variables", "body", "pre", "ret"]),
+        (Concept("foo", body="body"), [])  # only if eval_body_is_set
+    ])
+    def test_i_can_compute_metadata_to_eval(self, concept, expected):
+        sheerka, context, concept = self.init_concepts(concept)
+        service = sheerka.services[SheerkaEvaluateConcept.NAME]
+
+        service.initialize_concept_asts(context, concept)
+        assert service.compute_metadata_to_eval(context, concept) == expected
+
+    def test_i_can_compute_metadata_to_eval_for_where_and_body(self):
+        sheerka = self.get_sheerka()
+        service = sheerka.services[SheerkaEvaluateConcept.NAME]
+
+        context = self.get_context(sheerka, eval_where=True)
+        concept = Concept("foo", where="where")
+        service.initialize_concept_asts(context, concept)
+        assert service.compute_metadata_to_eval(context, concept) == ["where"]
+
+        concept = Concept("foo", where="where a").def_var("a")
+        service.initialize_concept_asts(context, concept)
+        assert service.compute_metadata_to_eval(context, concept) == ["variables", "where"]
+
+        concept = Concept("foo", where="where self")
+        service.initialize_concept_asts(context, concept)
+        assert service.compute_metadata_to_eval(context, concept) == ["body", "where"]
+
+        context = self.get_context(sheerka, eval_body=True)
+        concept = Concept("foo")
+        assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"]
+
+        context = self.get_context(sheerka, eval_body=True)
+        concept = Concept("foo").def_var("a")
+        assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"]
+
+        context = self.get_context(sheerka, eval_body=True)
+        concept = Concept("foo", body="body").def_var("a")
+        assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"]
+
+    @pytest.mark.parametrize("concept, expected", [
+        (Concept("foo"), True),
+    ])
+    def test_is_evaluated_is_correctly_set(self, concept, expected):
+        sheerka, context, concept = self.init_concepts(concept)
+
+        evaluated = sheerka.evaluate_concept(context, concept, eval_body=True)
+
+        assert evaluated.key == concept.key
+        assert concept.metadata.is_evaluated == expected
+
+    def test_i_only_compute_the_requested_metadata(self):
+        sheerka, context, concept = self.init_concepts(
+            Concept("foo", pre="'pre'", post="'post'", ret="'ret'", where="'where'", body="'body'").def_var("a", "'a'")
+        )
+        context.local_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)  # to prove that we do not care
+        context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)  # to prove that we do not care
+
+        evaluated = sheerka.evaluate_concept(context, concept, metadata=['pre'])
+        assert evaluated.values == {"a": Property("a", NotInit), ConceptParts.PRE: Property(ConceptParts.PRE, 'pre')}
+
+    # I cannot implement value cache for now
+    # def test_values_when_no_variables_are_computed_only_once(self):
+    #     sheerka, context, foo = self.init_concepts(Concept("foo", body="10"))
+    #
+    #     evaluated = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True)
+    #     assert evaluated.body == 10
+    #     assert len(evaluated.compiled) > 0
+    #
+    #     evaluated_2 = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True)
+    #     assert evaluated_2.body == 10
+    #     assert len(evaluated_2.compiled) == 0

tests/core/test_SheerkaSetsManager.py

@@ -239,8 +239,6 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
             Concept("foo"),
             Concept("bar"),
             Concept("baz"),
-            create_new=True
-
         )
 
         sheerka.set_isa(context, foo, bar)
@@ -250,6 +248,18 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isa(bar, baz)
         assert sheerka.isa(foo, baz)
 
+    def test_i_cannot_manage_isa_transitivity_when_using_body(self):
+        sheerka, context, one, another_one, number = self.init_concepts(
+            "one",
+            Concept("another one", body="one"),
+            "number"
+        )
+
+        sheerka.set_isa(context, one, number)
+
+        assert sheerka.isa(one, number)  # sanity
+        assert not sheerka.isa(another_one, number)  # Correct this misbehaviour when BuiltinConcepts.IS is implemented
+
     def test_concept_expression_recurse_id_is_updated(self):
         sheerka, context, one, number, twenties = self.init_concepts(
             "one",

tests/core/test_builtin_helpers.py

@@ -3,6 +3,7 @@ import ast
 import core.builtin_helpers
 import pytest
 from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -142,3 +143,73 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
         assert len(actual) == len(expected)
         for i in range(len(actual)):
             assert self.dump_ast(actual[i]) == self.dump_ast(expected[i])
+
+    @pytest.mark.parametrize("concepts, expected", [
+        ([], []),
+        ([Concept("foo", pre="True"), Concept("bar")], ["foo"]),
+        ([Concept("foo").def_var("a"), Concept("bar")], ["bar"]),
+    ])
+    def test_i_can_resolve_ambiguity_when_empty(self, concepts, expected):
+        context = self.get_context()
+        res = core.builtin_helpers.resolve_ambiguity(context, concepts)
+        assert [c.name for c in res] == expected
+
+    # @pytest.mark.parametrize("return_values", [
+    #     None,
+    #     []
+    # ])
+    # def test_i_can_resolve_simple_ambiguity_when_no_return_values(self, return_values):
+    #     context = self.get_context()
+    #
+    #     assert core.builtin_helpers.remove_ambiguity(context, return_values) == return_values
+
+    # def test_resolve_ambiguity_concepts_with_no_variable_take_precedence(self):
+    #     context = self.get_context()
+    #     return_values = [
+    #         self.pretval(Concept("hello a").def_var("a", "world"), "hello word"),
+    #         self.pretval(Concept("hello world"), "hello word"),
+    #         # self.pretval(Concept("hello world", pre="False"), "hello word"),
+    #         self.retval(Concept("not a parser result")),
+    #         self.retval(Concept("status is false"), status=False),
+    #         self.pretval(Concept("false parser result"), status=False),
+    #     ]
+    #
+    #     ret = core.builtin_helpers.remove_ambiguity(context, return_values)
+    #     assert ret.status
+    #     assert ret.parents == return_values
+    #
+    #     filtered = ret.body
+    #     assert context.sheerka.isinstance(ret.body, BuiltinConcepts.FILTERED)
+    #     assert filtered.body == [
+    #         return_values[2],
+    #         return_values[3],
+    #         return_values[4],
+    #         return_values[1],
+    #     ]
+    #     assert filtered.iterable == return_values
+    #     assert filtered.predicate == "remove_ambiguity(context, iterable)"
+    #
+    # def test_resolve_ambiguity_failed_pre_condition_are_discarded(self):
+    #     context = self.get_context()
+    #     return_values = [
+    #         self.pretval(Concept("hello world"), "hello word"),
+    #         self.pretval(Concept("hello world", pre="False"), "hello word"),
+    #     ]
+    #
+    #     ret = core.builtin_helpers.remove_ambiguity(context, return_values)
+    #     filtered = ret.body
+    #     assert context.sheerka.isinstance(ret.body, BuiltinConcepts.FILTERED)
+    #     assert filtered.body == [
+    #         return_values[0],
+    #     ]
+    #
+    # def test_resolve_ambiguity_original_return_value_is_returned_when_nothing_to_filter(self):
+    #     context = self.get_context()
+    #     return_values = [
+    #         self.pretval(Concept("hello a").def_var("a", "world"), "hello word"),
+    #         self.retval(Concept("not a parser result")),
+    #         self.retval(Concept("status is false"), status=False),
+    #         self.pretval(Concept("false parser result"), status=False),
+    #     ]
+    #
+    #     assert core.builtin_helpers.remove_ambiguity(context, return_values) == return_values

tests/core/test_sheerka_call_evaluators.py

@@ -165,10 +165,13 @@ class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority):
         return ret
 
 
-class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority):
+class EvaluatorAllReplaceFooEntry(EvaluatorAllWithPriority):
+    """
+    This evaluator replaces the concept 'foo' by another one
+    """
 
     def __init__(self):
-        super().__init__("suppress", 100)
+        super().__init__("replace", 100)
 
     def matches(self, context, return_values):
         super().matches(context, return_values)
@@ -187,6 +190,30 @@ class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority):
             return None
 
 
+class EvaluatorAllSuppressEntries(EvaluatorAllWithPriority):
+    """
+    This evaluator totally removes 'foo' and 'bar' the entries from the workflow
+    """
+
+    def __init__(self):
+        super().__init__("suppress", 100)
+
+    def matches(self, context, return_values):
+        super().matches(context, return_values)
+        to_remove = [r for r in return_values if r.body.name in ("foo", "bar")]
+        return len(to_remove) > 0
+
+    def eval(self, context, return_values):
+        super().eval(context, return_values)
+
+        to_remove = [r for r in return_values if r.body.name in ("foo", "bar")]
+        return context.sheerka.ret(
+            self.name,
+            True,
+            [],
+            parents=to_remove)
+
+
 class EvaluatorOneDoNotModifyExecutionFlow(EvaluatorOneWithPriority):
     """
     To test that when eval() returns the initial return_value, the execution flow is not modified
@@ -485,3 +512,22 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
 
         # check that 'foo' is no longer in res, but 'bar' is added
         assert res == [self.tretval(sheerka, Concept("bar"))]
+
+    def test_i_can_remove_return_values_from_the_execution_workflow(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorAllSuppressEntries]
+
+        entries = [self.tretval(sheerka, Concept("foo")),
+                   self.tretval(sheerka, Concept("bar")),
+                   self.tretval(sheerka, Concept("baz"))]
+        Out.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            "__EVALUATION [0] suppress - matches - target=['foo', 'bar', 'baz']",
+            "__EVALUATION [0] suppress - eval - target=['foo', 'bar', 'baz']",
+            "__EVALUATION [1] suppress - matches - target=['baz']",
+        ]
+
+        # check that 'foo' is no longer in res, but 'bar' is added
+        assert res == [self.tretval(sheerka, Concept("baz"))]

tests/evaluators/test_AddConceptEvaluator.py

@@ -19,7 +19,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
     @staticmethod
     def get_concept_part(part):
         if isinstance(part, str):
-            node = PythonNode(part, ast.parse(part, mode="eval"))
+            node = PythonNode(part, ast.parse(part, mode="exec"))
             return ReturnValueConcept(
                 who="parsers.Default",
                 status=True,
@@ -169,11 +169,16 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
 
         assert from_db.compiled == {}  # ast is not saved in db
 
-    def test_i_can_get_variables_from_python_node_when_long_name(self):
-        ret_val = self.get_concept_part("isinstance(a, str)")
+    @pytest.mark.parametrize("expression, name, expected", [
+        ("isinstance(a, str)", "a b", {"a"}),
+        ("a.location=b", "a is in b", {"a", "b"}),
+        ("a.location=b", "'a' is in b", {"b"}),
+    ])
+    def test_i_can_get_variables_from_python_node_when_long_name(self, expression, name, expected):
+        ret_val = self.get_concept_part(expression)
         context = self.get_context()
 
-        assert AddConceptEvaluator.get_variables(context.sheerka, ret_val, ["a", "b"]) == ["a"]
+        assert AddConceptEvaluator.get_variables(context.sheerka, ret_val, name.split()) == expected
 
     def test_i_can_get_variables_when_keywords(self):
         sheerka, context = self.init_concepts()
@@ -182,7 +187,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         name_to_use = AddConceptEvaluator.get_name_to_use(def_concept)
         concept_part = self.get_concept_part("pre")
 
-        assert AddConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == ["pre"]
+        assert AddConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == {"pre"}
 
     def test_i_cannot_get_variables_from_python_node_when_name_has_only_one_token(self):
         ret_val = self.get_concept_part("isinstance(a, str)")
@@ -196,14 +201,14 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
                                      status=True,
                                      value=ParserResultConcept(value=concept))
 
-        assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == ["a", "b"]
+        assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"a", "b"}
 
     def test_i_can_get_variables_from_definition(self):
         parsing_expression = Sequence(ConceptExpression('mult'),
                                       ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
         ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression)
 
-        assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == ["add", "mult"]
+        assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"add", "mult"}
 
     def test_concept_that_references_itself_is_correctly_created(self):
         context = self.get_context()

tests/evaluators/test_ConceptEvaluator.py

@@ -1,8 +1,8 @@
 import pytest
-
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.concept import Concept, ConceptParts
 from evaluators.ConceptEvaluator import ConceptEvaluator
+
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -20,11 +20,12 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_evaluate_concept(self):
         context = self.get_context()
-        context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        context.local_hints.update({BuiltinConcepts.EVAL_BODY_REQUESTED, BuiltinConcepts.EVAL_WHERE_REQUESTED})
         concept = Concept(name="foo",
                           where="True",
-                          pre="2",
-                          post="3").def_var("a", "4").def_var("b", "5")
+                          pre="2 > 1",
+                          ret="3",
+                          post="4").def_var("a", "5").def_var("b", "6")
 
         evaluator = ConceptEvaluator()
         item = self.pretval(concept)
@@ -34,10 +35,11 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert result.status
         assert result.value.name == "foo"
         assert result.value.get_value(ConceptParts.WHERE) == True
-        assert result.value.get_value(ConceptParts.PRE) == 2
-        assert result.value.get_value(ConceptParts.POST) == 3
-        assert result.value.get_value("a") == 4
-        assert result.value.get_value("b") == 5
+        assert result.value.get_value(ConceptParts.PRE) == True
+        assert result.value.get_value(ConceptParts.RET) == 3
+        assert result.value.get_value(ConceptParts.POST) == 4
+        assert result.value.get_value("a") == 5
+        assert result.value.get_value("b") == 6
         assert result.value.key == "foo"
         assert result.parents == [item]
 
@@ -118,4 +120,3 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert not context.sheerka.is_success(error_concept)  # it's indeed an error
         assert result.status
         assert result.value == error_concept
-

tests/evaluators/test_EvalEvaluator.py

@@ -15,7 +15,7 @@ def retval(obj, who="who", status=True):
 class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
     def test_i_can_match_and_eval(self):
         context = self.get_context()
-        context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED)
+        context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED)
 
         to_eval1 = ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init())
         to_eval2 = ReturnValueConcept("some_name", True, Concept(name="3", body="also to eval").auto_init())
@@ -49,12 +49,12 @@ class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         assert not EvalEvaluator().matches(context, [return_value])
 
-        context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED)
+        context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED)
         assert EvalEvaluator().matches(context, [return_value])
 
     def test_i_can_match_depending_on_builtin_concept_processing(self):
         context = self.get_context()
-        context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED)
+        context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED)
         return_values = [ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init())]
         evaluator = EvalEvaluator()
 

tests/evaluators/test_PrepareEvalEvaluator.py

@@ -45,4 +45,4 @@ class TestPrepareEvalEvaluator(TestUsingMemoryBasedSheerka):
         assert res.body.body == expected
 
         assert BuiltinConcepts.EVAL_BODY_REQUESTED in context.global_hints
-        assert BuiltinConcepts.CONCEPT_VALUE_REQUESTED in context.global_hints
+        assert BuiltinConcepts.RETURN_VALUE_REQUESTED in context.global_hints

tests/evaluators/test_PythonEvaluator.py

@@ -1,6 +1,6 @@
 import pytest
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import Concept, CB
+from core.concept import Concept, CB, NotInit
 from core.sheerka.services.SheerkaExecute import ParserInput
 from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError
 from parsers.PythonParser import PythonNode, PythonParser
@@ -137,7 +137,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert not evaluated.value  # the first test is between Concept(foo) and int(2)
 
-        context.local_hints.add(BuiltinConcepts.EVAL_SUCCESS_REQUESTED)
+        context.local_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
         evaluated = python_evaluator.eval(context, parsed)
         assert evaluated.status
         assert evaluated.value  # we test until we compare foo.body and 2
@@ -224,3 +224,18 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert isinstance(error1.error, TypeError)
         assert error1.error.args[0] == 'can only concatenate str (not "int") to str'
         assert error1.concepts == {'foo': 'string'}
+
+    def test_i_do_not_include_not_initialized_variables_when_evaluating(self):
+        sheerka, context, foo = self.init_concepts(
+            Concept("foo a", pre="a == 'True'").def_var("a", "'True'").def_var("b"))
+
+        foo.set_value("b", "'Initialized!'")
+        context.obj = foo
+
+        assert foo.get_value("a") == NotInit
+        assert foo.get_value("b") == "'Initialized!'"
+
+        my_globals = {}
+        PythonEvaluator().update_globals_with_context(my_globals, context)
+
+        assert my_globals == {"self": foo, "b": "'Initialized!'"}

tests/evaluators/test_ResolveAmbiguityEvaluator.py

@@ -0,0 +1,74 @@
+import pytest
+from core.concept import Concept
+from evaluators.ResolveAmbiguityEvaluator import ResolveAmbiguityEvaluator
+
+from tests.BaseTest import BaseTest
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+pretval = BaseTest.pretval
+
+
+class TestResolveAmbiguityEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("return_values, expected", [
+        ([pretval(Concept("foo"), source="source"), pretval(Concept("bar"), source="source")], True),
+        ([pretval(Concept("foo"), source="source"), pretval(Concept("bar"), source="source", status=False)], False),
+        ([pretval(Concept("foo"), source="source1"), pretval(Concept("bar"), source="source2")], False),
+    ])
+    def test_i_can_match(self, return_values, expected):
+        context = self.get_context()
+        assert ResolveAmbiguityEvaluator().matches(context, return_values) == expected
+
+    def test_i_can_manage_when_no_source(self):
+        context = self.get_context()
+        return_values = [BaseTest.retval(Concept("foo"))]
+
+        assert not ResolveAmbiguityEvaluator().matches(context, return_values)
+
+    def test_i_can_eval(self):
+        context = self.get_context()
+        return_values = [
+            self.pretval(Concept("hello a").def_var("a", "world"), "hello word"),
+            self.pretval(Concept("hello world"), "hello word"),
+            self.pretval(Concept("hello world", pre="False"), "hello word"),
+            self.retval(Concept("not a parser result")),
+            self.retval(Concept("status is false"), status=False),
+            self.pretval(Concept("false parser result"), status=False),
+        ]
+
+        evaluator = ResolveAmbiguityEvaluator()
+        evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert len(res) == 1
+        resolved = res[0]
+
+        assert resolved.who == evaluator.name
+        assert resolved.body == return_values[1].body
+        assert resolved.parents == [
+            return_values[0],
+            return_values[1],
+            return_values[2],
+        ]
+
+    def test_i_can_eval_all_fail(self):
+        context = self.get_context()
+        return_values = [
+            self.pretval(Concept("hello world", pre="2 < 1"), "hello word"),
+            self.pretval(Concept("hello world", pre="False"), "hello word"),
+        ]
+
+        evaluator = ResolveAmbiguityEvaluator()
+        evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert len(res) == 1
+        resolved = res[0]
+
+        assert resolved.who == evaluator.name
+        assert resolved.body == []
+        assert resolved.parents == [
+            return_values[0],
+            return_values[1],
+        ]

tests/non_reg/test_sheerka_non_reg.py

@@ -1,7 +1,8 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit, CC
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
+from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
 from evaluators.PythonEvaluator import PythonEvalError
 from parsers.BaseNodeParser import SyaAssociativity
 from parsers.BnfNodeParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression
@@ -216,6 +217,7 @@ as:
         assert evaluated.get_value("a").metadata.is_evaluated
 
     def test_i_can_recognize_duplicate_concepts_with_same_value(self):
+        # when multiple result, choose the one that is the more specific (that has the less variables)
         sheerka = self.get_sheerka()
         self.create_and_add_in_cache_concept(sheerka, Concept(name="hello a", body="'hello ' + a"), variables=["a"])
         self.create_and_add_in_cache_concept(sheerka, Concept(name="hello foo", body="'hello foo'"))
@@ -224,13 +226,14 @@ as:
         res = sheerka.evaluate_user_input("hello foo")
         assert len(res) == 1
         assert res[0].status
-        assert res[0].value.body == "hello foo"
-        assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
+        assert sheerka.isinstance(res[0].body, "hello foo")
+        assert res[0].value.body == BuiltinConcepts.NOT_INITIALIZED
+        assert res[0].who == sheerka.get_evaluator_name(OneSuccessEvaluator.NAME)
 
     def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different(self):
         sheerka = self.get_sheerka()
-        self.create_and_add_in_cache_concept(sheerka, Concept(name="hello a", body="'hello ' + a"), variables=["a"])
-        self.create_and_add_in_cache_concept(sheerka, Concept(name="hello foo", body="'hello foo'"))
+        self.create_and_add_in_cache_concept(sheerka, Concept(name="hello a", body="'hello ' + a").def_var("a"))
+        self.create_and_add_in_cache_concept(sheerka, Concept(name="hello b", body="'hello you ' + b").def_var("b"))
         self.create_and_add_in_cache_concept(sheerka, Concept(name="foo", body="'another value'"))
 
         res = sheerka.evaluate_user_input("hello foo")
@@ -242,7 +245,7 @@ as:
         assert len(concepts) == 2
         sorted_values = sorted(concepts, key=lambda x: x.value.body)
         assert sorted_values[0].value.body == "hello another value"
-        assert sorted_values[1].value.body == "hello foo"
+        assert sorted_values[1].value.body == "hello you another value"
 
     def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(self):
         sheerka = self.get_sheerka()
@@ -692,12 +695,12 @@ as:
         assert len(res) == 1
         assert not res[0].status
         assert sheerka.isinstance(res[0].body, BuiltinConcepts.MULTIPLE_ERRORS)
-        assert str(BuiltinConcepts.WHERE_CLAUSE_FAILED) in [error.key for error in sheerka.get_error(res[0].body.body)]
+        assert str(BuiltinConcepts.CONDITION_FAILED) in [error.key for error in sheerka.get_error(res[0].body.body)]
 
         res = sheerka.evaluate_user_input("eval twenty three")
         assert len(res) == 1
         assert not res[0].status
-        assert str(BuiltinConcepts.WHERE_CLAUSE_FAILED) in [error.key for error in sheerka.get_error(res[0].body.body)]
+        assert str(BuiltinConcepts.CONDITION_FAILED) in [error.key for error in sheerka.get_error(res[0].body.body)]
 
     def test_i_can_manage_some_type_of_infinite_recursion(self):
         sheerka = self.get_sheerka()
@@ -735,12 +738,12 @@ as:
         res = sheerka.evaluate_user_input("foo baz")
         assert len(res) == 1
         assert not res[0].status
-        assert sheerka.isinstance(res[0].body, BuiltinConcepts.WHERE_CLAUSE_FAILED)
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
 
         res = sheerka.evaluate_user_input("eval foo baz")
         assert len(res) == 1
         assert not res[0].status
-        assert sheerka.isinstance(res[0].body, BuiltinConcepts.WHERE_CLAUSE_FAILED)
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
 
         res = sheerka.evaluate_user_input("foobar")
         assert len(res) == 1
@@ -749,7 +752,7 @@ as:
         res = sheerka.evaluate_user_input("eval foobar")
         assert len(res) == 1  # error
         assert not res[0].status
-        assert sheerka.isinstance(res[0].body, BuiltinConcepts.WHERE_CLAUSE_FAILED)
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
 
     @pytest.mark.skip("Not ready for that")
     def test_i_can_manage_missing_variables_from_bnf_parsing(self):
@@ -998,6 +1001,20 @@ as:
         assert res[0].status
         assert res[0].body == 2
 
+    def test_i_can_evaluate_command(self):
+        init = [
+            "def concept command as 'Executed !'",
+            "set_isa(c:command:, __COMMAND)",
+        ]
+
+        # Since command is a __COMMAND, the body is auto evaluated
+        # and we return the body, not the concept
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("command")
+        assert res[0].status
+        assert res[0].body == "Executed !"
+
 
 class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
     def test_i_can_def_several_concepts(self):

tests/parsers/test_ExactConceptParser.py

@@ -1,7 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, CMV
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Tokenizer
 from parsers.ExactConceptParser import ExactConceptParser
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka

tests/sheerkapickle/test_sheerka_handlers.py

@@ -16,6 +16,14 @@ def set_full_serialization(concept):
 
 
 class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
+    user_input_id = 0
+    return_value_id = 0
+
+    @classmethod
+    def setup(cls):
+        sheerka = cls().get_sheerka()
+        cls.user_input_id = sheerka.get_by_key("__USER_INPUT").id
+        cls.return_value_id = sheerka.get_by_key("__RETURN_VALUE").id
 
     def test_i_can_encode_decode_unknown_concept_metadata(self):
         sheerka = self.get_sheerka()
@@ -201,7 +209,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, user_input)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == user_input
-        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "23"], "user_name": "my_user_name", "text": "my_text"}'
+        assert to_string == f'{{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "{self.user_input_id}"], "user_name": "my_user_name", "text": "my_text"}}'
 
     def test_i_can_encode_decode_user_input_when_tokens(self):
         sheerka = self.get_sheerka()
@@ -213,7 +221,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, user_input)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="my_user_name")
-        assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "23"], "user_name": "my_user_name", "text": "{text}"' + '}'
+        assert to_string == f'{{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "{self.user_input_id}"], "user_name": "my_user_name", "text": "{text}"}}'
 
     def test_i_can_encode_decode_return_value(self):
         sheerka = self.get_sheerka()
@@ -223,7 +231,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, ret_val)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == ret_val
-        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "28"], "who": "who", "status": true, "value": 10}'
+        assert to_string == f'{{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "{self.return_value_id}"], "who": "who", "status": true, "value": 10}}'
 
     def test_i_can_encode_decode_return_value_with_parent(self):
         sheerka = self.get_sheerka()
@@ -236,7 +244,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == ret_val
         assert decoded.parents == ret_val.parents
-        id_str = ', "concept/id": ["__RETURN_VALUE", "28"]'
+        id_str = f', "concept/id": ["__RETURN_VALUE", "{self.return_value_id}"]'
         parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]'
         assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}'