You must now use 'eval' to get the body of a concept

.editorconfig

@@ -9,6 +9,9 @@ indent_size=4
 indent_style=space
 indent_size=2
 
+[*.py]
+insert_final_newline=true
+
 # Tab indentation (no size specified)
 [Makefile]
 indent_style = tab

core/builtin_concepts.py

@@ -44,8 +44,10 @@ class BuiltinConcepts(Enum):
     CONCEPT_EVAL_ERROR = "concept evaluation error"  # cannot evaluate a property or metadata of a concept
     ENUMERATION = "enum"  # represents a list or a set
     LIST = "list"  # represents a list
-    CANNOT_RESOLVE_VALUE_ERROR = "value cannot be resolved"  # don't know how to find concept value
     CONCEPT_ALREADY_IN_SET = "concept already in set"
+    EVALUATOR_PRE_PROCESS = "evaluator pre process"  # used modify / tweak behaviour of evaluators
+    CONCEPT_EVAL_REQUESTED = "concept eval requested"
+    REDUCE_REQUESTED = "reduce requested"  # remove meaningless error when possible
 
     NODE = "node"
     GENERIC_NODE = "generic node"
@@ -68,7 +70,6 @@ BuiltinErrors = [str(e) for e in {
     BuiltinConcepts.INVALID_RETURN_VALUE,
     BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
     BuiltinConcepts.CONCEPT_EVAL_ERROR,
-    BuiltinConcepts.CANNOT_RESOLVE_VALUE_ERROR,
     BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
 }]
 
@@ -267,6 +268,15 @@ class AfterEvaluationConcept(Concept):
         super().__init__(BuiltinConcepts.AFTER_EVALUATION, True, True, BuiltinConcepts.AFTER_EVALUATION)
 
 
+class ConceptEvalRequested(Concept):
+    def __init__(self):
+        super().__init__(BuiltinConcepts.CONCEPT_EVAL_REQUESTED, True, True, BuiltinConcepts.CONCEPT_EVAL_REQUESTED)
+
+
+class ReduceRequested(Concept):
+    def __init__(self):
+        super().__init__(BuiltinConcepts.REDUCE_REQUESTED, True, True, BuiltinConcepts.REDUCE_REQUESTED)
+
 class ConceptEvalError(Concept):
     def __init__(self, error=None, concept=None, property_name=None):
         super().__init__(BuiltinConcepts.CONCEPT_EVAL_ERROR,

core/concept.py

@@ -12,6 +12,7 @@ PROPERTIES_FOR_DIGEST = ("name", "key",
                          "where", "pre", "post", "body",
                          "desc")
 PROPERTIES_TO_SERIALIZE = PROPERTIES_FOR_DIGEST + tuple(["id"])
+PROPERTIES_FOR_NEW = ("where", "pre", "post", "body", "desc")
 VARIABLE_PREFIX = "__var__"
 
 

core/sheerka.py

@@ -1,13 +1,13 @@
 from dataclasses import dataclass, field
 
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors
-from core.concept import Concept, ConceptParts, PROPERTIES_FOR_DIGEST
+from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
 from parsers.BaseParser import BaseParser
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderDuplicateKeyError
 import core.utils
 import core.builtin_helpers
 
-from core.sheerka_logger import console_handler, get_logger
+from core.sheerka_logger import console_handler
 
 import logging
 
@@ -85,8 +85,9 @@ class Sheerka(Concept):
             if self.sdp.first_time:
                 self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
 
-            evt_digest = self.sdp.save_event(Event("Initializing Sheerka."))
+            event = Event("Initializing Sheerka.")
-            exec_context = ExecutionContext(self.key, evt_digest, self)
+            self.sdp.save_event(event)
+            exec_context = ExecutionContext(self.key, event, self)
 
             self.initialize_builtin_concepts()
             self.initialize_builtin_parsers()
@@ -181,19 +182,24 @@ class Sheerka(Concept):
         :return:
         """
         self.log.debug(f"Processing user input '{text}', {user_name=}.")
-        evt_digest = self.sdp.save_event(Event(text, user_name))
+        event = Event(text, user_name)
+        evt_digest = self.sdp.save_event(event)
         self.log.debug(f"{evt_digest=}")
-        execution_context = ExecutionContext(self.key, evt_digest, self)
+        execution_context = ExecutionContext(self.key, event, self)
 
         user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
+        reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
+
         steps = [
             BuiltinConcepts.BEFORE_PARSING,
             BuiltinConcepts.PARSING,
+            BuiltinConcepts.AFTER_PARSING,
+            BuiltinConcepts.BEFORE_EVALUATION,
             BuiltinConcepts.EVALUATION,
             BuiltinConcepts.AFTER_EVALUATION
         ]
 
-        return self.execute(execution_context, user_input, steps)
+        return self.execute(execution_context, [user_input, reduce_requested], steps)
 
     def _call_parsers(self, execution_context, return_values, logger=None):
 
@@ -205,6 +211,7 @@ class Sheerka(Concept):
         for return_value in return_values:
             # make sure we only parse user input
             if not return_value.status or not self.isinstance(return_value.body, BuiltinConcepts.USER_INPUT):
+                result.append(return_value)
                 continue
 
             to_parse = self.value(return_value)
@@ -232,6 +239,26 @@ class Sheerka(Concept):
 
     def _call_evaluators(self, execution_context, return_values, process_step, evaluation_context=None, logger=None):
 
+        def _preprocess_evaluators(context, evaluators):
+            if not context.preprocess:
+                return evaluators
+
+            if not hasattr(evaluators, "__iter__"):
+                single_one = True
+                evaluators = [evaluators]
+            else:
+                single_one = False
+
+            for preprocess in context.preprocess:
+                for e in evaluators:
+                    if preprocess.props["name"].value == e.name:
+                        for prop, value in preprocess.props.items():
+                            if prop == "name":
+                                continue
+                            if hasattr(e, prop):
+                                setattr(e, prop, value.value)
+            return evaluators[0] if single_one else evaluators
+
         # return_values must be a list
         if not isinstance(return_values, list):
             return_values = [return_values]
@@ -255,6 +282,10 @@ class Sheerka(Concept):
         # The first one to be applied will be the one with the highest priority
         grouped_evaluators = {}
         instantiated_evaluators = [e_class() for e_class in self.evaluators]
+
+        # pre-process evaluators if needed
+        instantiated_evaluators = _preprocess_evaluators(execution_context, instantiated_evaluators)
+
         for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]:
             if logger:
                 evaluator.log = logger
@@ -273,6 +304,7 @@ class Sheerka(Concept):
                 evaluated_items = []
                 to_delete = []
                 for evaluator in grouped_evaluators[priority]:
+                    evaluator = _preprocess_evaluators(execution_context, evaluator.__class__())  # fresh copy
 
                     # process evaluators that work on return value
                     from evaluators.BaseEvaluator import OneReturnValueEvaluator
@@ -334,12 +366,16 @@ class Sheerka(Concept):
         for step in execution_steps:
             sub_context = execution_context.push(step=step)
             sub_context.log(logger or self.log, f"{step=}, context='{sub_context}'")
+
+            copy = return_values[:] if hasattr(return_values, "__iter__") else return_values
+
             if step == BuiltinConcepts.PARSING:
                 return_values = self._call_parsers(sub_context, return_values, logger)
             else:
                 return_values = self._call_evaluators(sub_context, return_values, step, None, logger)
 
-            sub_context.log_result(logger or self.log, return_values)
+            if copy != return_values:
+                sub_context.log_result(logger or self.log, return_values)
 
         return return_values
 
@@ -374,7 +410,11 @@ class Sheerka(Concept):
         # TODO checks if it exists in cache first
         if self.sdp.exists(self.CONCEPTS_ENTRY, concept.key, concept.get_digest()):
             error = SheerkaDataProviderDuplicateKeyError(self.CONCEPTS_ENTRY + "." + concept.key, concept)
-            return self.ret(self.create_new_concept.__name__, False, ErrorConcept(error), error.args[0])
+            return self.ret(
+                self.create_new_concept.__name__,
+                False,
+                self.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
+                error.args[0])
 
         # set id before saving in db
         self.set_id_if_needed(concept, False)
@@ -394,12 +434,18 @@ class Sheerka(Concept):
 
         # save the new context in sdp
         try:
-            self.sdp.add(context.event_digest, self.CONCEPTS_ENTRY, concept, use_ref=True)
+            self.sdp.add(context.event.get_digest(), self.CONCEPTS_ENTRY, concept, use_ref=True)
             if concepts_definitions is not None:
-                self.sdp.set(context.event_digest, self.CONCEPTS_DEFINITIONS_ENTRY, concepts_definitions, use_ref=True)
+                self.sdp.set(context.event.get_digest(),
+                             self.CONCEPTS_DEFINITIONS_ENTRY,
+                             concepts_definitions, use_ref=True)
         except SheerkaDataProviderDuplicateKeyError as error:
             context.log_error(logger, "Failed to create a new concept.", who=self.create_new_concept.__name__)
-            return self.ret(self.create_new_concept.__name__, False, ErrorConcept(error), error.args[0])
+            return self.ret(
+                self.create_new_concept.__name__,
+                False,
+                self.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
+                error.args[0])
 
         # Updates the caches
         self.concepts_cache[concept.key] = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.key)
@@ -427,8 +473,8 @@ class Sheerka(Concept):
         assert concept_set.id
 
         try:
-            ret = self.sdp.add_unique(context.event_digest, "All_" + str(concept_set.id), concept.id)
+            ret = self.sdp.add_unique(context.event.get_digest(), "All_" + str(concept_set.id), concept.id)
-            if ret == (None, None): # concept already in set
+            if ret == (None, None):  # concept already in set
                 return self.ret(
                     self.add_concept_to_set.__name__,
                     False,
@@ -506,12 +552,13 @@ class Sheerka(Concept):
 
         # to make sure of the order, it don't use ConceptParts.get_parts()
         # props must be evaluated first
-        properties_to_eval = ["props", "where", "pre", "post", "body"]
+        all_metadata_to_eval = ["props", "where", "pre", "post", "body"]
 
-        for prop_to_eval in properties_to_eval:
+        for metadata_to_eval in all_metadata_to_eval:
-            if prop_to_eval == "props":
+            if metadata_to_eval == "props":
                 for prop_name in (p for p in concept.props if p in concept.cached_asts):
                     sub_context = context.push(desc=f"Evaluating property '{prop_name}'")
+                    sub_context.add_preprocess(self.get_evaluator_name("Concept"), return_body=True)
                     res = _resolve(sub_context, concept.cached_asts[prop_name])
                     if res.status:
                         concept.set_prop(prop_name, res.value)
@@ -521,12 +568,14 @@ class Sheerka(Concept):
                                         concept=concept,
                                         property_name=prop_name)
             else:
-                part_key = ConceptParts(prop_to_eval)
+                part_key = ConceptParts(metadata_to_eval)
+
                 if part_key in concept.cached_asts and concept.cached_asts[part_key] is not None:
                     sub_context = context.push(desc=f"Evaluating '{part_key}'", obj=concept)
+                    sub_context.add_preprocess(self.get_evaluator_name("Concept"), return_body=True)
                     res = _resolve(sub_context, concept.cached_asts[part_key])
                     if res.status:
-                        setattr(concept.metadata, prop_to_eval, res.value)
+                        setattr(concept.metadata, metadata_to_eval, res.value)
                     else:
                         return self.new(BuiltinConcepts.CONCEPT_EVAL_ERROR,
                                         body=res.value,
@@ -611,7 +660,7 @@ class Sheerka(Concept):
             for k, v in kwargs_.items():
                 if k in concept.props:
                     concept.set_prop(k, v)
-                elif k in PROPERTIES_FOR_DIGEST:
+                elif k in PROPERTIES_FOR_NEW:
                     setattr(concept.metadata, k, v)
                 elif hasattr(concept, k):
                     setattr(concept, k, v)
@@ -651,28 +700,25 @@ class Sheerka(Concept):
             message=message,
             parents=parents)
 
-    def value(self, obj, allow_none_body=False):
+    def value(self, obj, reduce_simple_list=False):
         if obj is None:
             return None
 
-        if self.isinstance(obj, BuiltinConcepts.RETURN_VALUE) and \
-            obj.status and \
-            self.isinstance(obj.value, BuiltinConcepts.USER_INPUT):
-            return obj.value.body
-
-        if not isinstance(obj, Concept):
-            return obj
-
         if hasattr(obj, "get_value"):
             return obj.get_value()
 
-        if obj.body is not None:
+        if not isinstance(obj, Concept):
-            if (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1:
+            return obj
-                return obj.body[0]
-            else:
-                return obj.body
 
-        return obj if allow_none_body else self.new(BuiltinConcepts.CANNOT_RESOLVE_VALUE_ERROR, body=obj)
+        if obj.body is None:
+            return obj
+
+        if reduce_simple_list and (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1:
+            body_to_use = obj.body[0]
+        else:
+            body_to_use = obj.body
+
+        return self.value(body_to_use)
 
     def values(self, objs):
         if not (isinstance(objs, list) or
@@ -786,6 +832,17 @@ class Sheerka(Concept):
         defs = self.sdp.get(self.CONCEPTS_DEFINITIONS_ENTRY)
         self.log.info(defs)
 
+    def dump_desc(self, concept_name):
+        c = self.get(concept_name)
+        if self.isinstance(c, BuiltinConcepts.UNKNOWN_CONCEPT):
+            self.log.error("Concept unknown")
+            return False
+
+        self.log.info(f"name : {c.name}")
+        self.log.info(f"bnf  : {c.metadata.definition}")
+        self.log.info(f"key  : {c.key}")
+        self.log.info(f"body : {c.body}")
+
     @staticmethod
     def get_builtins_classes_as_dict():
         res = {}
@@ -817,7 +874,7 @@ class ExecutionContext:
 
     def __init__(self,
                  who,
-                 event_digest: str,
+                 event: Event,
                  sheerka: Sheerka,
                  /,
                  desc: str = None,
@@ -827,20 +884,32 @@ class ExecutionContext:
                  concepts: dict = None):
 
         self.who = who  # who is asking
-        self.event_digest = event_digest  # what was the (original) trigger
+        self.event = event  # what was the (original) trigger
         self.sheerka = sheerka  # sheerka
 
         self.step = step
         self.iteration = iteration
+        self.preprocess = None
 
         self.desc = desc  # human description of what is going on
         self.obj = obj  # what is the subject of the execution context (if known)
 
-        self.concepts = concepts or {}
+        self.concepts = concepts or {}  # cache for concepts that are specific to this execution
 
-        self._id = ExecutionContextIdManager.get_id(event_digest)
+        self._id = ExecutionContextIdManager.get_id(event.get_digest())
         self._tab = ""
 
+    def add_preprocess(self, name, **kwargs):
+        preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
+        preprocess.set_prop("name", name)
+        for k, v in kwargs.items():
+            preprocess.set_prop(k, v)
+
+        if not self.preprocess:
+            self.preprocess = set()
+        self.preprocess.add(preprocess)
+        return self
+
     @property
     def id(self):
         return self._id
@@ -854,7 +923,7 @@ class ExecutionContext:
         iteration = kwargs.get("iteration", self.iteration)
         new = ExecutionContext(
             who,
-            self.event_digest,
+            self.event,
             self.sheerka,
             desc=desc,
             obj=obj,
@@ -863,6 +932,7 @@ class ExecutionContext:
             iteration=iteration,
         )
         new._tab = self._tab + " " * DEBUG_TAB_SIZE
+        new.preprocess = self.preprocess
         return new
 
     def log_new(self, logger):

evaluators/BaseEvaluator.py

@@ -19,6 +19,9 @@ class BaseEvaluator:
         self.priority = priority
         self.enabled = enabled
 
+    def __repr__(self):
+        return f"{self.name} ({self.priority})"
+
 
 class OneReturnValueEvaluator(BaseEvaluator):
     """
@@ -37,8 +40,13 @@ class AllReturnValuesEvaluator(BaseEvaluator):
     Evaluates the groups of ReturnValues
     """
 
+    def __init__(self, name, steps, priority: int, enabled=True):
+        super().__init__(name, steps, priority, enabled)
+        self.eaten = []
+
     def matches(self, context: ExecutionContext, return_values):
         pass
 
     def eval(self, context: ExecutionContext, return_values):
         pass
+

evaluators/ConceptEvaluator.py

@@ -17,8 +17,9 @@ class ConceptEvaluator(OneReturnValueEvaluator):
         BuiltinConcepts.AFTER_EVALUATION
     ]
 
-    def __init__(self):
+    def __init__(self, return_body=False):
         super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
+        self.return_body = return_body
 
     def matches(self, context, return_value):
         return return_value.status and \
@@ -35,7 +36,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
         # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
         # The body should be 'property_a', and not a concept called a in our universe
         if context.obj and concept.name in context.obj.props:
-            return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME), parents=[return_value])
+            return sheerka.ret(self.name, True, context.obj.props[concept.name].value, parents=[return_value])
 
         evaluated = sheerka.evaluate_concept(context, concept, self.verbose_log)
 
@@ -48,7 +49,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
                 evaluated,
                 parents=[return_value])
 
-        if ConceptParts.BODY not in evaluated.cached_asts:
+        if not self.return_body or ConceptParts.BODY not in evaluated.cached_asts:
             return sheerka.ret(self.name, True, evaluated, parents=[return_value])
         else:
             return sheerka.ret(self.name, True, evaluated.body, parents=[return_value])

evaluators/DuplicateConceptEvaluator.py

@@ -1,45 +0,0 @@
-from core.builtin_concepts import BuiltinConcepts
-from evaluators.AddConceptEvaluator import AddConceptEvaluator
-from evaluators.BaseEvaluator import AllReturnValuesEvaluator
-from parsers.BaseParser import BaseParser
-from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
-
-
-class DuplicateConceptEvaluator(AllReturnValuesEvaluator):
-    """
-    Use to recognize when we tried to add the same concept twice
-    """
-
-    NAME = "DuplicateConcept"
-
-    def __init__(self):
-        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 10)
-        self.already_defined = None
-
-    def matches(self, context, return_values):
-        sheerka = context.sheerka
-        parsing = False
-        add_concept_in_error = False
-        only_parsers = True
-
-        for ret in return_values:
-            if sheerka.isinstance(ret.value, BuiltinConcepts.AFTER_EVALUATION):
-                if ret.status:
-                    parsing = True
-            elif ret.who == sheerka.get_evaluator_name(AddConceptEvaluator.NAME):
-                if not ret.status and isinstance(ret.value.body, SheerkaDataProviderDuplicateKeyError):
-                    add_concept_in_error = True
-                    self.already_defined = ret.value.body.obj
-            else:
-                if not ret.who.startswith(BaseParser.PREFIX):
-                    only_parsers = False
-
-        return parsing and add_concept_in_error and only_parsers
-
-    def eval(self, context, return_values):
-        sheerka = context.sheerka
-        return sheerka.ret(
-            self.name,
-            False,
-            sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=self.already_defined),
-            parents=return_values)

evaluators/EvalEvaluator.py

@@ -0,0 +1,38 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from evaluators.BaseEvaluator import AllReturnValuesEvaluator
+
+
+class EvalEvaluator(AllReturnValuesEvaluator):
+    """
+    Returns the body of all successful concepts
+    """
+
+    NAME = "Eval"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 80)
+        self.successful_return_value = None
+        self.to_eval = []
+        self.eval_requested = None
+
+    def matches(self, context, return_values):
+        sheerka = context.sheerka
+        for ret in return_values:
+            if ret.status and sheerka.isinstance(ret.body, BuiltinConcepts.CONCEPT_EVAL_REQUESTED):
+                self.eval_requested = ret
+            elif ret.status and isinstance(ret.body, Concept) and ret.body.body:
+                self.to_eval.append(ret)
+
+        return self.eval_requested is not None and len(self.to_eval) > 0
+
+    def eval(self, context, return_value):
+        sheerka = context.sheerka
+        result = []
+        context.log(self.verbose_log, f"{len(self.to_eval)} return value(s) to eval", who=self)
+
+        for ret_val in self.to_eval:
+            context.log(self.verbose_log, f"{ret_val}", who=self)
+            result.append(sheerka.ret(self.name, True, ret_val.body.body, parents=[ret_val, self.eval_requested]))
+
+        return result

evaluators/MutipleSameSuccessEvaluator.py

@@ -1,6 +1,9 @@
 from core.builtin_concepts import BuiltinConcepts
 import core.builtin_helpers
+from core.concept import Concept
 from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator
+from evaluators.ConceptEvaluator import ConceptEvaluator
+from evaluators.PythonEvaluator import PythonEvaluator
 from parsers.BaseParser import BaseParser
 
 
@@ -15,38 +18,66 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator):
     NAME = "MultipleSameSuccess"
 
     def __init__(self):
-        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 10)
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 50)
         self.success = []
 
     def matches(self, context, return_values):
-        sheerka = context.sheerka
-        after_evaluation = False
         nb_successful_evaluators = 0
         only_parsers_in_error = True
-        unlisted = False
+        to_process = False
 
         for ret in return_values:
 
-            if sheerka.isinstance(ret.value, BuiltinConcepts.AFTER_EVALUATION):
+            if ret.status and context.sheerka.isinstance(ret.body, BuiltinConcepts.REDUCE_REQUESTED):
-                if ret.status:
+                to_process = True
-                    after_evaluation = True
+                self.eaten.append(ret)
-
             elif ret.who.startswith(BaseEvaluator.PREFIX):
                 if ret.status:
                     nb_successful_evaluators += 1
                     self.success.append(ret)
+                    self.eaten.append(ret)
             elif ret.who.startswith(BaseParser.PREFIX):
+                self.eaten.append(ret)
                 if ret.status:
                     only_parsers_in_error = False
-            else:
-                unlisted = True
 
-        return after_evaluation and nb_successful_evaluators > 1 and only_parsers_in_error and not unlisted
+        return to_process and nb_successful_evaluators > 1 and only_parsers_in_error
 
     def eval(self, context, return_values):
         sheerka = context.sheerka
-        if core.builtin_helpers.is_same_success(sheerka, self.success):
+        context.log(self.verbose_log, f"{len(self.success)} successful return value(s)", who=self)
-            reference = sheerka.value(self.success[0].value, allow_none_body=True)
+        for s in self.success:
-            return sheerka.ret(self.name, True, reference, parents=return_values)
+            context.log(self.verbose_log, f"{s}", who=self)
+
+        if not core.builtin_helpers.is_same_success(sheerka, self.success):
+            return None
+
+        # ######################################
+        #   !!!!!  W A R N I N G  !!!!!!!!
+        # I have a massive issue with how I implement this feature
+        # I have forced an arbitrary order between Concept evaluator and Python evaluator
+        # I gave a random order to the other
+        #
+        # I guess that we need a proper algorithm to elect which return value to use if they have the same result
+        # I guts feeling is that, it will depend on the intent of the user
+        # So it depends on the context
+
+        # try to return a concept if possible
+        # give the priority to the ConceptEvaluator
+        for s in self.success:
+            if isinstance(s.value, Concept) and s.who == ConceptEvaluator().name:
+                return sheerka.ret(self.name, True, s.value, parents=self.eaten)
+
+        # Then the PythonEvaluator
+        for s in self.success:
+            if isinstance(s.value, Concept) and s.who == PythonEvaluator().name:
+                return sheerka.ret(self.name, True, s.value, parents=self.eaten)
+
+        # Then the first concept.
+        # It's not predictable, so I guess that it's not a good implementation choice
+        for s in self.success:
+            if isinstance(s.value, Concept):
+                return sheerka.ret(self.name, True, s.value, parents=self.eaten)
+
+        return sheerka.ret(self.name, True, self.success[0].value, parents=self.eaten)
 
-        return None

evaluators/OneErrorEvaluator.py

@@ -0,0 +1,42 @@
+from core.builtin_concepts import BuiltinConcepts
+from evaluators.BaseEvaluator import AllReturnValuesEvaluator
+from parsers.BaseParser import BaseParser
+
+
+class OneErrorEvaluator(AllReturnValuesEvaluator):
+    """
+    Use to reduce when there is only one evaluator in error
+    The rest of the return values must be parsers in error
+    """
+
+    NAME = "OneError"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 40)
+        self.return_value_in_error = None
+
+    def matches(self, context, return_values):
+        nb_evaluators_in_error = 0
+        to_process = False
+
+        for ret in return_values:
+            if ret.status and (ret.who.startswith(self.PREFIX) or ret.who.startswith(BaseParser.PREFIX)):
+                return False
+            elif ret.status and context.sheerka.isinstance(ret.body, BuiltinConcepts.REDUCE_REQUESTED):
+                to_process = True
+                self.eaten.append(ret)
+            elif not ret.status and ret.who.startswith(self.PREFIX):
+                nb_evaluators_in_error += 1
+                self.return_value_in_error = ret
+                self.eaten.append(ret)
+            elif not ret.status and ret.who.startswith(BaseParser.PREFIX):
+                self.eaten.append(ret)
+
+        return to_process and nb_evaluators_in_error == 1
+
+    def eval(self, context, return_values):
+        context.log(self.verbose_log, f"1 return value in error, {len(self.eaten)} item(s) eaten", who=self)
+        context.log(self.verbose_log, f"{self.return_value_in_error}", who=self)
+
+        sheerka = context.sheerka
+        return sheerka.ret(self.name, False, self.return_value_in_error.value, parents=self.eaten)

evaluators/OneSuccessEvaluator.py

@@ -14,28 +14,31 @@ class OneSuccessEvaluator(AllReturnValuesEvaluator):
     NAME = "OneSuccess"
 
     def __init__(self):
-        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 10)
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 60)  # before MultipleSameSuccess
         self.successful_return_value = None
 
     def matches(self, context, return_values):
-        sheerka = context.sheerka
-        after_evaluation = False
         nb_successful_evaluators = 0
-        only_parsers = True
+        to_process = False
-        for ret in return_values:
-            if sheerka.isinstance(ret.value, BuiltinConcepts.AFTER_EVALUATION):
-                if ret.status:
-                    after_evaluation = True
-            elif ret.who.startswith(self.PREFIX):
-                if ret.status:
-                    nb_successful_evaluators += 1
-                    self.successful_return_value = ret
-            else:
-                if not ret.who.startswith(BaseParser.PREFIX):
-                    only_parsers = False
 
-        return after_evaluation and nb_successful_evaluators == 1 and only_parsers
+        for ret in return_values:
+            if ret.status and ret.who.startswith(BaseParser.PREFIX):
+                return False
+            elif ret.status and context.sheerka.isinstance(ret.body, BuiltinConcepts.REDUCE_REQUESTED):
+                to_process = True
+                self.eaten.append(ret)
+            elif ret.status and ret.who.startswith(self.PREFIX):
+                nb_successful_evaluators += 1
+                self.successful_return_value = ret
+                self.eaten.append(ret)
+            elif not ret.status:
+                self.eaten.append(ret)
+
+        return to_process and nb_successful_evaluators == 1
 
     def eval(self, context, return_values):
+        context.log(self.verbose_log, f"1 successful return value, {len(self.eaten)} item(s) eaten", who=self)
+        context.log(self.verbose_log, f"{self.successful_return_value}", who=self)
+
         sheerka = context.sheerka
-        return sheerka.ret(self.name, True, self.successful_return_value.value, parents=return_values)
+        return sheerka.ret(self.name, True, self.successful_return_value.value, parents=self.eaten)

evaluators/PrepareEvalEvaluator.py

@@ -0,0 +1,40 @@
+from core.builtin_concepts import BuiltinConcepts
+from evaluators.BaseEvaluator import OneReturnValueEvaluator
+
+
+class PrepareEvalEvaluator(OneReturnValueEvaluator):
+    """
+    To parse evaluation requests
+    """
+
+    NAME = "PrepareEval"
+
+    def __init__(self, **kwargs):
+        super().__init__(self.NAME, [BuiltinConcepts.BEFORE_PARSING], 90)
+        self.text = None
+
+    def matches(self, context, return_value):
+        if not (return_value.status and
+                context.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) and
+                isinstance(return_value.body.body, str)):
+            return False
+
+        text = return_value.body.body.strip()
+        if not text.startswith("eval "):
+            return False
+
+        self.text = text
+        return True
+
+    def eval(self, context, return_value):
+        sheerka = context.sheerka
+
+        new_text_to_parse = sheerka.ret(
+            self.name,
+            True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.text[5:], user_name=context.event.user))
+
+        evaluation_requested = sheerka.ret(
+            self.name,
+            True, sheerka.new(BuiltinConcepts.CONCEPT_EVAL_REQUESTED))
+
+        return [new_text_to_parse, evaluation_requested]

evaluators/PythonEvaluator.py

@@ -2,6 +2,7 @@ import copy
 
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
+from core.concept import ConceptParts
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.PythonParser import PythonNode
 import ast
@@ -29,6 +30,14 @@ class PythonEvaluator(OneReturnValueEvaluator):
         try:
             context.log(self.verbose_log, f"Evaluating python node {node}.", self.name)
 
+            # Do not evaluate if the ast refers to a concept (leave it to ConceptEvaluator)
+            if isinstance(node.ast_, ast.Expression) and isinstance(node.ast_.body, ast.Name):
+                c = context.sheerka.get(node.ast_.body.id)
+                if not context.sheerka.isinstance(c, BuiltinConcepts.UNKNOWN_CONCEPT):
+                    context.log(self.verbose_log, "It's a simple concept. Not for me.", self.name)
+                    not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
+                    return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
+
             my_locals = self.get_locals(context, node.ast_)
             context.log(self.verbose_log, f"locals={my_locals}", self.name)
 
@@ -78,7 +87,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
             evaluated = context.sheerka.evaluate_concept(sub_context, concept, self.verbose_log)
 
             if evaluated.key == concept.key:
-                my_locals[name] = evaluated.body or evaluated
+                my_locals[name] = evaluated.body or evaluated # if ConceptParts.BODY not in evaluated.cached_asts else evaluated
 
         return my_locals
 

evaluators/TooManySuccessEvaluator.py

@@ -17,33 +17,25 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
     NAME = "TooManySuccess"
 
     def __init__(self):
-        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 10)
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 60)
         self.success = []
 
     def matches(self, context, return_values):
-        sheerka = context.sheerka
+        to_process = False
-        after_evaluation = False
-        nb_successful_evaluators = 0
-        only_parsers_in_error = True
-        unlisted = False
 
         for ret in return_values:
+            if ret.status and ret.who.startswith(BaseParser.PREFIX):
+                return False
+            elif ret.status and context.sheerka.isinstance(ret.body, BuiltinConcepts.REDUCE_REQUESTED):
+                to_process = True
+                self.eaten.append(ret)
+            elif ret.status and ret.who.startswith(self.PREFIX):
+                self.success.append(ret)
+                self.eaten.append(ret)
+            elif not ret.status:
+                self.eaten.append(ret)
 
-            if sheerka.isinstance(ret.value, BuiltinConcepts.AFTER_EVALUATION):
+        return to_process and len(self.success) > 1
-                if ret.status:
-                    after_evaluation = True
-
-            elif ret.who.startswith(BaseEvaluator.PREFIX):
-                if ret.status:
-                    nb_successful_evaluators += 1
-                    self.success.append(ret)
-            elif ret.who.startswith(BaseParser.PREFIX):
-                if ret.status:
-                    only_parsers_in_error = False
-            else:
-                unlisted = True
-
-        return after_evaluation and nb_successful_evaluators > 1 and only_parsers_in_error and not unlisted
 
     def eval(self, context, return_values):
         sheerka = context.sheerka
@@ -56,8 +48,7 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
             context.log(self.verbose_log,
                         f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name)
             too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success)
-            return sheerka.ret(self.name, False, too_many_success, parents=return_values)
+            return sheerka.ret(self.name, False, too_many_success, parents=self.eaten)
 
-        context.log(self.verbose_log,
+        context.log(self.verbose_log, f"Values are the same. Nothing to do.", self.name)
-                    f"Values are the same. Nothing to do.", self.name)
         return None

parsers/EmptyStringParser.py

@@ -1,8 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
 from parsers.BaseParser import BaseParser
-import logging
-
-log = logging.getLogger(__name__)
 
 
 class EmptyStringParser(BaseParser):

sdp/sheerkaDataProvider.py

@@ -25,21 +25,31 @@ class Event(object):
     Class that represents something that modifies the state of the system
     """
 
-    def __init__(self, message="", user="kodjo", date=datetime.now()):
+    def __init__(self, message="", user="", date=datetime.now()):
         self.version = 1
         self.user = user
         self.date = date
         self.message = message
+        self._digest = None
 
     def get_digest(self):
         """
         Returns the digest of the event
         :return: hexa form of the sha256
         """
+
+        if self._digest:
+            return self._digest
+
+        if self.message == "" and self.user == "":
+            self._digest = "xxx"  # to speed unit tests
+            return self._digest
+
         if not isinstance(self.message, str):
             raise NotImplementedError
 
-        return hashlib.sha256(f"Event:{self.user}{self.date}{self.message}".encode("utf-8")).hexdigest()
+        self._digest = hashlib.sha256(f"Event:{self.user}{self.date}{self.message}".encode("utf-8")).hexdigest()
+        return self._digest
 
     def to_dict(self):
         return self.__dict__

tests/test_AddConceptEvaluator.py

@@ -12,12 +12,13 @@ from parsers.ConceptLexerParser import Sequence, StrMatch, ZeroOrMore, ConceptMa
 from parsers.BnfParser import BnfParser
 from parsers.DefaultParser import DefConceptNode, NameNode
 from parsers.PythonParser import PythonNode, PythonParser
+from sdp.sheerkaDataProvider import Event
 
 
 def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def get_concept(name, where=None, pre=None, post=None, body=None, definition=None):

tests/test_AddConceptInSetEvaluator.py

@@ -6,12 +6,13 @@ from core.sheerka import Sheerka, ExecutionContext
 from core.tokenizer import Tokenizer
 from evaluators.AddConceptInSetEvaluator import AddConceptInSetEvaluator
 from parsers.DefaultParser import IsaConceptNode, NameNode
+from sdp.sheerkaDataProvider import Event
 
 
 def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def get_ret_val(concept_name, concept_set_name):
@@ -74,6 +75,23 @@ def test_i_can_add_concept_to_a_set_of_concept():
     assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
 
 
+def test_i_can_add_concept_with_a_body_to_a_set_of_concept():
+    context = get_context()
+    foo = Concept("foo", body="1")
+    context.sheerka.set_id_if_needed(foo, False)
+    context.sheerka.add_in_cache(foo)
+
+    bar = Concept("bar")
+    context.sheerka.set_id_if_needed(bar, False)
+    context.sheerka.add_in_cache(bar)
+
+    ret_val = get_ret_val("foo", "bar")
+    res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+    assert res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
+
+
 def test_i_cannot_add_the_same_concept_twice():
     context = get_context()
     foo = Concept("foo")

tests/test_BnfParser.py

@@ -7,13 +7,14 @@ from parsers.BaseParser import UnexpectedTokenErrorNode
 from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
 from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
     ConceptLexerParser, ConceptNode, ConceptMatch
+from sdp.sheerkaDataProvider import Event
 
 
 def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
 
-    return ExecutionContext("sheerka", "xxxx", sheerka)
+    return ExecutionContext("sheerka", Event(), sheerka)
 
 
 @pytest.mark.parametrize("expression, expected", [

tests/test_ConceptEvaluator.py

@@ -6,12 +6,13 @@ from core.sheerka import Sheerka, ExecutionContext
 from evaluators.ConceptEvaluator import ConceptEvaluator
 from parsers.BaseParser import BaseParser
 from parsers.ExactConceptParser import ExactConceptParser
+from sdp.sheerkaDataProvider import Event
 
 
 def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def get_return_value(concept, source=None):
@@ -55,7 +56,7 @@ def test_i_can_evaluate_concept():
     assert result.parents == [item]
 
 
-def test_body_is_returned_when_defined():
+def test_body_is_returned_when_defined_and_requested():
     context = get_context()
     concept = Concept(name="foo",
                       body="'I have a value'",
@@ -63,7 +64,7 @@ def test_body_is_returned_when_defined():
                       pre="2",
                       post="3").set_prop("a", "4").set_prop("b", "5")
 
-    evaluator = ConceptEvaluator()
+    evaluator = ConceptEvaluator(return_body=True)
     item = get_return_value(concept)
     result = evaluator.eval(context, item)
 
@@ -73,7 +74,25 @@ def test_body_is_returned_when_defined():
     assert result.parents == [item]
 
 
-def test_i_cannot_eval_if_with_the_same_name_is_defined_in_the_context():
+def test_body_is_not_returned_if_not_requested():
+    context = get_context()
+    concept = Concept(name="foo",
+                      body="'I have a value'",
+                      where="1",
+                      pre="2",
+                      post="3").set_prop("a", "4").set_prop("b", "5")
+
+    evaluator = ConceptEvaluator(return_body=False)  # which is the default behaviour
+    item = get_return_value(concept)
+    result = evaluator.eval(context, item)
+
+    assert result.who == evaluator.name
+    assert result.status
+    assert result.value == concept
+    assert result.parents == [item]
+
+
+def test_i_can_eval_if_with_the_same_name_is_defined_in_the_context():
     # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
     # ConceptEvaluator will be called to resolve 'a' while we know that 'a' refers to the string 'property_a'
 
@@ -84,8 +103,8 @@ def test_i_cannot_eval_if_with_the_same_name_is_defined_in_the_context():
     item = get_return_value(concept)
     result = ConceptEvaluator().eval(context, item)
 
-    assert not result.status
+    assert result.status
-    assert context.sheerka.isinstance(result.value, BuiltinConcepts.NOT_FOR_ME)
+    assert result.value == "'some_other_value'"
 
 
 def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown():

tests/test_ConceptLexerParser.py

@@ -4,6 +4,7 @@ from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
 from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \
     ParsingExpressionVisitor, TerminalNode, NonTerminalNode, LexerNode, ConceptMatch, ZeroOrMore, OneOrMore
+from sdp.sheerkaDataProvider import Event
 
 
 class ConceptVisitor(ParsingExpressionVisitor):
@@ -849,4 +850,4 @@ def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
 
-    return ExecutionContext("sheerka", "xxxx", sheerka)
+    return ExecutionContext("sheerka", Event(), sheerka)

tests/test_ConceptNodeEvaluator.py

@@ -4,14 +4,15 @@ from core.builtin_concepts import ReturnValueConcept, ParserResultConcept
 from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
 from evaluators.ConceptNodeEvaluator import ConceptNodeEvaluator
-from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, NonTerminalNode, Sequence, TerminalNode, \
+from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, Sequence, TerminalNode, \
     StrMatch, Optional, OrderedChoice, ZeroOrMore
+from sdp.sheerkaDataProvider import Event
 
 
 def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def get_return_value(nodes, source):

tests/test_DefaultParser.py

@@ -11,49 +11,8 @@ from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode
 from parsers.BnfParser import BnfParser
 
 
-# def nop():
+from sdp.sheerkaDataProvider import Event
-#     return NopNode()
-#
-#
-# def n(number):
-#     return NumberNode([], number)
-#
-#
-# def s(string, quote="'"):
-#     return StringNode([], string, quote)
-#
-#
-# def v(name):
-#     return VariableNode([], name)
-#
-#
-# def t():
-#     return TrueNode([])
-#
-#
-# def f():
-#     return FalseNode([])
-#
-#
-# def null():
-#     return NullNode([])
-#
-#
-# def b(operator, left, right):
-#     return BinaryNode([], operator, left, right)
 
-#
-# def compare_ast(left, right):
-#     left_as_string = ast.dump(left)
-#     left_as_string = left_as_string.replace(", ctx=Load()", "")
-#     left_as_string = left_as_string.replace(", kind=None", "")
-#
-#     right_as_string = right if isinstance(right, str) else ast.dump(right)
-#     right_as_string = right_as_string.replace(", ctx=Load()", "")
-#     right_as_string = right_as_string.replace(", kind=None", "")
-#
-#     return left_as_string == right_as_string
-#
 
 def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None):
     def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
@@ -78,7 +37,7 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition
 def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def get_concept_part(part):
@@ -104,45 +63,6 @@ def get_concept_part(part):
     if isinstance(part, ReturnValueConcept):
         return part
 
-    # @pytest.mark.parametrize("text, expected", [
-    #     ("1", n(1)),
-    #     ("+1", n(1)),
-    #     ("-1", n(-1)),
-    #     ("'foo'", s("foo")),
-    #     ("identifier", v("identifier")),
-    #     ("true", t()),
-    #     ("false", f()),
-    #     ("null", null()),
-    #     ("1 * 2", b(TokenKind.STAR, n(1), n(2))),
-    #     ("1 * 2/3", b(TokenKind.STAR, n(1), b(TokenKind.SLASH, n(2), n(3)))),
-    #     ("1 + 2", b(TokenKind.PLUS, n(1), n(2))),
-    #     ("1 + 2 - 3", b(TokenKind.PLUS, n(1), b(TokenKind.MINUS, n(2), n(3)))),
-    #     ("1 + 2-3", b(TokenKind.PLUS, n(1), b(TokenKind.PLUS, n(2), n(-3)))),
-    #     ("1 + 2 +-3", b(TokenKind.PLUS, n(1), b(TokenKind.PLUS, n(2), n(-3)))),
-    #     ("1 + 2 * 3", b(TokenKind.PLUS, n(1), b(TokenKind.STAR, n(2), n(3)))),
-    #     ("1 * 2 + 3", b(TokenKind.PLUS, b(TokenKind.STAR, n(1), n(2)), n(3))),
-    #     ("(1 + 2) * 3", b(TokenKind.STAR, b(TokenKind.PLUS, n(1), n(2)), n(3))),
-    #     ("1 * (2 + 3)", b(TokenKind.STAR, n(1), b(TokenKind.PLUS, n(2), n(3)))),
-    # ])
-    # def test_i_can_parse_simple_expression(text, expected):
-    #     parser = DefaultParser(text, None)
-    #     ast = parser.parse()
-    #     assert ast.is_same(expected)
-    #
-    #
-    # @pytest.mark.parametrize("text, token_found, expected_tokens", [
-    #     ("1+", TokenKind.EOF,
-    #      [TokenKind.NUMBER, TokenKind.STRING, TokenKind.IDENTIFIER, 'true', 'false', 'null', TokenKind.LPAR]),
-    #     ("(1+1", TokenKind.EOF, [TokenKind.RPAR])
-    # ])
-    # def test_i_can_detect_unexpected_end_of_code(text, token_found, expected_tokens):
-    #     parser = DefaultParser(text, None)
-    #     parser.parse()
-    #
-    #     assert parser.has_error
-    #     assert parser.error_sink[0].tokens[0].type == token_found
-    #     assert parser.error_sink[0].expected_tokens == expected_tokens
-
 
 @pytest.mark.parametrize("text, expected", [
     ("def concept hello", get_def_concept(name="hello")),

tests/test_EvalEvaluator.py

@@ -0,0 +1,59 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from core.sheerka import Sheerka, ExecutionContext
+from evaluators.EvalEvaluator import EvalEvaluator
+from sdp.sheerkaDataProvider import Event
+
+
+def get_context():
+    sheerka = Sheerka(skip_builtins_in_db=True)
+    sheerka.initialize("mem://")
+    return ExecutionContext("test", Event(), sheerka)
+
+
+def r(value, status=True):
+    return ReturnValueConcept("some_name", status, value)
+
+
+eval_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.CONCEPT_EVAL_REQUESTED))
+
+
+def test_i_can_match_and_eval():
+    context = get_context()
+
+    to_eval1 = ReturnValueConcept("some_name", True, Concept(name="2", body="to eval"))
+    to_eval2 = ReturnValueConcept("some_name", True, Concept(name="3", body="also to eval"))
+
+    return_values = [
+        ReturnValueConcept("some_name", True, "not to eval"),
+        ReturnValueConcept("some_name", True, Concept(name="not to eval")),
+        ReturnValueConcept("some_name", False, Concept(name="1", body="not to eval")),
+        to_eval1,
+        to_eval2,
+        eval_requested
+    ]
+
+    evaluator = EvalEvaluator()
+    assert evaluator.matches(context, return_values)
+
+    evaluated = evaluator.eval(context, return_values)
+    assert len(evaluated) == 2
+    assert evaluated[0].value == to_eval1.body.body
+    assert evaluated[0].parents == [to_eval1, eval_requested]
+
+    assert evaluated[1].value == to_eval2.body.body
+    assert evaluated[1].parents == [to_eval2, eval_requested]
+
+
+@pytest.mark.parametrize("return_values, expected", [
+    ([r(Concept("foo", body="bar")), eval_requested], True),
+    ([r(Concept("status is false", body="bar"), False), eval_requested], False),
+    ([r("string_value"), eval_requested], False),
+    ([r(Concept("no body")), eval_requested], False),
+    ([r(Concept("eval requested missing", body="bar"))], False),
+])
+def test_i_cannot_match_if_eval_request_is_not_present(return_values, expected):
+    context = get_context()
+    assert EvalEvaluator().matches(context, return_values) == expected

tests/test_ExactConceptParser.py

@@ -3,6 +3,7 @@ from core.concept import Concept, Property
 from core.sheerka import Sheerka, ExecutionContext
 from core.tokenizer import Tokenizer
 from parsers.ExactConceptParser import ExactConceptParser
+from sdp.sheerkaDataProvider import Event
 
 
 def test_i_can_compute_combinations():
@@ -130,7 +131,7 @@ def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
 
-    return ExecutionContext("sheerka", "xxxx", sheerka)
+    return ExecutionContext("sheerka", Event(), sheerka)
 
 
 def get_concept(name, variables):

tests/test_ExecutionContext.py

@@ -1,12 +1,13 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka import ExecutionContext
+from sdp.sheerkaDataProvider import Event
 
 
 def test_id_is_incremented_by_event_digest():
-    a = ExecutionContext("foo", "event_1", None)
+    a = ExecutionContext("foo", Event("event_1"), None)
-    b = ExecutionContext("foo", "event_1", None)
+    b = ExecutionContext("foo", Event("event_1"), None)
-    c = ExecutionContext("foo", "event_2", None)
+    c = ExecutionContext("foo", Event("event_2"), None)
     d = b.push()
     e = c.push()
 
@@ -18,17 +19,19 @@ def test_id_is_incremented_by_event_digest():
 
 
 def test_some_properties_are_given_to_the_child():
-    a = ExecutionContext("foo", "event_1", "fake_sheerka",
+    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka",
                          desc="some description",
                          obj=Concept("foo"),
                          step=BuiltinConcepts.EVALUATION,
                          iteration=15,
                          concepts={"bar": Concept("bar")})
+    a.preprocess = set()
+    a.preprocess.add("preprocess")
 
     b = a.push()
 
     assert b.who == a.who
-    assert b.event_digest == a.event_digest
+    assert b.event == a.event
     assert b.sheerka == a.sheerka
     assert b.desc == ""
     assert b.obj == a.obj
@@ -37,3 +40,4 @@ def test_some_properties_are_given_to_the_child():
     assert b.concepts == a.concepts
     assert b.id == a.id + 1
     assert b._tab == a._tab + "    "
+    assert b.preprocess == a.preprocess

tests/test_MultipleSameSuccessEvaluator.py

@@ -4,12 +4,13 @@ from core.sheerka import Sheerka, ExecutionContext
 from evaluators.BaseEvaluator import BaseEvaluator
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from parsers.BaseParser import BaseParser
+from sdp.sheerkaDataProvider import Event
 
 
 def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def test_i_can_match_and_eval():
@@ -21,7 +22,7 @@ def test_i_can_match_and_eval():
         ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION))
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
     ]
 
     evaluator = MultipleSameSuccessEvaluator()
@@ -29,7 +30,7 @@ def test_i_can_match_and_eval():
 
     evaluated = evaluator.eval(context, return_values)
     assert evaluated.status
-    assert evaluated.value == "value"
+    assert evaluated.value == Concept(name="1", body="value")  # the first concept is returned
 
 
 def test_i_can_match_and_eval_when_no_body():
@@ -41,7 +42,7 @@ def test_i_can_match_and_eval_when_no_body():
         ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1")),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION))
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
     ]
 
     evaluator = MultipleSameSuccessEvaluator()
@@ -61,7 +62,7 @@ def test_i_can_match_and_eval_when_value_is_not_a_concept():
         ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION))
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
     ]
 
     evaluator = MultipleSameSuccessEvaluator()
@@ -72,6 +73,54 @@ def test_i_can_match_and_eval_when_value_is_not_a_concept():
     assert evaluated.value == "value"
 
 
+def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_concept_evaluator_first():
+    context = get_context()
+    sheerka = context.sheerka
+
+    return_values = [
+        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="2", body="value")),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "Concept", True, Concept(name="1", body="value")),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="3", body="value")),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+    ]
+
+    evaluator = MultipleSameSuccessEvaluator()
+    assert evaluator.matches(context, return_values)
+
+    evaluated = evaluator.eval(context, return_values)
+    assert evaluated.status
+    assert evaluated.value == Concept(name="1", body="value")  # the concept is returned, not the value
+
+
+def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_python_evaluator_first():
+    context = get_context()
+    sheerka = context.sheerka
+
+    return_values = [
+        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="2", body="value")),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="1", body="value")),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="3", body="value")),
+        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+    ]
+
+    evaluator = MultipleSameSuccessEvaluator()
+    assert evaluator.matches(context, return_values)
+
+    evaluated = evaluator.eval(context, return_values)
+    assert evaluated.status
+    assert evaluated.value == Concept(name="1", body="value")  # the concept is returned, not the value
+
+
 def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail():
     context = get_context()
     sheerka = context.sheerka
@@ -81,7 +130,7 @@ def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail():
         ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value2")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION))
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
     ]
 
     evaluator = MultipleSameSuccessEvaluator()
@@ -98,7 +147,7 @@ def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail_when_
         ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1")),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION))
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
     ]
 
     evaluator = MultipleSameSuccessEvaluator()
@@ -113,13 +162,13 @@ def test_i_can_match_if_no_parser():
     return_values = [
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION))
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
     ]
 
     assert MultipleSameSuccessEvaluator().matches(context, return_values)
 
 
-def test_i_cannot_match_if_not_after_evaluation():
+def test_i_cannot_match_if_not_reduced_requested():
     context = get_context()
     sheerka = context.sheerka
 
@@ -129,7 +178,7 @@ def test_i_cannot_match_if_not_after_evaluation():
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value")),
         ReturnValueConcept("some_name", True, Concept(name="2", body="value")),
-        # ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION))
+        # ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
     ]
 
     assert not MultipleSameSuccessEvaluator().matches(context, return_values)
@@ -145,7 +194,7 @@ def test_i_cannot_match_if_only_one_successful_evaluator():
         # ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value")),
         ReturnValueConcept("some_name", True, Concept(name="2", body="value")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION))
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
     ]
 
     assert not MultipleSameSuccessEvaluator().matches(context, return_values)
@@ -160,41 +209,8 @@ def test_i_cannot_match_if_at_least_one_parser_is_successful():
         ReturnValueConcept(BaseParser.PREFIX + "some_name2", True, "Not relevant"),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION))
+        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
     ]
 
     assert not MultipleSameSuccessEvaluator().matches(context, return_values)
 
-
-def test_i_cannot_match_if_i_have_unlisted_return_value_in_success():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION)),
-        ReturnValueConcept("some_name", True, "not relevant"),
-
-    ]
-
-    assert not MultipleSameSuccessEvaluator().matches(context, return_values)
-
-
-def test_i_cannot_match_if_i_have_unlisted_return_value_in_error():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.AFTER_EVALUATION)),
-        ReturnValueConcept("some_name", False, "not relevant"),
-
-    ]
-
-    assert not MultipleSameSuccessEvaluator().matches(context, return_values)

tests/test_OneErrorEvaluator.py

@@ -0,0 +1,80 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from core.sheerka import Sheerka, ExecutionContext
+from evaluators.OneErrorEvaluator import OneErrorEvaluator
+from sdp.sheerkaDataProvider import Event
+
+
+def get_context():
+    sheerka = Sheerka(skip_builtins_in_db=True)
+    sheerka.initialize("mem://")
+    return ExecutionContext("test", Event(), sheerka)
+
+
+def r(value, status=True):
+    return ReturnValueConcept(value, status, value)
+
+
+reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
+
+
+@pytest.mark.parametrize("return_values, expected", [
+    ([r("evaluators.one error", False), reduce_requested], True),
+    ([r("evaluators.one error", False), r("failed", False), r("failed", False), reduce_requested], True),
+    ([r("evaluators.error", False), r("not a parser in success"), reduce_requested], True),
+    ([r("evaluators.no reduce required", False), r("failed", False), r("failed", False)], False),
+    ([r("evaluators.no reduce required", False)], False),
+    ([r("evaluators.error", False), r("evaluators.success"), reduce_requested], False),
+    ([r("evaluators.error", False), r("parsers.success"), reduce_requested], False),
+    ([r("evaluators.success"), r("not an evaluator in error", False), reduce_requested], False),
+    ([r("evaluators.error", False), r("evaluators.another error", False), reduce_requested], False),
+])
+def test_i_can_match(return_values, expected):
+    context = get_context()
+    assert OneErrorEvaluator().matches(context, return_values) == expected
+
+
+def test_i_can_eval():
+    context = get_context()
+
+    return_values = [
+        r("evaluators.one error", False),
+        r("parsers.failed", False),
+        r("parsers.failed", False),
+        reduce_requested
+    ]
+
+    evaluator = OneErrorEvaluator()
+    evaluator.matches(context, return_values)
+    res = evaluator.eval(context, return_values)
+
+    assert not res.status
+    assert res.body == "evaluators.one error"
+    assert len(res.parents) == 4
+
+
+def test_unwanted_return_values_are_not_eaten():
+    context = get_context()
+
+    a_successful_concept = r("successful concept")
+    a_concept_in_error = r("concept in error", False)
+    return_values = [
+        r("evaluators.one error", False),
+        r("parsers.failed", False),
+        r("parsers.failed", False),
+        a_successful_concept,
+        a_concept_in_error,
+        reduce_requested
+    ]
+
+    evaluator = OneErrorEvaluator()
+    evaluator.matches(context, return_values)
+    res = evaluator.eval(context, return_values)
+
+    assert not res.status
+    assert res.body == "evaluators.one error"
+    assert len(res.parents) == 4
+
+    assert a_successful_concept not in res.parents

tests/test_OneSuccessEvaluator.py

@@ -0,0 +1,79 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from core.sheerka import Sheerka, ExecutionContext
+from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
+from sdp.sheerkaDataProvider import Event
+
+
+def get_context():
+    sheerka = Sheerka(skip_builtins_in_db=True)
+    sheerka.initialize("mem://")
+    return ExecutionContext("test", Event(), sheerka)
+
+
+def r(value, status=True):
+    return ReturnValueConcept(value, status, value)
+
+
+reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
+
+
+@pytest.mark.parametrize("return_values, expected", [
+    ([r("evaluators.one success"), reduce_requested], True),
+    ([r("evaluators.one success"), r("failed", False), r("failed", False), reduce_requested], True),
+    ([r("evaluators.no reduce required"), r("failed", False), r("failed", False)], False),
+    ([r("evaluators.no reduce required")], False),
+    ([r("evaluators.failed", False), r("failed", False), r("failed", False), reduce_requested], False),
+    ([r("evaluators.failed", False), r("not evaluator success"), reduce_requested], False),
+    ([r("evaluators.success"), r("evaluators.another success"), reduce_requested], False),
+    ([r("evaluators.one success"), r("other success"), r("failed", False), reduce_requested], True),
+    ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False),
+])
+def test_i_can_match(return_values, expected):
+    context = get_context()
+    assert OneSuccessEvaluator().matches(context, return_values) == expected
+
+
+def test_i_can_eval():
+    context = get_context()
+
+    return_values = [
+        r("evaluators.one success"),
+        r("failed", False),
+        r("failed", False),
+        reduce_requested
+    ]
+
+    evaluator = OneSuccessEvaluator()
+    matches = evaluator.matches(context, return_values)
+    res = evaluator.eval(context, return_values)
+
+    assert matches
+    assert res.status
+    assert res.body == "evaluators.one success"
+    assert len(res.parents) == 4
+
+
+def test_i_do_not_eat_the_other_success():
+    context = get_context()
+
+    not_a_parser_success = r("other success")
+    return_values = [
+        r("evaluators.one success"),
+        not_a_parser_success,
+        r("failed", False),
+        reduce_requested
+    ]
+
+    evaluator = OneSuccessEvaluator()
+    matches = evaluator.matches(context, return_values)
+    res = evaluator.eval(context, return_values)
+
+    assert matches
+    assert res.status
+    assert res.body == "evaluators.one success"
+    assert len(res.parents) == 3
+
+    assert not_a_parser_success not in res.parents

tests/test_PrepareEvalEvaluator.py

@@ -0,0 +1,52 @@
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
+from core.concept import Concept
+from core.sheerka import Sheerka, ExecutionContext
+from evaluators.PrepareEvalEvaluator import PrepareEvalEvaluator
+from sdp.sheerkaDataProvider import Event
+
+
+def get_context():
+    sheerka = Sheerka(skip_builtins_in_db=True)
+    sheerka.initialize("mem://")
+    return ExecutionContext("test", Event(), sheerka)
+
+
+@pytest.mark.parametrize("ret_val, expected", [
+    (ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.USER_INPUT, body="eval 1 + 1")), True),
+    (ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.USER_INPUT, body="  eval 1 + 1")), True),
+    (ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.USER_INPUT, body="eval")), False),
+    (ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.USER_INPUT, body="1+1")), False),
+    (ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.USER_INPUT, body="")), False),
+    (ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.USER_INPUT, body="eval   ")), False),
+    (ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.USER_INPUT, body=[])), False),
+    (ReturnValueConcept("some_name", True, Concept("foo")), False),
+    (ReturnValueConcept("some_name", True, "not a concept"), False),
+    (ReturnValueConcept("some_name", False, Concept(key=BuiltinConcepts.USER_INPUT, body="eval 1 + 1")), False),
+    (ReturnValueConcept("some_name", False, Concept(key=BuiltinConcepts.USER_INPUT, body="  eval 1 + 1")), False),
+])
+def test_i_can_match(ret_val, expected):
+    context = get_context()
+    assert PrepareEvalEvaluator().matches(context, ret_val) == expected
+
+
+@pytest.mark.parametrize("ret_val, expected", [
+    (ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.USER_INPUT, body="eval 1 + 1")), "1 + 1"),
+    (ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.USER_INPUT, body="   eval 1 + 1")), "1 + 1"),
+])
+def test_i_can_eval(ret_val, expected):
+    context = get_context()
+    sheerka = context.sheerka
+
+    prepare_evaluator = PrepareEvalEvaluator()
+    prepare_evaluator.matches(context, ret_val)
+    res = prepare_evaluator.eval(context, ret_val)
+
+    assert len(res) == 2
+    assert res[0].status
+    assert sheerka.isinstance(res[0].body, BuiltinConcepts.USER_INPUT)
+    assert res[0].body.body == expected
+
+    assert res[1].status
+    assert sheerka.isinstance(res[1].body, BuiltinConcepts.CONCEPT_EVAL_REQUESTED)

tests/test_PythonEvaluator.py

@@ -5,12 +5,13 @@ from core.sheerka import Sheerka, ExecutionContext
 from core.concept import Concept
 from evaluators.PythonEvaluator import PythonEvaluator
 from parsers.PythonParser import PythonNode, PythonParser
+from sdp.sheerkaDataProvider import Event
 
 
 def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 @pytest.mark.parametrize("ret_val, expected", [
@@ -39,30 +40,62 @@ def test_i_can_eval(text, expected):
     assert evaluated.value == expected
 
 
-def test_i_can_eval_expression_that_references_concepts():
+@pytest.mark.parametrize("concept", [
+    Concept("foo"),
+    Concept("foo", body="2"),
+    Concept("foo").set_prop("prop", "'a'"),
+    Concept("foo", body="bar")
+])
+def test_i_cannot_eval_simple_concept(concept):
     context = get_context()
     context.sheerka.add_in_cache(Concept("foo"))
 
     parsed = PythonParser().parse(context, "foo")
     evaluated = PythonEvaluator().eval(context, parsed)
 
-    assert evaluated.status
+    assert not evaluated.status
-    assert evaluated.value == Concept("foo").init_key()
+    assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.NOT_FOR_ME)
 
 
-def test_i_can_eval_expression_that_references_concepts_with_body():
+#
+# def test_i_can_eval_expression_that_references_concepts():
+#     context = get_context()
+#     context.sheerka.add_in_cache(Concept("foo"))
+#
+#     parsed = PythonParser().parse(context, "foo")
+#     evaluated = PythonEvaluator().eval(context, parsed)
+#
+#     assert evaluated.status
+#     assert evaluated.value == Concept("foo").init_key()
+#
+#
+# def test_i_can_eval_expression_that_references_concepts_with_body():
+#     """
+#     I can test expression with variables
+#     :return:
+#     """
+#     context = get_context()
+#     context.sheerka.add_in_cache(Concept("foo", body="2"))
+#
+#     parsed = PythonParser().parse(context, "foo")
+#     evaluated = PythonEvaluator().eval(context, parsed)
+#
+#     assert evaluated.status
+#     assert evaluated.value == 2
+
+def test_i_can_eval_expression_with_that_references_concepts():
     """
-    I can test expression with variables
+    I can test modules with variables
     :return:
     """
     context = get_context()
-    context.sheerka.add_in_cache(Concept("foo", body="2"))
+    context.sheerka.add_in_cache(Concept("foo", body=1))
 
-    parsed = PythonParser().parse(context, "foo")
+    parsed = PythonParser().parse(context, "foo + 2")
     evaluated = PythonEvaluator().eval(context, parsed)
 
     assert evaluated.status
-    assert evaluated.value == 2
+    assert evaluated.value == 3
 
 
 def test_i_can_eval_module_with_that_references_concepts():
@@ -94,24 +127,24 @@ def test_i_can_eval_module_with_that_references_concepts_with_body():
     assert evaluated.status
     assert evaluated.value == 2
 
-
+#
-def test_i_can_eval_concept_with_props():
+# def test_i_can_eval_concept_with_props():
-    context = get_context()
+#     context = get_context()
-    context.sheerka.add_in_cache(Concept("foo").set_prop("prop", "'a'"))
+#     context.sheerka.add_in_cache(Concept("foo").set_prop("prop", "'a'"))
-
+#
-    parsed = PythonParser().parse(context, "foo")
+#     parsed = PythonParser().parse(context, "foo")
-    evaluated = PythonEvaluator().eval(context, parsed)
+#     evaluated = PythonEvaluator().eval(context, parsed)
-
+#
-    assert evaluated.status
+#     assert evaluated.status
-    assert evaluated.value == Concept("foo").set_prop("prop", "a").init_key()  # evaluated version of foo
+#     assert evaluated.value == Concept("foo").set_prop("prop", "a").init_key()  # evaluated version of foo
-
+#
-
+#
-def test_i_cannot_eval_when_body_references_unknown_concept():
+# def test_i_cannot_eval_when_body_references_unknown_concept():
-    context = get_context()
+#     context = get_context()
-    context.sheerka.add_in_cache(Concept("foo", body="bar"))
+#     context.sheerka.add_in_cache(Concept("foo", body="bar"))
-
+#
-    parsed = PythonParser().parse(context, "foo")
+#     parsed = PythonParser().parse(context, "foo")
-    evaluated = PythonEvaluator().eval(context, parsed)
+#     evaluated = PythonEvaluator().eval(context, parsed)
-
+#
-    assert not evaluated.status
+#     assert not evaluated.status
-    assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.ERROR)
+#     assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.ERROR)

tests/test_PythonParser.py

@@ -6,12 +6,13 @@ from core.builtin_concepts import ParserResultConcept
 from core.sheerka import Sheerka, ExecutionContext
 from core.tokenizer import Tokenizer
 from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
+from sdp.sheerkaDataProvider import Event
 
 
 def get_context():
     sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 @pytest.mark.parametrize("text, expected", [

tests/test_TooManySucessEvaluator.py

@@ -0,0 +1,112 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from core.sheerka import Sheerka, ExecutionContext
+from evaluators.TooManySuccessEvaluator import TooManySuccessEvaluator
+from sdp.sheerkaDataProvider import Event
+
+
+def get_context():
+    sheerka = Sheerka(skip_builtins_in_db=True)
+    sheerka.initialize("mem://")
+    return ExecutionContext("test", Event(), sheerka)
+
+
+def r(name, status=True, value=None):
+    value = value or name
+    return ReturnValueConcept(name, status, value)
+
+
+reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
+
+
+@pytest.mark.parametrize("return_values, expected", [
+    ([r("evaluators.success1"), r("evaluators.success2"), reduce_requested], True),
+    ([r("evaluators.success1"), r("evaluators.success2"), r("other"), reduce_requested], True),
+    ([r("evaluators.success1"), r("evaluators.success2"), r("other", False), reduce_requested], True),
+    ([r("evaluators.a", value=Concept("c1", body="1")),
+      r("evaluators.a", value=Concept("c2", body="1")),
+      r("other"),
+      reduce_requested], True),
+    ([r("evaluators.a", value=Concept("c1", body="1")),
+      r("evaluators.a", value=Concept("c2", body="1")),
+      r("parsers.other", False),
+      reduce_requested], True),
+    ([r("evaluators.a", value=Concept("c1", body="1")),
+      r("evaluators.a", value=Concept("c2", body="1")),
+      r("parsers.other"),
+      reduce_requested], False),
+    ([r("evaluators.success1"), reduce_requested], False),
+    ([reduce_requested], False),
+    ([r("evaluators.success1"), r("evaluators.success2")], False),
+])
+def test_i_can_match(return_values, expected):
+    context = get_context()
+    assert TooManySuccessEvaluator().matches(context, return_values) == expected
+
+
+def test_i_can_eval():
+    context = get_context()
+
+    value1 = r("evaluators.a", value=Concept("c1", body="1"))
+    value2 = r("evaluators.a", value=Concept("c2", body="2"))
+    return_values = [
+        value1,
+        value2,
+        r("other", False),
+        reduce_requested
+    ]
+
+    evaluator = TooManySuccessEvaluator()
+    matches = evaluator.matches(context, return_values)
+    res = evaluator.eval(context, return_values)
+
+    assert matches
+    assert not res.status
+    assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS)
+    assert res.body.body == [value1, value2]
+    assert len(res.parents) == 4
+
+
+def test_i_can_eval_when_same_success():
+    context = get_context()
+
+    return_values = [
+        r("evaluators.a", value=Concept("c1", body="1")),
+        r("evaluators.a", value=Concept("c2", body="1")),
+        r("other", False),
+        reduce_requested
+    ]
+
+    evaluator = TooManySuccessEvaluator()
+    matches = evaluator.matches(context, return_values)
+    res = evaluator.eval(context, return_values)
+
+    assert matches
+    assert res is None
+
+
+def test_other_success_are_not_reduced():
+    context = get_context()
+
+    value1 = r("evaluators.a", value=Concept("c1", body="1"))
+    value2 = r("evaluators.a", value=Concept("c2", body="2"))
+    other_success = r("other")
+    return_values = [
+        value1,
+        value2,
+        other_success,
+        reduce_requested
+    ]
+
+    evaluator = TooManySuccessEvaluator()
+    matches = evaluator.matches(context, return_values)
+    res = evaluator.eval(context, return_values)
+
+    assert matches
+    assert not res.status
+    assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS)
+    assert res.body.body == [value1, value2]
+    assert len(res.parents) == 3
+    assert other_success not in res.parents

tests/test_builtin_helpers.py

@@ -5,6 +5,7 @@ import pytest
 from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
 from core.sheerka import Sheerka, ExecutionContext
 import core.builtin_helpers
+from sdp.sheerkaDataProvider import Event
 
 
 def test_i_can_use_expect_one_when_empty():
@@ -160,7 +161,7 @@ def get_sheerka():
 
 
 def get_context(sheerka):
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def dump_ast(node):

tests/test_sheerka.py

@@ -9,7 +9,7 @@ from core.sheerka import Sheerka, ExecutionContext
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from parsers.ConceptLexerParser import Sequence, ZeroOrMore, StrMatch, OrderedChoice, Optional, ConceptMatch, \
     ConceptLexerParser
-from sdp.sheerkaDataProvider import SheerkaDataProvider
+from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 
 tests_root = path.abspath("../build/tests")
 root_folder = "init_folder"
@@ -39,7 +39,7 @@ def get_sheerka(use_dict=True, skip_builtins_in_db=True):
 
 
 def get_context(sheerka):
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def get_default_concept():
@@ -131,8 +131,8 @@ def test_i_cannot_add_the_same_concept_twice():
     res = sheerka.create_new_concept(get_context(sheerka), concept)
 
     assert not res.status
-    assert sheerka.isinstance(res.value, BuiltinConcepts.ERROR)
+    assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
-    assert res.value.body.args[0] == "Duplicate object."
+    assert res.value.body == concept
 
 
 def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string():
@@ -317,33 +317,27 @@ def test_i_cannot_instantiate_when_properties_are_not_recognized():
     assert sheerka.isinstance(new.concept, concept)
 
 
-@pytest.mark.parametrize("concept, allow_non_body, expected", [
+@pytest.mark.parametrize("concept, reduce_simple_list, expected", [
     (None, False, None),
     (3.14, False, 3.14),
     ("foo", False, "foo"),
     (True, False, True),
     (Concept("name", body="foo"), False, "foo"),
-    (Concept("name"), True, Concept("name")),
+    (Concept("name"), False, Concept("name")),
-    (ConceptWithGetValue("name").set_prop("my_prop", "my_value"), True, "my_value"),
+    (ConceptWithGetValue("name").set_prop("my_prop", "my_value"), False, "my_value"),
     (ReturnValueConcept(value="return_value"), False, "return_value"),
     (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
     (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
     (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]),
-    (Concept("name", body=["foo"]), False, "foo"),
+    (Concept("name", body=["foo"]), True, "foo"),
+    (Concept("name", body=Concept("foo")), False, Concept("foo")),
+    (Concept("name", body=Concept("foo", body="value")), False, "value"),
+    (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"),
 ])
-def test_i_can_get_value(concept, allow_non_body, expected):
+def test_i_can_get_value(concept, reduce_simple_list, expected):
     sheerka = get_sheerka()
 
-    assert sheerka.value(concept, allow_non_body) == expected
+    assert sheerka.value(concept, reduce_simple_list) == expected
-
-
-def test_i_cannot_get_value_when_no_body_and_allow_none_body_is_false():
-    sheerka = get_sheerka()
-    concept = Concept("name")
-    allow_none_body = False
-
-    assert sheerka.value(concept, allow_none_body) == sheerka.new(BuiltinConcepts.CANNOT_RESOLVE_VALUE_ERROR,
-                                                                  body=concept)
 
 
 def test_list_of_concept_is_sorted_by_id():
@@ -569,6 +563,17 @@ def test_properties_values_takes_precedence_over_the_outside_world():
     assert evaluated.body == 'concept_b'
 
 
+def test_properties_values_takes_precedence():
+    sheerka = get_sheerka()
+    sheerka.add_in_cache(Concept(name="a", body="'concept_a'").init_key())
+    sheerka.add_in_cache(Concept(name="b", body="'concept_b'").init_key())
+
+    concept = Concept("foo", body="a + b").set_prop("a", "'prop_a'").init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
+    assert evaluated.key == concept.key
+    assert evaluated.body == 'prop_aconcept_b'
+
+
 def test_i_can_reference_sub_property_of_a_property():
     sheerka = get_sheerka()
     sheerka.add_in_cache(Concept(name="concept_a").set_prop("subProp", "'sub_a'").init_key())
@@ -666,5 +671,3 @@ def test_i_cannot_add_the_same_concept_twice_in_a_set():
     all_entries = sheerka.sdp.get("All_" + all_foos.id, None, False)
     assert len(all_entries) == 1
     assert foo.id in all_entries
-
-

tests/test_sheerka_call_evaluators.py

@@ -3,6 +3,7 @@ from core.builtin_concepts import BuiltinConcepts, SuccessConcept
 from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
 from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator
+from sdp.sheerkaDataProvider import Event
 
 
 def get_sheerka():
@@ -12,7 +13,7 @@ def get_sheerka():
 
 
 def get_context(sheerka):
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def get_ret_val(sheerka, concept, who="who"):
@@ -142,7 +143,7 @@ class EvaluatorOnePreEvaluation(OneReturnValueEvaluatorForTestingPurpose):
 
 class EvaluatorOneMultiSteps(OneReturnValueEvaluatorForTestingPurpose):
     def __init__(self):
-        super().__init__("multiStep", [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION], 10)
+        super().__init__("multiStep", [BuiltinConcepts.EVALUATION, BuiltinConcepts.BEFORE_EVALUATION], 10)
 
 
 class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority):
@@ -321,10 +322,10 @@ def test_evaluation_steps_are_respected():
     sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation]
 
     entries = [get_ret_val(sheerka, Concept("foo"))]
-    BaseEvaluator.debug_out = []
+    Out.debug_out = []
     sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION])
 
-    assert BaseEvaluator.debug_out == [
+    assert Out.debug_out == [
         '__BEFORE_EVALUATION [0] preEval - matches - target=foo',
         '__BEFORE_EVALUATION [0] preEval - eval - target=foo',
         '__BEFORE_EVALUATION [0] preEval - matches - target=__BEFORE_EVALUATION',
@@ -336,10 +337,10 @@ def test_evaluation_multi_steps_are_respected():
     sheerka.evaluators = [EvaluatorOneMultiSteps]
 
     entries = [get_ret_val(sheerka, Concept("foo"))]
-    BaseEvaluator.debug_out = []
+    Out.debug_out = []
     sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION])
 
-    assert BaseEvaluator.debug_out == [
+    assert Out.debug_out == [
         '__BEFORE_EVALUATION [0] multiStep - matches - target=foo',
         '__BEFORE_EVALUATION [0] multiStep - eval - target=foo',
         '__BEFORE_EVALUATION [0] multiStep - matches - target=__BEFORE_EVALUATION',
@@ -349,3 +350,28 @@ def test_evaluation_multi_steps_are_respected():
         '__EVALUATION [0] multiStep - matches - target=__EVALUATION',
         '__EVALUATION [0] multiStep - eval - target=__EVALUATION'
     ]
+
+
+def test_evaluators_can_be_pre_processed():
+    sheerka = get_sheerka()
+    sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+    entries = [get_ret_val(sheerka, Concept("foo"))]
+
+    # disable evaluator
+    context = get_context(sheerka)
+    context.add_preprocess(EvaluatorOneModifyFoo().name, enabled=False)  # disabled  for this exec context
+    Out.debug_out = []
+    sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION])
+    assert Out.debug_out == []
+
+    # other contextes are not impacted
+    Out.debug_out = []
+    sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+    assert Out.debug_out == [
+        '__EVALUATION [0] modifyFoo - matches - target=foo',
+        '__EVALUATION [0] modifyFoo - eval - target=foo',
+        '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+        '__EVALUATION [1] modifyFoo - matches - target=bar',
+        '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
+    ]

tests/test_sheerka_non_reg.py

@@ -9,7 +9,7 @@ from core.sheerka import Sheerka, ExecutionContext
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from parsers.ConceptLexerParser import Sequence, ZeroOrMore, StrMatch, OrderedChoice, Optional, ConceptMatch, \
     ConceptLexerParser
-from sdp.sheerkaDataProvider import SheerkaDataProvider
+from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 
 tests_root = path.abspath("../build/tests")
 root_folder = "init_folder"
@@ -39,7 +39,7 @@ def get_sheerka(use_dict=True, skip_builtins_in_db=True):
 
 
 def get_context(sheerka):
-    return ExecutionContext("test", "xxx", sheerka)
+    return ExecutionContext("test", Event(), sheerka)
 
 
 def get_default_concept():
@@ -79,7 +79,7 @@ def test_i_can_eval_concept_with_python_body():
     res = sheerka.evaluate_user_input(text)
     assert len(res) == 1
     assert res[0].status
-    assert res[0].value == 1
+    assert res[0].value == Concept(name="one", body=1).init_key()  # by default, the concept is returned
 
 
 def test_i_can_eval_concept_with_concept_body():
@@ -93,7 +93,7 @@ def test_i_can_eval_concept_with_concept_body():
     return_value = res[0].value
     assert len(res) == 1
     assert res[0].status
-    assert sheerka.isinstance(return_value, concept_one)
+    assert return_value == Concept(name="un", body=Concept(name="one").init_key()).init_key()
 
 
 def test_i_can_eval_concept_with_no_body():
@@ -207,23 +207,6 @@ as:
     assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
 
 
-# def test_i_can_disable_an_evaluator():
-#     sheerka = get_sheerka()
-#     concept = Concept(name="one", body="1")
-#     sheerka.add_in_cache(concept)
-#
-#     text = "one"
-#     p = next(e for e in sheerka.evaluators if e.__name__ == "PythonEvaluator")
-#     p.enabled = False  # not that you disable the class, not the instance
-#
-#     res = sheerka.evaluate_user_input(text)
-#     assert len(res) == 1
-#     assert res[0].status
-#     assert sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
-#
-#     p.enabled = True  # put back for the remaining unit tests
-
-
 @pytest.mark.parametrize("text", [
     "",
     "   ",
@@ -293,9 +276,9 @@ def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different():
 
     concepts = res[0].value.body
     assert len(concepts) == 2
-    sorted_values = sorted(concepts, key=lambda x: x.value)
+    sorted_values = sorted(concepts, key=lambda x: x.value.body)
-    assert sorted_values[0].value == "hello another value"
+    assert sorted_values[0].value.body == "hello another value"
-    assert sorted_values[1].value == "hello foo"
+    assert sorted_values[1].value.body == "hello foo"
 
 
 def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same():
@@ -308,7 +291,7 @@ def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same():
     res = sheerka.evaluate_user_input("hello 'foo'")
     assert len(res) == 1
     assert res[0].status
-    assert res[0].value == "hello foo"
+    assert res[0].value.body == "hello foo" # I don't know yet the one to choose
     assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
 
 
@@ -321,7 +304,7 @@ def test_i_can_create_concepts_with_python_code_as_body():
 
     assert len(res) == 1
     assert res[0].status
-    assert isinstance(res[0].value, list)
+    assert isinstance(res[0].value.body, list)
 
 
 def test_i_can_create_concept_with_bnf_definition():