First implementation of questions management

_concepts.txt

@@ -81,6 +81,20 @@ def concept thousands from bnf number=n1 'thousand' 'and' number=n2 as n1 * 1000
 last_created_concept() is number
 def concept history as history()
 def concept plus from a plus b as a + b
-def concept mult from a mult b as a * b
+def concept minus from a plus b as a - b
+def concept multiplied from a multiplied by b as a * b
+def concept divided from a divided by b as a * b
+set_is_greater_than(BuiltinConcepts.PRECEDENCE, multiplied, plus)
+set_is_greater_than(BuiltinConcepts.PRECEDENCE, divided, plus)
+set_is_greater_than(BuiltinConcepts.PRECEDENCE, multiplied, minus)
+set_is_greater_than(BuiltinConcepts.PRECEDENCE, divided, minus)
 def concept explain as get_results() | filter("id == 0") | recurse(2)
 def concept explain last as get_last_results() | filter("id == 0") | recurse(2)
+set_isa(c:explain:, __COMMAND)
+set_isa(c:explain last:, __COMMAND)
+def concept precedence a > precedence b as set_is_greater_than(BuiltinConcepts.PRECEDENCE, a, b)
+set_isa(c:precedence a > precedence b:, __COMMAND)
+def concept x is a command as set_isa(x, __COMMAND)
+set_isa(c:x is a command:, __COMMAND)
+def concept q from q ? as question(q) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+def concept x is a 'concept' as isinstance(x, Concept) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)

_concepts_lite.txt

@@ -9,3 +9,5 @@ def concept precedence a > precedence b as set_is_greater_than(BuiltinConcepts.P
 set_isa(c:precedence a > precedence b:, __COMMAND)
 def concept x is a command as set_isa(x, __COMMAND)
 set_isa(c:x is a command:, __COMMAND)
+def concept q from q ? as question(q) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+def concept x is a 'concept' as isinstance(x, Concept) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)

src/cache/BaseCache.py

@@ -147,6 +147,26 @@ class BaseCache:
 
         return nb_to_delete
 
+    def evict_by_key(self, predicate):
+        """
+        Remove entries that matches the predicate
+        :param predicate:
+        :return:
+        """
+        to_delete = []
+        with self._lock:
+            for key in self._cache:
+                if predicate(key):
+                    to_delete.append(key)
+
+            for key in to_delete:
+                del (self._cache[key])
+                try:
+                    self._initialized_keys.remove(key)
+                except KeyError:
+                    pass
+        return len(to_delete)
+
     def clear(self):
         with self._lock:
             self._cache.clear()

src/core/builtin_concepts.py

@@ -18,10 +18,10 @@ class BuiltinConcepts(Enum):
     # processing instructions during sheerka.execute()
     EVAL_BODY_REQUESTED = "eval body"  # to evaluate the body
     EVAL_WHERE_REQUESTED = "eval where"  # to evaluate the where clause
-    RETURN_VALUE_REQUESTED = "return value"  # returns the body of the concept instead of the concept itself
+    RETURN_BODY_REQUESTED = "return body"  # returns the body of the concept instead of the concept itself
     REDUCE_REQUESTED = "reduce"  # remove meaningless error when possible
     EVAL_UNTIL_SUCCESS_REQUESTED = "eval until success"  # PythonEvaluator tries combination until True is found
-    QUESTION_REQUESTED = "question"  # a question is asked
+    EVAL_QUESTION_REQUESTED = "question"  # the user input must be treated as question
 
     # possible actions during sheerka.execute()
     INIT_SHEERKA = "init sheerka"  #
@@ -36,6 +36,7 @@ class BuiltinConcepts(Enum):
     BEFORE_RENDERING = "before rendering"  # activate before the output is rendered
     RENDERING = "rendering"  # rendering the response from sheerka
     AFTER_RENDERING = "after rendering"  # rendering the response from sheerka
+    EVALUATE_SOURCE = "evaluate source"  #
     EVALUATE_CONCEPT = "evaluate concept"  # a concept will be evaluated
     EVALUATING_CONCEPT = "evaluating concept"  # a concept will be evaluated
     EVALUATING_ATTRIBUTE = "evaluating concept attribute"  #
@@ -45,7 +46,7 @@ class BuiltinConcepts(Enum):
     INIT_BNF = "initialize bnf"
     MANAGE_INFINITE_RECURSION = "manage infinite recursion"
     PARSE_CODE = "execute source code"
-    EXEC_CODE = "execute source code"
+    EXEC_CODE = "execute source code"  # to use when executing Python or other language compiled code
     TESTING = "testing"
 
     # builtin attributes
@@ -69,6 +70,7 @@ class BuiltinConcepts(Enum):
     MULTIPLE_ERRORS = "multiple errors"  # filter the result, only keep evaluator in error
     NOT_FOR_ME = "not for me"  # a parser recognize that the entry is not meant for it
     IS_EMPTY = "is empty"  # when a set is empty
+    NO_RESULT = "no result"  # no return value returned
     INVALID_RETURN_VALUE = "invalid return value"  # the return value of an evaluator is not correct
     CONCEPT_ALREADY_DEFINED = "concept already defined"  # when you try to add the same concept twice
     NOP = "no operation"  # no operation concept. Does nothing
@@ -119,10 +121,10 @@ class BuiltinConcepts(Enum):
 BuiltinUnique = [
     BuiltinConcepts.EVAL_BODY_REQUESTED,
     BuiltinConcepts.EVAL_WHERE_REQUESTED,
-    BuiltinConcepts.RETURN_VALUE_REQUESTED,
+    BuiltinConcepts.RETURN_BODY_REQUESTED,
     BuiltinConcepts.REDUCE_REQUESTED,
     BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED,
-    BuiltinConcepts.QUESTION_REQUESTED,
+    BuiltinConcepts.EVAL_QUESTION_REQUESTED,
 
     BuiltinConcepts.INIT_SHEERKA,
     BuiltinConcepts.PROCESS_INPUT,
@@ -341,8 +343,11 @@ class ParserResultConcept(Concept):
         if not isinstance(other, ParserResultConcept):
             return False
 
+        self_parser_name = self.get_parser_name(self.parser)
+        other_parser_name = self.get_parser_name(other.parser)
+
         return self.source == other.source and \
-               self.parser == other.parser and \
+               self_parser_name == other_parser_name and \
                self.body == other.body and \
                self.try_parsed == other.try_parsed
 
@@ -365,6 +370,11 @@ class ParserResultConcept(Concept):
     def parser(self):
         return self.get_value("parser")
 
+    @staticmethod
+    def get_parser_name(parser):
+        from parsers.BaseParser import BaseParser
+        return parser.name if isinstance(parser, BaseParser) else str(parser)
+
 
 class InvalidReturnValueConcept(Concept):
     """

src/core/builtin_helpers.py

@@ -6,9 +6,15 @@ from core.ast.nodes import CallNodeConcept
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, NotInit, ConceptParts
+from core.tokenizer import Keywords
+# from evaluators.BaseEvaluator import BaseEvaluator
 from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode
 from parsers.BaseParser import BaseParser, ErrorNode
 
+PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
+EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION,
+                            BuiltinConcepts.AFTER_EVALUATION]
+
 
 def is_same_success(context, return_values):
     """
@@ -105,7 +111,7 @@ def expect_one(context, return_values):
                 sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=successful_results),
                 parents=return_values)
 
-    # only errors, i cannot help you
+    # number_of_successful == 0, only errors, i cannot help you
     if context.logger and context.logger.isEnabledFor(logging.DEBUG):
         context.log(f"Too many errors found by expect_one()", context.who)
         for s in successful_results:
@@ -170,88 +176,11 @@ def only_successful(context, return_values):
         sheerka.new(BuiltinConcepts.ONLY_SUCCESSFUL, body=successful_results),
         parents=return_values)
 
-#
-# def remove_ambiguity(context, return_values):
-#     """
-#     When multiple concepts are matching, try to find the one(s) which is/are the more accurate
-#     :param context:
-#     :param return_values:
-#     :return:
-#     """
-#     # example :
-#     # Concept("x is a y", PRE=QUESTION)
-#     # Concept("x is a y")
-#     # Concept("x is a command")
-#     # with the source 'foo is a command'
-#     #   The first one do not respect the pre condition (assuming that QUESTION is not in context)
-#     #   The second one is Ok, but the third is more specific (as it's a concept that explicitly asks for 'command')
-#     #   The third one will remain
-#     if not return_values:
-#         return return_values
-#
-#     sheerka = context.sheerka
-#     by_number_of_vars = {}  # sorted by number of variables
-#     to_keep = []  # return values values that are not eligible (ex non parser result)
-#     passed_validation = []
-#
-#     # sort by number of variables
-#     # The less variables there are, the more accurate is the concept
-#     for r in return_values:
-#         if (r.status and
-#             sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT) and
-#             isinstance(r.body.body, Concept)):
-#             by_number_of_vars.setdefault(len(r.body.body.metadata.variables), []).append(r)
-#         else:
-#             to_keep.append(r)
-#
-#         # Check the concepts, starting with the ones that have the less variables
-#     # Once a concept with a given number of variables is found, we do not process the concepts with an higher
-#     #   number of variables
-#     for nb_vars in sorted(by_number_of_vars.keys()):
-#         for r in by_number_of_vars[nb_vars]:
-#             concept = r.body.body
-#             if concept.metadata.pre is None or concept.metadata.pre.strip() == "":
-#                 passed_validation.append(r)
-#             else:
-#                 evaluated = context.sheerka.evaluate_concept(context, concept, metadata=["pre"])
-#                 if evaluated.key == concept.key and evaluated.get_value(ConceptParts.PRE):
-#                     passed_validation.append(r)
-#         if len(passed_validation) > 0:
-#             break
-#
-#     # Nothing was filtered, return the original return values
-#     if len(passed_validation) + len(to_keep) == len(return_values):
-#         return return_values  # nothing to filter
-#
-#     # All return_values fail, return empty list
-#     if len(passed_validation) == 0:
-#         return sheerka.ret(
-#             context.who,
-#             True,
-#             sheerka.new(BuiltinConcepts.FILTERED,
-#                         body=to_keep,
-#                         iterable=return_values,
-#                         predicate="remove_ambiguity(context, iterable)"),
-#             parents=return_values)
-#
-#     # Final check, we consider that the concepts that match a PRE condition are better than concepts with no condition
-#     by_condition_complexity = {}
-#     for r in passed_validation:
-#         by_condition_complexity.setdefault(get_condition_complexity(r.body.body, "pre"), []).append(r)
-#
-#     return sheerka.ret(
-#         context.who,
-#         True,
-#         sheerka.new(BuiltinConcepts.FILTERED,
-#                     body=to_keep + by_condition_complexity[max(by_condition_complexity.keys())],
-#                     iterable=return_values,
-#                     predicate="remove_ambiguity(context, iterable)"),
-#         parents=return_values)
-
 
 def resolve_ambiguity(context, concepts):
     """
-    From the list of concept, elect the one(s) that best suit(s) the context
+    From the list of concepts, elect the one(s) that best suit(s) the context
+    Use the PRE metadata to choose the correct concepts
     :param context:
     :param concepts:
     :return:
@@ -265,6 +194,9 @@ def resolve_ambiguity(context, concepts):
 
     remaining_concepts = []
     for complexity in sorted(by_complexity.keys(), reverse=True):
+        if complexity == 0:
+            remaining_concepts.extend(by_complexity[complexity])
+        else:
             for c in by_complexity[complexity]:
                 evaluated = context.sheerka.evaluate_concept(context, c, metadata=["pre"])
                 if evaluated.key == c.key:
@@ -349,30 +281,48 @@ def only_parsers_results(context, return_values):
         parents=return_values)
 
 
-def parse_unrecognized(context, source, parsers):
+def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_func=None):
     """
     Try to recognize concepts or code from source using the given parsers
     :param context:
     :param source:
     :param parsers:
+    :param who: who is asking the parsing ?
+    :param prop: Extra info, when parsing a property
+    :param filter_func: filter function to call is provided
     :return:
     """
-    steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
     sheerka = context.sheerka
 
-    with context.push(BuiltinConcepts.PARSING, source, desc=f"Parsing unrecognized '{source}'") as sub_context:
+    if prop:
-        # disable all parsers but the following ones
+        action_context = {"prop": prop, "source": source}
+        desc = f"Parsing attribute '{prop}'"
+    else:
+        action_context = source
+        desc = f"Parsing '{source}'"
+
+    with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context:
+        # disable all parsers but the requested ones
+        if parsers != "all":
             sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False)
             for parser in parsers:
                 sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True)
 
+        if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE):
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+
         sub_context.add_inputs(source=source)
-        to_parse = sheerka.ret(
+        to_parse = sheerka.ret(context.who,
-            context.who,
                                True,
                                sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
-        res = sheerka.execute(sub_context, to_parse, steps)
+        res = sheerka.execute(sub_context, to_parse, PARSE_STEPS)
+
+        if filter_func:
+            res = filter_func(sub_context, res)
+
         sub_context.add_values(return_values=res)
+        if not hasattr(res, "__iter__"):
+            return res
 
         # discard Python response if accepted by AtomNode
         is_concept = False
@@ -383,13 +333,65 @@ def parse_unrecognized(context, source, parsers):
         if not is_concept:
             return res
 
-        filtered = []
+        no_python = []
         for r in res:
             if r.who == "parsers.Python":
                 continue
-            filtered.append(r)
+            no_python.append(r)
 
-        return filtered
+        return no_python
+
+
+def evaluate(context,
+             source,
+             evaluators="all",
+             desc=None,
+             eval_body=True,
+             eval_where=True,
+             expect_success=False,
+             stm=None):
+    """
+
+    :param context:
+    :param source:
+    :param evaluators:
+    :param desc:
+    :param eval_body:
+    :param eval_where:
+    :param expect_success:
+    :param stm: short term memories entries
+    :return:
+    """
+
+    sheerka = context.sheerka
+    desc = desc or f"Eval '{source}'"
+    with context.push(BuiltinConcepts.EVALUATE_SOURCE, source, desc=desc) as sub_context:
+        if eval_body:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+
+        if eval_where:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+
+        if expect_success:
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+
+        if stm:
+            for k, v in stm.items():
+                sub_context.add_to_short_term_memory(k, v)
+
+        # disable all evaluators but the requested ones
+        if evaluators != "all":
+            from evaluators.BaseEvaluator import BaseEvaluator
+            sub_context.add_preprocess(BaseEvaluator.PREFIX + "*", enabled=False)
+            for evaluator in evaluators:
+                sub_context.add_preprocess(BaseEvaluator.PREFIX + evaluator, enabled=True)
+
+        user_input = sheerka.ret(context.who, True, sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
+        ret = sheerka.execute(sub_context, [user_input], EVAL_STEPS)
+        sub_context.add_values(return_values=ret)
+
+        return ret
 
 
 def get_lexer_nodes(return_values, start, tokens):

src/core/concept.py

@@ -119,7 +119,8 @@ class Concept:
         self.original_definition_hash = None  # concept hash before any alteration of the metadata
 
     def __repr__(self):
-        return f"({self.metadata.id}){self.metadata.name}"
+        text = f"({self.metadata.id}){self.metadata.name}"
+        return text + " (" + self.metadata.pre + ")" if self.metadata.pre else text
 
     def __eq__(self, other):
 
@@ -641,15 +642,25 @@ class CV:
     Test class that tests all the values (not the metadata, so not the properties) of a concept
     """
 
-    def __init__(self, concept, body, **kwargs):
+    def __init__(self, concept, **kwargs):
         self.concept_key = concept.key if isinstance(concept, Concept) else concept
         self.concept = concept if isinstance(concept, Concept) else None
-        self.values = kwargs
+        self.values = {}
-        self.values[ConceptParts.BODY] = body
+        for k, v in kwargs.items():
+            try:
+                concept_part = ConceptParts(k)
+                self.values[concept_part] = v
+            except ValueError:
+                self.values[k] = v
 
     def __eq__(self, other):
         if isinstance(other, Concept):
-            return self.concept_key == other.key and self.values == other.values
+            if self.concept_key != other.key:
+                return False
+            for k, v in self.values.items():
+                if self.values[k] != other.get_value(k):
+                    return False
+            return True
 
         if not isinstance(other, CV):
             return False

src/core/sheerka/ExecutionContext.py

@@ -4,6 +4,7 @@ import time
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka.services.SheerkaExecute import NO_MATCH
+from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory
 from core.sheerka_logger import get_logger
 from sdp.sheerkaDataProvider import Event
 
@@ -11,11 +12,13 @@ DEBUG_TAB_SIZE = 4
 
 PROPERTIES_TO_SERIALIZE = ("_id",
                            "_bag",
+                           "_children",
                            "_start",
                            "_stop",
                            "who",
+                           "action",
+                           "action_context",
                            "desc",
-                           "children",
                            "inputs",
                            "values",
                            "obj",
@@ -46,16 +49,20 @@ class ExecutionContext:
                  desc: str = None,
                  logger=None,
                  global_hints=None,
-                 global_errors=None,
+                 errors=None,
                  **kwargs):
 
-        self._parent = None
         self._id = ExecutionContext.get_id(event.get_digest()) if event else None
+        self._parent = None
+        self._children = []
         self._tab = ""
         self._bag = {}  # context variables
         self._start = 0  # when the execution starts (to measure elapsed time)
         self._stop = 0  # when the execution stops (to measure elapses time)
+        self._logger = logger
         self._format_instructions = None  # how to print the execution context
+        self._stat_log = get_logger("stats")
+        self._show_stats = False
 
         self.who = who  # who is asking
         self.event = event  # what was the (original) trigger
@@ -63,26 +70,24 @@ class ExecutionContext:
         self.action = action
         self.action_context = action_context
         self.desc = desc  # human description of what is going on
-        self.children = []
         self.preprocess = None
-        self.logger = logger
+        self.stm = False  # True if the context has short term memory entries
-        self.local_hints = set()
+
+        self.private_hints = set()
+        self.protected_hints = set()
         self.global_hints = set() if global_hints is None else global_hints
-        self.global_errors = [] if global_errors is None else global_errors
+        self.errors = [] if errors is None else errors  # error are global
 
-        self.inputs = {}  # what was the parameters of the execution context
+        self.inputs = {}  # what were the parameters of the execution context
         self.values = {}  # what was produced by the execution context
-
         self.obj = kwargs.pop("obj", None)  # current obj we are working on
+
         self.concepts = kwargs.pop("concepts", {})  # known concepts specific to this context
 
         # update the other elements
         for k, v in kwargs.items():
             self._bag[k] = v
 
-        self.stat_log = get_logger("stats")
-        self.show_stats = False
-
     @property
     def elapsed(self):
         if self._start == 0:
@@ -96,10 +101,22 @@ class ExecutionContext:
         dt = nano_sec / 1e6
         return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
 
+    @property
+    def logger(self):
+        return self._logger
+
     @property
     def id(self):
         return self._id
 
+    @property
+    def achildren(self):
+        """
+        I prefixed with an 'a' to make it appear on the top when debugging
+        :return:
+        """
+        return self._children
+
     def __getattr__(self, item):
         if item in self._bag:
             return self._bag[item]
@@ -112,9 +129,12 @@ class ExecutionContext:
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.stm:
+            self.sheerka.services[SheerkaShortTermMemory.NAME].remove_context(self)
+
         self._stop = time.time_ns()
-        if self.show_stats:
+        if self._show_stats:
-            self.stat_log.debug(f"[{self._id:2}]" + self._tab + "Execution time: " + self.elapsed_str)
+            self._stat_log.debug(f"[{self._id:2}]" + self._tab + "Execution time: " + self.elapsed_str)
 
     def __repr__(self):
         msg = f"ExecutionContext(who={self.who}, id={self._id}, action={self.action}, context={self.action_context}"
@@ -136,7 +156,7 @@ class ExecutionContext:
             return False
 
         for prop in PROPERTIES_TO_SERIALIZE:
-            if prop == "who":
+            if prop in ("who", "action", "action_context"):
                 value = str(getattr(self, prop))
                 other_value = str(getattr(other, prop))
             else:
@@ -150,7 +170,7 @@ class ExecutionContext:
 
     def push(self, action: BuiltinConcepts, action_context, who=None, desc=None, logger=None, **kwargs):
         who = who or self.who
-        logger = logger or self.logger
+        logger = logger or self._logger
         _kwargs = {"obj": self.obj, "concepts": self.concepts}
         _kwargs.update(self._bag)
         _kwargs.update(kwargs)
@@ -163,14 +183,14 @@ class ExecutionContext:
             desc,
             logger,
             self.global_hints,
-            self.global_errors,
+            self.errors,
             **_kwargs)
         new._parent = self
         new._tab = self._tab + " " * DEBUG_TAB_SIZE
         new.preprocess = self.preprocess
-        new.local_hints.update(self.local_hints)
+        new.protected_hints.update(self.protected_hints)
 
-        self.children.append(new)
+        self._children.append(new)
         return new
 
     def add_preprocess(self, name, **kwargs):
@@ -194,6 +214,23 @@ class ExecutionContext:
             self.values[k] = v
         return self
 
+    def add_to_short_term_memory(self, key, concept):
+        """
+        Add a concept to the short term memory (relative to the current execution context)
+        :param key:
+        :param concept:
+        :return:
+        """
+        self.sheerka.add_to_short_term_memory(self, key, concept)
+
+    def get_from_short_term_memory(self, key):
+        """
+
+        :param key:
+        :return:
+        """
+        return self.sheerka.get_from_short_term_memory(self, key)
+
     def get_concept(self, key):
         # search in obj
         if isinstance(self.obj, Concept):
@@ -234,44 +271,43 @@ class ExecutionContext:
         return self.sheerka.new(key, **kwargs)
 
     def log_new(self):
-        if self.logger and not self.logger.disabled:
+        if self._logger and not self._logger.disabled:
-            self.logger.debug(f"[{self._id:2}]" + self._tab + str(self))
+            self._logger.debug(f"[{self._id:2}]" + self._tab + str(self))
-            self.show_stats = True
+            self._show_stats = True
 
     def log(self, message, who=None):
-        if self.logger and not self.logger.disabled:
+        if self._logger and not self._logger.disabled:
-            self.logger.debug(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
+            self._logger.debug(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
 
     def log_error(self, message, who=None, exc=None):
-        self.global_errors.append(exc or message)
+        self.errors.append(exc or message)
-        if self.logger and not self.logger.disabled:
+        if self._logger and not self._logger.disabled:
-            self.logger.exception(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
+            self._logger.exception(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
 
     def log_result(self, return_values):
-        if not self.logger or not self.logger.isEnabledFor(logging.DEBUG):
+        if not self._logger or not self._logger.isEnabledFor(logging.DEBUG):
             return
 
         if len(return_values) == 0:
-            self.logger.debug(self._tab + "No return value")
+            self._logger.debug(self._tab + "No return value")
 
         for r in return_values:
             to_str = self.return_value_to_str(r)
-            self.logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
+            self._logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
     def get_parent(self):
         return self._parent
 
     def in_context(self, concept_key):
-        if concept_key in self.local_hints:
+        return concept_key in self.protected_hints or \
-            return True
+               concept_key in self.global_hints or \
-
+               concept_key in self.private_hints
-        if concept_key in self.global_hints:
-            return True
-
-        return False
 
     def in_current_context(self, concept_key):
-        return concept_key in self.local_hints
+        return concept_key in self.protected_hints or concept_key in self.private_hints
+
+    def in_private_context(self, concept_key):
+        return concept_key in self.private_hints
 
     @staticmethod
     def _is_return_value(obj):
@@ -336,7 +372,7 @@ class ExecutionContext:
             bag["bag." + k] = v
         for prop in ("id", "who", "action", "desc", "obj", "inputs", "values", "concepts"):
             bag[prop] = getattr(self, prop)
-        bag["action"] = self.action_context
+        bag["context"] = self.action_context
         for prop in ("desc", "obj", "inputs", "values", "concepts"):
             bag[prop] = getattr(self, prop)
         bag["status"] = self.get_status()

src/core/sheerka/Sheerka.py

@@ -21,6 +21,14 @@ from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 
 BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
 EXIT_COMMANDS = ("quit", "exit", "bye")
+EXECUTE_STEPS = [
+    BuiltinConcepts.BEFORE_PARSING,
+    BuiltinConcepts.PARSING,
+    BuiltinConcepts.AFTER_PARSING,
+    BuiltinConcepts.BEFORE_EVALUATION,
+    BuiltinConcepts.EVALUATION,
+    BuiltinConcepts.AFTER_EVALUATION
+]
 
 
 @dataclass
@@ -124,11 +132,11 @@ class Sheerka(Concept):
 
     @property
     def concepts_grammars(self):
-        return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
+        return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
 
     @property
     def chicken_and_eggs(self):
-        return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
+        return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
 
     def bind_service_method(self, bound_method, has_side_effect, as_name=None):
         """
@@ -406,16 +414,7 @@ class Sheerka(Concept):
             user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
             reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
 
-            steps = [
+            ret = self.execute(execution_context, [user_input, reduce_requested], EXECUTE_STEPS)
-                BuiltinConcepts.BEFORE_PARSING,
-                BuiltinConcepts.PARSING,
-                BuiltinConcepts.AFTER_PARSING,
-                BuiltinConcepts.BEFORE_EVALUATION,
-                BuiltinConcepts.EVALUATION,
-                BuiltinConcepts.AFTER_EVALUATION
-            ]
-
-            ret = self.execute(execution_context, [user_input, reduce_requested], steps)
             execution_context.add_values(return_values=ret)
 
             if self.cache_manager.is_dirty:

src/core/sheerka/services/SheerkaAdmin.py

@@ -49,9 +49,11 @@ class SheerkaAdmin(BaseService):
 
         try:
             start = time.time_ns()
+            nb_lines = 0
             self.sheerka.during_restore = True
             with open(concept_file, "r") as f:
                 for line in f.readlines():
+                    nb_lines += 1
                     line = line.strip()
                     if line == "" or line.startswith("#"):
                         continue
@@ -65,7 +67,7 @@ class SheerkaAdmin(BaseService):
             nano_sec = stop - start
             dt = nano_sec / 1e6
             elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
-            print(f"Execution time: {elapsed}")
+            print(f"Imported {nb_lines} line(s) in {elapsed}.")
         except IOError:
             pass
 

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -1,9 +1,13 @@
+from dataclasses import dataclass
+
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import expect_one, only_successful
+from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
+from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit
+from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
 from core.utils import unstr_concept
+from parsers.ExpressionParser import ExpressionParser, TrueifyVisitor
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,
@@ -11,6 +15,15 @@ CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.AFTER_EVALUATION]
 
 
+@dataclass
+class WhereClauseDef:
+    concept: Concept  # concept on which the where clause is applied
+    clause: str  # original where clause
+    trueified: str  # modified where clause (where unresolvable variables are removed)
+    prop: str  # variable to test
+    compiled: object  # trueified where clause Python compiled
+
+
 class SheerkaEvaluateConcept(BaseService):
     NAME = "EvaluateConcept"
 
@@ -61,6 +74,112 @@ class SheerkaEvaluateConcept(BaseService):
         """
         return concept.get_value(ConceptParts.RET) if ConceptParts.RET in concept.values else concept
 
+    @staticmethod
+    def get_needed_metadata(concept, concept_part, check_vars, check_body):
+        """
+        Check if the concept_part has to be evaluated
+        It also checks if the variables and the body need to be evaluated prior to it
+        :param concept:
+        :param concept_part:
+        :param check_vars:
+        :param check_body:
+        :return:
+        """
+        ret = []
+        vars_needed = False
+        body_needed = False
+
+        if concept_part in concept.compiled and concept.compiled[concept_part] is not None:
+            concept_part_source = getattr(concept.metadata, concept_part.value)
+
+            assert concept_part_source is not None
+
+            tokens = [t.str_value for t in Tokenizer(concept_part_source)]
+
+            if check_vars:
+                for var_name in (v[0] for v in concept.metadata.variables):
+                    if var_name in tokens:
+                        vars_needed = True
+                        ret.append("variables")
+                        break
+
+            if check_body and "self" in tokens:
+                body_needed = True
+                ret.append("body")
+
+            ret.append(concept_part.value)
+
+        return ret, vars_needed, body_needed
+
+    @staticmethod
+    def get_where_clause_def(context, concept, var_name):
+        """
+        Returns the compiled code to be executed
+        :param context:
+        :param concept:
+        :param var_name:
+        :return:
+        """
+        if concept.metadata.where is None or concept.metadata.where.strip() == "":
+            return None
+
+        ret = ExpressionParser().parse(context, ParserInput(concept.metadata.where))
+        if not ret.status:
+            # TODO: manage invalid where clause
+            return None
+        expr = ret.body.body
+
+        to_trueify = [v[0] for v in concept.metadata.variables if v[0] != var_name]
+        trueified_where = str(TrueifyVisitor(to_trueify, [var_name]).visit(expr))
+
+        tokens = [t.str_value for t in Tokenizer(trueified_where)]
+        if var_name in tokens:
+            compiled = None
+            try:
+                compiled = compile(trueified_where, "<where clause>", "eval")
+            except Exception:
+                pass
+            return WhereClauseDef(concept, concept.metadata.where, trueified_where, var_name, compiled)
+        else:
+            return None
+
+    def apply_where_clause(self, context, where_clause_def, return_values):
+        """
+        Apply intermediate where clause when evaluating concept variables
+        :param context:
+        :param where_clause_def:
+        :param return_values:
+        :return:
+        """
+        ret = []
+        for r in [r for r in return_values if r.status]:
+            if where_clause_def.compiled:
+                try:
+                    if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}):
+                        ret.append(r)
+                except NameError:
+                    ret.append(r)  # it cannot be solved unitary, let's give a chance to the global where condition
+            else:
+                # it means that the where condition is an expression that needs to be executed
+                evaluation_res = evaluate(context,
+                                          where_clause_def.trueified,
+                                          desc=f"Apply where clause on '{where_clause_def.prop}'",
+                                          expect_success=True,
+                                          stm={where_clause_def.prop: r.body})
+                one_res = expect_one(context, evaluation_res)
+                if one_res.status:
+                    value = context.sheerka.objvalue(one_res)
+                    if isinstance(value, bool) and value:
+                        ret.append(r)
+
+        if len(ret) > 0:
+            return ret
+
+        return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED,
+                                body=where_clause_def.clause,
+                                concept=where_clause_def.concept,
+                                prop=where_clause_def.prop)
+
     def manage_infinite_recursion(self, context):
         """
         We look for the fist parent that has a body that means something
@@ -105,7 +224,6 @@ class SheerkaEvaluateConcept(BaseService):
                 return self.sheerka.resolve(identifier)
             return None
 
-        steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
         for part_key in ConceptParts:
             if part_key in concept.compiled:
                 continue
@@ -122,16 +240,12 @@ class SheerkaEvaluateConcept(BaseService):
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
                     concept.compiled[part_key] = concept_found
                 else:
-                    with context.push(BuiltinConcepts.INIT_COMPILED,
+                    res = parse_unrecognized(context,
-                                      {"part": part_key, "source": source},
+                                             source,
-                                      desc=f"Initializing *compiled* for {part_key}") as sub_context:
+                                             parsers="all",
-                        sub_context.add_inputs(source=source)
+                                             prop=part_key,
-                        to_parse = self.sheerka.ret(context.who, True,
+                                             filter_func=only_successful)
-                                                    self.sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
+                    concept.compiled[part_key] = res.body.body if is_only_successful(res) else res
-                        res = self.sheerka.execute(sub_context, to_parse, steps)
-                        only_success = only_successful(sub_context, res)
-                        concept.compiled[part_key] = only_success.body.body if is_only_successful(only_success) else res
-                        sub_context.add_values(return_values=res)
 
         for var_name, default_value in concept.metadata.variables:
             if var_name in concept.compiled:
@@ -148,22 +262,36 @@ class SheerkaEvaluateConcept(BaseService):
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
                     concept.compiled[var_name] = concept_found
                 else:
-                    with context.push(BuiltinConcepts.INIT_COMPILED,
+                    res = parse_unrecognized(context,
-                                      {"property": var_name, "source": default_value},
+                                             default_value,
-                                      desc=f"Initializing *compiled* for property {var_name}") as sub_context:
+                                             parsers="all",
-                        sub_context.add_inputs(source=default_value)
+                                             prop=var_name,
-                        to_parse = self.sheerka.ret(context.who, True,
+                                             filter_func=only_successful)
-                                                    self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value))
+                    concept.compiled[var_name] = res.body.body if is_only_successful(res) else res
-                        res = self.sheerka.execute(sub_context, to_parse, steps)
-                        only_success = only_successful(sub_context, res)
-                        concept.compiled[var_name] = only_success.body.body if is_only_successful(only_success) else res
-                        sub_context.add_values(return_values=res)
 
         # Updates the cache of concepts when possible
         if self.sheerka.has_id(concept.id):
             self.sheerka.get_by_id(concept.id).compiled = concept.compiled
 
-    def resolve(self, context, to_resolve, current_prop, current_concept, force_evaluation, expect_success):
+    def resolve(self,
+                context,
+                to_resolve,
+                current_prop,
+                current_concept,
+                force_evaluation,
+                expect_success,
+                where_clause_def):
+        """
+        Resolve a variable or a Concept
+        :param context: current execution context
+        :param to_resolve: Concept or list of ReturnValueConcept to resolve
+        :param current_prop: current property or ConceptPart
+        :param current_concept: current concept
+        :param force_evaluation: Force body evaluation
+        :param expect_success: for PythonEvaluator, try all possibilities to find a positive result
+        :param where_clause_def: intermediate where clause for variables
+        :return:
+        """
 
         def get_path(context_, prop_name):
             concept_name = f'"{context_.action_context.name}"' if isinstance(context_.action_context, Concept) \
@@ -195,10 +323,11 @@ class SheerkaEvaluateConcept(BaseService):
                           path=path) as sub_context:
 
             if force_evaluation:
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             if expect_success:
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
 
             # when it's a concept, evaluate it
             if isinstance(to_resolve, Concept) and \
@@ -212,9 +341,22 @@ class SheerkaEvaluateConcept(BaseService):
 
             # otherwise, execute all return values to find out what is the value
             else:
+                # update short term memory with current concept variables
+                if current_concept:
+                    for var in current_concept.metadata.variables:
+                        value = current_concept.get_value(var[0])
+                        if value != NotInit:
+                            sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0]))
                 use_copy = [r for r in to_resolve] if hasattr(to_resolve, "__iter__") else to_resolve
                 r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS)
 
+                if where_clause_def:
+                    # apply intermediate where clause
+                    r = self.apply_where_clause(context, where_clause_def, r)
+
+                if self.sheerka.isinstance(r, BuiltinConcepts.CONDITION_FAILED):
+                    return r
+                else:
                     one_r = expect_one(context, r)
                     sub_context.add_values(return_values=one_r)
                     if one_r.status:
@@ -228,7 +370,14 @@ class SheerkaEvaluateConcept(BaseService):
                                       concept=current_concept,
                                       property_name=current_prop)
 
-    def resolve_list(self, context, list_to_resolve, current_prop, current_concept, force_evaluation, expect_success):
+    def resolve_list(self,
+                     context,
+                     list_to_resolve,
+                     current_prop,
+                     current_concept,
+                     force_evaluation,
+                     expect_success,
+                     where_clause_def):
         """When dealing with a list, there are two possibilities"""
         # It may be a list of ReturnValueConcept to execute (always the case for metadata)
         # or a list of single values (may be the case for properties)
@@ -242,7 +391,8 @@ class SheerkaEvaluateConcept(BaseService):
                                 current_prop,
                                 current_concept,
                                 force_evaluation,
-                                expect_success)
+                                expect_success,
+                                where_clause_def)
 
         res = []
         for to_resolve in list_to_resolve:
@@ -253,7 +403,13 @@ class SheerkaEvaluateConcept(BaseService):
                                         concept=current_concept,
                                         property_name=current_prop)
 
-            r = self.resolve(context, to_resolve, current_prop, current_concept, force_evaluation, expect_success)
+            r = self.resolve(context,
+                             to_resolve,
+                             current_prop,
+                             current_concept,
+                             force_evaluation,
+                             expect_success,
+                             where_clause_def)
             if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
                 return r
             res.append(r)
@@ -263,7 +419,7 @@ class SheerkaEvaluateConcept(BaseService):
     def evaluate_concept(self, context, concept: Concept, eval_body=False, metadata=None):
         """
         Evaluation a concept
-        It means that if the where clause is True, will evaluate the body
+        ie : resolve its body
         :param context:
         :param concept:
         :param eval_body:
@@ -275,7 +431,7 @@ class SheerkaEvaluateConcept(BaseService):
             return concept
 
         # I cannot use cache because of concept like 'number'.
-        # They don't have variables, but their values change every time they are instanciated
+        # They don't have variables, but their values change every time they are instantiated
         # TODO: Need to find a way to cache despite of them
         # need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)
         # if need_body and len(concept.metadata.variables) == 0 and context.sheerka.has_id(concept.id):
@@ -290,11 +446,11 @@ class SheerkaEvaluateConcept(BaseService):
 
             if eval_body:
                 # ask for body evaluation
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             # auto evaluate commands
             if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)):
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             self.initialize_concept_asts(sub_context, concept)
 
@@ -307,12 +463,15 @@ class SheerkaEvaluateConcept(BaseService):
                     for var_name in (v for v in concept.variables() if v in concept.compiled):
                         prop_ast = concept.compiled[var_name]
 
+                        w_clause = self.get_where_clause_def(context, concept, var_name)
+                        # TODO, manage when the where clause cannot be parsed
+
                         if isinstance(prop_ast, list):
                             # Do not send the current concept for the properties
-                            resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False)
+                            resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False, w_clause)
                         else:
                             # Do not send the current concept for the properties
-                            resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False)
+                            resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False, w_clause)
 
                         if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
                             resolved.set_value("concept", concept)  # since current concept was not sent
@@ -347,7 +506,8 @@ class SheerkaEvaluateConcept(BaseService):
                                             part_key,
                                             concept,
                                             force_concept_eval,
-                                            expect_success)
+                                            expect_success,
+                                            None)
 
                     # 'FATAL' error is detected, let's stop
                     if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
@@ -411,40 +571,3 @@ class SheerkaEvaluateConcept(BaseService):
                 to_eval.append("body")
 
         return to_eval
-
-    @staticmethod
-    def get_needed_metadata(concept, concept_part, check_vars, check_body):
-        """
-        Check if the concept_part has to be evaluated
-        It also checks if the variables and the body need to be evaluated prior to it
-        :param concept:
-        :param concept_part:
-        :param check_vars:
-        :param check_body:
-        :return:
-        """
-        ret = []
-        vars_needed = False
-        body_needed = False
-
-        if concept_part in concept.compiled and concept.compiled[concept_part] is not None:
-            concept_part_source = getattr(concept.metadata, concept_part.value)
-
-            assert concept_part_source is not None
-
-            tokens = [t.str_value for t in Tokenizer(concept_part_source)]
-
-            if check_vars:
-                for var_name in (v[0] for v in concept.metadata.variables):
-                    if var_name in tokens:
-                        vars_needed = True
-                        ret.append("variables")
-                        break
-
-            if check_body and "self" in tokens:
-                body_needed = True
-                ret.append("body")
-
-            ret.append(concept_part.value)
-
-        return ret, vars_needed, body_needed

src/core/sheerka/services/SheerkaExecute.py

@@ -362,7 +362,7 @@ class SheerkaExecute(BaseService):
                                     if not isinstance(results, list):
                                         results = [results]
                                     for result in results:
-                                        if result.body:
+                                        if result.body != BuiltinConcepts.NO_RESULT:
                                             evaluated_items.append(result)
                                         to_delete.extend(result.parents)
                                     sub_context.add_values(return_values=results)

src/core/sheerka/services/SheerkaFilter.py

@@ -404,11 +404,11 @@ class SheerkaFilter(BaseService):
             yield item
 
     @staticmethod
-    def pipe_recurse(iterable, depth, prop_name="children", when=None):
+    def pipe_recurse(iterable, depth, prop_name="_children", when=None):
         """
         When printing an object that has sub properties,
         indicate the depth of recursion to apply to a specific properties
-        Quick and dirty version because the prop name is not taken from the item (but set to 'children' by default)
+        Quick and dirty version because the prop name is not taken from the item (but set to '_children' by default)
         :param iterable:
         :param depth:
         :param prop_name:

src/core/sheerka/services/SheerkaQuestion.py

@@ -0,0 +1,40 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.sheerka_service import BaseService
+
+
+class SheerkaQuestion(BaseService):
+    NAME = "Question"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.question, False)
+        self.sheerka.bind_service_method(self.is_question, False)
+
+    def question(self, context, q):
+        """
+        Evaluate q in the context in a question
+        :param context:
+        :param q:
+        :return:
+        """
+
+        if isinstance(q, Concept):
+            with context.push(BuiltinConcepts.EVALUATE_CONCEPT, q, desc=f"Evaluating question '{q}'") as sub_context:
+                sub_context.global_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+                sub_context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+                sub_context.global_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
+
+                evaluated = self.sheerka.evaluate_concept(sub_context, q)
+
+            return evaluated
+
+    def is_question(self, context):
+        """
+        Returns True if a question is asked
+        :return:
+        """
+        return context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)

src/core/sheerka/services/SheerkaResultManager.py

@@ -119,7 +119,7 @@ class SheerkaResultConcept(BaseService):
             for e in lst:
                 yield e
 
-                if e.children:
+                if e._children:
-                    yield from _yield_result(e.children)
+                    yield from _yield_result(e._children)
 
         return _yield_result([execution_context])

src/core/sheerka/services/SheerkaSetsManager.py

@@ -267,7 +267,7 @@ for x in xx__concepts__xx:
                           {"ids": ids},
                           desc=f"Evaluating concepts of a set") as sub_context:
             sub_context.add_inputs(ids=ids)
-            sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
             errors = []
             for element_id in ids:
                 concept = self.sheerka.get_by_id(element_id)

src/core/sheerka/services/SheerkaShortTermMemory.py

@@ -0,0 +1,37 @@
+from cache.ListIfNeededCache import ListIfNeededCache
+from core.sheerka.services.sheerka_service import BaseService
+
+
+class SheerkaShortTermMemory(BaseService):
+    NAME = "ShortTermMemory"
+
+    SHORT_TERM_MEMORY_ENTRY = "ShortTermMemory:Objects"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.objects = ListIfNeededCache()
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.get_from_short_term_memory, False)
+        self.sheerka.bind_service_method(self.add_to_short_term_memory, True)
+        self.sheerka.cache_manager.register_cache(self.SHORT_TERM_MEMORY_ENTRY, self.objects, persist=False)
+
+    def get_from_short_term_memory(self, context, key):
+        while True:
+            key_to_use = (str(context.id) if context else "") + ":" + key
+            if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_MEMORY_ENTRY, key_to_use)) is not None:
+                return obj
+
+            if context is None:
+                return None
+
+            context = context.get_parent()
+
+    def add_to_short_term_memory(self, context, key, concept):
+        if context:
+            context.stm = True
+        key_to_use = (str(context.id) if context else "") + ":" + key
+        return self.sheerka.cache_manager.put(self.SHORT_TERM_MEMORY_ENTRY, key_to_use, concept)
+
+    def remove_context(self, context):
+        self.objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))

src/evaluators/AddConceptEvaluator.py

@@ -1,14 +1,11 @@
-from core.ast.nodes import python_to_concept
+import core.utils
 from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
-from core.builtin_helpers import get_names
 from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
 from core.tokenizer import TokenKind, Tokenizer
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.BaseParser import NotInitializedNode
 from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
 from parsers.DefaultParser import DefConceptNode, NameNode
-from parsers.PythonParser import PythonNode
-import core.utils
 
 
 class ConceptOrRuleNameVisitor(ParsingExpressionVisitor):
@@ -132,30 +129,6 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
             variables = filter(lambda x: x in concept_name, names)
             return set(variables)
 
-        #
-        # Case of python code
-        #
-        if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode):
-            if len(concept_name) > 1:
-                # tokens from ParserResult or source from python node
-                variables = set()
-                tokens = ret_value.value.tokens or list(Tokenizer(ret_value.value.value.source))
-                tokens = [t.str_value for t in tokens]
-                for identifier in [i for i in concept_name if str(i).isalnum()]:
-                    if identifier in tokens:
-                        variables.add(identifier)
-                # python_node = ret_value.value.value
-                # as_concept_node = python_to_concept(python_node.ast_)
-                # names = get_names(sheerka, as_concept_node)
-                # variables = filter(lambda x: x in concept_name, names)
-                return variables
-
-        #
-        # case of concept
-        #
-        if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, Concept):
-            return set(ret_value.value.value.values.keys())
-
         #
         # case of BNF
         #
@@ -164,4 +137,16 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
             visitor.visit(ret_value.value.value)
             return set(visitor.names)
 
+        #
+        # other (python code and concept)
+        #
+        if isinstance(ret_value.value, ParserResultConcept) and len(concept_name) > 1:
+            variables = set()
+            tokens = ret_value.value.tokens or list(Tokenizer(ret_value.value.source, yield_eof=False))
+            tokens = [t.str_value for t in tokens]
+            for identifier in [i for i in concept_name if str(i).isalnum()]:
+                if identifier in tokens:
+                    variables.add(identifier)
+            return variables
+
         return []

src/evaluators/ConceptEvaluator.py

@@ -21,11 +21,11 @@ class ConceptEvaluator(OneReturnValueEvaluator):
         self.return_body = return_body
 
     # def init_evaluator(self, context, return_values):
-    #     if BuiltinConcepts.EVAL_BODY_REQUESTED in context.local_hints:
+    #     if BuiltinConcepts.EVAL_BODY_REQUESTED in context.protected_hints:
     #         self.evaluate_body = True
     #
     #     for r in return_values:
-    #         if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.RETURN_VALUE_REQUESTED):
+    #         if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.RETURN_BODY_REQUESTED):
     #             self.evaluate_body = True
     #             break
     #

src/evaluators/PrepareEvalBodyEvaluator.py

@@ -2,12 +2,12 @@ from core.builtin_concepts import BuiltinConcepts
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 
 
-class PrepareEvalEvaluator(OneReturnValueEvaluator):
+class PrepareEvalBodyEvaluator(OneReturnValueEvaluator):
     """
-    To parse evaluation requests
+    To recognize when the user input is an (body) evaluation
     """
 
-    NAME = "PrepareEval"
+    NAME = "PrepareEvalBody"
 
     def __init__(self, **kwargs):
         super().__init__(self.NAME, [BuiltinConcepts.BEFORE_PARSING], 90)
@@ -33,8 +33,10 @@ class PrepareEvalEvaluator(OneReturnValueEvaluator):
             self.name,
             True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.text[5:], user_name=context.event.user_id))
 
-        context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        root = context.get_parents(lambda ec: ec.action == BuiltinConcepts.PROCESS_INPUT)
-        context.global_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+        root = root[0] if root else context
-        context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED)
+        root.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        root.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+        root.protected_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
 
         return new_text_to_parse

src/evaluators/PrepareEvalQuestionEvaluator.py

@@ -0,0 +1,45 @@
+from core.builtin_concepts import BuiltinConcepts
+from evaluators.BaseEvaluator import OneReturnValueEvaluator
+
+
+class PrepareEvalQuestionEvaluator(OneReturnValueEvaluator):
+    """
+    To recognize when the user input is a question
+    """
+
+    NAME = "PrepareEvalQuestion"
+
+    def __init__(self, **kwargs):
+        super().__init__(self.NAME, [BuiltinConcepts.BEFORE_PARSING], 90)
+        self.question = None
+
+    def matches(self, context, return_value):
+        if not (return_value.status and
+                context.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) and
+                isinstance(return_value.body.body, str)):
+            return False
+
+        text = return_value.body.body.strip()
+        if not (text.startswith("question(") and text.endswith(")")):
+            return False
+
+        self.question = text[9:-1].strip()
+        if self.question == "":
+            return False
+
+        return True
+
+    def eval(self, context, return_value):
+        sheerka = context.sheerka
+
+        new_text_to_parse = sheerka.ret(
+            self.name,
+            True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.question, user_name=context.event.user_id))
+
+        root = context.get_parents(lambda ec: ec.action == BuiltinConcepts.PROCESS_INPUT)
+        root = root[0] if root else context
+        root.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        root.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        root.protected_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
+
+        return new_text_to_parse

src/evaluators/PythonEvaluator.py

@@ -12,6 +12,9 @@ from core.sheerka.services.SheerkaFilter import Pipe
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.PythonParser import PythonNode
 
+TO_DISABLED = ["breakpoint", "callable", "compile", "delattr", "eval", "exec", "exit", "input", "locals", "open",
+               "print", "quit", "setattr"]
+
 
 def inject_context(context):
     """
@@ -95,7 +98,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
         concepts_entries = None
         evaluated = BuiltinConcepts.NOT_INITIALIZED
         errors = []
-        expect_success = BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED in context.local_hints
+        expect_success = context.in_context(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
         for globals_ in all_possible_globals:
             try:
                 # eval
@@ -142,12 +145,13 @@ class PythonEvaluator(OneReturnValueEvaluator):
         my_globals = {
             "Concept": core.concept.Concept,
             "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts,
-            "in_context": context.in_context
+            "in_context": context.in_context,
         }
 
         if expression_only:
-            # disable builtin
+            # disable some builtin
-            my_globals["__builtins__"] = None
+            for statement in TO_DISABLED:
+                my_globals[statement] = None
 
         # has to be the first, to allow override
         method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context, expression_only)
@@ -164,17 +168,6 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
     @staticmethod
     def update_globals_with_sheerka_methods(my_locals, context, expression_only):
-        # methods_from_sheerka = {}
-        #
-        # # Make sure that methods that need the concept are correctly wrapped
-        # for method_name, method in context.sheerka.sheerka_methods.items():
-        #     if expression_only and method.has_side_effect:
-        #         continue
-        #
-        #     if method_name in context.sheerka.methods_with_context:
-        #         methods_from_sheerka[method_name] = inject_context(context)(method.method)
-        #     else:
-        #         methods_from_sheerka[method_name] = method.method
         methods_from_sheerka = {}
 
         # Add all the methods as a direct access
@@ -238,6 +231,8 @@ class PythonEvaluator(OneReturnValueEvaluator):
             elif name in already_known:
                 context.log(f"Already known. Skipping.", self.name)
                 continue
+            elif (concept := context.get_from_short_term_memory(name)) is not None:
+                context.log(f"Using from STM known.", self.name)
             else:
                 context.log(f"Instantiating new concept with {name}.", self.name)
                 concept = self.resolve_concept(context, name)

src/evaluators/ResolveAmbiguityEvaluator.py

@@ -38,12 +38,13 @@ class ResolveAmbiguityEvaluator(AllReturnValuesEvaluator):
             parser_results = {id(r.body.body): r.body for r in ret_vals}
             resolved = resolve_ambiguity(context, [r.body.body for r in ret_vals])
             if len(resolved) == 0:
-                ret.append(context.sheerka.ret(self.name, True, [], parents=ret_vals))
+                ret.append(context.sheerka.ret(self.name, True, BuiltinConcepts.NO_RESULT, parents=ret_vals))
             else:
+                if len(resolved) < len(ret_vals):
                     for c in resolved:
                         ret.append(context.sheerka.ret(self.name, True, parser_results[id(c)], parents=ret_vals))
 
-        return ret
+        return None if len(ret) == 0 else ret
 
     @staticmethod
     def get_source(context, return_value):

src/evaluators/ReturnBodyEvaluator.py

@@ -3,12 +3,12 @@ from core.concept import Concept
 from evaluators.BaseEvaluator import AllReturnValuesEvaluator
 
 
-class EvalEvaluator(AllReturnValuesEvaluator):
+class ReturnBodyEvaluator(AllReturnValuesEvaluator):
     """
     Returns the body of all successful concepts
     """
 
-    NAME = "Eval"
+    NAME = "ReturnBody"
 
     def __init__(self):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 80)
@@ -16,7 +16,7 @@ class EvalEvaluator(AllReturnValuesEvaluator):
     def matches(self, context, return_values):
         evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING)
         is_root = len(evaluation_parents) <= 1
-        return context.in_context(BuiltinConcepts.RETURN_VALUE_REQUESTED) and is_root
+        return context.in_context(BuiltinConcepts.RETURN_BODY_REQUESTED) and is_root
 
     def eval(self, context, return_values):
         sheerka = context.sheerka

src/evaluators/UpdateFunctionsParametersEvaluator.py

@@ -26,6 +26,7 @@ class UpdateFunctionsParametersEvaluator(OneReturnValueEvaluator):
 
     def __init__(self):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 79)
+        self.enabled = False
 
     def matches(self, context, return_value):
         """

src/parsers/BaseNodeParser.py

@@ -616,6 +616,10 @@ class UTN(HelperWithPos):
 
 
 class BaseNodeParser(BaseParser):
+    """
+    Parser that return LexerNode
+    """
+
     def __init__(self, name, priority, **kwargs):
         super().__init__(name, priority)
         if 'sheerka' in kwargs:

src/parsers/DefaultParser.py

@@ -416,21 +416,38 @@ class DefaultParser(BaseParser):
                 continue
 
             # ask the other parsers if they recognize the tokens
-            with self.context.push(BuiltinConcepts.PARSING, keyword, who=self.name, desc=f"Parsing {keyword}") as sub_context:
+            source = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens)
-                parser_input = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens)
+            parsed = core.builtin_helpers.parse_unrecognized(self.context,
-                to_parse = self.sheerka.ret(
+                                                             source,
-                    sub_context.who,
+                                                             parsers="all",
-                    True,
+                                                             who=self.name,
-                    self.sheerka.new(BuiltinConcepts.USER_INPUT, body=parser_input))
+                                                             prop=keyword,
-                steps = [BuiltinConcepts.PARSING]
+                                                             filter_func=core.builtin_helpers.expect_one)
-                parsed = self.sheerka.execute(sub_context, to_parse, steps)
-                parsing_result = core.builtin_helpers.expect_one(sub_context, parsed)
-                sub_context.add_values(return_values=parsing_result)
 
-                if not parsing_result.status:
+            if not parsed.status:
-                    self.add_error(parsing_result.value)
+                self.add_error(parsed.value)
                 continue
 
-                asts_found_by_parts[keyword] = parsing_result
+            asts_found_by_parts[keyword] = parsed
+
+            #
+            # with self.context.push(BuiltinConcepts.PARSING, keyword, who=self.name, desc=f"Parsing {keyword}") as sub_context:
+            #     parser_input = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens)
+            #     to_parse = self.sheerka.ret(
+            #         sub_context.who,
+            #         True,
+            #         self.sheerka.new(BuiltinConcepts.USER_INPUT, body=parser_input))
+            #     steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
+            #     if keyword in (Keywords.WHERE, Keywords.PRE):
+            #         sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+            #     parsed = self.sheerka.execute(sub_context, to_parse, steps)
+            #     parsing_result = core.builtin_helpers.expect_one(sub_context, parsed)
+            #     sub_context.add_values(return_values=parsing_result)
+            #
+            #     if not parsing_result.status:
+            #         self.add_error(parsing_result.value)
+            #         continue
+            #
+            #     asts_found_by_parts[keyword] = parsing_result
 
         return asts_found_by_parts

src/parsers/ExpressionParser.py

@@ -3,7 +3,9 @@ from typing import List, Tuple, Callable
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from parsers.BaseParser import Node
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import LexerError, TokenKind, Token
+from parsers.BaseParser import Node, BaseParser, UnexpectedTokenErrorNode, UnexpectedEof, ErrorNode
 
 
 class ExprNode(Node):
@@ -16,6 +18,29 @@ class ExprNode(Node):
         return True
 
 
+@dataclass()
+class LeftPartNotFoundError(ErrorNode):
+    """
+    When the expression starts with 'or' or 'and'
+    """
+    pass
+
+
+class NameExprNode(ExprNode):
+    def __init__(self, tokens):
+        self.tokens = tokens
+        self.value = "".join([t.str_value for t in self.tokens])
+
+    def eval(self, obj):
+        return self.value
+
+    def __repr__(self):
+        return f"NameExprNode('{self.value}')"
+
+    def __str__(self):
+        return self.value
+
+
 @dataclass
 class PropertyEqualsNode(ExprNode):
     prop: str
@@ -110,6 +135,12 @@ class AndNode(ExprNode):
             res &= part.eval(obj)
         return res
 
+    def __repr__(self):
+        return f"AndNode(" + ", ".join([repr(p) for p in self.parts]) + ")"
+
+    def __str__(self):
+        return " and ".join([str(p) for p in self.parts])
+
 
 @dataclass(init=False)
 class OrNode(ExprNode):
@@ -124,6 +155,11 @@ class OrNode(ExprNode):
             res |= part.eval(obj)
         return res
 
+    def __repr__(self):
+        return f"OrNode(" + ", ".join([repr(p) for p in self.parts]) + ")"
+
+    def __str__(self):
+        return " or ".join([str(p) for p in self.parts])
 
 @dataclass()
 class NotNode(ExprNode):
@@ -143,7 +179,7 @@ class TrueNode(ExprNode):
         return True
 
 
-class ExpressionParser:
+class ExpressionParser(BaseParser):
     """
     will parser logic expression
     like not (a and b or c)
@@ -151,7 +187,140 @@ class ExpressionParser:
     The nodes can be used for custom filtering (ex with ExplanationConcept)
     Or to help to understand why a python expression returns True or False
     """
-    pass
+
+    def __init__(self, **kwargs):
+        super().__init__("Expression", 50, False)
+
+    def reset_parser(self, context, parser_input: ParserInput):
+        self.context = context
+        self.sheerka = context.sheerka
+        self.parser_input = parser_input
+        self.error_sink.clear()
+
+        try:
+            self.parser_input.reset(False)
+            self.parser_input.next_token()
+        except LexerError as e:
+            self.add_error(self.sheerka.new(BuiltinConcepts.ERROR, body=e), False)
+            return False
+        return True
+
+    def parse(self, context, parser_input: ParserInput):
+        """
+        parser_input can be string, but text can also be an list of tokens
+        :param context:
+        :param parser_input:
+        :return:
+        """
+
+        if not isinstance(parser_input, ParserInput):
+            return None
+
+        context.log(f"Parsing '{parser_input}' with ExpressionParser", self.name)
+        sheerka = context.sheerka
+
+        if parser_input.is_empty():
+            return context.sheerka.ret(self.name,
+                                       False,
+                                       sheerka.new(BuiltinConcepts.IS_EMPTY))
+
+        if not self.reset_parser(context, parser_input):
+            return self.sheerka.ret(
+                self.name,
+                False,
+                context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))
+
+        tree = self.parse_or()
+        token = self.parser_input.token
+        if token and token.type != TokenKind.EOF:
+            self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, []))
+
+        value = self.get_return_value_body(context.sheerka, self.parser_input.as_text(), tree, tree)
+
+        ret = self.sheerka.ret(
+            self.name,
+            not self.has_error,
+            value)
+
+        return ret
+
+    def parse_or(self):
+        expr = self.parse_and()
+        token = self.parser_input.token
+        if token.type != TokenKind.IDENTIFIER or token.value != "or":
+            return expr
+
+        parts = [expr]
+        while token.type == TokenKind.IDENTIFIER and token.value == "or":
+            self.parser_input.next_token()
+            expr = self.parse_and()
+            if expr is None:
+                self.add_error(UnexpectedEof("When parsing 'or'"))
+                return OrNode(*parts)
+            parts.append(expr)
+            token = self.parser_input.token
+
+        return OrNode(*parts)
+
+    def parse_and(self):
+        expr = self.parse_names()
+        token = self.parser_input.token
+        if token.type != TokenKind.IDENTIFIER or token.value != "and":
+            return expr
+
+        parts = [expr]
+        while token.type == TokenKind.IDENTIFIER and token.value == "and":
+            self.parser_input.next_token()
+            expr = self.parse_names()
+            if expr is None:
+                self.add_error(UnexpectedEof("When parsing 'and'"))
+                return AndNode(*parts)
+            parts.append(expr)
+            token = self.parser_input.token
+
+        return AndNode(*parts)
+
+    def parse_names(self):
+
+        def stop():
+            return token.type == TokenKind.EOF or \
+                   paren_count == 0 and token.type == TokenKind.RPAR or \
+                   token.type == TokenKind.IDENTIFIER and token.value in ("and", "or")
+
+        token = self.parser_input.token
+        if token.type == TokenKind.EOF:
+            return None
+
+        if token.type == TokenKind.LPAR:
+            self.parser_input.next_token()
+            expr = self.parse_or()
+            token = self.parser_input.token
+            if token.type != TokenKind.RPAR:
+                self.error_sink.append(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, [TokenKind.RPAR]))
+                return expr
+            self.parser_input.next_token()
+            return expr
+
+        buffer = []
+        paren_count = 0
+        while not stop():
+            buffer.append(token)
+            if token.type == TokenKind.LPAR:
+                paren_count += 1
+            if token.type == TokenKind.RPAR:
+                paren_count -= 1
+            self.parser_input.next_token(False)
+            token = self.parser_input.token
+
+        if len(buffer) == 0:
+            if token.type != TokenKind.RPAR:
+                self.error_sink.append(LeftPartNotFoundError())
+            return None
+
+        if buffer[-1].type == TokenKind.WHITESPACE:
+            buffer.pop()
+
+        return NameExprNode(buffer)
 
 
 class ExpressionVisitor:
@@ -175,3 +344,38 @@ class ExpressionVisitor:
                         self.visit(item)
             elif isinstance(value, ExprNode):
                 self.visit(value)
+
+
+class TrueifyVisitor(ExpressionVisitor):
+    """
+    Visit an ExprNode
+    replace all the nodes containing a variable to 'trueify' with True
+    The node containing both variables to trueify and to skip are skipped
+    """
+
+    def __init__(self, to_trueify, to_skip):
+        self.to_trueify = to_trueify
+        self.to_skip = to_skip
+
+    def visit_AndNode(self, expr_node):
+        parts = []
+        for part in expr_node.parts:
+            parts.append(self.visit(part))
+        return AndNode(*parts)
+
+    def visit_OrNode(self, expr_node):
+        parts = []
+        for part in expr_node.parts:
+            parts.append(self.visit(part))
+        return OrNode(*parts)
+
+    def visit_NameExprNode(self, expr_node):
+        return_true = False
+        for t in expr_node.tokens:
+            if t.type == TokenKind.IDENTIFIER:
+                if t.value in self.to_skip:
+                    return expr_node
+                if t.value in self.to_trueify:
+                    return_true = True
+
+        return NameExprNode([Token(TokenKind.IDENTIFIER, "True", -1, -1, -1)]) if return_true else expr_node

src/parsers/ShortTermMemoryParser.py

@@ -0,0 +1,44 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.sheerka.services.SheerkaExecute import ParserInput
+from parsers.BaseParser import BaseParser
+
+
+class ShortTermMemoryParser(BaseParser):
+    """
+    This parser is used to recognize concept that are already instantiated
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__("shortTermMemory", 85)
+
+    def parse(self, context, parser_input):
+        """
+        Browse context.instance in order to find the parser_input (which is the name of the concept)
+        If not found, browse the parent
+        :param context:
+        :param parser_input:
+        :return:
+        """
+
+        if not isinstance(parser_input, ParserInput):
+            return None
+
+        context.log(f"Parsing '{parser_input}' with ShortTermMemory", self.name)
+        sheerka = context.sheerka
+
+        if parser_input.is_empty():
+            return context.sheerka.ret(self.name,
+                                       False,
+                                       sheerka.new(BuiltinConcepts.IS_EMPTY))
+
+        concept_name = parser_input.as_text()
+        concept = sheerka.get_from_short_term_memory(context, concept_name)
+
+        if concept:
+            # Unlike what is usually done, we directly return the concept, not a ParsingResult of the concept
+            # This is to save the evaluation time cost
+            return sheerka.ret(self.name, True, concept)
+
+        else:
+            body = sheerka.new(BuiltinConcepts.NOT_FOUND, body=concept_name)
+            return sheerka.ret(self.name, False, body)

tests/BaseTest.py

@@ -15,9 +15,9 @@ class BaseTest:
     def get_context(self, sheerka=None, eval_body=False, eval_where=False):
         context = ExecutionContext("test", Event(), sheerka or self.get_sheerka(), BuiltinConcepts.TESTING, None)
         if eval_body:
-            context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+            context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
         if eval_where:
-            context.local_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+            context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
 
         return context
 
@@ -78,6 +78,15 @@ class BaseTest:
 
         return sheerka, context, *result
 
+    @staticmethod
+    def get_concept_instance(sheerka, concept, **kwargs):
+        instance = sheerka.new(concept.key if isinstance(concept, Concept) else concept)
+        for i, var in enumerate(instance.metadata.variables):
+            if var[0] in kwargs:
+                instance.metadata.variables[i] = (var[0], kwargs[var[0]])
+
+        return instance
+
     @staticmethod
     def retval(obj, who="who", status=True):
         """ret_val"""

tests/core/test_ExecutionContext.py

@@ -35,7 +35,7 @@ def test_i_can_push():
                          concepts={"bar": Concept("bar")})
     a.preprocess = set()
     a.preprocess.add("preprocess")
-    a.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+    a.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
     a.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
     b = a.push(BuiltinConcepts.EVALUATION, "sub action context", desc="sub description")
@@ -54,7 +54,7 @@ def test_i_can_push():
     assert b.id == a.id + 1
     assert b._tab == a._tab + "    "
     assert b.preprocess == a.preprocess
-    assert b.local_hints == {BuiltinConcepts.EVAL_BODY_REQUESTED}
+    assert b.protected_hints == {BuiltinConcepts.EVAL_BODY_REQUESTED}
     assert b.global_hints == a.global_hints
 
 
@@ -64,10 +64,10 @@ def test_children_i_created_when_i_push():
     e.push(BuiltinConcepts.NOP, None, who="b", desc="oups! I did a again")
     e.push(BuiltinConcepts.NOP, None, who="c", desc="I do something else")
 
-    assert len(e.children) == 3
+    assert len(e._children) == 3
-    assert e.children[0].who, e.children[0].who == ("a", "I do something")
+    assert e._children[0].who, e._children[0].who == ("a", "I do something")
-    assert e.children[1].who, e.children[1].who == ("b", "oups! I did a again")
+    assert e._children[1].who, e._children[1].who == ("b", "oups! I did a again")
-    assert e.children[2].who, e.children[2].who == ("c", "I do something else")
+    assert e._children[2].who, e._children[2].who == ("c", "I do something else")
 
 
 def test_i_can_add_variable_when_i_push():
@@ -81,17 +81,17 @@ def test_i_can_add_variable_when_i_push():
 
 def test_local_hints_are_local_and_global_hints_are_global():
     a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None)
-    a.local_hints.add("local hint 1")
+    a.protected_hints.add("local hint 1")
     a.global_hints.add("global hint 1")
 
     b = a.push(BuiltinConcepts.NOP, None)
-    b.local_hints.add("local hint 2")
+    b.protected_hints.add("local hint 2")
     b.global_hints.add("global hint 2")
 
-    assert a.local_hints == {"local hint 1"}
+    assert a.protected_hints == {"local hint 1"}
     assert a.global_hints == {"global hint 1", "global hint 2"}
 
-    assert b.local_hints == {"local hint 1", "local hint 2"}
+    assert b.protected_hints == {"local hint 1", "local hint 2"}
     assert b.global_hints == {"global hint 1", "global hint 2"}
 
 

tests/core/test_SheerkaEvaluateConcept.py

@@ -284,6 +284,49 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka, True), concept)
         assert evaluated.get_value("prop") == 2
 
+    def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables(self):
+        sheerka, context, plus, add = self.init_concepts(
+            Concept("a plus b", body="a + b").def_var("a").def_var("b"),
+            Concept("add a b", body="a plus b").def_var("a").def_var("b"),
+            eval_body=True)
+
+        add_instance = self.get_concept_instance(sheerka, add, a="1", b="2")
+
+        evaluated = sheerka.evaluate_concept(context, add_instance)
+
+        assert evaluated.key == add_instance.key
+        assert evaluated.metadata.is_evaluated
+        assert sheerka.objvalue(evaluated) == 3
+
+    def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_and_different_names(self):
+        sheerka, context, plus, add = self.init_concepts(
+            Concept("a plus b", body="a + b").def_var("a").def_var("b"),
+            Concept("add x y", body="x plus y").def_var("x").def_var("y"),
+            eval_body=True)
+
+        add_instance = self.get_concept_instance(sheerka, add, x="1", y="2")
+
+        evaluated = sheerka.evaluate_concept(context, add_instance)
+
+        assert evaluated.key == add_instance.key
+        assert evaluated.metadata.is_evaluated
+        assert sheerka.objvalue(evaluated) == 3
+
+    def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_multiple_levels(self):
+        sheerka, context, plus, add, inc = self.init_concepts(
+            Concept("a plus b", body="a + b").def_var("a").def_var("b"),
+            Concept("add x y", body="x plus y").def_var("x").def_var("y"),
+            Concept("inc number", body="add number 1").def_var("number"),
+            eval_body=True)
+
+        inc_instance = self.get_concept_instance(sheerka, inc, number="1")
+
+        evaluated = sheerka.evaluate_concept(context, inc_instance)
+
+        assert evaluated.key == inc_instance.key
+        assert evaluated.metadata.is_evaluated
+        assert sheerka.objvalue(evaluated) == 2
+
     def test_i_can_reference_sheerka(self):
         sheerka = self.get_sheerka()
 
@@ -354,21 +397,24 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         assert evaluated.key == concept.init_key().key
 
-    @pytest.mark.parametrize("where_clause, expected, expected_body", [
+    @pytest.mark.parametrize("where_clause, expected, expected_prop, expected_body", [
-        ("True", True, BuiltinConcepts.NOT_INITIALIZED),
+        ("True", True, None, BuiltinConcepts.NOT_INITIALIZED),
-        ("False", False, BuiltinConcepts.NOT_INITIALIZED),
+        ("False", False, ConceptParts.WHERE, BuiltinConcepts.NOT_INITIALIZED),
-        ("self < 10", False, 10),
+        ("self < 10", False, ConceptParts.WHERE, 10),
-        ("self < 11", True, 10),
+        ("self < 11", True, None, 10),
-        ("a < 20", False, BuiltinConcepts.NOT_INITIALIZED),
+        ("a < 20", False, "a", BuiltinConcepts.NOT_INITIALIZED),
-        ("a > 19", True, BuiltinConcepts.NOT_INITIALIZED),
+        ("a > 19", True, None, BuiltinConcepts.NOT_INITIALIZED),
-        ("a + self > 20", True, 10),
+        ("a + self > 20", True, None, 10),
+        ("a + self < 20", False, ConceptParts.WHERE, 10),
     ])
-    def test_i_can_evaluate_simple_where(self, where_clause, expected, expected_body):
+    def test_i_can_evaluate_simple_where(self, where_clause, expected, expected_prop, expected_body):
         # We check that the WHERE condition is correctly evaluated
         # We also check that the body is evaluated only when it's mandatory
 
         sheerka, context, concept = self.init_concepts(
             Concept("foo", body="10", where=where_clause).def_var("a", "20"),
+            eval_body=False,  # to check when the evaluation is forced
+            eval_where=True,
         )
 
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept, eval_body=False)
@@ -380,7 +426,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
             assert sheerka.isinstance(evaluated, BuiltinConcepts.CONDITION_FAILED)
             assert evaluated.body == where_clause
             assert evaluated.concept == concept
-            assert evaluated.prop == ConceptParts.WHERE
+            assert evaluated.prop == expected_prop
             assert concept.body == expected_body
 
     def test_i_can_evaluate_where_when_using_other_concept(self):
@@ -402,6 +448,116 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, False), concept)
         assert evaluated.key == concept.key
 
+    def test_i_can_apply_intermediate_where_condition_using_python(self):
+        sheerka, context, one_1, one_str, plus = self.init_concepts(
+            Concept("one", body="1"),
+            Concept("one", body="'one'"),
+            Concept("a plus b", body="a + b", where="isinstance(a, int)").def_var("a", "one").def_var("b", "2"),
+            eval_body=True,
+            eval_where=True,
+        )
+
+        evaluated = sheerka.evaluate_concept(context, plus)
+        assert evaluated.key == plus.key
+        assert evaluated.body == 3
+
+    def test_i_can_apply_intermediate_where_condition_when_multiple_variables_using_python(self):
+        sheerka, context, one_1, one_str, two_2, two_str, plus = self.init_concepts(
+            Concept("one", body="1"),
+            Concept("one", body="'one'"),
+            Concept("two", body="2"),
+            Concept("two", body="'two'"),
+            Concept("a plus b",
+                    body="a + b",
+                    where="isinstance(a, int) and isinstance(b, int)").def_var("a", "one").def_var("b", "two"),
+            eval_body=True,
+            eval_where=True,
+        )
+
+        evaluated = sheerka.evaluate_concept(context, plus)
+        assert evaluated.key == plus.key
+        assert evaluated.body == 3
+
+    def test_i_can_apply_intermediate_where_condition_using_other_concepts(self):
+        sheerka, context, one_1, one_str, is_an_int, plus = self.init_concepts(
+            Concept("one", body="1"),
+            Concept("one", body="'one'"),
+            Concept("x is an int", body="isinstance(x, int)").def_var("x"),
+            Concept("a plus b", body="a + b", where="a is an int").def_var("a", "one").def_var("b", "2"),
+            eval_body=True,
+            eval_where=True,
+        )
+
+        evaluated = sheerka.evaluate_concept(context, plus)
+        assert evaluated.key == plus.key
+        assert evaluated.body == 3
+
+    def test_i_can_apply_intermediate_where_condition_when_multiple_levels(self):
+        sheerka, context, one_1, one_str, is_an_int, is_an_integer, plus = self.init_concepts(
+            Concept("one", body="1"),
+            Concept("one", body="'one'"),
+            Concept("x is an int", body="isinstance(x, int)").def_var("x"),
+            Concept("y is an integer", body="y is an int").def_var("y"),
+            Concept("a plus b", body="a + b", where="a is an integer").def_var("a", "one").def_var("b", "2"),
+            eval_body=True,
+            eval_where=True,
+        )
+
+        evaluated = sheerka.evaluate_concept(context, plus)
+        assert evaluated.key == plus.key
+        assert evaluated.body == 3
+
+    def test_i_can_apply_intermediate_where_condition_choosing_the_correct_where_concept(self):
+        # in this test, there are two where conditions with the same name
+        # We need to use the 'PRE' condition to select the correct one
+        sheerka, context, one_1, one_str, is_an_int, is_an_integer, plus = self.init_concepts(
+            Concept("one", body="1"),
+            Concept("one", body="'one'"),
+            Concept("x is an int",
+                    body="isinstance(x, int)",
+                    pre="in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)").def_var("x"),
+            Concept("x is an int", body="False").def_var("x"),
+            Concept("a plus b", body="a + b", where="a is an int").def_var("a", "one").def_var("b", "2"),
+            eval_body=True,
+            eval_where=True,
+        )
+
+        evaluated = sheerka.evaluate_concept(context, plus)
+        assert evaluated.key == plus.key
+        assert evaluated.body == 3
+
+    @pytest.mark.skip("Not ready for that")
+    def test_i_can_apply_intermediate_where_condition_when_multiple_variables(self):
+        sheerka, context, one_1, one_str, two_2, two_str, is_an_int, plus = self.init_concepts(
+            Concept("one", body="1"),
+            Concept("one", body="'one'"),
+            Concept("two", body="2"),
+            Concept("two", body="'two'"),
+            Concept("x is an int", body="isinstance(x, int)").def_var("x"),
+            Concept("a plus b",
+                    body="a + b",
+                    where="a is an int and isinstance(b, int)").def_var("a", "one").def_var("b", "two"),
+            eval_body=True,
+            eval_where=True,
+        )
+
+        evaluated = sheerka.evaluate_concept(context, plus)
+        assert evaluated.key == plus.key
+        assert evaluated.body == 3
+
+    def test_i_cannot_evaluate_when_intermediate_where_condition_fails(self):
+        sheerka, context, one_1, is_an_int, plus = self.init_concepts(
+            Concept("one", body="'one'"),
+            Concept("x is an int", body="isinstance(x, int)").def_var("x"),
+            Concept("a plus b", body="a + b", where="a is an int").def_var("a", "one").def_var("b", "2"),
+            eval_body=True,
+            eval_where=True,
+        )
+
+        evaluated = sheerka.evaluate_concept(context, plus)
+        assert evaluated.key == BuiltinConcepts.CONDITION_FAILED
+        assert evaluated.body == "a is an int"
+
     def test_i_can_enable_disable_where_clause_evaluation(self):
         sheerka, context, concept = self.init_concepts(
             Concept("foo", body="10", where="a > 10").def_var("a", None),
@@ -409,7 +565,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept)
         assert evaluated.key == concept.key
 
-        context.local_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
+        context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
         evaluated = sheerka.evaluate_concept(context, concept)
         assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
 
@@ -495,6 +651,16 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(evaluated, BuiltinConcepts.CHICKEN_AND_EGG)
         assert evaluated.body == {foo}
 
+    # def test_i_can_detect_auto_recursion_when_evaluating_another_concept(self):
+    #     sheerka, context, one_1, one_str, plus = self.init_concepts(
+    #         Concept("one", body="1"),
+    #         Concept("one", body="one"),
+    #         Concept("a plus b", body="a + b", where="isinstance(a, int)").def_var("a", "one").def_var("b", "2")
+    #     )
+    #
+    #     evaluated = sheerka.evaluate_concept(context, plus, eval_body=True)
+    #     sheerka.isinstance(evaluated, BuiltinConcepts.CHICKEN_AND_EGG)
+
     def test_i_can_manage_auto_recursion_when_constant(self):
         sheerka = self.get_sheerka()
 
@@ -566,7 +732,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert captured.out == ""
 
     def test_python_builtin_function_are_forbidden_in_where_pre_post_ret(self, capsys):
-        # I do the test only for PRE, as it will be the same for the other CounceptPart
+        # I do the test only for PRE, as it will be the same for the other ConceptPart
         sheerka, context, foo, bar = self.init_concepts(
             Concept("foo", body="print('10')"),  # print will be executed
             Concept("bar", pre="print('10')"),  # print won't be executed
@@ -658,8 +824,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         sheerka, context, concept = self.init_concepts(
             Concept("foo", pre="'pre'", post="'post'", ret="'ret'", where="'where'", body="'body'").def_var("a", "'a'")
         )
-        context.local_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)  # to prove that we do not care
+        context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)  # to prove that we do not care
-        context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)  # to prove that we do not care
+        context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)  # to prove that we do not care
 
         evaluated = sheerka.evaluate_concept(context, concept, metadata=['pre'])
         assert evaluated.values == {"a": Property("a", NotInit), ConceptParts.PRE: Property(ConceptParts.PRE, 'pre')}

tests/core/test_SheerkaFilter.py

@@ -197,11 +197,11 @@ class TestSheerkaFilter(TestUsingMemoryBasedSheerka):
         assert len(res) == 2
         format_instructions = res[0].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS)
         assert isinstance(format_instructions, FormatInstructions)
-        assert format_instructions.recursive_props["children"] == 10
+        assert format_instructions.recursive_props["_children"] == 10
 
         format_instructions = res[1].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS)
         assert isinstance(format_instructions, FormatInstructions)
-        assert format_instructions.recursive_props["children"] == 10
+        assert format_instructions.recursive_props["_children"] == 10
 
         res = lst | Pipe(SheerkaFilter.pipe_recurse)(15, "other_prop")
         res = list(res)
@@ -209,7 +209,7 @@ class TestSheerkaFilter(TestUsingMemoryBasedSheerka):
         assert len(res) == 2
         format_instructions = res[0].get_prop(BuiltinConcepts.FORMAT_INSTRUCTIONS)
         assert isinstance(format_instructions, FormatInstructions)
-        assert format_instructions.recursive_props["children"] == 10
+        assert format_instructions.recursive_props["_children"] == 10
         assert format_instructions.recursive_props["other_prop"] == 15
 
     def test_i_can_inspect_obj(self):

tests/core/test_SheerkaShortTermMemory.py

@@ -0,0 +1,52 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaShortTermMemory(TestUsingMemoryBasedSheerka):
+    def test_i_can_add_to_global_short_term_memory(self):
+        sheerka = self.get_sheerka()
+        service = sheerka.services[SheerkaShortTermMemory.NAME]
+
+        foo = Concept("foo")
+        sheerka.add_to_short_term_memory(None, "a", foo)
+
+        assert service.objects.copy() == {":a": foo}
+        assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo)
+
+    def test_i_can_add_context_short_term_memory(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaShortTermMemory.NAME]
+
+        foo = Concept("foo")
+        sheerka.add_to_short_term_memory(context, "a", foo)
+
+        context_id = ExecutionContext.ids[context.event.get_digest()]
+        assert service.objects.copy() == {f"{context_id}:a": foo}
+        assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
+        assert sheerka.get_from_short_term_memory(None, "a") is None
+
+    def test_i_can_get_obj_from_parents(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaShortTermMemory.NAME]
+        foo = Concept("foo")
+        sheerka.add_to_short_term_memory(None, "a", foo)
+
+        with context.push(BuiltinConcepts.TESTING, None) as sub_context:
+            assert service.objects.copy() == {":a": foo}
+            assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
+            assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
+            assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo)
+
+    def test_entry_are_removed_on_context_exit(self):
+        sheerka, context = self.init_concepts()
+
+        with context.push(BuiltinConcepts.TESTING, None) as sub_context:
+            foo = Concept("foo")
+            sheerka.add_to_short_term_memory(sub_context, "a", foo)
+            assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
+
+        assert sheerka.get_from_short_term_memory(sub_context, "a") is None

tests/core/test_sheerka.py

@@ -242,16 +242,17 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(new.concept, concept)
 
     @pytest.mark.parametrize("concept, reduce_simple_list, expected", [
-        (None, False, None),
+        # (None, False, None),
-        (3.14, False, 3.14),
+        # (3.14, False, 3.14),
-        ("foo", False, "foo"),
+        # ("foo", False, "foo"),
-        (True, False, True),
+        # (True, False, True),
-        (Concept("name", body="foo"), False, "foo"),
+        # (Concept("name", body="foo"), False, "foo"),
-        (Concept("name"), False, Concept("name")),
+        # (Concept("name"), False, Concept("name")),
-        (ConceptWithGetObjValue("name").set_value("my_prop", "my_value"), False, "my_value"),
+        # (ConceptWithGetObjValue("name").set_value("my_prop", "my_value"), False, "my_value"),
-        (ReturnValueConcept(value="return_value"), False, "return_value"),
+        # (ReturnValueConcept(value="return_value"), False, "return_value"),
-        (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
+        # (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
-        (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
+        # (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
+        (ReturnValueConcept(value=Concept("foo", body=False).auto_init(), status=True), False, False),
         (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]),
         (Concept("name", body=["foo"]), True, "foo"),
         (Concept("name", body=Concept("foo")), False, Concept("foo")),

tests/core/test_sheerka_call_evaluators.py

@@ -210,7 +210,7 @@ class EvaluatorAllSuppressEntries(EvaluatorAllWithPriority):
         return context.sheerka.ret(
             self.name,
             True,
-            [],
+            BuiltinConcepts.NO_RESULT,
             parents=to_remove)
 
 

tests/evaluators/test_AddConceptEvaluator.py

@@ -114,6 +114,21 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert created_concept.metadata.definition == "hello a"
         assert created_concept.metadata.definition_type == "bnf"
 
+    def test_i_can_add_concept_with_the_correct_variables_when_referencing_other_concepts(self):
+        context = self.get_context()
+        def_concept_return_value = self.get_def_concept(
+            name="x plus y",
+            where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"),
+            body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"),)
+
+        evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
+
+        assert evaluated.status
+        assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
+
+        created_concept = evaluated.body.body
+        assert created_concept.metadata.variables == [("x", None), ("y", None)]
+
     def test_that_the_source_is_correctly_set_for_concept_with_simple_definition(self):
         context = self.get_context()
         def_concept_return_value = self.get_def_concept(
@@ -195,14 +210,6 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
 
         assert AddConceptEvaluator.get_variables(context.sheerka, ret_val, ["a"]) == []
 
-    def test_i_can_get_variables_from_another_concept(self):
-        concept = Concept("hello").def_var("a").def_var("b")
-        ret_val = ReturnValueConcept(who="some_parser",
-                                     status=True,
-                                     value=ParserResultConcept(value=concept))
-
-        assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"a", "b"}
-
     def test_i_can_get_variables_from_definition(self):
         parsing_expression = Sequence(ConceptExpression('mult'),
                                       ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))

tests/evaluators/test_ConceptEvaluator.py

@@ -20,7 +20,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_evaluate_concept(self):
         context = self.get_context()
-        context.local_hints.update({BuiltinConcepts.EVAL_BODY_REQUESTED, BuiltinConcepts.EVAL_WHERE_REQUESTED})
+        context.protected_hints.update({BuiltinConcepts.EVAL_BODY_REQUESTED, BuiltinConcepts.EVAL_WHERE_REQUESTED})
         concept = Concept(name="foo",
                           where="True",
                           pre="2 > 1",
@@ -45,7 +45,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_body_is_returned_when_defined_and_requested(self):
         context = self.get_context()
-        context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
         concept = Concept(name="foo",
                           body="'I have a value'",
                           where="True",
@@ -94,7 +94,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(self):
         context = self.get_context()
-        context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
         context.sheerka.add_in_cache(Concept(name="one").init_key())
         concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b")
                                                     .def_var("a", "one")

tests/evaluators/test_EvalEvaluator.py

@@ -2,7 +2,7 @@ import pytest
 from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
 from core.concept import Concept
 from core.sheerka.services.SheerkaSetsManager import SheerkaSetsManager
-from evaluators.EvalEvaluator import EvalEvaluator
+from evaluators.ReturnBodyEvaluator import ReturnBodyEvaluator
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -15,7 +15,7 @@ def retval(obj, who="who", status=True):
 class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
     def test_i_can_match_and_eval(self):
         context = self.get_context()
-        context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED)
+        context.global_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
 
         to_eval1 = ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init())
         to_eval2 = ReturnValueConcept("some_name", True, Concept(name="3", body="also to eval").auto_init())
@@ -28,7 +28,7 @@ class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
             to_eval2,
         ]
 
-        evaluator = EvalEvaluator()
+        evaluator = ReturnBodyEvaluator()
         assert evaluator.matches(context, return_values)
 
         evaluated = evaluator.eval(context, return_values)
@@ -47,16 +47,16 @@ class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
     ])
     def test_i_cannot_match_if_eval_request_is_not_present(self, return_value):
         context = self.get_context()
-        assert not EvalEvaluator().matches(context, [return_value])
+        assert not ReturnBodyEvaluator().matches(context, [return_value])
 
-        context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED)
+        context.global_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
-        assert EvalEvaluator().matches(context, [return_value])
+        assert ReturnBodyEvaluator().matches(context, [return_value])
 
     def test_i_can_match_depending_on_builtin_concept_processing(self):
         context = self.get_context()
-        context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED)
+        context.global_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
         return_values = [ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init())]
-        evaluator = EvalEvaluator()
+        evaluator = ReturnBodyEvaluator()
 
         # i match when no BuiltinConcepts.PROCESSING is found
         assert evaluator.matches(context, return_values)
@@ -80,7 +80,7 @@ class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
             ReturnValueConcept("some_name", False, Concept(name="1", body="not to eval")),  # no evaluated body
         ]
 
-        evaluated = EvalEvaluator().eval(context, return_values)
+        evaluated = ReturnBodyEvaluator().eval(context, return_values)
         assert evaluated is None
 
     def test_i_can_evaluate_sets(self):
@@ -92,7 +92,7 @@ class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
         sets_handler = sheerka.services[SheerkaSetsManager.NAME]
         sets_handler.add_concepts_to_set(context, [foo, bar, baz], number)
 
-        evaluated = EvalEvaluator().eval(context, [retval(number)])
+        evaluated = ReturnBodyEvaluator().eval(context, [retval(number)])
 
         assert len(evaluated) == 1
         assert evaluated[0].status

tests/evaluators/test_PrepareEvalBodyEvaluator.py

@@ -2,14 +2,14 @@ import pytest
 
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
 from core.concept import Concept
-from evaluators.PrepareEvalEvaluator import PrepareEvalEvaluator
+from evaluators.PrepareEvalBodyEvaluator import PrepareEvalBodyEvaluator
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 r = ReturnValueConcept
 
 
-class TestPrepareEvalEvaluator(TestUsingMemoryBasedSheerka):
+class TestPrepareEvalBodyEvaluator(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("ret_val, expected", [
         (r("name", True, UserInputConcept("eval 1 + 1")), True),
@@ -26,7 +26,7 @@ class TestPrepareEvalEvaluator(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_match(self, ret_val, expected):
         context = self.get_context()
-        assert PrepareEvalEvaluator().matches(context, ret_val) == expected
+        assert PrepareEvalBodyEvaluator().matches(context, ret_val) == expected
 
     @pytest.mark.parametrize("ret_val, expected", [
         (r("name", True, UserInputConcept("eval 1 + 1")), "1 + 1"),
@@ -36,7 +36,7 @@ class TestPrepareEvalEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         sheerka = context.sheerka
 
-        prepare_evaluator = PrepareEvalEvaluator()
+        prepare_evaluator = PrepareEvalBodyEvaluator()
         prepare_evaluator.matches(context, ret_val)
         res = prepare_evaluator.eval(context, ret_val)
 
@@ -44,5 +44,6 @@ class TestPrepareEvalEvaluator(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.USER_INPUT)
         assert res.body.body == expected
 
-        assert BuiltinConcepts.EVAL_BODY_REQUESTED in context.global_hints
+        assert BuiltinConcepts.EVAL_BODY_REQUESTED in context.protected_hints
-        assert BuiltinConcepts.RETURN_VALUE_REQUESTED in context.global_hints
+        assert BuiltinConcepts.EVAL_WHERE_REQUESTED in context.protected_hints
+        assert BuiltinConcepts.RETURN_BODY_REQUESTED in context.protected_hints

tests/evaluators/test_PrepareEvalQuestionEvaluator.py

@@ -0,0 +1,49 @@
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
+from core.concept import Concept
+from evaluators.PrepareEvalQuestionEvaluator import PrepareEvalQuestionEvaluator
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+r = ReturnValueConcept
+
+
+class TestPrepareEvalQuestionEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (r("name", True, UserInputConcept("question(1 + 1)")), True),
+        (r("name", True, UserInputConcept("  question(1 + 1)  ")), True),
+        (r("name", True, UserInputConcept("question()")), False),
+        (r("name", True, UserInputConcept("1+1")), False),
+        (r("name", True, UserInputConcept("")), False),
+        (r("name", True, UserInputConcept("question(")), False),
+        (r("name", True, UserInputConcept(")")), False),
+        (r("name", True, UserInputConcept([])), False),
+        (r("name", True, Concept("foo")), False),
+        (r("name", True, "not a concept"), False),
+        (r("name", False, UserInputConcept("question(1 + 1)")), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert PrepareEvalQuestionEvaluator().matches(context, ret_val) == expected
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (r("name", True, UserInputConcept("question(1 + 1)")), "1 + 1"),
+        (r("name", True, UserInputConcept("   question( 1 + 1 )  ")), "1 + 1"),
+    ])
+    def test_i_can_eval(self, ret_val, expected):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        prepare_evaluator = PrepareEvalQuestionEvaluator()
+        prepare_evaluator.matches(context, ret_val)
+        res = prepare_evaluator.eval(context, ret_val)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.USER_INPUT)
+        assert res.body.body == expected
+
+        assert BuiltinConcepts.EVAL_QUESTION_REQUESTED in context.protected_hints
+        assert BuiltinConcepts.EVAL_BODY_REQUESTED in context.protected_hints
+        assert BuiltinConcepts.RETURN_BODY_REQUESTED in context.protected_hints

tests/evaluators/test_PythonEvaluator.py

@@ -137,7 +137,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert not evaluated.value  # the first test is between Concept(foo) and int(2)
 
-        context.local_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+        context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
         evaluated = python_evaluator.eval(context, parsed)
         assert evaluated.status
         assert evaluated.value  # we test until we compare foo.body and 2

tests/evaluators/test_ResolveAmbiguityEvaluator.py

@@ -1,4 +1,5 @@
 import pytest
+from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from evaluators.ResolveAmbiguityEvaluator import ResolveAmbiguityEvaluator
 
@@ -67,8 +68,26 @@ class TestResolveAmbiguityEvaluator(TestUsingMemoryBasedSheerka):
         resolved = res[0]
 
         assert resolved.who == evaluator.name
-        assert resolved.body == []
+        assert resolved.body == BuiltinConcepts.NO_RESULT
         assert resolved.parents == [
             return_values[0],
             return_values[1],
         ]
+
+    def test_i_can_eval_all_pass(self):
+        """
+        If they all pass, that means that no concept was reduced
+        -> We need to act like resolve ambiguity was not called
+        :return:
+        """
+        context = self.get_context()
+        return_values = [
+            self.pretval(Concept("hello world 1"), "hello word"),
+            self.pretval(Concept("hello world 2"), "hello word"),
+        ]
+
+        evaluator = ResolveAmbiguityEvaluator()
+        evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert res is None

tests/non_reg/test_sheerka_non_reg.py

@@ -1015,6 +1015,82 @@ as:
         assert res[0].status
         assert res[0].body == "Executed !"
 
+    def test_i_can_manage_question(self):
+        init = [
+            "def concept one",
+            "def concept foo",
+            "def concept number",
+            "one isa number",
+            "def concept x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
+            "def concept x is a y as set_isa(x,y)",
+        ]
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("question(one is a number)")  # automatically evaluated
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body
+
+        res = sheerka.evaluate_user_input("question(foo is a number)")  # automatically evaluated
+        assert len(res) == 1
+        assert res[0].status
+        assert not res[0].body
+
+    def test_where_clause_implicitly_infer_to_question(self):
+        init = [
+            "def concept one as 1",
+            "def concept number",
+            "one isa number",
+            "def concept one as 10",  # to make sure that it won't be rejected because of the cast
+            "def concept x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
+            "def concept x is a y as set_isa(x,y)",
+        ]
+
+        # Explanations
+        # There are two concepts 'one'. The first one is tagged as a number while the second one is not
+        # So the first one should be picked.
+        # the second concept 'one' 's value is an integer, to make sure that it won't be rejected because of its type
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("def concept x plus y where x is a number as x + y")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.NEW_CONCEPT)
+
+        # Let's check that the concept is correctly understood
+        res = sheerka.evaluate_user_input("eval one plus 2")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 3  # it means that the first 'one' was chosen
+
+    def test_i_can_manage_concept_that_refers_to_question(self):
+        init = [
+            "def concept one",
+            "def concept foo",
+            "def concept number",
+            "one isa number",
+            "def concept q from q ? as question(q)",
+            "def concept is_a from x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
+            "set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:)"
+        ]
+
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("one is a number ?")  # automatically evaluated
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body
+
+        res = sheerka.evaluate_user_input("foo is a number ?")  # automatically evaluated
+        assert len(res) == 1
+        assert res[0].status
+        assert not res[0].body
+
+        # Sanity, when there is only one 'is a' concept. It's chosen regardless of the PRE condition
+        res = sheerka.evaluate_user_input("one is a number")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body
+
 
 class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
     def test_i_can_def_several_concepts(self):

tests/parsers/test_DefaultParser.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass
 
 import pytest
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
-from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
+from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch
@@ -87,9 +87,9 @@ class PN:
 class TestDefaultParser(TestUsingMemoryBasedSheerka):
 
     def init_parser(self, *concepts):
-        sheerka, concept, *updated = self.init_concepts(*concepts, singleton=True)
+        sheerka, context, *updated = self.init_concepts(*concepts, singleton=True)
         parser = DefaultParser()
-        return sheerka, concept, parser, *updated
+        return sheerka, context, parser, *updated
 
     @pytest.mark.parametrize("text, expected", [
         ("def concept hello", get_def_concept(name="hello")),
@@ -313,6 +313,38 @@ def concept add one to a as
         assert isinstance(res.value, ParserResultConcept)
         assert node == expected
 
+    def test_i_can_parse_when_ambiguity_in_where_pre_clause(self):
+        sheerka, context, parser, *concepts = self.init_parser(
+            Concept("x is a y", pre="in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)"),
+            Concept("x is a y")
+        )
+
+        text = "def concept foo x y where x is a y"
+        res = parser.parse(context, ParserInput(text))
+        expected_body = self.pretval(CV(concepts[0], pre=True), source="x is a y", who="parsers.Default",
+                                     parser="parsers.ExactConcept")
+        expected = get_def_concept("foo x y", where=expected_body)
+        node = res.value.value
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
+        text = "def concept foo x y pre x is a y"
+        res = parser.parse(context, ParserInput(text))
+        expected_body = self.pretval(CV(concepts[0], pre=True), source="x is a y", who="parsers.Default",
+                                     parser="parsers.ExactConcept")
+        expected = get_def_concept("foo x y", pre=expected_body)
+        node = res.value.value
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
     def test_i_can_detect_not_for_me(self):
         text = "hello world"
         sheerka, context, parser = self.init_parser()

tests/parsers/test_ExpressionParser.py

@@ -1,9 +1,13 @@
 from dataclasses import dataclass
 
+import pytest
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import Tokenizer, TokenKind
+from parsers.BaseParser import UnexpectedEof, UnexpectedTokenErrorNode
 from parsers.ExpressionParser import PropertyEqualsNode, PropertyEqualsSequenceNode, PropertyContainsNode, AndNode, \
-    OrNode, NotNode, LambdaNode, IsaNode
+    OrNode, NotNode, LambdaNode, IsaNode, NameExprNode, ExpressionParser, LeftPartNotFoundError, TrueifyVisitor
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -16,8 +20,93 @@ class Obj:
     parent: object = None
 
 
+def n(value):
+    return NameExprNode(Tokenizer(value, yield_eof=False))
+
+
 class TestExpressionParser(TestUsingMemoryBasedSheerka):
 
+    def init_parser(self):
+        sheerka, context = self.init_concepts()
+        parser = ExpressionParser()
+        return sheerka, context, parser
+
+    @pytest.mark.parametrize("expression, expected", [
+        ("one complicated expression", n("one complicated expression")),
+        ("function_call(a,b,c)", n("function_call(a,b,c)")),
+        ("one expression or another expression", OrNode(n("one expression"), n("another expression"))),
+        ("one expression and another expression", AndNode(n("one expression"), n("another expression"))),
+        ("one or two or three", OrNode(n("one"), n("two"), n("three"))),
+        ("one and two and three", AndNode(n("one"), n("two"), n("three"))),
+        ("one or two and three", OrNode(n("one"), AndNode(n("two"), n("three")))),
+        ("one and two or three", OrNode(AndNode(n("one"), n("two")), n("three"))),
+        ("one and (two or three)", AndNode(n("one"), OrNode(n("two"), n("three")))),
+    ])
+    def test_i_can_parse_expression(self, expression, expected):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(expression))
+        wrapper = res.body
+        expressions = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert expressions == expected
+
+    @pytest.mark.parametrize("expression, expected_errors", [
+        ("one or", [UnexpectedEof("When parsing 'or'")]),
+        ("one and", [UnexpectedEof("When parsing 'and'")]),
+        ("and one", [LeftPartNotFoundError()]),
+        ("or one", [LeftPartNotFoundError()]),
+        ("or", [LeftPartNotFoundError(), UnexpectedEof("When parsing 'or'")]),
+        ("and", [LeftPartNotFoundError(), UnexpectedEof("When parsing 'and'")]),
+    ])
+    def test_i_can_detect_error(self, expression, expected_errors):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(expression))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == expected_errors
+
+    def test_i_can_detect_unbalanced_parenthesis(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput("("))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
+        assert res.body.body[0].token.type == TokenKind.EOF
+        assert res.body.body[0].expected_tokens == [TokenKind.RPAR]
+
+        res = parser.parse(context, ParserInput(")"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
+        assert res.body.body[0].token.type == TokenKind.RPAR
+        assert res.body.body[0].expected_tokens == []
+
+        res = parser.parse(context, ParserInput("one and two)"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
+        assert res.body.body[0].token.type == TokenKind.RPAR
+        assert res.body.body[0].expected_tokens == []
+
+        res = parser.parse(context, ParserInput("one and two)"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
+        assert res.body.body[0].token.type == TokenKind.RPAR
+        assert res.body.body[0].expected_tokens == []
+
+    def test_i_can_detect_empty_expression(self):
+        sheerka, context, parser = self.init_parser()
+        res = parser.parse(context, ParserInput(""))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
+
     def test_i_can_test_property_equals(self):
         node = PropertyEqualsNode("prop_a", "good value")
 
@@ -101,3 +190,21 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         assert concept_node2.eval(Concept("foo").init_key())
         assert not concept_node2.eval(Obj)
         assert not concept_node2.eval(Concept())
+
+    @pytest.mark.parametrize("expression, to_trueify, to_skip, expected", [
+        ("a", ["b"], ["a"], "a"),
+        ("b", ["b"], ["a"], "True"),
+        ("a and b", ["b"], ["a"], "a and True"),
+        ("b or a", ["b"], ["a"], "True or a"),
+        ("isinstance(b, str)", ["b"], ["a"], "True"),
+        ("isinstance(b, str) or instance(a, str)", ["b"], ["a"], "True or instance(a, str)"),
+        ("a and b or c", ["b", "c"], ["a"], "a and True or True"),
+        ("a + b or a + c", ["b", "c"], ["a"], "a + b or a + c"),
+    ])
+    def test_i_can_trueify(self, expression, to_trueify, to_skip, expected):
+        sheerka, context, parser = self.init_parser()
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+
+        translated_node = TrueifyVisitor(to_trueify, to_skip).visit(expr_node)
+
+        assert str(translated_node) == expected

tests/parsers/test_TestShortTermMemoryParser.py

@@ -0,0 +1,69 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.SheerkaExecute import ParserInput
+from parsers.ShortTermMemoryParser import ShortTermMemoryParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestShortTermMemoryParser(TestUsingMemoryBasedSheerka):
+    def init_parser(self):
+        sheerka, context = self.init_concepts()
+        parser = ShortTermMemoryParser()
+        return sheerka, context, parser
+
+    def test_i_cannot_parse_empty_string(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(""))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
+
+    def test_i_can_get_concept_from_sheerka(self):
+        sheerka, context, parser = self.init_parser()
+        foo = Concept("foo")
+
+        sheerka.add_to_short_term_memory(None, "test", foo)
+
+        res = parser.parse(context, ParserInput("test"))
+
+        assert res.status
+        assert id(res.body) == id(foo)
+
+    def test_i_can_get_concept_from_the_context(self):
+        sheerka, context, parser = self.init_parser()
+        foo = Concept("foo")
+
+        sheerka.add_to_short_term_memory(context, "test", foo)
+
+        res = parser.parse(context, ParserInput("test"))
+
+        assert res.status
+        assert id(res.body) == id(foo)
+
+    def test_i_can_get_concept_from_parent_context(self):
+        sheerka, context, parser = self.init_parser()
+        foo = Concept("foo")
+
+        sheerka.add_to_short_term_memory(context, "test", foo)
+
+        with context.push(BuiltinConcepts.TESTING, None) as sub_context:
+            res = parser.parse(sub_context, ParserInput("test"))
+
+        assert res.status
+        assert id(res.body) == id(foo)
+
+    def test_i_cannot_get_concept_when_no_concept_in_memory(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput("test"))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOUND)
+        assert res.body.body == "test"
+
+    def test_parser_can_only_be_called_with_parser_input(self):
+        sheerka, context, parser = self.init_parser()
+
+        assert parser.parse(context, "not_a_parser_input") is None

tests/sheerkapickle/test_sheerka_handlers.py

@@ -279,14 +279,14 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
 
     def test_i_can_encode_decode_execution_context(self):
         sheerka = self.get_sheerka()
-
+        c = Concept("foo").def_var("a")
-        context = ExecutionContext("who", Event("xxx"), sheerka, BuiltinConcepts.NOP, None, "my desc")
+        context = ExecutionContext("who", Event("xxx"), sheerka, BuiltinConcepts.EVALUATE_CONCEPT, c, "my desc")
         input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)]
         context.inputs = {"a": input_list, "b": set_full_serialization(Concept("foo"))}
         context.values = {"c": input_list, "d": set_full_serialization(Concept("bar"))}
         context.obj = set_full_serialization(Concept("baz"))
-        context.push(BuiltinConcepts.NOP, None, who="who3", desc="sub_child1")
+        context.push(BuiltinConcepts.EVALUATING_CONCEPT, c, who="who3", desc="sub_child1")
-        context.push(BuiltinConcepts.NOP, None, who="who4", desc="sub_child2")
+        context.push(BuiltinConcepts.EVALUATING_ATTRIBUTE, "a", who="who4", desc="sub_child2")
 
         to_string = sheerkapickle.encode(sheerka, context)
         decoded = sheerkapickle.decode(sheerka, to_string)