First implementation of questions management

src/core/sheerka/ExecutionContext.py

@@ -4,6 +4,7 @@ import time
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka.services.SheerkaExecute import NO_MATCH
+from core.sheerka.services.SheerkaShortTermMemory import SheerkaShortTermMemory
 from core.sheerka_logger import get_logger
 from sdp.sheerkaDataProvider import Event
 
@@ -11,11 +12,13 @@ DEBUG_TAB_SIZE = 4
 
 PROPERTIES_TO_SERIALIZE = ("_id",
                            "_bag",
+                           "_children",
                            "_start",
                            "_stop",
                            "who",
+                           "action",
+                           "action_context",
                            "desc",
-                           "children",
                            "inputs",
                            "values",
                            "obj",
@@ -46,16 +49,20 @@ class ExecutionContext:
                  desc: str = None,
                  logger=None,
                  global_hints=None,
-                 global_errors=None,
+                 errors=None,
                  **kwargs):
 
-        self._parent = None
         self._id = ExecutionContext.get_id(event.get_digest()) if event else None
+        self._parent = None
+        self._children = []
         self._tab = ""
         self._bag = {}  # context variables
         self._start = 0  # when the execution starts (to measure elapsed time)
         self._stop = 0  # when the execution stops (to measure elapses time)
+        self._logger = logger
         self._format_instructions = None  # how to print the execution context
+        self._stat_log = get_logger("stats")
+        self._show_stats = False
 
         self.who = who  # who is asking
         self.event = event  # what was the (original) trigger
@@ -63,26 +70,24 @@ class ExecutionContext:
         self.action = action
         self.action_context = action_context
         self.desc = desc  # human description of what is going on
-        self.children = []
         self.preprocess = None
-        self.logger = logger
-        self.local_hints = set()
+        self.stm = False  # True if the context has short term memory entries
+
+        self.private_hints = set()
+        self.protected_hints = set()
         self.global_hints = set() if global_hints is None else global_hints
-        self.global_errors = [] if global_errors is None else global_errors
+        self.errors = [] if errors is None else errors  # error are global
 
-        self.inputs = {}  # what was the parameters of the execution context
+        self.inputs = {}  # what were the parameters of the execution context
         self.values = {}  # what was produced by the execution context
-
         self.obj = kwargs.pop("obj", None)  # current obj we are working on
+
         self.concepts = kwargs.pop("concepts", {})  # known concepts specific to this context
 
         # update the other elements
         for k, v in kwargs.items():
             self._bag[k] = v
 
-        self.stat_log = get_logger("stats")
-        self.show_stats = False
-
     @property
     def elapsed(self):
         if self._start == 0:
@@ -96,10 +101,22 @@ class ExecutionContext:
         dt = nano_sec / 1e6
         return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
 
+    @property
+    def logger(self):
+        return self._logger
+
     @property
     def id(self):
         return self._id
 
+    @property
+    def achildren(self):
+        """
+        I prefixed with an 'a' to make it appear on the top when debugging
+        :return:
+        """
+        return self._children
+
     def __getattr__(self, item):
         if item in self._bag:
             return self._bag[item]
@@ -112,9 +129,12 @@ class ExecutionContext:
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.stm:
+            self.sheerka.services[SheerkaShortTermMemory.NAME].remove_context(self)
+
         self._stop = time.time_ns()
-        if self.show_stats:
-            self.stat_log.debug(f"[{self._id:2}]" + self._tab + "Execution time: " + self.elapsed_str)
+        if self._show_stats:
+            self._stat_log.debug(f"[{self._id:2}]" + self._tab + "Execution time: " + self.elapsed_str)
 
     def __repr__(self):
         msg = f"ExecutionContext(who={self.who}, id={self._id}, action={self.action}, context={self.action_context}"
@@ -136,7 +156,7 @@ class ExecutionContext:
             return False
 
         for prop in PROPERTIES_TO_SERIALIZE:
-            if prop == "who":
+            if prop in ("who", "action", "action_context"):
                 value = str(getattr(self, prop))
                 other_value = str(getattr(other, prop))
             else:
@@ -150,7 +170,7 @@ class ExecutionContext:
 
     def push(self, action: BuiltinConcepts, action_context, who=None, desc=None, logger=None, **kwargs):
         who = who or self.who
-        logger = logger or self.logger
+        logger = logger or self._logger
         _kwargs = {"obj": self.obj, "concepts": self.concepts}
         _kwargs.update(self._bag)
         _kwargs.update(kwargs)
@@ -163,14 +183,14 @@ class ExecutionContext:
             desc,
             logger,
             self.global_hints,
-            self.global_errors,
+            self.errors,
             **_kwargs)
         new._parent = self
         new._tab = self._tab + " " * DEBUG_TAB_SIZE
         new.preprocess = self.preprocess
-        new.local_hints.update(self.local_hints)
+        new.protected_hints.update(self.protected_hints)
 
-        self.children.append(new)
+        self._children.append(new)
         return new
 
     def add_preprocess(self, name, **kwargs):
@@ -194,6 +214,23 @@ class ExecutionContext:
             self.values[k] = v
         return self
 
+    def add_to_short_term_memory(self, key, concept):
+        """
+        Add a concept to the short term memory (relative to the current execution context)
+        :param key:
+        :param concept:
+        :return:
+        """
+        self.sheerka.add_to_short_term_memory(self, key, concept)
+
+    def get_from_short_term_memory(self, key):
+        """
+
+        :param key:
+        :return:
+        """
+        return self.sheerka.get_from_short_term_memory(self, key)
+
     def get_concept(self, key):
         # search in obj
         if isinstance(self.obj, Concept):
@@ -234,44 +271,43 @@ class ExecutionContext:
         return self.sheerka.new(key, **kwargs)
 
     def log_new(self):
-        if self.logger and not self.logger.disabled:
-            self.logger.debug(f"[{self._id:2}]" + self._tab + str(self))
-            self.show_stats = True
+        if self._logger and not self._logger.disabled:
+            self._logger.debug(f"[{self._id:2}]" + self._tab + str(self))
+            self._show_stats = True
 
     def log(self, message, who=None):
-        if self.logger and not self.logger.disabled:
-            self.logger.debug(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
+        if self._logger and not self._logger.disabled:
+            self._logger.debug(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
 
     def log_error(self, message, who=None, exc=None):
-        self.global_errors.append(exc or message)
-        if self.logger and not self.logger.disabled:
-            self.logger.exception(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
+        self.errors.append(exc or message)
+        if self._logger and not self._logger.disabled:
+            self._logger.exception(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
 
     def log_result(self, return_values):
-        if not self.logger or not self.logger.isEnabledFor(logging.DEBUG):
+        if not self._logger or not self._logger.isEnabledFor(logging.DEBUG):
             return
 
         if len(return_values) == 0:
-            self.logger.debug(self._tab + "No return value")
+            self._logger.debug(self._tab + "No return value")
 
         for r in return_values:
             to_str = self.return_value_to_str(r)
-            self.logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
+            self._logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
     def get_parent(self):
         return self._parent
 
     def in_context(self, concept_key):
-        if concept_key in self.local_hints:
-            return True
-
-        if concept_key in self.global_hints:
-            return True
-
-        return False
+        return concept_key in self.protected_hints or \
+               concept_key in self.global_hints or \
+               concept_key in self.private_hints
 
     def in_current_context(self, concept_key):
-        return concept_key in self.local_hints
+        return concept_key in self.protected_hints or concept_key in self.private_hints
+
+    def in_private_context(self, concept_key):
+        return concept_key in self.private_hints
 
     @staticmethod
     def _is_return_value(obj):
@@ -336,7 +372,7 @@ class ExecutionContext:
             bag["bag." + k] = v
         for prop in ("id", "who", "action", "desc", "obj", "inputs", "values", "concepts"):
             bag[prop] = getattr(self, prop)
-        bag["action"] = self.action_context
+        bag["context"] = self.action_context
         for prop in ("desc", "obj", "inputs", "values", "concepts"):
             bag[prop] = getattr(self, prop)
         bag["status"] = self.get_status()

src/core/sheerka/Sheerka.py

@@ -21,6 +21,14 @@ from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 
 BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
 EXIT_COMMANDS = ("quit", "exit", "bye")
+EXECUTE_STEPS = [
+    BuiltinConcepts.BEFORE_PARSING,
+    BuiltinConcepts.PARSING,
+    BuiltinConcepts.AFTER_PARSING,
+    BuiltinConcepts.BEFORE_EVALUATION,
+    BuiltinConcepts.EVALUATION,
+    BuiltinConcepts.AFTER_EVALUATION
+]
 
 
 @dataclass
@@ -124,11 +132,11 @@ class Sheerka(Concept):
 
     @property
     def concepts_grammars(self):
-        return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
+        return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
 
     @property
     def chicken_and_eggs(self):
-        return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache
+        return self.cache_manager.caches[self.CHICKEN_AND_EGG_CONCEPTS_ENTRY].cache
 
     def bind_service_method(self, bound_method, has_side_effect, as_name=None):
         """
@@ -406,16 +414,7 @@ class Sheerka(Concept):
             user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
             reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
 
-            steps = [
-                BuiltinConcepts.BEFORE_PARSING,
-                BuiltinConcepts.PARSING,
-                BuiltinConcepts.AFTER_PARSING,
-                BuiltinConcepts.BEFORE_EVALUATION,
-                BuiltinConcepts.EVALUATION,
-                BuiltinConcepts.AFTER_EVALUATION
-            ]
-
-            ret = self.execute(execution_context, [user_input, reduce_requested], steps)
+            ret = self.execute(execution_context, [user_input, reduce_requested], EXECUTE_STEPS)
             execution_context.add_values(return_values=ret)
 
             if self.cache_manager.is_dirty:

src/core/sheerka/services/SheerkaAdmin.py

@@ -49,9 +49,11 @@ class SheerkaAdmin(BaseService):
 
         try:
             start = time.time_ns()
+            nb_lines = 0
             self.sheerka.during_restore = True
             with open(concept_file, "r") as f:
                 for line in f.readlines():
+                    nb_lines += 1
                     line = line.strip()
                     if line == "" or line.startswith("#"):
                         continue
@@ -65,7 +67,7 @@ class SheerkaAdmin(BaseService):
             nano_sec = stop - start
             dt = nano_sec / 1e6
             elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
-            print(f"Execution time: {elapsed}")
+            print(f"Imported {nb_lines} line(s) in {elapsed}.")
         except IOError:
             pass
 

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -1,9 +1,13 @@
+from dataclasses import dataclass
+
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import expect_one, only_successful
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
+from core.builtin_helpers import expect_one, only_successful, parse_unrecognized, evaluate
+from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, NotInit
+from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import BaseService
 from core.tokenizer import Tokenizer
 from core.utils import unstr_concept
+from parsers.ExpressionParser import ExpressionParser, TrueifyVisitor
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,
@@ -11,6 +15,15 @@ CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.AFTER_EVALUATION]
 
 
+@dataclass
+class WhereClauseDef:
+    concept: Concept  # concept on which the where clause is applied
+    clause: str  # original where clause
+    trueified: str  # modified where clause (where unresolvable variables are removed)
+    prop: str  # variable to test
+    compiled: object  # trueified where clause Python compiled
+
+
 class SheerkaEvaluateConcept(BaseService):
     NAME = "EvaluateConcept"
 
@@ -61,6 +74,112 @@ class SheerkaEvaluateConcept(BaseService):
         """
         return concept.get_value(ConceptParts.RET) if ConceptParts.RET in concept.values else concept
 
+    @staticmethod
+    def get_needed_metadata(concept, concept_part, check_vars, check_body):
+        """
+        Check if the concept_part has to be evaluated
+        It also checks if the variables and the body need to be evaluated prior to it
+        :param concept:
+        :param concept_part:
+        :param check_vars:
+        :param check_body:
+        :return:
+        """
+        ret = []
+        vars_needed = False
+        body_needed = False
+
+        if concept_part in concept.compiled and concept.compiled[concept_part] is not None:
+            concept_part_source = getattr(concept.metadata, concept_part.value)
+
+            assert concept_part_source is not None
+
+            tokens = [t.str_value for t in Tokenizer(concept_part_source)]
+
+            if check_vars:
+                for var_name in (v[0] for v in concept.metadata.variables):
+                    if var_name in tokens:
+                        vars_needed = True
+                        ret.append("variables")
+                        break
+
+            if check_body and "self" in tokens:
+                body_needed = True
+                ret.append("body")
+
+            ret.append(concept_part.value)
+
+        return ret, vars_needed, body_needed
+
+    @staticmethod
+    def get_where_clause_def(context, concept, var_name):
+        """
+        Returns the compiled code to be executed
+        :param context:
+        :param concept:
+        :param var_name:
+        :return:
+        """
+        if concept.metadata.where is None or concept.metadata.where.strip() == "":
+            return None
+
+        ret = ExpressionParser().parse(context, ParserInput(concept.metadata.where))
+        if not ret.status:
+            # TODO: manage invalid where clause
+            return None
+        expr = ret.body.body
+
+        to_trueify = [v[0] for v in concept.metadata.variables if v[0] != var_name]
+        trueified_where = str(TrueifyVisitor(to_trueify, [var_name]).visit(expr))
+
+        tokens = [t.str_value for t in Tokenizer(trueified_where)]
+        if var_name in tokens:
+            compiled = None
+            try:
+                compiled = compile(trueified_where, "<where clause>", "eval")
+            except Exception:
+                pass
+            return WhereClauseDef(concept, concept.metadata.where, trueified_where, var_name, compiled)
+        else:
+            return None
+
+    def apply_where_clause(self, context, where_clause_def, return_values):
+        """
+        Apply intermediate where clause when evaluating concept variables
+        :param context:
+        :param where_clause_def:
+        :param return_values:
+        :return:
+        """
+        ret = []
+        for r in [r for r in return_values if r.status]:
+            if where_clause_def.compiled:
+                try:
+                    if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}):
+                        ret.append(r)
+                except NameError:
+                    ret.append(r)  # it cannot be solved unitary, let's give a chance to the global where condition
+            else:
+                # it means that the where condition is an expression that needs to be executed
+                evaluation_res = evaluate(context,
+                                          where_clause_def.trueified,
+                                          desc=f"Apply where clause on '{where_clause_def.prop}'",
+                                          expect_success=True,
+                                          stm={where_clause_def.prop: r.body})
+                one_res = expect_one(context, evaluation_res)
+                if one_res.status:
+                    value = context.sheerka.objvalue(one_res)
+                    if isinstance(value, bool) and value:
+                        ret.append(r)
+
+        if len(ret) > 0:
+            return ret
+
+        return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED,
+                                body=where_clause_def.clause,
+                                concept=where_clause_def.concept,
+                                prop=where_clause_def.prop)
+
     def manage_infinite_recursion(self, context):
         """
         We look for the fist parent that has a body that means something
@@ -105,7 +224,6 @@ class SheerkaEvaluateConcept(BaseService):
                 return self.sheerka.resolve(identifier)
             return None
 
-        steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
         for part_key in ConceptParts:
             if part_key in concept.compiled:
                 continue
@@ -122,16 +240,12 @@ class SheerkaEvaluateConcept(BaseService):
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
                     concept.compiled[part_key] = concept_found
                 else:
-                    with context.push(BuiltinConcepts.INIT_COMPILED,
-                                      {"part": part_key, "source": source},
-                                      desc=f"Initializing *compiled* for {part_key}") as sub_context:
-                        sub_context.add_inputs(source=source)
-                        to_parse = self.sheerka.ret(context.who, True,
-                                                    self.sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
-                        res = self.sheerka.execute(sub_context, to_parse, steps)
-                        only_success = only_successful(sub_context, res)
-                        concept.compiled[part_key] = only_success.body.body if is_only_successful(only_success) else res
-                        sub_context.add_values(return_values=res)
+                    res = parse_unrecognized(context,
+                                             source,
+                                             parsers="all",
+                                             prop=part_key,
+                                             filter_func=only_successful)
+                    concept.compiled[part_key] = res.body.body if is_only_successful(res) else res
 
         for var_name, default_value in concept.metadata.variables:
             if var_name in concept.compiled:
@@ -148,22 +262,36 @@ class SheerkaEvaluateConcept(BaseService):
                     context.log(f"Recognized concept '{concept_found}'", self.NAME)
                     concept.compiled[var_name] = concept_found
                 else:
-                    with context.push(BuiltinConcepts.INIT_COMPILED,
-                                      {"property": var_name, "source": default_value},
-                                      desc=f"Initializing *compiled* for property {var_name}") as sub_context:
-                        sub_context.add_inputs(source=default_value)
-                        to_parse = self.sheerka.ret(context.who, True,
-                                                    self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value))
-                        res = self.sheerka.execute(sub_context, to_parse, steps)
-                        only_success = only_successful(sub_context, res)
-                        concept.compiled[var_name] = only_success.body.body if is_only_successful(only_success) else res
-                        sub_context.add_values(return_values=res)
+                    res = parse_unrecognized(context,
+                                             default_value,
+                                             parsers="all",
+                                             prop=var_name,
+                                             filter_func=only_successful)
+                    concept.compiled[var_name] = res.body.body if is_only_successful(res) else res
 
         # Updates the cache of concepts when possible
         if self.sheerka.has_id(concept.id):
             self.sheerka.get_by_id(concept.id).compiled = concept.compiled
 
-    def resolve(self, context, to_resolve, current_prop, current_concept, force_evaluation, expect_success):
+    def resolve(self,
+                context,
+                to_resolve,
+                current_prop,
+                current_concept,
+                force_evaluation,
+                expect_success,
+                where_clause_def):
+        """
+        Resolve a variable or a Concept
+        :param context: current execution context
+        :param to_resolve: Concept or list of ReturnValueConcept to resolve
+        :param current_prop: current property or ConceptPart
+        :param current_concept: current concept
+        :param force_evaluation: Force body evaluation
+        :param expect_success: for PythonEvaluator, try all possibilities to find a positive result
+        :param where_clause_def: intermediate where clause for variables
+        :return:
+        """
 
         def get_path(context_, prop_name):
             concept_name = f'"{context_.action_context.name}"' if isinstance(context_.action_context, Concept) \
@@ -195,10 +323,11 @@ class SheerkaEvaluateConcept(BaseService):
                           path=path) as sub_context:
 
             if force_evaluation:
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             if expect_success:
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
 
             # when it's a concept, evaluate it
             if isinstance(to_resolve, Concept) and \
@@ -212,15 +341,28 @@ class SheerkaEvaluateConcept(BaseService):
 
             # otherwise, execute all return values to find out what is the value
             else:
+                # update short term memory with current concept variables
+                if current_concept:
+                    for var in current_concept.metadata.variables:
+                        value = current_concept.get_value(var[0])
+                        if value != NotInit:
+                            sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0]))
                 use_copy = [r for r in to_resolve] if hasattr(to_resolve, "__iter__") else to_resolve
                 r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS)
 
-                one_r = expect_one(context, r)
-                sub_context.add_values(return_values=one_r)
-                if one_r.status:
-                    return one_r.value
+                if where_clause_def:
+                    # apply intermediate where clause
+                    r = self.apply_where_clause(context, where_clause_def, r)
+
+                if self.sheerka.isinstance(r, BuiltinConcepts.CONDITION_FAILED):
+                    return r
                 else:
-                    error = one_r.value
+                    one_r = expect_one(context, r)
+                    sub_context.add_values(return_values=one_r)
+                    if one_r.status:
+                        return one_r.value
+                    else:
+                        error = one_r.value
 
             return error if self.sheerka.isinstance(error, BuiltinConcepts.CHICKEN_AND_EGG) \
                 else self.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR,
@@ -228,7 +370,14 @@ class SheerkaEvaluateConcept(BaseService):
                                       concept=current_concept,
                                       property_name=current_prop)
 
-    def resolve_list(self, context, list_to_resolve, current_prop, current_concept, force_evaluation, expect_success):
+    def resolve_list(self,
+                     context,
+                     list_to_resolve,
+                     current_prop,
+                     current_concept,
+                     force_evaluation,
+                     expect_success,
+                     where_clause_def):
         """When dealing with a list, there are two possibilities"""
         # It may be a list of ReturnValueConcept to execute (always the case for metadata)
         # or a list of single values (may be the case for properties)
@@ -242,7 +391,8 @@ class SheerkaEvaluateConcept(BaseService):
                                 current_prop,
                                 current_concept,
                                 force_evaluation,
-                                expect_success)
+                                expect_success,
+                                where_clause_def)
 
         res = []
         for to_resolve in list_to_resolve:
@@ -253,7 +403,13 @@ class SheerkaEvaluateConcept(BaseService):
                                         concept=current_concept,
                                         property_name=current_prop)
 
-            r = self.resolve(context, to_resolve, current_prop, current_concept, force_evaluation, expect_success)
+            r = self.resolve(context,
+                             to_resolve,
+                             current_prop,
+                             current_concept,
+                             force_evaluation,
+                             expect_success,
+                             where_clause_def)
             if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
                 return r
             res.append(r)
@@ -263,7 +419,7 @@ class SheerkaEvaluateConcept(BaseService):
     def evaluate_concept(self, context, concept: Concept, eval_body=False, metadata=None):
         """
         Evaluation a concept
-        It means that if the where clause is True, will evaluate the body
+        ie : resolve its body
         :param context:
         :param concept:
         :param eval_body:
@@ -275,7 +431,7 @@ class SheerkaEvaluateConcept(BaseService):
             return concept
 
         # I cannot use cache because of concept like 'number'.
-        # They don't have variables, but their values change every time they are instanciated
+        # They don't have variables, but their values change every time they are instantiated
         # TODO: Need to find a way to cache despite of them
         # need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)
         # if need_body and len(concept.metadata.variables) == 0 and context.sheerka.has_id(concept.id):
@@ -290,11 +446,11 @@ class SheerkaEvaluateConcept(BaseService):
 
             if eval_body:
                 # ask for body evaluation
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             # auto evaluate commands
             if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)):
-                sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
             self.initialize_concept_asts(sub_context, concept)
 
@@ -307,12 +463,15 @@ class SheerkaEvaluateConcept(BaseService):
                     for var_name in (v for v in concept.variables() if v in concept.compiled):
                         prop_ast = concept.compiled[var_name]
 
+                        w_clause = self.get_where_clause_def(context, concept, var_name)
+                        # TODO, manage when the where clause cannot be parsed
+
                         if isinstance(prop_ast, list):
                             # Do not send the current concept for the properties
-                            resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False)
+                            resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False, w_clause)
                         else:
                             # Do not send the current concept for the properties
-                            resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False)
+                            resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False, w_clause)
 
                         if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
                             resolved.set_value("concept", concept)  # since current concept was not sent
@@ -347,7 +506,8 @@ class SheerkaEvaluateConcept(BaseService):
                                             part_key,
                                             concept,
                                             force_concept_eval,
-                                            expect_success)
+                                            expect_success,
+                                            None)
 
                     # 'FATAL' error is detected, let's stop
                     if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
@@ -411,40 +571,3 @@ class SheerkaEvaluateConcept(BaseService):
                 to_eval.append("body")
 
         return to_eval
-
-    @staticmethod
-    def get_needed_metadata(concept, concept_part, check_vars, check_body):
-        """
-        Check if the concept_part has to be evaluated
-        It also checks if the variables and the body need to be evaluated prior to it
-        :param concept:
-        :param concept_part:
-        :param check_vars:
-        :param check_body:
-        :return:
-        """
-        ret = []
-        vars_needed = False
-        body_needed = False
-
-        if concept_part in concept.compiled and concept.compiled[concept_part] is not None:
-            concept_part_source = getattr(concept.metadata, concept_part.value)
-
-            assert concept_part_source is not None
-
-            tokens = [t.str_value for t in Tokenizer(concept_part_source)]
-
-            if check_vars:
-                for var_name in (v[0] for v in concept.metadata.variables):
-                    if var_name in tokens:
-                        vars_needed = True
-                        ret.append("variables")
-                        break
-
-            if check_body and "self" in tokens:
-                body_needed = True
-                ret.append("body")
-
-            ret.append(concept_part.value)
-
-        return ret, vars_needed, body_needed

src/core/sheerka/services/SheerkaExecute.py

@@ -362,7 +362,7 @@ class SheerkaExecute(BaseService):
                                     if not isinstance(results, list):
                                         results = [results]
                                     for result in results:
-                                        if result.body:
+                                        if result.body != BuiltinConcepts.NO_RESULT:
                                             evaluated_items.append(result)
                                         to_delete.extend(result.parents)
                                     sub_context.add_values(return_values=results)

src/core/sheerka/services/SheerkaFilter.py

@@ -404,11 +404,11 @@ class SheerkaFilter(BaseService):
             yield item
 
     @staticmethod
-    def pipe_recurse(iterable, depth, prop_name="children", when=None):
+    def pipe_recurse(iterable, depth, prop_name="_children", when=None):
         """
         When printing an object that has sub properties,
         indicate the depth of recursion to apply to a specific properties
-        Quick and dirty version because the prop name is not taken from the item (but set to 'children' by default)
+        Quick and dirty version because the prop name is not taken from the item (but set to '_children' by default)
         :param iterable:
         :param depth:
         :param prop_name:

src/core/sheerka/services/SheerkaQuestion.py

@@ -0,0 +1,40 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.sheerka_service import BaseService
+
+
+class SheerkaQuestion(BaseService):
+    NAME = "Question"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.question, False)
+        self.sheerka.bind_service_method(self.is_question, False)
+
+    def question(self, context, q):
+        """
+        Evaluate q in the context in a question
+        :param context:
+        :param q:
+        :return:
+        """
+
+        if isinstance(q, Concept):
+            with context.push(BuiltinConcepts.EVALUATE_CONCEPT, q, desc=f"Evaluating question '{q}'") as sub_context:
+                sub_context.global_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+                sub_context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+                sub_context.global_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+                sub_context.protected_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
+
+                evaluated = self.sheerka.evaluate_concept(sub_context, q)
+
+            return evaluated
+
+    def is_question(self, context):
+        """
+        Returns True if a question is asked
+        :return:
+        """
+        return context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)

src/core/sheerka/services/SheerkaResultManager.py

@@ -119,7 +119,7 @@ class SheerkaResultConcept(BaseService):
             for e in lst:
                 yield e
 
-                if e.children:
-                    yield from _yield_result(e.children)
+                if e._children:
+                    yield from _yield_result(e._children)
 
         return _yield_result([execution_context])

src/core/sheerka/services/SheerkaSetsManager.py

@@ -267,7 +267,7 @@ for x in xx__concepts__xx:
                           {"ids": ids},
                           desc=f"Evaluating concepts of a set") as sub_context:
             sub_context.add_inputs(ids=ids)
-            sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+            sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
             errors = []
             for element_id in ids:
                 concept = self.sheerka.get_by_id(element_id)

src/core/sheerka/services/SheerkaShortTermMemory.py

@@ -0,0 +1,37 @@
+from cache.ListIfNeededCache import ListIfNeededCache
+from core.sheerka.services.sheerka_service import BaseService
+
+
+class SheerkaShortTermMemory(BaseService):
+    NAME = "ShortTermMemory"
+
+    SHORT_TERM_MEMORY_ENTRY = "ShortTermMemory:Objects"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka)
+        self.objects = ListIfNeededCache()
+
+    def initialize(self):
+        self.sheerka.bind_service_method(self.get_from_short_term_memory, False)
+        self.sheerka.bind_service_method(self.add_to_short_term_memory, True)
+        self.sheerka.cache_manager.register_cache(self.SHORT_TERM_MEMORY_ENTRY, self.objects, persist=False)
+
+    def get_from_short_term_memory(self, context, key):
+        while True:
+            key_to_use = (str(context.id) if context else "") + ":" + key
+            if (obj := self.sheerka.cache_manager.get(self.SHORT_TERM_MEMORY_ENTRY, key_to_use)) is not None:
+                return obj
+
+            if context is None:
+                return None
+
+            context = context.get_parent()
+
+    def add_to_short_term_memory(self, context, key, concept):
+        if context:
+            context.stm = True
+        key_to_use = (str(context.id) if context else "") + ":" + key
+        return self.sheerka.cache_manager.put(self.SHORT_TERM_MEMORY_ENTRY, key_to_use, concept)
+
+    def remove_context(self, context):
+        self.objects.evict_by_key(lambda k: k.startswith(str(context.id) + ":"))