Concepts bodies are now evaluated on demand

src/core/builtin_concepts.py

@@ -53,6 +53,7 @@ class BuiltinConcepts(Enum):
     WHERE_CLAUSE_FAILED = "where clause failed"  # failed to validate where clause during evaluation
     CHICKEN_AND_EGG = "chicken and egg"  # infinite recursion when declaring concept
     ISA = "is a"  # builtin concept to express that a concept is an instance of another one
+    CONCEPT_VALUE_REQUESTED = "concept value requested"  # returns the body of the concept instead of the concept itself
 
     NODE = "node"
     GENERIC_NODE = "generic node"

src/core/builtin_helpers.py

@@ -5,12 +5,13 @@ import core.ast.nodes
 from core.ast.nodes import CallNodeConcept, GenericNodeConcept
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
 
 
-def is_same_success(sheerka, return_values):
+def is_same_success(context, return_values):
     """
     Returns True if all returns values are successful and have the same value
-    :param sheerka:
+    :param context:
     :param return_values:
     :return:
     """
@@ -19,13 +20,27 @@ def is_same_success(sheerka, return_values):
     if not return_values[0].status:
         return False
 
-    reference = sheerka.value(return_values[0].value)
+    if isinstance(return_values[0].body, Concept):
+        evaluated = context.sheerka.evaluate_concept(context, return_values[0].body, True)
+        if evaluated.key != return_values[0].body.key:
+            return False
+        reference = context.sheerka.value(evaluated)
+    else:
+        reference = context.sheerka.value(return_values[0])
 
     for return_value in return_values[1:]:
         if not return_value.status:
             return False
 
-        actual = sheerka.value(return_value.value)
+        if isinstance(return_value.body, Concept):
+            evaluated = context.sheerka.evaluate_concept(context, return_value.body, True)
+            if evaluated.key != return_value.body.key:
+                return False
+
+            actual = context.sheerka.value(evaluated)
+        else:
+            actual = context.sheerka.value(return_value)
+
         if actual != reference:
             return False
 
@@ -67,7 +82,7 @@ def expect_one(context, return_values):
 
     # too many winners, which one to choose ?
     if number_of_successful > 1:
-        if is_same_success(sheerka, successful_results):
+        if is_same_success(context, successful_results):
             return sheerka.ret(
                 context.who,
                 True,
@@ -218,3 +233,22 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
                 predicates.append(res)
 
     return predicates
+
+
+def add_to_ret_val(sheerka, context, return_values, concept_key):
+    concept = sheerka.new(concept_key)
+    ret_val = sheerka.ret(context.who, True, concept)
+    return_values.append(ret_val)
+    return return_values
+
+
+def remove_from_ret_val(sheerka, return_values, concept_key):
+    to_remove = []
+    for ret_val in return_values:
+        if ret_val.status and sheerka.isinstance(ret_val.body, concept_key):
+            to_remove.append(ret_val)
+
+    for item in to_remove:
+        return_values.remove(item)
+
+    return return_values

src/core/sheerka/ExecutionContext.py

@@ -60,6 +60,7 @@ class ExecutionContext:
         self.children = []
         self.preprocess = None
         self.logger = logger
+        self.extra_info = []
 
         self.inputs = {}  # what was the parameters of the execution context
         self.values = {}  # what was produced by the execution context
@@ -210,11 +211,11 @@ class ExecutionContext:
             self.sheerka,
             desc,
             logger,
-            **_kwargs,
-        )
+            **_kwargs)
         new._parent = self
         new._tab = self._tab + " " * DEBUG_TAB_SIZE
         new.preprocess = self.preprocess
+        new.extra_info.extend(self.extra_info)
 
         self.children.append(new)
         return new

src/core/sheerka/Services/SheerkaEvaluateConcept.py

@@ -1,6 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
-import core.builtin_helpers
+from core.builtin_helpers import add_to_ret_val, remove_from_ret_val, expect_one
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,
@@ -118,7 +118,7 @@ class SheerkaEvaluateConcept:
             else:
                 self.sheerka.cache_by_key[concept.key].compiled = concept.compiled
 
-    def resolve(self, context, to_resolve, current_prop, current_concept):
+    def resolve(self, context, to_resolve, current_prop, current_concept, evaluate_body):
         if isinstance(to_resolve, DoNotResolve):
             return to_resolve.value
 
@@ -137,7 +137,7 @@ class SheerkaEvaluateConcept:
             # when it's a concept, evaluate it
             if isinstance(to_resolve, Concept) and \
                 not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
-                evaluated = self.evaluate_concept(sub_context, to_resolve)
+                evaluated = self.evaluate_concept(sub_context, to_resolve, evaluate_body)
                 sub_context.add_values(return_values=evaluated)
                 if evaluated.key == to_resolve.key:
                     return evaluated
@@ -147,8 +147,11 @@ class SheerkaEvaluateConcept:
             # otherwise, execute all return values to find out what is the value
             else:
                 use_copy = [r for r in to_resolve] if hasattr(to_resolve, "__iter__") else to_resolve
+                if evaluate_body:
+                    sub_context.extra_info.append(BuiltinConcepts.CONCEPT_EVAL_REQUESTED)
                 r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS)
-                one_r = core.builtin_helpers.expect_one(context, r)
+
+                one_r = expect_one(context, r)
                 sub_context.add_values(return_values=one_r)
                 if one_r.status:
                     return one_r.value
@@ -161,7 +164,7 @@ class SheerkaEvaluateConcept:
                                       concept=current_concept,
                                       property_name=current_prop)
 
-    def resolve_list(self, context, list_to_resolve, current_prop, current_concept):
+    def resolve_list(self, context, list_to_resolve, current_prop, current_concept, evaluate_body):
         """When dealing with a list, there are two possibilities"""
         # It may be a list of ReturnValueConcept to execute (always the case for metadata)
         # or a list of single values (may be the case for properties)
@@ -170,7 +173,7 @@ class SheerkaEvaluateConcept:
             return []
 
         if self.sheerka.isinstance(list_to_resolve[0], BuiltinConcepts.RETURN_VALUE):
-            return self.resolve(context, list_to_resolve, current_prop, current_concept)
+            return self.resolve(context, list_to_resolve, current_prop, current_concept, evaluate_body)
 
         res = []
         for to_resolve in list_to_resolve:
@@ -181,35 +184,31 @@ class SheerkaEvaluateConcept:
                                         concept=current_concept,
                                         property_name=current_prop)
 
-            r = self.resolve(context, to_resolve, current_prop, current_concept)
+            r = self.resolve(context, to_resolve, current_prop, current_concept, evaluate_body)
             if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
                 return r
             res.append(r)
 
         return res
 
-    def evaluate_concept(self, context, concept: Concept):
+    def evaluate_concept(self, context, concept: Concept, evaluate_body=False):
         """
         Evaluation a concept
         It means that if the where clause is True, will evaluate the body
         :param context:
         :param concept:
-        :param logger:
+        :param evaluate_body: If false, only evaluate body when necessary
         :return: value of the evaluation or error
         """
 
         if concept.metadata.is_evaluated:
             return concept
 
-        #
-        # TODO : Validate the PRE condition
-        #
-
         self.initialize_concept_asts(context, concept)
 
         # to make sure of the order, it don't use ConceptParts.get_parts()
         # props must be evaluated first, body must be evaluated before where
-        all_metadata_to_eval = ["pre", "post", "props", "body", "where"]
+        all_metadata_to_eval = self.choose_metadata_to_eval(concept, evaluate_body)
 
         for metadata_to_eval in all_metadata_to_eval:
             if metadata_to_eval == "props":
@@ -218,10 +217,10 @@ class SheerkaEvaluateConcept:
 
                     if isinstance(prop_ast, list):
                         # Do not send the current concept for the properties
-                        resolved = self.resolve_list(context, prop_ast, prop_name, None)
+                        resolved = self.resolve_list(context, prop_ast, prop_name, None, True)
                     else:
                         # Do not send the current concept for the properties
-                        resolved = self.resolve(context, prop_ast, prop_name, None)
+                        resolved = self.resolve(context, prop_ast, prop_name, None, True)
                     if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
                         resolved.set_prop("concept", concept)  # since current concept was not sent
                         return resolved
@@ -237,12 +236,16 @@ class SheerkaEvaluateConcept:
 
                 if part_key in concept.compiled and concept.compiled[part_key] is not None:
                     metadata_ast = concept.compiled[part_key]
-                    resolved = self.resolve(context, metadata_ast, part_key, concept)
+                    resolved = self.resolve(context, metadata_ast, part_key, concept, evaluate_body)
                     if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved):
                         return resolved
                     else:
                         concept.values[part_key] = self.get_infinite_recursion_resolution(resolved) or resolved
 
+        #
+        # TODO : Validate the PRE condition
+        #
+
         # validate where clause
         if ConceptParts.WHERE in concept.values:
             where_value = concept.values[ConceptParts.WHERE]
@@ -254,5 +257,52 @@ class SheerkaEvaluateConcept:
         #
 
         concept.init_key()  # only does it if needed
-        concept.metadata.is_evaluated = True
+        concept.metadata.is_evaluated = "body" in all_metadata_to_eval
         return concept
+
+    def choose_metadata_to_eval(self, concept, evaluate_body):
+        if evaluate_body:
+            return ["pre", "post", "props", "body", "where"]
+
+        metadata = ["pre", "post"] + self.needed_metadata(concept) + ["where"]
+        return metadata
+
+    def needed_metadata(self, concept):
+        """
+        Tries to find out if the evaluation of the body is necessary
+        It's a very basic approach that will need to be improved
+        :param concept:
+        :return:
+        """
+
+        needed = []
+
+        for metadata in (ConceptParts.PRE, ConceptParts.POST, ConceptParts.WHERE):
+            if metadata not in concept.compiled:
+                continue
+
+            return_values = concept.compiled[metadata]
+            if not isinstance(return_values, list):
+                continue
+
+            for return_value in return_values:
+                if not self.sheerka.isinstance(return_value, BuiltinConcepts.RETURN_VALUE):
+                    continue
+
+                if not return_value.status:
+                    continue
+
+                if not self.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT):
+                    continue
+
+                if not isinstance(return_value.body.source, str):
+                    continue
+
+                for prop_name in (p[0] for p in concept.metadata.props):
+                    if prop_name in return_value.body.source:
+                        needed.append("props")
+                        break
+
+                if "self" in return_value.body.source:
+                    needed.append("body")
+        return needed

src/core/sheerka/Services/SheerkaExecute.py

@@ -150,6 +150,11 @@ class SheerkaExecute:
                                 debug_result = []
                                 for item in original_items:
                                     if evaluator.matches(sub_context, item):
+
+                                        # init the evaluator is possible
+                                        if hasattr(evaluator, "init_evaluator") and not evaluator.is_initialized:
+                                            evaluator.init_evaluator(sub_context, original_items)
+
                                         result = evaluator.eval(sub_context, item)
                                         if result is None:
                                             debug_result.append({"input": item, "return_value": None})

src/core/sheerka/Services/SheerkaSetsManager.py

@@ -108,7 +108,7 @@ class SheerkaSetsManager:
             # it may be a concept that references a set
             if not sub_concept.metadata.is_evaluated:
                 with context.push(desc=f"Evaluating concept {sub_concept}") as sub_context:
-                    evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept)
+                    evaluated = self.sheerka.evaluate_concept(sub_context, sub_concept, True)
                     if evaluated.key != concept.key:
                         return False
             return _get_set_elements(context, concept, sub_concept.body)
@@ -167,7 +167,7 @@ class SheerkaSetsManager:
         # it may be a concept that references a set
         if not concept.metadata.is_evaluated:
             with context.push(desc=f"Evaluating concept {concept}") as sub_context:
-                evaluated = self.sheerka.evaluate_concept(sub_context, concept)
+                evaluated = self.sheerka.evaluate_concept(sub_context, concept, True)
                 if evaluated.key != concept.key:
                     return False
 
@@ -218,7 +218,7 @@ for x in xx__concepts__xx:
         with context.push(desc=f"Evaluating concepts of a set") as sub_context:
             for element_id in ids:
                 concept = self.sheerka.get_by_id(element_id)
-                evaluated = self.sheerka.evaluate_concept(context, concept)
+                evaluated = self.sheerka.evaluate_concept(sub_context, concept, True)
                 result.append(evaluated)
 
         return result

src/core/sheerka/Sheerka.py

@@ -269,7 +269,6 @@ class Sheerka(Concept):
         :param execution_context:
         :param return_values:
         :param execution_steps:
-        :param logger: logger to use (if not directly called by sheerka)
         :return:
         """
         return self.execute_handler.execute(execution_context, return_values, execution_steps)
@@ -294,7 +293,6 @@ class Sheerka(Concept):
         Adds a new concept to the system
         :param context:
         :param concept:  DefConceptNode
-        :param logger
         :return: digest of the new concept
         """
 
@@ -309,7 +307,6 @@ class Sheerka(Concept):
         :param context:
         :param concept:
         :param concept_set:
-        :param logger:
         :return:
         """
         return self.sets_handler.add_concept_to_set(context, concept, concept_set)
@@ -320,7 +317,6 @@ class Sheerka(Concept):
         :param context:
         :param concept:
         :param concept_set:
-        :param logger:
         :return:
         """
         return self.sets_handler.set_isa(context, concept, concept_set)
@@ -336,16 +332,17 @@ class Sheerka(Concept):
 
         return self.sets_handler.get_set_elements(context, concept)
 
-    def evaluate_concept(self, context, concept: Concept):
+    def evaluate_concept(self, context, concept: Concept, evaluate_body=False):
         """
         Evaluation a concept
         It means that if the where clause is True, will evaluate the body
+        :param evaluate_body:
         :param context:
         :param concept:
-        :param logger:
+        :param evaluate_body:
         :return: value of the evaluation or error
         """
-        return self.evaluate_concept_handler.evaluate_concept(context, concept)
+        return self.evaluate_concept_handler.evaluate_concept(context, concept, evaluate_body)
 
     def add_in_cache(self, concept: Concept):
         """
@@ -393,11 +390,13 @@ class Sheerka(Concept):
             if result is None:
                 metadata = [("key", concept_key), ("id", concept_id)] if concept_id else ("key", concept_key)
                 result = self._get_unknown(metadata)
-
-            self.cache_by_key[concept_key] = result
-            for r in (result if isinstance(result, list) else [result]):
-                if r.id:
-                    self.cache_by_id[r.id] = r
+                # Do not put in cache_by_key or cache_by_id unknown concept
+                # TODO: implement an MRU cache for them
+            else:
+                self.cache_by_key[concept_key] = result
+                for r in (result if isinstance(result, list) else [result]):
+                    if r.id:
+                        self.cache_by_id[r.id] = r
 
         if not (isinstance(result, list) and concept_id):
             return result
@@ -640,7 +639,9 @@ class Sheerka(Concept):
                     if line == "" or line.startswith("#"):
                         continue
                     self.log.info(line)
-                    self.evaluate_user_input(line)
+                    res = self.evaluate_user_input(line)
+                    if len(res) > 1 or not res[0].status:
+                        self.log.error("Error detected !")
             self.during_restore = False
         except IOError:
             pass

src/core/sheerka_logger.py

@@ -8,6 +8,7 @@ console_handler = logging.StreamHandler(sys.stdout)
 
 all_loggers = {}
 
+
 def init_config(loggers):
     if loggers is None:
         return
@@ -55,7 +56,7 @@ def get_logger(logger_name):
             logger.disabled = True
 
     for e in enabled:
-        if logger_name.startswith("verbose." + e):
+        if logger_name.startswith("verbose." + e) or logger_name == e:
             logger.disabled = False
 
     return logger

src/evaluators/ConceptEvaluator.py

@@ -5,7 +5,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator
 
 class ConceptEvaluator(OneReturnValueEvaluator):
     """
-    The concept evaluatuor is the main class that know what to do with a concept
+    The concept evaluator is the main class that know what to do with a concept
     It verifies the PRE
     If ok, can execute or not the BODY
     Then checks the POST conditions
@@ -15,6 +15,19 @@ class ConceptEvaluator(OneReturnValueEvaluator):
     def __init__(self, return_body=False):
         super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
         self.return_body = return_body
+        self.evaluate_body = False
+        self.is_initialized = False
+
+    def init_evaluator(self, context, return_values):
+        if BuiltinConcepts.CONCEPT_EVAL_REQUESTED in context.extra_info:
+            self.evaluate_body = True
+
+        for r in return_values:
+            if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.CONCEPT_VALUE_REQUESTED):
+                self.evaluate_body = True
+                break
+
+        self.is_initialized = True
 
     def matches(self, context, return_value):
         return return_value.status and \
@@ -36,7 +49,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
 
             return sheerka.ret(self.name, True, value, parents=[return_value])
 
-        evaluated = sheerka.evaluate_concept(context, concept)
+        evaluated = sheerka.evaluate_concept(context, concept, self.evaluate_body)
 
         if evaluated.key != concept.key:
             # evaluated.key != concept.key means that we have transformed the concept

src/evaluators/EvalEvaluator.py

@@ -17,7 +17,7 @@ class EvalEvaluator(AllReturnValuesEvaluator):
     def matches(self, context, return_values):
         sheerka = context.sheerka
         for ret in return_values:
-            if ret.status and sheerka.isinstance(ret.body, BuiltinConcepts.CONCEPT_EVAL_REQUESTED):
+            if ret.status and sheerka.isinstance(ret.body, BuiltinConcepts.CONCEPT_VALUE_REQUESTED):
                 self.eval_requested = ret
                 return True
 
@@ -47,5 +47,5 @@ class EvalEvaluator(AllReturnValuesEvaluator):
             return sheerka.ret(
                 self.name,
                 False,
-                sheerka.new(BuiltinConcepts.CONCEPT_EVAL_REQUESTED),
+                sheerka.new(BuiltinConcepts.CONCEPT_VALUE_REQUESTED),
                 parents=[self.eval_requested])

src/evaluators/MutipleSameSuccessEvaluator.py

@@ -49,7 +49,7 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator):
         for s in self.success:
             context.log(f"{s}", who=self)
 
-        if not core.builtin_helpers.is_same_success(sheerka, self.success):
+        if not core.builtin_helpers.is_same_success(context, self.success):
             return None
 
         # ######################################

src/evaluators/PrepareEvalEvaluator.py

@@ -35,6 +35,6 @@ class PrepareEvalEvaluator(OneReturnValueEvaluator):
 
         evaluation_requested = sheerka.ret(
             self.name,
-            True, sheerka.new(BuiltinConcepts.CONCEPT_EVAL_REQUESTED))
+            True, sheerka.new(BuiltinConcepts.CONCEPT_VALUE_REQUESTED))
 
         return [new_text_to_parse, evaluation_requested]

src/evaluators/PythonEvaluator.py

@@ -112,7 +112,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
             context.log(f"Evaluating '{concept}'", self.name)
             with context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept) as sub_context:
-                evaluated = context.sheerka.evaluate_concept(sub_context, concept)
+                evaluated = context.sheerka.evaluate_concept(sub_context, concept, True)
                 sub_context.add_values(return_values=evaluated)
 
                 if evaluated.key == concept.key:

src/evaluators/TooManySuccessEvaluator.py

@@ -44,7 +44,7 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator):
                 context.log(s, self.name)
                 context.log(f"value={sheerka.value(s.value)}", self.name)
 
-        if not core.builtin_helpers.is_same_success(sheerka, self.success):
+        if not core.builtin_helpers.is_same_success(context, self.success):
             context.log(f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name)
             too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success)
             return sheerka.ret(self.name, False, too_many_success, parents=self.eaten)