Fixed some misbehaviours regarding question() + added #import functionality when restoring

_concepts_admin.txt

@@ -0,0 +1,12 @@
+# admin helpers
+def concept explain as get_results() | filter("id == 0") | recurse(2)
+set_isa(c:explain:, __AUTO_EVAL)
+
+def concept explain last as get_last_results() | filter("id == 0") | recurse(2)
+set_isa(c:explain last:, __AUTO_EVAL)
+
+def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x
+set_isa(c:explain x:, __AUTO_EVAL)
+
+def concept explain x values where x as get_results() | filter(f"id=={x}") | format_d
+set_isa(c:explain x values:, __AUTO_EVAL)

_concepts_lite.txt

@@ -1,12 +1,10 @@
+#import admin
 def concept one as 1
 def concept two as 2
-def concept explain as get_results() | filter("id == 0") | recurse(2)
+def concept number
-set_isa(c:explain:, __AUTO_EVAL)
-def concept explain last as get_last_results() | filter("id == 0") | recurse(2)
-set_isa(c:explain last:, __AUTO_EVAL)
-def concept explain x as get_results() | filter(f"id == {x}") | recurse(3) where x
-set_isa(c:explain x:, __AUTO_EVAL)
 def concept apple
 def concept table
 def concept location
 def concept x is on y as set_attr(x, location, y)
+def concept x is a y as set_isa(x, y)
+def concept x is a y as isa(x, y) pre is_question()

src/core/ast/visitors.py

@@ -63,8 +63,8 @@ class UnreferencedNamesVisitor(ConceptNodeVisitor):
         if ("Call", "func") in parents:  # name of the function
             return
 
-        if ("Assign", "targets") in parents:  # variable which is assigned
+        # if ("Assign", "targets") in parents:  # variable which is assigned
-            return
+        #     return
 
         if self.can_be_discarded(self.sheerka.objvalue(node), parents):
             return

src/core/builtin_concepts.py

@@ -181,6 +181,7 @@ BuiltinErrors = [str(e) for e in {
     BuiltinConcepts.NOT_FOUND,
     BuiltinConcepts.INVALID_LESSER_OPERATION,
     BuiltinConcepts.INVALID_GREATEST_OPERATION,
+    # DO NOT PUT NOT_INITIALIZED. It's not an error
 }]
 
 """
@@ -534,3 +535,15 @@ class PythonSecurityError(Concept):
         self.set_value("column", column)  # column number
         self.set_value(ConceptParts.BODY, source_code)  # code being executed
         self.metadata.is_evaluated = True
+
+
+class NotFound(Concept):
+    def __init__(self, body=None):
+        super().__init__(BuiltinConcepts.NOT_FOUND,
+                         True,
+                         False,
+                         BuiltinConcepts.NOT_FOUND)
+        self.set_value(ConceptParts.BODY, body)
+
+    def __repr__(self):
+        return f"({self.metadata.id}){self.metadata.name}, body={self.get_value(ConceptParts.BODY)}"

src/core/builtin_helpers.py

@@ -194,7 +194,7 @@ def resolve_ambiguity(context, concepts):
         else:
             for c in by_complexity[complexity]:
                 evaluated = context.sheerka.evaluate_concept(context, c, metadata=["pre"])
-                if evaluated.key == c.key:
+                if context.sheerka.is_success(evaluated) or evaluated.key == c.key:
                     remaining_concepts.append(c)
 
         if len(remaining_concepts) > 0:
@@ -214,6 +214,13 @@ def resolve_ambiguity(context, concepts):
 
 
 def get_condition_complexity(concept, concept_part_str):
+    """
+    Need to find a proper algorithm to compute the complexity of a concept
+    So far, the concept is considered as complex if it has pre
+    :param concept:
+    :param concept_part_str:
+    :return:
+    """
     concept_part_value = getattr(concept.metadata, concept_part_str)
     if concept_part_value is None or concept_part_value.strip() == 0:
         return 0
@@ -374,6 +381,7 @@ def evaluate(context,
              desc=None,
              eval_body=True,
              eval_where=True,
+             is_question=False,
              expect_success=False,
              stm=None):
     """
@@ -384,6 +392,7 @@ def evaluate(context,
     :param desc:
     :param eval_body:
     :param eval_where:
+    :param is_question:
     :param expect_success:
     :param stm: short term memories entries
     :return:
@@ -400,6 +409,8 @@ def evaluate(context,
 
         if expect_success:
             sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+
+        if is_question:
             sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
 
         if stm:

src/core/sheerka/ExecutionContext.py

@@ -191,6 +191,7 @@ class ExecutionContext:
         new.protected_hints.update(self.protected_hints)
 
         self._children.append(new)
+
         return new
 
     def add_preprocess(self, name, **kwargs):
@@ -295,6 +296,9 @@ class ExecutionContext:
             to_str = self.return_value_to_str(r)
             self._logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
+    def debug(self, text):
+        print(text)
+
     def get_parent(self):
         return self._parent
 

src/core/sheerka/services/SheerkaAdmin.py

@@ -1,4 +1,5 @@
 import time
+from os import path
 
 from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.services.sheerka_service import BaseService
@@ -50,6 +51,38 @@ class SheerkaAdmin(BaseService):
         :return:
         """
 
+        def restore_from_file(file_name):
+            _nb_lines, _nb_instructions, _nb_lines_in_error = 0, 0, 0
+            if not path.exists(file_name):
+                self.sheerka.log.error(f"\u001b[31mFile '{file_name}' is not found !\u001b[0m")
+                return 0, 0, 1
+
+            with open(file_name, "r") as f:
+                for line in f.readlines():
+                    _nb_lines += 1
+                    line = line.strip()
+
+                    if line.startswith("#import "):
+                        to_import = "_concepts_" + line[8:] + ".txt"
+                        self.sheerka.log.info(f"Importing {to_import}")
+                        res = restore_from_file(to_import)
+                        _nb_lines += res[0]
+                        _nb_instructions += res[1]
+                        _nb_lines_in_error += res[2]
+                        continue
+
+                    if line == "" or line.startswith("#"):
+                        continue
+
+                    self.sheerka.log.info(line)
+                    _nb_instructions += 1
+                    res = self.sheerka.evaluate_user_input(line)
+                    if len(res) > 1 or not res[0].status:
+                        _nb_lines_in_error += 1
+                        self.sheerka.log.error("\u001b[31mError detected !\u001b[0m")
+
+            return _nb_lines, _nb_instructions, _nb_lines_in_error
+
         if concept_file == "full":
             concept_file = CONCEPTS_FILE_ALL_CONCEPTS
 
@@ -58,20 +91,8 @@ class SheerkaAdmin(BaseService):
 
         try:
             start = time.time_ns()
-            nb_lines = 0
-            nb_lines_in_error = 0
             self.sheerka.during_restore = True
-            with open(concept_file, "r") as f:
+            nb_lines, nb_instructions, nb_lines_in_error = restore_from_file(concept_file)
-                for line in f.readlines():
-                    nb_lines += 1
-                    line = line.strip()
-                    if line == "" or line.startswith("#"):
-                        continue
-                    self.sheerka.log.info(line)
-                    res = self.sheerka.evaluate_user_input(line)
-                    if len(res) > 1 or not res[0].status:
-                        nb_lines_in_error += 1
-                        self.sheerka.log.error("\u001b[31mError detected !\u001b[0m")
             self.sheerka.during_restore = False
             stop = time.time_ns()
 
@@ -79,12 +100,13 @@ class SheerkaAdmin(BaseService):
             dt = nano_sec / 1e6
             elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
             self.sheerka.log.info(f"Imported {nb_lines} line(s) in {elapsed}.")
+            self.sheerka.log.info(f"{nb_instructions} instruction(s).")
             if nb_lines_in_error > 0:
                 self.sheerka.log.info(f"\u001b[31m{nb_lines_in_error} errors(s) found.\u001b[0m")
             else:
                 self.sheerka.log.info(f"No error.")
-        except IOError:
+        except IOError as e:
-            pass
+            raise e
 
     def concepts(self):
         return self.sheerka.sdp.list(self.sheerka.CONCEPTS_BY_ID_ENTRY)

src/core/sheerka/services/SheerkaCreateNewConcept.py

@@ -4,13 +4,12 @@ from core.concept import Concept, DEFINITION_TYPE_DEF, ensure_concept, DEFINITIO
 from core.sheerka.services.sheerka_service import BaseService
 from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
 
-BNF_NODE_PARSER_CLASS = "parsers.BnfNodeParser_Old.BnfNodeParser"
 BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
 
 
 class SheerkaCreateNewConcept(BaseService):
     """
-    Manage the creation of a new concept
+    Manages the creation of a new concept
     """
 
     NAME = "CreateNewConcept"

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -166,6 +166,7 @@ class SheerkaEvaluateConcept(BaseService):
                                           where_clause_def.trueified,
                                           desc=f"Apply where clause on '{where_clause_def.prop}'",
                                           expect_success=True,
+                                          is_question=True,
                                           stm={where_clause_def.prop: r.body})
                 one_res = expect_one(context, evaluation_res)
                 if one_res.status:
@@ -286,7 +287,6 @@ class SheerkaEvaluateConcept(BaseService):
                 current_prop,
                 current_concept,
                 force_evaluation,
-                expect_success,
                 where_clause_def):
         """
         Resolve a variable or a Concept
@@ -295,7 +295,6 @@ class SheerkaEvaluateConcept(BaseService):
         :param current_prop: current property or ConceptPart
         :param current_concept: current concept
         :param force_evaluation: Force body evaluation
-        :param expect_success: for PythonEvaluator, try all possibilities to find a positive result
         :param where_clause_def: intermediate where clause for variables
         :return:
         """
@@ -332,8 +331,10 @@ class SheerkaEvaluateConcept(BaseService):
             if force_evaluation:
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
-            if expect_success:
+            if current_prop in (ConceptParts.WHERE, ConceptParts.PRE):
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+
+            if current_prop == ConceptParts.WHERE:
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
 
             # when it's a concept, evaluate it
@@ -384,7 +385,6 @@ class SheerkaEvaluateConcept(BaseService):
                      current_prop,
                      current_concept,
                      force_evaluation,
-                     expect_success,
                      where_clause_def):
         """When dealing with a list, there are two possibilities"""
         # It may be a list of ReturnValueConcept to execute (always the case for metadata)
@@ -399,7 +399,6 @@ class SheerkaEvaluateConcept(BaseService):
                                 current_prop,
                                 current_concept,
                                 force_evaluation,
-                                expect_success,
                                 where_clause_def)
 
         res = []
@@ -416,7 +415,6 @@ class SheerkaEvaluateConcept(BaseService):
                              current_prop,
                              current_concept,
                              force_evaluation,
-                             expect_success,
                              where_clause_def)
             if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
                 return r
@@ -476,10 +474,10 @@ class SheerkaEvaluateConcept(BaseService):
 
                         if isinstance(prop_ast, list):
                             # Do not send the current concept for the properties
-                            resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False, w_clause)
+                            resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, w_clause)
                         else:
                             # Do not send the current concept for the properties
-                            resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False, w_clause)
+                            resolved = self.resolve(sub_context, prop_ast, var_name, None, True, w_clause)
 
                         if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
                             resolved.set_value("concept", concept)  # since current concept was not sent
@@ -504,18 +502,8 @@ class SheerkaEvaluateConcept(BaseService):
                     # otherwise no need to force
                     force_concept_eval = False if part_key == ConceptParts.BODY else True
 
-                    # when resolving predicate (where or pre), we need to make sure that PythonEvaluator
-                    # will try every possibilities before returning False
-                    expect_success = part_key in (ConceptParts.WHERE, ConceptParts.PRE)
-
                     # resolve
-                    resolved = self.resolve(sub_context,
+                    resolved = self.resolve(sub_context, metadata_ast, part_key, concept, force_concept_eval, None)
-                                            metadata_ast,
-                                            part_key,
-                                            concept,
-                                            force_concept_eval,
-                                            expect_success,
-                                            None)
 
                     # 'FATAL' error is detected, let's stop
                     if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):

src/core/sheerka/services/SheerkaModifyConcept.py

@@ -100,4 +100,9 @@ class SheerkaModifyConcept(BaseService):
         :return:
         """
         ensure_concept()
-        return concept.get_value(attribute)
+        if not self.sheerka.is_success(concept):
+            return concept
+
+        if (value := concept.get_value(attribute)) == BuiltinConcepts.NOT_INITIALIZED:
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#concept": concept, "#attr": attribute})
+        return value

src/core/sheerka/services/SheerkaQuestion.py

@@ -10,27 +10,25 @@ class SheerkaQuestion(BaseService):
         super().__init__(sheerka)
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.question, False)
+       # self.sheerka.bind_service_method(self.question, False)
         self.sheerka.bind_service_method(self.is_question, False)
 
-    def question(self, context, q):
+    # def question(self, context, q):
-        """
+    #     """
-        Evaluate q in the context in a question
+    #     Evaluate q in the context in a question
-        :param context:
+    #     :param context:
-        :param q:
+    #     :param q:
-        :return:
+    #     :return:
-        """
+    #     """
-
+    #
-        if isinstance(q, Concept):
+    #     if isinstance(q, Concept):
-            with context.push(BuiltinConcepts.EVALUATE_CONCEPT, q, desc=f"Evaluating question '{q}'") as sub_context:
+    #         with context.push(BuiltinConcepts.EVALUATE_CONCEPT, q, desc=f"Evaluating question '{q}'") as sub_context:
-                sub_context.global_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+    #             sub_context.global_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
-                sub_context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+    #             sub_context.global_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
-                sub_context.global_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
+    #
-                sub_context.protected_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
+    #             evaluated = self.sheerka.evaluate_concept(sub_context, q)
-
+    #
-                evaluated = self.sheerka.evaluate_concept(sub_context, q)
+    #         return evaluated
-
-            return evaluated
 
     def is_question(self, context):
         """

src/evaluators/AddConceptEvaluator.py

@@ -1,5 +1,7 @@
 import core.utils
+from core.ast.nodes import python_to_concept
 from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
+from core.builtin_helpers import get_names
 from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import TokenKind, Tokenizer
@@ -7,6 +9,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.BaseParser import NotInitializedNode
 from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
 from parsers.DefaultParser import DefConceptNode, NameNode
+from parsers.PythonParser import PythonNode
 
 
 class ConceptOrRuleNameVisitor(ParsingExpressionVisitor):
@@ -141,7 +144,18 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
             return set(visitor.names)
 
         #
-        # other (python code and concept)
+        # Case of python code
+        #
+        if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode):
+            if len(concept_name) > 1:
+                python_node = ret_value.value.value
+                as_concept_node = python_to_concept(python_node.ast_)
+                names = get_names(sheerka, as_concept_node)
+                variables = filter(lambda x: x in concept_name, names)
+                return set(variables)
+
+        #
+        # Concept
         #
         if isinstance(ret_value.value, ParserResultConcept) and len(concept_name) > 1:
             variables = set()

src/evaluators/PostExecutionEvaluator.py

@@ -26,7 +26,8 @@ class PostExecutionEvaluator(OneReturnValueEvaluator):
 
     def eval(self, context, return_value):
         # only support the rule for the COMMANDS
-        body = return_value.body.body
+        #body = return_value.body.body
+        body = context.sheerka.objvalue(return_value)
         return context.sheerka.ret(
             self.name,
             True,

src/evaluators/PrepareEvalQuestionEvaluator.py

@@ -36,10 +36,12 @@ class PrepareEvalQuestionEvaluator(OneReturnValueEvaluator):
             self.name,
             True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.question, user_name=context.event.user_id))
 
-        root = context.get_parents(lambda ec: ec.action == BuiltinConcepts.PROCESS_INPUT)
+        root = context.get_parents(lambda ec: ec.action in (BuiltinConcepts.EVALUATING_CONCEPT,
+                                                            BuiltinConcepts.PROCESS_INPUT))
         root = root[0] if root else context
-        root.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        root.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
-        root.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        root.add_to_protected_hints(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED)
-        root.protected_hints.add(BuiltinConcepts.RETURN_BODY_REQUESTED)
+        root.add_to_protected_hints(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        root.add_to_protected_hints(BuiltinConcepts.RETURN_BODY_REQUESTED)
 
         return new_text_to_parse

src/parsers/SyaNodeParser.py

@@ -1184,6 +1184,14 @@ class SyaNodeParser(BaseNodeParser):
             infix_to_postfix.finalize(self.parser_input.pos)
         _add_forked_to_res()
 
+        if context.in_context(BuiltinConcepts.DEBUG):
+            context.debug(f"Parsing {parser_input}")
+            context.debug(f"{len(res)} InfixToPostFix(s) found")
+            for i, r in enumerate(res):
+                context.debug(f"#{i}")
+                for line in r.debug:
+                    context.debug(line)
+
         return res
 
     def postfix_to_item(self, sheerka, postfixed):

tests/core/test_SheerkaCreateNewConcept.py

@@ -215,7 +215,6 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
         assert sheerka.cache_manager.get(sheerka.CONCEPTS_REFERENCES_ENTRY, twenties.id) is None
 
 
-
 class TestSheerkaCreateNewConceptFileBased(TestUsingFileBasedSheerka):
     def test_i_can_add_several_concepts(self):
         sheerka = self.get_sheerka()

tests/core/test_builtin_helpers.py

@@ -146,8 +146,11 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("concepts, expected", [
         ([], []),
+        ([Concept("foo", pre="False"), Concept("bar")], ["bar"]),
         ([Concept("foo", pre="True"), Concept("bar")], ["foo"]),
-        ([Concept("foo").def_var("a"), Concept("bar")], ["bar"]),
+        ([Concept("foo").def_var("a"), Concept("bar")], ["bar"]),  # less variables is better
+        ([Concept("foo"), Concept("bar")], ["foo", "bar"]),
+        ([Concept("foo", pre="is_question()"), Concept("bar")], ["bar"]),
     ])
     def test_i_can_resolve_ambiguity_when_empty(self, concepts, expected):
         context = self.get_context()

tests/evaluators/test_AddConceptEvaluator.py

@@ -188,6 +188,8 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         ("isinstance(a, str)", "a b", {"a"}),
         ("a.location = b", "a is in b", {"a", "b"}),
         ("a.location = b", "'a' is in b", {"b"}),
+        ("date.today()", "what is the date", set()),
+        ("a.location", "where is a", {"a"})
     ])
     def test_i_can_get_variables_from_python_node_when_long_name(self, expression, name, expected):
         ret_val = self.get_concept_part(expression)

tests/evaluators/test_PrepareEvalQuestionEvaluator.py

@@ -45,5 +45,6 @@ class TestPrepareEvalQuestionEvaluator(TestUsingMemoryBasedSheerka):
         assert res.body.body == expected
 
         assert BuiltinConcepts.EVAL_QUESTION_REQUESTED in context.protected_hints
+        assert BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED in context.protected_hints
         assert BuiltinConcepts.EVAL_BODY_REQUESTED in context.protected_hints
         assert BuiltinConcepts.RETURN_BODY_REQUESTED in context.protected_hints

tests/non_reg/test_sheerka_non_reg.py

@@ -1061,26 +1061,27 @@ as:
             "def concept number",
             "set_isa(one, number)",
             "def concept q from q ? as question(q)",
+            "set_auto_eval(q)",
             "def concept is_a from x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
-            "set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:)"
+            "set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:)",
         ]
 
         sheerka = self.init_scenario(init)
         res = sheerka.evaluate_user_input("one is a number ?")  # automatically evaluated
         assert len(res) == 1
         assert res[0].status
-        assert res[0].body
+        assert res[0].body == True  # the body MUST be a boolean
 
         res = sheerka.evaluate_user_input("foo is a number ?")  # automatically evaluated
         assert len(res) == 1
         assert res[0].status
-        assert not res[0].body
+        assert res[0].body == False  # the body MUST be a boolean
 
-        # Sanity, when there is only one 'is a' concept. It's chosen regardless of the PRE condition
+        # x is a y is supposed to be a question. It cannot be used if not in a context of a question
         res = sheerka.evaluate_user_input("one is a number")
         assert len(res) == 1
-        assert res[0].status
+        assert not res[0].status
-        assert res[0].body
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
 
     def test_i_can_evaluate_source_code_with_concept(self):
         init = [