In PythonEvaluator, I now evaluate concept and/or concept body

src/core/sheerka/Sheerka.py

@@ -340,6 +340,7 @@ class Sheerka(Concept):
             self.cache_manager.clear()
         self.printer_handler.reset()
         self.sdp.reset()
+        self.locals = {}
 
     def evaluate_user_input(self, text: str, user_name="kodjo"):
         """

src/core/utils.py

@@ -214,6 +214,34 @@ def product(a, b):
     return res
 
 
+def dict_product(a, b):
+    """
+    Cartesian product like where a and b are list of dictionaries
+    >>> a = [{"a": "a", "b":"b", "c":"c"}]
+    >>> b = [{"d":"d1"}, {"d":"d2"}]
+    >>>
+    >>> assert dict_product(a, b) == [{"a": "a", "b":"b", "c":"c", "d":"d1"}, {"a": "a", "b":"b", "c":"c", "d":"d2"}]
+
+    :param a:
+    :param b:
+    :return:
+    """
+    if a is None or len(a) == 0:
+        return b
+    if b is None or len(b) == 0:
+        return a
+
+    res = []
+    for item_a in a:
+        for item_b in b:
+            items = item_a.copy()
+            items.update(item_b)
+            res.append(items)
+
+    return res
+
+
+
 def strip_quotes(text):
     if not isinstance(text, str):
         return text

src/evaluators/PythonEvaluator.py

@@ -1,14 +1,14 @@
 import ast
 import copy
 import traceback
-from functools import partial, update_wrapper
+from dataclasses import dataclass, field
 
 import core.ast.nodes
-from core.sheerka.services.SheerkaFilter import Pipe
 import core.utils
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
 from core.concept import ConceptParts, Concept
+from core.sheerka.services.SheerkaFilter import Pipe
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.PythonParser import PythonNode
 
@@ -35,6 +35,13 @@ class Expando:
             setattr(self, k, v)
 
 
+@dataclass
+class PythonEvalError:
+    error: Exception
+    traceback: str = field(repr=False)
+    concepts: dict = field(repr=False)
+
+
 class PythonEvaluator(OneReturnValueEvaluator):
     NAME = "Python"
 
@@ -54,37 +61,59 @@ class PythonEvaluator(OneReturnValueEvaluator):
     def eval(self, context, return_value):
         sheerka = context.sheerka
         node = return_value.value.value
-        try:
-            context.log(f"Evaluating python node {node}.", self.name)
 
-            # Do not evaluate if the ast refers to a concept (leave it to ConceptEvaluator)
+        context.log(f"Evaluating python node {node}.", self.name)
-            if isinstance(node.ast_, ast.Expression) and isinstance(node.ast_.body, ast.Name):
-                c = context.sheerka.resolve(node.ast_.body.id)
-                if c is not None:
-                    context.log("It's a simple concept. Not for me.", self.name)
-                    not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
-                    return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
 
-            # get globals
+        # Do not evaluate if the ast refers to a concept (leave it to ConceptEvaluator)
-            my_globals = self.get_globals(context, node)
+        # TODO: Remove this section when this check will be implemented in the AFTER_PARSING step
-            context.log(f"globals={my_globals}", self.name)
+        if isinstance(node.ast_, ast.Expression) and isinstance(node.ast_.body, ast.Name):
+            c = context.sheerka.resolve(node.ast_.body.id)
+            if c is not None:
+                context.log("It's a simple concept. Not for me.", self.name)
+                not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
+                return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
 
-            # eval
+        # get globals
-            if isinstance(node.ast_, ast.Expression):
+        my_globals = self.get_globals(context, node)
-                context.log("Evaluating using 'eval'.", self.name)
+        context.log(f"globals={my_globals}", self.name)
-                compiled = compile(node.ast_, "<string>", "eval")
-                evaluated = eval(compiled, my_globals, sheerka.locals)
-            else:
-                context.log("Evaluating using 'exec'.", self.name)
-                evaluated = self.exec_with_return(node.ast_, my_globals, sheerka.locals)
 
-            context.log(f"{evaluated=}", self.name)
+        all_possible_globals = self.get_all_possible_globals(context, my_globals)
-            return sheerka.ret(self.name, True, evaluated, parents=[return_value])
+        concepts_entries = None
+        evaluated = BuiltinConcepts.NOT_INITIALIZED
+        errors = []
+        for globals_ in all_possible_globals:
+            try:
+                # eval
+                if isinstance(node.ast_, ast.Expression):
+                    context.log("Evaluating using 'eval'.", self.name)
+                    compiled = compile(node.ast_, "<string>", "eval")
+                    evaluated = eval(compiled, globals_, sheerka.locals)
+                else:
+                    context.log("Evaluating using 'exec'.", self.name)
+                    evaluated = self.exec_with_return(node.ast_, globals_, sheerka.locals)
 
-        except Exception as error:
+                break  # in this first version, we stop once a success is found
-            context.log_error(error, who=self.name, exc=traceback.format_exc())
+            except Exception as ex:
-            error = sheerka.new(BuiltinConcepts.ERROR, body=error)
+                if concepts_entries is None:
-            return sheerka.ret(self.name, False, error, parents=[return_value])
+                    concepts_entries = self.get_concepts_entries_from_globals(my_globals)
+                errors.append(PythonEvalError(ex,
+                                              traceback.format_exc(),
+                                              self.get_concepts_values_from_globals(globals_, concepts_entries)))
+
+        if evaluated == BuiltinConcepts.NOT_INITIALIZED:
+            if len(errors) == 1:
+                context.log_error(errors[0].error, who=self.name, exc=errors[0].traceback)
+                one_error = sheerka.new(BuiltinConcepts.ERROR, body=errors[0])
+                return sheerka.ret(self.name, False, one_error, parents=[return_value])
+
+            if len(errors) > 1:
+                for eval_error in errors:
+                    context.log_error(eval_error.error, who=self.name, exc=eval_error.traceback)
+                too_many_errors = sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=errors)
+                return sheerka.ret(self.name, False, too_many_errors, parents=[return_value])
+
+        context.log(f"{evaluated=}", self.name)
+        return sheerka.ret(self.name, True, evaluated, parents=[return_value])
 
     def get_globals(self, context, node):
         my_globals = {
@@ -94,9 +123,9 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         # has to be the first, to allow override
         method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context)
-
         self.update_globals_with_context(my_globals, context)
-        self.update_globals_with_node(my_globals, context, node)
+        already_know = set(my_globals.keys())
+        self.update_globals_with_node(my_globals, context, node, already_know)
 
         if self.globals:  # when extra values are given. Add them
             my_globals.update(self.globals)
@@ -125,20 +154,23 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         return methods_from_sheerka  # to allow access using prefix "sheerka."
 
-    def update_globals_with_context(self, my_locals, context):
+    def update_globals_with_context(self, my_globals, context):
         if context.obj:
             context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name)
 
             for prop_name in context.obj.variables():
-                prop_value = context.obj.get_value(prop_name)
+                my_globals[prop_name] = context.obj.get_value(prop_name)
-                if isinstance(prop_value, Concept):
+            my_globals["self"] = context.obj
-                    my_locals[prop_name] = context.sheerka.objvalue(prop_value)
-                else:
-                    my_locals[prop_name] = prop_value
 
-            my_locals["self"] = context.obj.body
+    def update_globals_with_node(self, my_globals, context, node, already_known):
-
+        """
-    def update_globals_with_node(self, my_locals, context, node):
+        Try to find concepts using the names that appear in the AST of the node.
+        :param my_globals: dictionary to update
+        :param context:
+        :param node:
+        :param already_known: if the name is in this list, do no try to instantiate it again
+        :return:
+        """
         node_concept = core.ast.nodes.python_to_concept(node.ast_)
         unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)
         unreferenced_names_visitor.visit(node_concept)
@@ -146,14 +178,15 @@ class PythonEvaluator(OneReturnValueEvaluator):
         for name in unreferenced_names_visitor.names:
             context.log(f"Resolving '{name}'.", self.name)
 
+            # get the concept
             if name in node.concepts:
+                # use it, even if it already in already_known
+                # This concept take precedence other the outer world
                 context.log(f"Using value from node.", self.name)
                 concept = self.resolve_concept(context, node.concepts[name])
-
+            elif name in already_known:
-            elif name in my_locals:
+                context.log(f"Already known. Skipping.", self.name)
-                context.log(f"Using value from property.", self.name)
                 continue
-
             else:
                 context.log(f"Instantiating new concept with {name}.", self.name)
                 concept = self.resolve_concept(context, name)
@@ -162,9 +195,9 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 context.log(f"Concept '{name}' is not found or cannot be instantiated. Skipping.", self.name)
                 continue
 
+            # evaluate it if needed
             if concept.metadata.is_evaluated:
                 context.log(f"Concept {name} is already evaluated.", self.name)
-
             else:
                 context.log(f"Evaluating '{concept}'", self.name)
                 with context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept) as sub_context:
@@ -177,7 +210,49 @@ class PythonEvaluator(OneReturnValueEvaluator):
                         continue
                     concept = evaluated
 
-            my_locals[name] = context.sheerka.objvalue(concept)
+            my_globals[name] = concept
+
+    @staticmethod
+    def get_all_possible_globals(context, my_globals):
+        """
+        From a dictionary of globals (str, obj)
+        Creates as many globals as there are combination between a concept and its body
+        Example:
+            if the entry 'foo': Concept("foo", body="something")
+            2 globals will be created
+            one with foo: Concept("foo")  # we keep the concept as an object
+            one with foo: 'something'   # we substitute its value
+        :param context:
+        :param my_globals:
+        :return:
+        """
+
+        # first pass, get all the non concept or concept with no body
+        # Note that we consider that all concepts are evaluated
+        # In the future, it may be a good optimisation to defer the evaluation of the body
+        # until the python evaluation fails
+        fixed_values = {}
+        concepts_with_body = {}
+        for k, v in my_globals.items():
+            if not isinstance(v, Concept) or context.sheerka.objvalue(v) == v:
+                fixed_values[k] = v
+            else:
+                concepts_with_body[k] = v
+
+        # make the product the rest as cartesian product
+        res = [fixed_values]
+        for k, v in concepts_with_body.items():
+            res = core.utils.dict_product(res, [{k: context.sheerka.objvalue(v)}, {k: v}])
+
+        return res
+
+    @staticmethod
+    def get_concepts_entries_from_globals(my_globals):
+        return [k for k, v in my_globals.items() if isinstance(v, Concept)]
+
+    @staticmethod
+    def get_concepts_values_from_globals(my_globals, names):
+        return {name: my_globals[name] for name in names}
 
     @staticmethod
     def resolve_concept(context, concept_hint):
@@ -189,30 +264,13 @@ class PythonEvaluator(OneReturnValueEvaluator):
             return None
         new_instance = context.sheerka.new_from_template(concept, concept.key)
         if isinstance(concept_hint, tuple):
-            # It's means that it comes from PythonParser which have found a concept token (c:xxx:)
+            # It's means that it was requested by PythonParser which have found a concept token (c:xxx:)
             # So a concept was explicitly required, not its value
             # We mark the concept as already evaluated, so it's body will not be evaluated
             new_instance.metadata.is_evaluated = True
 
         return new_instance
 
-    @staticmethod
-    def resolve_name(to_resolve):
-        """
-        Try to match
-        __C__concept_key__C__
-        or
-        __C__concept_key__concept_id__C__
-
-        :param to_resolve:
-        :return:
-        """
-        key, id_, use_concept = core.utils.decode_concept(to_resolve)
-        if key or id_:
-            return key, id_, use_concept
-        else:
-            return to_resolve, None, False
-
     @staticmethod
     def expr_to_expression(expr):
         expr.lineno = 0

tests/core/test_utils.py

@@ -1,7 +1,6 @@
 import core.utils
 import pytest
 from core.concept import ConceptParts, Concept
-
 from core.tokenizer import Token, TokenKind
 
 
@@ -212,3 +211,21 @@ def test_decode_concept_key_id():
     assert core.utils.decode_concept("__C__KEY_key__ID_id__C__") == ("key", "id")
     assert core.utils.decode_concept("__C__KEY_00None00__ID_id__C__") == (None, "id")
     assert core.utils.decode_concept("__C__KEY_key__ID_00None00__C__") == ("key", None)
+
+
+@pytest.mark.parametrize("a,b,expected", [
+    ([], [], []),
+    ([{"a": "a", "b": "b"}], [], [{"a": "a", "b": "b"}]),
+    ([], [{"a": "a", "b": "b"}], [{"a": "a", "b": "b"}]),
+    ([{"a": "a", "b": "b"}], [{"d": "d1"}, {"d": "d2"}], [{"a": "a", "b": "b", "d": "d1"},
+                                                          {"a": "a", "b": "b", "d": "d2"}]),
+    ([{"d": "d1"}, {"d": "d2"}], [{"a": "a", "b": "b"}], [{"a": "a", "b": "b", "d": "d1"},
+                                                          {"a": "a", "b": "b", "d": "d2"}]),
+    ([{"a": "a", "b": "b"}], [{"d": "d", "e": "e"}], [{"a": "a", "b": "b", "d": "d", "e": "e"}]),
+    ([{"a": "a"}, {"b": "b"}], [{"d": "d"}, {"e": "e"}], [{"a": "a", "d": "d"},
+                                                          {"a": "a", "e": "e"},
+                                                          {"b": "b", "d": "d"},
+                                                          {"b": "b", "e": "e"}])
+])
+def test_dict_product(a, b, expected):
+    assert core.utils.dict_product(a, b) == expected

tests/evaluators/test_PythonEvaluator.py

@@ -1,10 +1,10 @@
 import pytest
-
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import Concept, DEFINITION_TYPE_DEF
+from core.concept import Concept, CB
 from core.sheerka.services.SheerkaExecute import ParserInput
-from evaluators.PythonEvaluator import PythonEvaluator
+from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError
 from parsers.PythonParser import PythonNode, PythonParser
+
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -63,7 +63,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         Concept("foo").def_var("prop", "'a'"),
         Concept("foo", body="bar")
     ])
-    def test_i_cannot_eval_simple_concept(self, concept):
+    def test_simple_concepts_are_not_for_me(self, concept):
         context = self.get_context()
         context.sheerka.add_in_cache(Concept("foo"))
 
@@ -73,7 +73,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert not evaluated.status
         assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.NOT_FOR_ME)
 
-    def test_i_can_eval_expression_with_that_references_concepts(self):
+    def test_i_can_eval_ast_expression_that_references_concepts(self):
         """
         I can test modules with variables
         :return:
@@ -87,7 +87,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.value == 3
 
-    def test_i_can_eval_module_with_that_references_concepts(self):
+    def test_i_can_eval_ast_module_that_references_concepts(self):
         """
         I can test modules with variables
         :return:
@@ -101,13 +101,12 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.value == Concept("foo").init_key()
 
-    def test_i_can_eval_module_with_that_references_concepts_with_body(self):
+    def test_i_can_eval_ast_module_that_references_concepts_with_body(self):
         """
         I can test modules with variables
         :return:
         """
-        context = self.get_context()
+        sheerka, context, foo = self.init_concepts(Concept("foo", body="2"))
-        context.sheerka.add_in_cache(Concept("foo", body="2"))
 
         parsed = PythonParser().parse(context, ParserInput("def a(b):\n    return b\na(foo)"))
         evaluated = PythonEvaluator().eval(context, parsed)
@@ -127,15 +126,6 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.value == "foo"
 
-        # sanity, does not work otherwise
-        parsed = PythonParser().parse(context, ParserInput("get_concept_name(foo)"))
-        python_evaluator = PythonEvaluator()
-        python_evaluator.globals["get_concept_name"] = get_concept_name
-        evaluated = python_evaluator.eval(context, parsed)
-
-        assert not evaluated.status
-        assert evaluated.body.body.args[0] == "'int' object has no attribute 'name'"
-
     def test_i_can_call_function_with_complex_concepts(self):
         sheerka, context, plus, mult = self.init_concepts(
             self.from_def_concept("plus", "a plus b", ["a", "b"]),
@@ -163,23 +153,52 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.body == 10
 
-    # @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [
+    def test_i_can_get_all_possibles_globals(self):
-    #     ("__C__key__C__", "key", None, False),
+        sheerka, context, foo = self.init_concepts(Concept("foo", body="foo").auto_init())
-    #     ("__C__key__id__C__", "key", "id", False),
+        python_evaluator = PythonEvaluator()
-    #     ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+        my_globals = {
-    #     ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+            "a": "a string",
-    # ])
+            "b": self.test_i_can_get_all_possibles_globals,
-    # def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept):
+            "foo": foo
-    #     context = self.get_context()
+        }
-    #     assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept)
+
-    #
+        all_globals = python_evaluator.get_all_possible_globals(context, my_globals)
-    # @pytest.mark.parametrize("text", [
+        assert len(all_globals) == 2
-    #     "__C__",
+        assert all_globals[0]["foo"] == 'foo'
-    #     "__C__key",
+        assert all_globals[1]["foo"] == CB(foo, "foo")  # body is evaluated
-    #     "__C__key____",
+
-    #     "__C____",
+    def test_i_can_detect_one_error(self):
-    #     "__C__USE_CONCEPT__",
+        sheerka, context, foo = self.init_concepts("foo")
-    # ])
+
-    # def test_i_cannot_resolve_name(self, text):
+        parsed = PythonParser().parse(context, ParserInput("foo + 1"))
-    #     context = self.get_context()
+        evaluated = PythonEvaluator().eval(context, parsed)
-    #     assert PythonEvaluator().resolve_name(context, text) is None
+
+        assert not evaluated.status
+        assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.ERROR)
+
+        error = evaluated.body.body
+        assert isinstance(error, PythonEvalError)
+        assert isinstance(error.error, TypeError)
+        assert error.error.args[0] == "unsupported operand type(s) for +: 'Concept' and 'int'"
+        assert error.concepts == {'foo': foo}
+
+    def test_i_can_detect_multiple_errors(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo", body="'string'"))
+
+        parsed = PythonParser().parse(context, ParserInput("foo + 1"))
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert not evaluated.status
+        assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.TOO_MANY_ERRORS)
+
+        error0 = evaluated.body.body[0]
+        assert isinstance(error0, PythonEvalError)
+        assert isinstance(error0.error, TypeError)
+        assert error0.error.args[0] == 'can only concatenate str (not "int") to str'
+        assert error0.concepts == {'foo': 'string'}
+
+        error1 = evaluated.body.body[1]
+        assert isinstance(error1, PythonEvalError)
+        assert isinstance(error1.error, TypeError)
+        assert error1.error.args[0] == "unsupported operand type(s) for +: 'Concept' and 'int'"
+        assert error1.concepts == {'foo': CB(foo, 'string')}

tests/non_reg/test_sheerka_non_reg.py

@@ -921,6 +921,7 @@ as:
         assert not res[0].status
 
 
+
 class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
     def test_i_can_def_several_concepts(self):
         sheerka = self.get_sheerka()