Fixed BNF concept evaluations

evaluators/AddConceptInSetEvaluator.py

@@ -65,6 +65,10 @@ class AddConceptInSetEvaluator(OneReturnValueEvaluator):
         else:
             context.log(self.log, f"Concept added.", self.name)
 
-        return res
+        return sheerka.ret(
+            self.name,
+            res.status,
+            res.body,
+            parents=[return_value])
 
 

evaluators/ConceptComposerEvaluator.py

@@ -0,0 +1,110 @@
+from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
+from core.concept import Concept
+from core.tokenizer import TokenKind
+from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator
+from parsers.BaseParser import BaseParser
+from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, ConceptLexerParser
+import core.utils
+
+
+class ConceptComposerEvaluator(AllReturnValuesEvaluator):
+    """
+    Try to reassemble parts of concepts from different evaluators
+    """
+
+    NAME = "ConceptComposer"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 40)
+
+    def matches(self, context, return_values):
+        concept_lexer_parser_name = ConceptLexerParser().name
+
+        for return_value in return_values:
+            if return_value.who.startswith(BaseParser.PREFIX) and return_value.status:
+                return False
+
+            if return_value.who.startswith(BaseEvaluator.PREFIX):
+                return False
+
+            if return_value.who != concept_lexer_parser_name:
+                continue
+
+            if not isinstance(return_value.value, ParserResultConcept):
+                return False
+
+            if not (
+                isinstance(return_value.value.value, ConceptNode) or
+                isinstance(return_value.value.value, UnrecognizedTokensNode) or
+                (
+                    hasattr(return_value.value.value, "__iter__") and
+                    len(return_value.value.value) > 0 and
+                    (
+                        isinstance(return_value.value.value[0], ConceptNode) or
+                        isinstance(return_value.value.value[0], UnrecognizedTokensNode)
+                    ))):
+                return False
+
+            self.eaten = return_value
+            return True
+
+        return False
+
+    def eval(self, context, return_value):
+        sheerka = context.sheerka
+        nodes = self.eaten.value.value
+        temp_res = []
+        has_error = False
+        concepts_only = True
+
+        for node in nodes:
+            if isinstance(node, UnrecognizedTokensNode):
+                tokens = core.utils.strip_tokens(node.tokens, True)
+                for token in tokens:
+                    if token.type == TokenKind.IDENTIFIER:
+                        concept = context.new_concept(token.value)
+                        if sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
+                            has_error = True
+                        else:
+                            sub_context = context.push(self.name, desc=f"Evaluating '{concept}'")
+                            sub_context.log_new(self.verbose_log)
+                            concept = sheerka.evaluate_concept(sub_context, concept, self.verbose_log)
+                        temp_res.append(concept)
+                    else:
+                        temp_res.append(core.utils.strip_quotes(token.value))
+                        concepts_only &= token.type == TokenKind.WHITESPACE or token.type == TokenKind.NEWLINE
+            else:
+                sub_context = context.push(self.name, desc=f"Evaluating '{node.concept}'")
+                sub_context.log_new(self.verbose_log)
+                concept = sheerka.evaluate_concept(sub_context, node.concept, self.verbose_log)
+                temp_res.append(concept)
+
+        if has_error:
+            return sheerka.ret(
+                self.name,
+                False,
+                temp_res,
+                parents=[self.eaten])
+
+        if concepts_only:
+            res = []
+            for r in temp_res:
+                if isinstance(r, Concept):
+                    res.append(r)
+        else:
+            res = ""
+            for r in temp_res:
+                if isinstance(r, Concept):
+                    res += sheerka.value(r)
+                else:
+                    res += r
+
+        return sheerka.ret(
+            self.name,
+            True,
+            res,
+            parents=[self.eaten])
+
+
+
+

evaluators/ConceptEvaluator.py

@@ -11,11 +11,6 @@ class ConceptEvaluator(OneReturnValueEvaluator):
     Then checks the POST conditions
     """
     NAME = "Concept"
-    evaluation_steps = [
-        BuiltinConcepts.BEFORE_EVALUATION,
-        BuiltinConcepts.EVALUATION,
-        BuiltinConcepts.AFTER_EVALUATION
-    ]
 
     def __init__(self, return_body=False):
         super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
@@ -34,7 +29,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
         # If the concept that is requested is in the context(at least its name), drop the call.
         # Why ?
         # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
-        # The body should be 'property_a', and not a concept called a in our universe
+        # The body should be 'property_a', and not a concept called 'a'
         if context.obj and concept.name in context.obj.props:
             value = context.obj.props[concept.name].value
             context.log(self.verbose_log, f"{concept.name} is a property. Returning value '{value}'.", self.name)

evaluators/ConceptNodeEvaluator.py

@@ -1,7 +1,7 @@
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
-
-from parsers.ConceptLexerParser import ConceptNode, NonTerminalNode, ConceptMatch, UnrecognizedTokensNode
+import core.utils
+from parsers.ConceptLexerParser import ConceptNode, NonTerminalNode, ConceptMatch, UnrecognizedTokensNode, TerminalNode
 
 
 class ConceptNodeEvaluator(OneReturnValueEvaluator):
@@ -46,10 +46,12 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
 
         concepts = []
         error_found = False
+        source = ""
         for node in nodes:
             if isinstance(node, ConceptNode):
+                source += node.source if source == "" else (" " + node.source)
                 concept = sheerka.new(node.concept.key)
-                concept = self.update_concept(sheerka, concept, node.underlying)
+                concept = self.finalize_concept(sheerka, concept, node.underlying)
                 concepts.append(concept)
             else:
                 error_found = True
@@ -58,12 +60,17 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
             return sheerka.ret(
                 self.name,
                 not error_found,
-                concepts[0],
+                context.sheerka.new(
+                    BuiltinConcepts.PARSER_RESULT,
+                    parser=self,
+                    source=source,
+                    body=concepts[0],
+                    try_parsed=None),
                 parents=[return_value])
 
         return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME), parents=[return_value])
 
-    def update_concept(self, sheerka, concept, underlying, init_empty_body=True):
+    def finalize_concept(self, sheerka, concept, underlying, init_empty_body=True):
         """
         Updates the properties of the concept
         Goes in recursion if the property is a concept
@@ -74,9 +81,12 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
             Adds a new entry,
             makes a list if the property already exists
             """
+
             if prop_name not in c.props or c.props[prop_name].value is None:
+                # new entry
                 c.set_prop(prop_name, value)
             else:
+                # make a list if there was a value
                 previous_value = c.props[prop_name].value
                 if isinstance(previous_value, list):
                     previous_value.append(value)
@@ -87,11 +97,12 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
         parsing_expression = underlying.parsing_expression
 
         if parsing_expression.rule_name:
-            _add_prop(concept, parsing_expression.rule_name, underlying.source)
+            _add_prop(concept, parsing_expression.rule_name, self.get_underlying_as_string(underlying))
 
         # the update of the body must come BEFORE the recursion
+        # otherwise it will be updated by a children and it won't be possible to modify the value
         if init_empty_body and concept.body is None:
-            concept.metadata.body = underlying.source
+            concept.metadata.body = self.get_underlying_as_string(underlying) # self.escape_if_needed(underlying.source)
 
         if isinstance(underlying, NonTerminalNode):
             for child in underlying.children:
@@ -101,8 +112,75 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
                     if sheerka.isinstance(new_concept, BuiltinConcepts.UNKNOWN_CONCEPT):
                         continue
                     else:
-                        self.update_concept(sheerka, new_concept, child.children[0], init_empty_body)
+                        self.finalize_concept(sheerka, new_concept, child.children[0], init_empty_body)
                 else:
-                    self.update_concept(sheerka, concept, child, init_empty_body)
+                    self.finalize_concept(sheerka, concept, child, init_empty_body)
 
         return concept
+
+    @staticmethod
+    def escape_if_needed(value):
+        if not isinstance(value, str):
+            return value
+
+        return "'" + core.utils.escape_char(value, "'") + "'"
+
+    def get_underlying_as_string(self, underlying):
+        """
+        Return the sequence of the recognized character
+        When a concept is recognized, return the string version of the concept eg c:concept name:
+        :param underlying:
+        :return:
+        """
+
+        # Example
+        #     grammar = {
+        #         foo: Sequence("one", "two", rule_name="var"),
+        #         bar: Sequence(foo, "three", rule_name="var")}
+        #
+        # we want bar.body and bar.prop["var"]
+        #  to be "foo 'three'"  (no quotes surrounding foo, as it is a concept, not a string)
+
+        if isinstance(underlying, TerminalNode):
+            return self.escape_if_needed(underlying.source)
+
+        res = ""
+        first = True
+        in_quote = ""
+        for node in underlying.children:
+            if isinstance(node.parsing_expression, ConceptMatch):
+                if in_quote != "":
+                    res += in_quote + "'"
+                if not first:
+                    res += " "
+                res += node.parsing_expression.concept.key
+                in_quote = ""
+            else:
+                if in_quote == "":
+                    in_quote = ("'" if first else " '") + core.utils.escape_char(node.source, "'")
+                else:
+                    in_quote += ("" if first else " ") + core.utils.escape_char(node.source, "'")
+
+            first = False
+
+        if in_quote:
+            res += in_quote + "'"
+        return res
+
+# - - -  E X P L A N A T I O N S - - -
+# why do we need to update the body ?
+# cf test_concept_property_is_correctly_updated_when_concept_recursion_using_zero_or_more()
+# def concept number from bnf one | two | three
+# def concept add from bnf number plus number
+#
+# the expression 'one plus two plus three' will match concept add
+# add.props["number"] is a list of concepts 'number'
+# But which one is 'one', which one is 'two' which one is 'three' ?
+#
+# That's the reason why we update the body
+# add.props["number"] is a list of concepts 'number' but they won't have the same body
+#
+# !!! C A U T I O N !!!
+# In the current implementation, the body is the sequence of char found
+# If a concept is recognized, we don't put this information in the body
+# Use get_body_as_string() instead of escape_if_needed() if we need this information

evaluators/OneErrorEvaluator.py

@@ -12,7 +12,7 @@ class OneErrorEvaluator(AllReturnValuesEvaluator):
     NAME = "OneError"
 
     def __init__(self):
-        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 40)
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 30)
         self.return_value_in_error = None
 
     def matches(self, context, return_values):

evaluators/PythonEvaluator.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
-from core.concept import ConceptParts
+from core.concept import ConceptParts, Concept
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.PythonParser import PythonNode
 import ast
@@ -65,7 +65,10 @@ class PythonEvaluator(OneReturnValueEvaluator):
                         f"Concept '{context.obj}' is in context. Adding its properties to locals if any.", self.name)
 
             for prop_name, prop_value in context.obj.props.items():
-                my_locals[prop_name] = prop_value.value
+                if not isinstance(prop_value.value, Concept):
+                    my_locals[prop_name] = prop_value.value
+                else:
+                    my_locals[prop_name] = context.sheerka.value(prop_value.value)
 
         node_concept = core.ast.nodes.python_to_concept(ast_)
         unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)