Fixed BNF concept evaluations

core/builtin_concepts.py

@@ -60,6 +60,22 @@ class BuiltinConcepts(Enum):
         return "__" + self.name
 
 
+BuiltinUnique = [
+    BuiltinConcepts.BEFORE_PARSING,
+    BuiltinConcepts.PARSING,
+    BuiltinConcepts.AFTER_PARSING,
+    BuiltinConcepts.BEFORE_EVALUATION,
+    BuiltinConcepts.EVALUATION,
+    BuiltinConcepts.AFTER_EVALUATION,
+    BuiltinConcepts.BEFORE_RENDERING,
+    BuiltinConcepts.RENDERING,
+    BuiltinConcepts.AFTER_RENDERING,
+    BuiltinConcepts.SUCCESS,
+    BuiltinConcepts.NOP,
+    BuiltinConcepts.CONCEPT_EVAL_REQUESTED,
+    BuiltinConcepts.REDUCE_REQUESTED,
+]
+
 BuiltinErrors = [str(e) for e in {
     BuiltinConcepts.ERROR,
     BuiltinConcepts.UNKNOWN_CONCEPT,
@@ -75,7 +91,7 @@ BuiltinErrors = [str(e) for e in {
 
 """
 Some concepts have a specific implementation
-It's mainly to a have proper __repr__ implementation, or because they are singleton (is_unique=True)
+It's mainly to ease the usage
 """
 
 
@@ -96,11 +112,6 @@ class UserInputConcept(Concept):
         return f"({self.id}){self.name}: '{self.body}'"
 
 
-class SuccessConcept(Concept):
-    def __init__(self):
-        super().__init__(BuiltinConcepts.SUCCESS, True, True, BuiltinConcepts.SUCCESS)
-
-
 class ErrorConcept(Concept):
     def __init__(self, error=None):
         super().__init__(BuiltinConcepts.ERROR, True, False, BuiltinConcepts.ERROR, error)
@@ -256,30 +267,6 @@ class InvalidReturnValueConcept(Concept):
         self.set_prop("evaluator", evaluator)
 
 
-class BeforeParsingConcept(Concept):
-    def __init__(self):
-        super().__init__(BuiltinConcepts.BEFORE_PARSING, True, True, BuiltinConcepts.BEFORE_PARSING)
-
-
-class EvaluationConcept(Concept):
-    def __init__(self):
-        super().__init__(BuiltinConcepts.EVALUATION, True, True, BuiltinConcepts.EVALUATION)
-
-
-class AfterEvaluationConcept(Concept):
-    def __init__(self):
-        super().__init__(BuiltinConcepts.AFTER_EVALUATION, True, True, BuiltinConcepts.AFTER_EVALUATION)
-
-
-class ConceptEvalRequested(Concept):
-    def __init__(self):
-        super().__init__(BuiltinConcepts.CONCEPT_EVAL_REQUESTED, True, True, BuiltinConcepts.CONCEPT_EVAL_REQUESTED)
-
-
-class ReduceRequested(Concept):
-    def __init__(self):
-        super().__init__(BuiltinConcepts.REDUCE_REQUESTED, True, True, BuiltinConcepts.REDUCE_REQUESTED)
-
 class ConceptEvalError(Concept):
     def __init__(self, error=None, concept=None, property_name=None):
         super().__init__(BuiltinConcepts.CONCEPT_EVAL_ERROR,

core/sheerka.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 
-from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors
+from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
 from parsers.BaseParser import BaseParser
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderDuplicateKeyError
@@ -114,6 +114,10 @@ class Sheerka(Concept):
                 else builtins_classes[str(key)]() if str(key) in builtins_classes \
                 else Concept(key, True, False, key)
 
+            if key in BuiltinUnique:
+                concept.metadata.is_unique = True
+                concept.metadata.is_evaluated = True
+
             if not concept.metadata.is_unique and str(key) in builtins_classes:
                 self.builtin_cache[key] = builtins_classes[str(key)]
 
@@ -214,7 +218,7 @@ class Sheerka(Concept):
                 result.append(return_value)
                 continue
 
-            to_parse = self.value(return_value)
+            to_parse = return_value.body.body  # get the underlying text
 
             if self.log.isEnabledFor(logging.DEBUG):
                 debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \
@@ -509,11 +513,15 @@ class Sheerka(Concept):
                 concept.cached_asts[part_key] = self.execute(sub_context, to_parse, steps, logger)
 
         for prop in concept.props:
-            if concept.props[prop].value:
+            value = concept.props[prop].value
+            if value:
+                if isinstance(value, Concept):
+                    concept.cached_asts[prop] = value
+                else:
                     to_parse = self.ret(
                         context.who,
                         True,
-                    self.new(BuiltinConcepts.USER_INPUT, body=concept.props[prop].value))
+                        self.new(BuiltinConcepts.USER_INPUT, body=value))
                     sub_context = context.push(desc=f"Initializing AST for property {prop}")
                     sub_context.log_new(logger)
                     concept.cached_asts[prop] = self.execute(context, to_parse, steps)
@@ -545,7 +553,6 @@ class Sheerka(Concept):
         def _resolve(return_value, desc, obj):
             context.log(logger, desc, self.evaluate_concept.__name__)
             sub_context = context.push(desc=desc, obj=obj)
-            sub_context.add_preprocess(self.get_evaluator_name("Concept"), return_body=True)
             sub_context.log_new(logger)
             r = self.execute(sub_context, return_value, CONCEPT_EVALUATION_STEPS, logger)
             return core.builtin_helpers.expect_one(context, r)
@@ -568,7 +575,16 @@ class Sheerka(Concept):
         for metadata_to_eval in all_metadata_to_eval:
             if metadata_to_eval == "props":
                 for prop_name in (p for p in concept.props if p in concept.cached_asts):
-                    res = _resolve(concept.cached_asts[prop_name], f"Evaluating property '{prop_name}'", None)
+                    prop_ast = concept.cached_asts[prop_name]
+                    if isinstance(concept.cached_asts[prop_name], Concept):
+                        context.log(
+                            logger, f"Evaluation prop={prop_name}, value={prop_ast}", self.evaluate_concept.__name__)
+                        sub_context = context.push(f"Evaluation property '{prop_name}', value='{prop_ast}'")
+                        sub_context.log_new(logger)
+                        evaluated = self.evaluate_concept(sub_context, prop_ast)
+                        concept.set_prop(prop_name, evaluated)
+                    else:
+                        res = _resolve(prop_ast, f"Evaluating property '{prop_name}'", None)
                         if res.status:
                             concept.set_prop(prop_name, res.value)
                         else:
@@ -654,41 +670,41 @@ class Sheerka(Concept):
         """
         template = self.get(concept_key)
 
-        def new_from_template(t, k, **kwargs_):
-            # manage singleton
-            if t.metadata.is_unique:
-                return t
-
-            # otherwise, create another instance
-            concept = self.builtin_cache[k]() if k in self.builtin_cache else Concept()
-            concept.update_from(t)
-
-            # update the properties
-            for k, v in kwargs_.items():
-                if k in concept.props:
-                    concept.set_prop(k, v)
-                elif k in PROPERTIES_FOR_NEW:
-                    setattr(concept.metadata, k, v)
-                elif hasattr(concept, k):
-                    setattr(concept, k, v)
-                else:
-                    return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept)
-
-            # TODO : add the concept to the list of known concepts (self.instances)
-            return concept
-
         # manage concept not found
         if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \
             concept_key != BuiltinConcepts.UNKNOWN_CONCEPT:
             return template
 
         if not isinstance(template, list):
-            return new_from_template(template, concept_key, **kwargs)
+            return self.new_from_template(template, concept_key, **kwargs)
 
         # if template is a list, it means that there a multiple concepts under the same key
-        concepts = [new_from_template(t, concept_key, **kwargs) for t in template]
+        concepts = [self.new_from_template(t, concept_key, **kwargs) for t in template]
         return self.new(BuiltinConcepts.ENUMERATION, body=concepts)
 
+    def new_from_template(self, template, key, **kwargs):
+        # manage singleton
+        if template.metadata.is_unique:
+            return template
+
+        # otherwise, create another instance
+        concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
+        concept.update_from(template)
+
+        # update the properties
+        for key, v in kwargs.items():
+            if key in concept.props:
+                concept.set_prop(key, v)
+            elif key in PROPERTIES_FOR_NEW:
+                setattr(concept.metadata, key, v)
+            elif hasattr(concept, key):
+                setattr(concept, key, v)
+            else:
+                return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=key, concept=concept)
+
+        # TODO : add the concept to the list of known concepts (self.instances)
+        return concept
+
     def ret(self, who: str, status: bool, value, message=None, parents=None):
         """
         Creates and returns a ReturnValue concept
@@ -841,17 +857,25 @@ class Sheerka(Concept):
 
     def dump_desc(self, concept_name):
         if isinstance(concept_name, Concept):
-            c = concept_name
+            concepts = concept_name
         else:
-            c = self.get(concept_name)
-            if self.isinstance(c, BuiltinConcepts.UNKNOWN_CONCEPT):
+            concepts = self.get(concept_name)
+            if self.isinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT):
                 self.log.error("Concept unknown")
                 return False
 
+        if not hasattr(concepts, "__iter__"):
+            concepts = [concepts]
+
+        first = True
+        for c in concepts:
+            if not first:
+                self.log.info("")
             self.log.info(f"name : {c.name}")
             self.log.info(f"bnf  : {c.metadata.definition}")
             self.log.info(f"key  : {c.key}")
             self.log.info(f"body : {c.body}")
+            first = False
 
     @staticmethod
     def get_builtins_classes_as_dict():
@@ -920,6 +944,26 @@ class ExecutionContext:
         self.preprocess.add(preprocess)
         return self
 
+    def new_concept(self, key, **kwargs):
+        # search in obj
+        if self.obj:
+            if self.obj.key == key:
+                return self.sheerka.new_from_template(self.obj, key, **kwargs)
+            for prop in self.obj.props:
+                if prop == key:
+                    value = self.obj.props[prop].value
+                    if isinstance(value, Concept):
+                        return self.sheerka.new_from_template(value, key, **kwargs)
+                    else:
+                        return value
+
+        if self.concepts:
+            for k, c in self.concepts.items():
+                if k == key:
+                    return self.sheerka.new_from_template(c, key, **kwargs)
+
+        return self.sheerka.new(key, **kwargs)
+
     @property
     def id(self):
         return self._id

core/utils.py

@@ -220,6 +220,15 @@ def strip_tokens(tokens, strip_eof=False):
     return tokens[start: end + 1]
 
 
+def escape_char(text, to_escape):
+    res = ""
+
+    for c in text:
+        res += ("\\" + c) if c in to_escape else c
+
+    return res
+
+
 def pp(items):
     if not hasattr(items, "__iter__"):
         return str(items)

evaluators/AddConceptInSetEvaluator.py

@@ -65,6 +65,10 @@ class AddConceptInSetEvaluator(OneReturnValueEvaluator):
         else:
             context.log(self.log, f"Concept added.", self.name)
 
-        return res
+        return sheerka.ret(
+            self.name,
+            res.status,
+            res.body,
+            parents=[return_value])
 
 

evaluators/ConceptComposerEvaluator.py

@@ -0,0 +1,110 @@
+from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
+from core.concept import Concept
+from core.tokenizer import TokenKind
+from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator
+from parsers.BaseParser import BaseParser
+from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, ConceptLexerParser
+import core.utils
+
+
+class ConceptComposerEvaluator(AllReturnValuesEvaluator):
+    """
+    Try to reassemble parts of concepts from different evaluators
+    """
+
+    NAME = "ConceptComposer"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 40)
+
+    def matches(self, context, return_values):
+        concept_lexer_parser_name = ConceptLexerParser().name
+
+        for return_value in return_values:
+            if return_value.who.startswith(BaseParser.PREFIX) and return_value.status:
+                return False
+
+            if return_value.who.startswith(BaseEvaluator.PREFIX):
+                return False
+
+            if return_value.who != concept_lexer_parser_name:
+                continue
+
+            if not isinstance(return_value.value, ParserResultConcept):
+                return False
+
+            if not (
+                isinstance(return_value.value.value, ConceptNode) or
+                isinstance(return_value.value.value, UnrecognizedTokensNode) or
+                (
+                    hasattr(return_value.value.value, "__iter__") and
+                    len(return_value.value.value) > 0 and
+                    (
+                        isinstance(return_value.value.value[0], ConceptNode) or
+                        isinstance(return_value.value.value[0], UnrecognizedTokensNode)
+                    ))):
+                return False
+
+            self.eaten = return_value
+            return True
+
+        return False
+
+    def eval(self, context, return_value):
+        sheerka = context.sheerka
+        nodes = self.eaten.value.value
+        temp_res = []
+        has_error = False
+        concepts_only = True
+
+        for node in nodes:
+            if isinstance(node, UnrecognizedTokensNode):
+                tokens = core.utils.strip_tokens(node.tokens, True)
+                for token in tokens:
+                    if token.type == TokenKind.IDENTIFIER:
+                        concept = context.new_concept(token.value)
+                        if sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
+                            has_error = True
+                        else:
+                            sub_context = context.push(self.name, desc=f"Evaluating '{concept}'")
+                            sub_context.log_new(self.verbose_log)
+                            concept = sheerka.evaluate_concept(sub_context, concept, self.verbose_log)
+                        temp_res.append(concept)
+                    else:
+                        temp_res.append(core.utils.strip_quotes(token.value))
+                        concepts_only &= token.type == TokenKind.WHITESPACE or token.type == TokenKind.NEWLINE
+            else:
+                sub_context = context.push(self.name, desc=f"Evaluating '{node.concept}'")
+                sub_context.log_new(self.verbose_log)
+                concept = sheerka.evaluate_concept(sub_context, node.concept, self.verbose_log)
+                temp_res.append(concept)
+
+        if has_error:
+            return sheerka.ret(
+                self.name,
+                False,
+                temp_res,
+                parents=[self.eaten])
+
+        if concepts_only:
+            res = []
+            for r in temp_res:
+                if isinstance(r, Concept):
+                    res.append(r)
+        else:
+            res = ""
+            for r in temp_res:
+                if isinstance(r, Concept):
+                    res += sheerka.value(r)
+                else:
+                    res += r
+
+        return sheerka.ret(
+            self.name,
+            True,
+            res,
+            parents=[self.eaten])
+
+
+
+

evaluators/ConceptEvaluator.py

@@ -11,11 +11,6 @@ class ConceptEvaluator(OneReturnValueEvaluator):
     Then checks the POST conditions
     """
     NAME = "Concept"
-    evaluation_steps = [
-        BuiltinConcepts.BEFORE_EVALUATION,
-        BuiltinConcepts.EVALUATION,
-        BuiltinConcepts.AFTER_EVALUATION
-    ]
 
     def __init__(self, return_body=False):
         super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50)
@@ -34,7 +29,7 @@ class ConceptEvaluator(OneReturnValueEvaluator):
         # If the concept that is requested is in the context(at least its name), drop the call.
         # Why ?
         # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
-        # The body should be 'property_a', and not a concept called a in our universe
+        # The body should be 'property_a', and not a concept called 'a'
         if context.obj and concept.name in context.obj.props:
             value = context.obj.props[concept.name].value
             context.log(self.verbose_log, f"{concept.name} is a property. Returning value '{value}'.", self.name)

evaluators/ConceptNodeEvaluator.py

@@ -1,7 +1,7 @@
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
-
-from parsers.ConceptLexerParser import ConceptNode, NonTerminalNode, ConceptMatch, UnrecognizedTokensNode
+import core.utils
+from parsers.ConceptLexerParser import ConceptNode, NonTerminalNode, ConceptMatch, UnrecognizedTokensNode, TerminalNode
 
 
 class ConceptNodeEvaluator(OneReturnValueEvaluator):
@@ -46,10 +46,12 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
 
         concepts = []
         error_found = False
+        source = ""
         for node in nodes:
             if isinstance(node, ConceptNode):
+                source += node.source if source == "" else (" " + node.source)
                 concept = sheerka.new(node.concept.key)
-                concept = self.update_concept(sheerka, concept, node.underlying)
+                concept = self.finalize_concept(sheerka, concept, node.underlying)
                 concepts.append(concept)
             else:
                 error_found = True
@@ -58,12 +60,17 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
             return sheerka.ret(
                 self.name,
                 not error_found,
-                concepts[0],
+                context.sheerka.new(
+                    BuiltinConcepts.PARSER_RESULT,
+                    parser=self,
+                    source=source,
+                    body=concepts[0],
+                    try_parsed=None),
                 parents=[return_value])
 
         return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME), parents=[return_value])
 
-    def update_concept(self, sheerka, concept, underlying, init_empty_body=True):
+    def finalize_concept(self, sheerka, concept, underlying, init_empty_body=True):
         """
         Updates the properties of the concept
         Goes in recursion if the property is a concept
@@ -74,9 +81,12 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
             Adds a new entry,
             makes a list if the property already exists
             """
+
             if prop_name not in c.props or c.props[prop_name].value is None:
+                # new entry
                 c.set_prop(prop_name, value)
             else:
+                # make a list if there was a value
                 previous_value = c.props[prop_name].value
                 if isinstance(previous_value, list):
                     previous_value.append(value)
@@ -87,11 +97,12 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
         parsing_expression = underlying.parsing_expression
 
         if parsing_expression.rule_name:
-            _add_prop(concept, parsing_expression.rule_name, underlying.source)
+            _add_prop(concept, parsing_expression.rule_name, self.get_underlying_as_string(underlying))
 
         # the update of the body must come BEFORE the recursion
+        # otherwise it will be updated by a children and it won't be possible to modify the value
         if init_empty_body and concept.body is None:
-            concept.metadata.body = underlying.source
+            concept.metadata.body = self.get_underlying_as_string(underlying) # self.escape_if_needed(underlying.source)
 
         if isinstance(underlying, NonTerminalNode):
             for child in underlying.children:
@@ -101,8 +112,75 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
                     if sheerka.isinstance(new_concept, BuiltinConcepts.UNKNOWN_CONCEPT):
                         continue
                     else:
-                        self.update_concept(sheerka, new_concept, child.children[0], init_empty_body)
+                        self.finalize_concept(sheerka, new_concept, child.children[0], init_empty_body)
                 else:
-                    self.update_concept(sheerka, concept, child, init_empty_body)
+                    self.finalize_concept(sheerka, concept, child, init_empty_body)
 
         return concept
+
+    @staticmethod
+    def escape_if_needed(value):
+        if not isinstance(value, str):
+            return value
+
+        return "'" + core.utils.escape_char(value, "'") + "'"
+
+    def get_underlying_as_string(self, underlying):
+        """
+        Return the sequence of the recognized character
+        When a concept is recognized, return the string version of the concept eg c:concept name:
+        :param underlying:
+        :return:
+        """
+
+        # Example
+        #     grammar = {
+        #         foo: Sequence("one", "two", rule_name="var"),
+        #         bar: Sequence(foo, "three", rule_name="var")}
+        #
+        # we want bar.body and bar.prop["var"]
+        #  to be "foo 'three'"  (no quotes surrounding foo, as it is a concept, not a string)
+
+        if isinstance(underlying, TerminalNode):
+            return self.escape_if_needed(underlying.source)
+
+        res = ""
+        first = True
+        in_quote = ""
+        for node in underlying.children:
+            if isinstance(node.parsing_expression, ConceptMatch):
+                if in_quote != "":
+                    res += in_quote + "'"
+                if not first:
+                    res += " "
+                res += node.parsing_expression.concept.key
+                in_quote = ""
+            else:
+                if in_quote == "":
+                    in_quote = ("'" if first else " '") + core.utils.escape_char(node.source, "'")
+                else:
+                    in_quote += ("" if first else " ") + core.utils.escape_char(node.source, "'")
+
+            first = False
+
+        if in_quote:
+            res += in_quote + "'"
+        return res
+
+# - - -  E X P L A N A T I O N S - - -
+# why do we need to update the body ?
+# cf test_concept_property_is_correctly_updated_when_concept_recursion_using_zero_or_more()
+# def concept number from bnf one | two | three
+# def concept add from bnf number plus number
+#
+# the expression 'one plus two plus three' will match concept add
+# add.props["number"] is a list of concepts 'number'
+# But which one is 'one', which one is 'two' which one is 'three' ?
+#
+# That's the reason why we update the body
+# add.props["number"] is a list of concepts 'number' but they won't have the same body
+#
+# !!! C A U T I O N !!!
+# In the current implementation, the body is the sequence of char found
+# If a concept is recognized, we don't put this information in the body
+# Use get_body_as_string() instead of escape_if_needed() if we need this information

evaluators/OneErrorEvaluator.py

@@ -12,7 +12,7 @@ class OneErrorEvaluator(AllReturnValuesEvaluator):
     NAME = "OneError"
 
     def __init__(self):
-        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 40)
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 30)
         self.return_value_in_error = None
 
     def matches(self, context, return_values):

evaluators/PythonEvaluator.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
-from core.concept import ConceptParts
+from core.concept import ConceptParts, Concept
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.PythonParser import PythonNode
 import ast
@@ -65,7 +65,10 @@ class PythonEvaluator(OneReturnValueEvaluator):
                         f"Concept '{context.obj}' is in context. Adding its properties to locals if any.", self.name)
 
             for prop_name, prop_value in context.obj.props.items():
+                if not isinstance(prop_value.value, Concept):
                     my_locals[prop_name] = prop_value.value
+                else:
+                    my_locals[prop_name] = context.sheerka.value(prop_value.value)
 
         node_concept = core.ast.nodes.python_to_concept(ast_)
         unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)

parsers/BaseParser.py

@@ -78,7 +78,7 @@ class BaseParser:
             context.log(self.log, f"  Recognized '{value}'", self.name)
 
     @staticmethod
-    def get_text_from_tokens(tokens):
+    def get_text_from_tokens(tokens, custom_switcher=None):
         if tokens is None:
             return ""
         res = ""
@@ -88,9 +88,12 @@ class BaseParser:
 
         switcher = {
             TokenKind.KEYWORD: lambda t: Keywords(t.value).value,
-            TokenKind.CONCEPT: lambda t: f"__C__{t.value}__C__"
+            TokenKind.CONCEPT: lambda t: "c:" + t.value + ":",
         }
 
+        if custom_switcher:
+            switcher.update(custom_switcher)
+
         for token in tokens:
             value = switcher.get(token.type, lambda t: t.value)(token)
             res += value

parsers/ConceptLexerParser.py

@@ -795,7 +795,7 @@ class ConceptLexerParser(BaseParser):
                 if not self.next_token():
                     break
 
-        # Fix the source if we were working on unrecognized tokens
+        # Fix the source for unrecognized tokens
         if unrecognized_tokens:
             unrecognized_tokens.fix_source()
 

parsers/PythonParser.py

@@ -1,5 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.tokenizer import Tokenizer, LexerError
+from core.tokenizer import Tokenizer, LexerError, TokenKind
 from parsers.BaseParser import BaseParser, Node, ErrorNode
 from dataclasses import dataclass
 import ast
@@ -66,13 +66,17 @@ class PythonParser(BaseParser):
         sheerka = context.sheerka
         tree = None
 
+        python_switcher = {
+            TokenKind.CONCEPT: lambda t: f"__C__{t.value}__C__"
+        }
+
         try:
             if isinstance(text, str) and "c:" in text:
-                source = self.get_text_from_tokens(list(Tokenizer(text)))
+                source = self.get_text_from_tokens(list(Tokenizer(text)), python_switcher)
             elif isinstance(text, str):
                 source = text
             else:
-                source = self.get_text_from_tokens(text)
+                source = self.get_text_from_tokens(text, python_switcher)
             source = source.strip()
 
             text = text if isinstance(text, str) else source

tests/test_BaseParser.py

@@ -9,9 +9,17 @@ from parsers.BaseParser import BaseParser
     ("'hello' 'world'", "'hello' 'world'"),
     ("def concept a from", "def concept a from"),
     ("()[]{}1=1.5+-/*><&é", "()[]{}1=1.5+-/*><&é"),
-    ("execute(c:concept_name:)", "execute(__C__concept_name__C__)")
+    ("execute(c:concept_name:)", "execute(c:concept_name:)")
 
 ])
 def test_i_can_get_text_from_tokens(text, expected_text):
     tokens = list(Tokenizer(text))
     assert BaseParser.get_text_from_tokens(tokens) == expected_text
+
+
+@pytest.mark.parametrize("text, custom, expected_text", [
+    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value}"}, "execute(__C__concept_name)")
+])
+def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
+    tokens = list(Tokenizer(text))
+    assert BaseParser.get_text_from_tokens(tokens, custom) == expected_text

tests/test_ConceptComposerEvaluator.py

@@ -0,0 +1,128 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept
+from core.concept import Concept
+from core.sheerka import Sheerka, ExecutionContext
+from evaluators.BaseEvaluator import BaseEvaluator
+from evaluators.ConceptComposerEvaluator import ConceptComposerEvaluator
+from parsers.BaseParser import BaseParser
+from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, Sequence
+from sdp.sheerkaDataProvider import Event
+
+concept_lexer_name = ConceptLexerParser().name
+
+
+def get_context():
+    sheerka = Sheerka(skip_builtins_in_db=True)
+    sheerka.initialize("mem://")
+    return ExecutionContext("test", Event(), sheerka)
+
+
+def get_return_values(context, grammar, expression):
+    parser = ConceptLexerParser()
+    parser.initialize(context, grammar)
+
+    ret_val = parser.parse(context, expression)
+    assert not ret_val.status
+    return [ret_val]
+
+
+def init(concepts, grammar, expression):
+    context = get_context()
+    for c in concepts:
+        context.sheerka.add_in_cache(c)
+    return_values = get_return_values(context, grammar, expression)
+
+    return context, return_values
+
+
+@pytest.mark.parametrize("return_values, expected", [
+    ([
+         ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"),
+         ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
+         ReturnValueConcept("not a parser", True, "some value"),
+     ], True),
+    ([
+         ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
+     ], True),
+    ([
+         ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not in error"),
+         ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
+     ], False),
+    ([
+         ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"),
+         ReturnValueConcept(concept_lexer_name, True, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
+     ], False),
+    ([
+         ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"),
+         ReturnValueConcept(concept_lexer_name, False, "some value"),
+     ], False),
+    ([
+         ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"),
+         ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=["not a concept"])),
+     ], False),
+    ([
+         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", False, "evaluator in error"),
+         ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
+         ReturnValueConcept("not a parser", True, "some value"),
+     ], False),
+    ([
+         ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "evaluator"),
+         ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
+         ReturnValueConcept("not a parser", True, "some value"),
+     ], False),
+])
+def test_i_can_match(return_values, expected):
+    context = get_context()
+    assert ConceptComposerEvaluator().matches(context, return_values) == expected
+
+
+def test_i_can_eval_simple_concepts():
+    foo = Concept("foo", body="'foo'")
+    bar = Concept("bar", body="'bar'")
+    grammar = {}
+    context, return_values = init([foo, bar], grammar, "bar foo")
+
+    composer = ConceptComposerEvaluator()
+    assert composer.matches(context, return_values)
+
+    ret_val = composer.eval(context, return_values)
+    assert ret_val.status
+    assert ret_val.who == composer.name
+    assert ret_val.value == [Concept("bar", body="bar").init_key(), Concept("foo", body="foo").init_key()]
+    assert ret_val.value[0].metadata.is_evaluated
+    assert ret_val.value[1].metadata.is_evaluated
+    assert ret_val.parents == [return_values[0]]
+
+
+def test_i_can_eval_simple_concepts_when_some_are_bnf():
+    foo = Concept("foo", body="'foo'")
+    bar = Concept("bar", body="'bar'")
+    grammar = {foo: "foo"}
+    context, return_values = init([foo, bar], grammar, "bar foo")
+
+    composer = ConceptComposerEvaluator()
+    assert composer.matches(context, return_values)
+
+    ret_val = composer.eval(context, return_values)
+    assert ret_val.status
+    assert ret_val.who == composer.name
+    assert ret_val.value == [Concept("bar", body="bar").init_key(), Concept("foo", body="foo").init_key()]
+    assert ret_val.value[0].metadata.is_evaluated
+    assert ret_val.value[1].metadata.is_evaluated
+    assert ret_val.parents == [return_values[0]]
+
+
+def test_i_can_eval_simple_concept_and_text():
+    foo = Concept("foo", body="'foo'")
+    grammar = {}
+    context, return_values = init([foo], grammar, "'bar' foo")
+
+    composer = ConceptComposerEvaluator()
+    assert composer.matches(context, return_values)
+
+    ret_val = composer.eval(context, return_values)
+    assert ret_val.status
+    assert ret_val.who == composer.name
+    assert ret_val.value == "bar foo"
+    assert ret_val.parents == [return_values[0]]

tests/test_ConceptNodeEvaluator.py

@@ -5,7 +5,7 @@ from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
 from evaluators.ConceptNodeEvaluator import ConceptNodeEvaluator
 from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, Sequence, TerminalNode, \
-    StrMatch, Optional, OrderedChoice, ZeroOrMore, UnrecognizedTokensNode
+    StrMatch, Optional, OrderedChoice, ZeroOrMore, UnrecognizedTokensNode, ConceptMatch
 from sdp.sheerkaDataProvider import Event
 
 
@@ -15,23 +15,26 @@ def get_context():
     return ExecutionContext("test", Event(), sheerka)
 
 
-def get_return_value(nodes, source):
-    return ReturnValueConcept(
-        "some_name",
-        True,
-        ParserResultConcept(parser=ConceptLexerParser(),
-                            source=source,
-                            value=nodes,
-                            try_parsed=nodes))
-
-
-def get_concept_node(context, grammar, expression):
+def get_return_value(context, grammar, expression):
     parser = ConceptLexerParser()
     parser.initialize(context, grammar)
 
-    res = parser.parse(context, expression)
-    assert res.status
-    return res.value.value[0]
+    ret_val = parser.parse(context, expression)
+    assert ret_val.status
+    return ret_val
+
+
+def init(concept, grammar, text):
+    context = get_context()
+    if isinstance(concept, list):
+        for c in concept:
+            context.sheerka.add_in_cache(c)
+    else:
+        context.sheerka.add_in_cache(concept)
+    ret_val = get_return_value(context, grammar, text)
+    node = ret_val.value.value[0]
+
+    return context, node
 
 
 @pytest.mark.parametrize("ret_val, expected", [
@@ -53,174 +56,172 @@ def test_i_can_match(ret_val, expected):
     assert ConceptNodeEvaluator().matches(context, ret_val) == expected
 
 
-def test_concept_is_returned_when_list_of_one_concept_node():
+def test_parser_result_of_concept_is_returned_when_list_of_one_concept_node():
     foo = Concept("foo")
     context = get_context()
     context.sheerka.add_in_cache(foo)
 
     evaluator = ConceptNodeEvaluator()
-    node = ConceptNode(foo, 0, 0, underlying=TerminalNode(StrMatch("foo"), 0, 0, "foo"))
+    ret_val = get_return_value(context, {foo: StrMatch("foo")}, "foo")
 
-    ret_val = get_return_value([node], "h")
     result = evaluator.eval(context, ret_val)
 
     assert result.who == evaluator.name
     assert result.status
-    assert result.value == Concept("foo", body="foo").init_key()
+    assert result.value == ParserResultConcept(
+        evaluator,
+        "foo",
+        Concept("foo", body="'foo'").init_key(),
+        None)
     assert result.parents == [ret_val]
 
 
 def test_concept_property_is_correctly_updated_for_str_match():
-    context = get_context()
-
     foo = Concept("foo")
-    concept_node = get_concept_node(context, {foo: StrMatch("foo", rule_name="variable")}, "foo")
-    updated = ConceptNodeEvaluator().update_concept(context.sheerka, concept_node.concept, concept_node.underlying)
+    grammar = {foo: StrMatch("foo", rule_name="variable")}
+    context, node = init(foo, grammar, "foo")
+
+    updated = ConceptNodeEvaluator().finalize_concept(context.sheerka, node.concept, node.underlying)
 
     assert "variable" in updated.props
-    assert updated.props["variable"].value == "foo"
+    assert updated.props["variable"].value == "'foo'"
+    assert updated.body == "'foo'"
 
 
 def test_concept_property_is_correctly_updated_for_sequence():
-    context = get_context()
-
     foo = Concept("foo")
     grammar = {foo: Sequence("one", "two", rule_name="variable")}
-    concept_node = get_concept_node(context, grammar, "one two")
-    updated = ConceptNodeEvaluator().update_concept(context.sheerka, concept_node.concept, concept_node.underlying)
+    context, node = init(foo, grammar, "one two")
+
+    updated = ConceptNodeEvaluator().finalize_concept(
+        context.sheerka,
+        context.sheerka.new(node.concept.key),
+        node.underlying)
 
     assert "variable" in updated.props
-    assert updated.props["variable"].value == "one two"
+    assert updated.props["variable"].value == "'one two'"
+    assert updated.body == "'one two'"
 
 
 def test_concept_property_is_updated_for_str_in_sequence():
-    context = get_context()
-
     foo = Concept("foo")
     grammar = {foo: Sequence(StrMatch("one", rule_name="s1"), StrMatch("two", rule_name="s2"), rule_name="variable")}
-    concept_node = get_concept_node(context, grammar, "one two")
+    context, node = init(foo, grammar, "one two")
 
-    updated = ConceptNodeEvaluator().update_concept(context.sheerka, concept_node.concept, concept_node.underlying)
+    updated = ConceptNodeEvaluator().finalize_concept(
+        context.sheerka,
+        context.sheerka.new(node.concept.key),
+        node.underlying)
 
-    assert updated.props["variable"].value == "one two"
-    assert updated.props["s1"].value == "one"
-    assert updated.props["s2"].value == "two"
+    assert updated.props["variable"].value == "'one two'"
+    assert updated.props["s1"].value == "'one'"
+    assert updated.props["s2"].value == "'two'"
+    assert updated.body == "'one two'"
 
 
 def test_concept_property_is_correctly_updated_for_optional():
-    context = get_context()
-
     foo = Concept("foo")
     grammar = {foo: Sequence("one", Optional("two", rule_name="o"), rule_name="variable")}
-    concept_node = get_concept_node(context, grammar, "one two")
+    context, node = init(foo, grammar, "one two")
 
-    updated = ConceptNodeEvaluator().update_concept(
+    updated = ConceptNodeEvaluator().finalize_concept(
         context.sheerka,
-        context.sheerka.new(concept_node.concept.key),
-        concept_node.underlying)
+        context.sheerka.new(node.concept.key),
+        node.underlying)
 
     assert "variable" in updated.props
-    assert updated.props["variable"].value == "one two"
-    assert updated.props["o"].value == "two"
+    assert updated.props["variable"].value == "'one two'"
+    assert updated.props["o"].value == "'two'"
+    assert updated.body == "'one two'"
 
 
 def test_concept_property_is_correctly_updated_for_zero_or_more():
-    context = get_context()
-
     foo = Concept("foo")
     grammar = {foo: ZeroOrMore("one", rule_name="variable")}
-    concept_node = get_concept_node(context, grammar, "one one one")
+    context, node = init(foo, grammar, "one one one")
 
-    updated = ConceptNodeEvaluator().update_concept(
+    updated = ConceptNodeEvaluator().finalize_concept(
         context.sheerka,
-        context.sheerka.new(concept_node.concept.key),
-        concept_node.underlying)
+        context.sheerka.new(node.concept.key),
+        node.underlying)
 
     assert "variable" in updated.props
-    assert updated.props["variable"].value == "one one one"
+    assert updated.props["variable"].value == "'one one one'"
+    assert updated.body == "'one one one'"
 
 
 def test_concept_property_is_correctly_updated_when_list_of_properties():
-    context = get_context()
-
     foo = Concept("foo")
     grammar = {foo: Sequence(StrMatch("one", rule_name="s"), StrMatch("two", rule_name="s"), rule_name="variable")}
-    concept_node = get_concept_node(context, grammar, "one two")
+    context, node = init(foo, grammar, "one two")
 
-    updated = ConceptNodeEvaluator().update_concept(
+    updated = ConceptNodeEvaluator().finalize_concept(
         context.sheerka,
-        context.sheerka.new(concept_node.concept.key),
-        concept_node.underlying)
+        context.sheerka.new(node.concept.key),
+        node.underlying)
 
-    assert updated.props["variable"].value == "one two"
-    assert updated.props["s"].value == ["one", "two"]
+    assert updated.props["variable"].value == "'one two'"
+    assert updated.props["s"].value == ["'one'", "'two'"]
+    assert updated.body == "'one two'"
 
 
 def test_concept_property_is_correctly_updated_when_another_concept():
-    context = get_context()
-
     foo = Concept("foo")
     bar = Concept("bar")
-    context.sheerka.add_in_cache(foo)
-    context.sheerka.add_in_cache(bar)
     grammar = {
         foo: Sequence("one", "two", rule_name="var"),
-        bar: Sequence(foo, "three", rule_name="var")}
-    concept_node = get_concept_node(context, grammar, "one two three")
+        bar: Sequence(foo, "three", "four", rule_name="var")}
+    context, node = init([foo, bar], grammar, "one two three four")
 
-    updated = ConceptNodeEvaluator().update_concept(
+    updated = ConceptNodeEvaluator().finalize_concept(
         context.sheerka,
-        context.sheerka.new(concept_node.concept.key),
-        concept_node.underlying)
+        context.sheerka.new(node.concept.key),
+        node.underlying)
 
-    assert updated.props["var"].value == "one two three"
-    assert updated.props["foo"].value == Concept("foo", body="one two").set_prop("var", "one two").init_key()
+    assert updated.body == "foo 'three four'"
+    assert updated.props["var"].value == "foo 'three four'"
+    assert updated.props["foo"].value == Concept("foo", body="'one two'").set_prop("var", "'one two'").init_key()
 
 
 def test_concept_property_is_correctly_updated_when_concept_recursion_using_optional():
-    context = get_context()
-
     number = Concept("number")
     add = Concept("add")
-    context.sheerka.add_in_cache(number)
-    context.sheerka.add_in_cache(add)
     grammar = {
         number: OrderedChoice("one", "two"),
         add: Sequence(number, Optional(Sequence(OrderedChoice("plus", "minus", rule_name="op"), add)))
     }
-    concept_node = get_concept_node(context, grammar, "one plus two")
+    context, node = init([number, add], grammar, "one plus two")
 
-    updated = ConceptNodeEvaluator().update_concept(
+    updated = ConceptNodeEvaluator().finalize_concept(
         context.sheerka,
-        context.sheerka.new(concept_node.concept.key),
-        concept_node.underlying)
+        context.sheerka.new(node.concept.key),
+        node.underlying)
 
-    assert updated.props["number"].value == Concept("number", body="one").init_key()
-    assert updated.props["op"].value == "plus"
-    expected_add = Concept("add", body="two").set_prop("number", Concept("number", body="two").init_key()).init_key()
+    assert updated.props["number"].value == Concept("number", body="'one'").init_key()
+    assert updated.props["op"].value == "'plus'"
+    expected_add = Concept("add", body="number"). \
+        set_prop("number", Concept("number", body="'two'").init_key()). \
+        init_key()
     assert updated.props["add"].value == expected_add
 
 
 def test_concept_property_is_correctly_updated_when_concept_recursion_using_zero_or_more():
-    context = get_context()
-
     number = Concept("number")
     add = Concept("add")
-    context.sheerka.add_in_cache(number)
-    context.sheerka.add_in_cache(add)
     grammar = {
         number: OrderedChoice("one", "two", 'three'),
         add: Sequence(number, ZeroOrMore(Sequence(OrderedChoice("plus", "minus", rule_name="op"), number)))
     }
-    concept_node = get_concept_node(context, grammar, "one plus two minus three")
+    context, node = init([number, add], grammar, "one plus two minus three")
 
-    updated = ConceptNodeEvaluator().update_concept(
+    updated = ConceptNodeEvaluator().finalize_concept(
         context.sheerka,
-        context.sheerka.new(concept_node.concept.key),
-        concept_node.underlying,
+        context.sheerka.new(node.concept.key),
+        node.underlying,
         init_empty_body=True)
 
-    assert updated.props["number"].value == [Concept("number", body="one").init_key(),
-                                             Concept("number", body="two").init_key(),
-                                             Concept("number", body="three").init_key()]
-    assert updated.props["op"].value == ["plus", "minus"]
+    assert updated.props["number"].value == [Concept("number", body="'one'").init_key(),
+                                             Concept("number", body="'two'").init_key(),
+                                             Concept("number", body="'three'").init_key()]
+    assert updated.props["op"].value == ["'plus'", "'minus'"]
+

tests/test_PythonEvaluator.py

@@ -118,7 +118,7 @@ def test_i_can_eval_concept_token():
     assert evaluated.status
     assert evaluated.value == "foo"
 
-    # sanity, to make sure that otherwise foo is resolved to '2'
+    # sanity, does not work otherwise
     parsed = PythonParser().parse(context, "get_context_name(foo)")
     python_evaluator = PythonEvaluator()
     python_evaluator.locals["get_context_name"] = get_context_name

tests/test_sheerka.py

@@ -332,6 +332,7 @@ def test_i_cannot_instantiate_when_properties_are_not_recognized():
     (Concept("name", body=["foo"]), True, "foo"),
     (Concept("name", body=Concept("foo")), False, Concept("foo")),
     (Concept("name", body=Concept("foo", body="value")), False, "value"),
+    (Concept("name", body=Concept("foo", body=Concept("bar", body="value"))), False, "value"),
     (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"),
 ])
 def test_i_can_get_value(concept, reduce_simple_list, expected):
@@ -404,7 +405,7 @@ def test_i_can_evaluate_the_other_metadata(expr, expected):
 
 
 @pytest.mark.parametrize("expr, expected", [
-    (None, None),
+    # (None, None),
     ("", ""),
     ("1", 1),
     ("1+1", 2),
@@ -450,6 +451,7 @@ def test_i_can_evaluate_when_another_concept_is_referenced():
     assert sheerka.isinstance(evaluated.body, concept_a)
     assert id(evaluated.body) != id(concept_a)
     assert evaluated.metadata.is_evaluated
+    assert evaluated.body.metadata.is_evaluated
 
 
 def test_i_can_evaluate_when_the_referenced_concept_has_a_body():
@@ -461,34 +463,50 @@ def test_i_can_evaluate_when_the_referenced_concept_has_a_body():
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
 
     assert evaluated.key == concept.key
-    assert evaluated.body == 1
-    assert not concept_a.metadata.is_evaluated  #
+    assert evaluated.body == Concept("a", body=1).init_key()
+    assert not concept_a.metadata.is_evaluated
+    assert evaluated.metadata.is_evaluated
 
 
 def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body():
     sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="a", body="'a'").init_key())
-    sheerka.add_in_cache(Concept(name="b", body="a").init_key())
-    sheerka.add_in_cache(Concept(name="c", body="b").init_key())
-    concept_d = sheerka.add_in_cache(Concept(name="d", body="c").init_key())
+    sheerka.add_in_cache(Concept(name="a", body="'a'"))
+    sheerka.add_in_cache(Concept(name="b", body="a"))
+    sheerka.add_in_cache(Concept(name="c", body="b"))
+    concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
 
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d)
 
     assert evaluated.key == concept_d.key
-    assert evaluated.body == 'a'
+    assert evaluated.body == Concept(
+        name="c",
+        body=Concept(
+            name="b",
+            body=Concept(
+                name="a",
+                body="a").init_key()).init_key()).init_key()
+    assert sheerka.value(evaluated) == 'a'
+    assert evaluated.metadata.is_evaluated
 
 
 def test_i_can_evaluate_concept_of_concept_does_not_have_a_body():
     sheerka = get_sheerka()
-    concept_a = sheerka.add_in_cache(Concept(name="a").init_key())
-    sheerka.add_in_cache(Concept(name="b", body="a").init_key())
-    sheerka.add_in_cache(Concept(name="c", body="b").init_key())
-    concept_d = sheerka.add_in_cache(Concept(name="d", body="c").init_key())
+    sheerka.add_in_cache(Concept(name="a"))
+    sheerka.add_in_cache(Concept(name="b", body="a"))
+    sheerka.add_in_cache(Concept(name="c", body="b"))
+    concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
 
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d)
 
     assert evaluated.key == concept_d.key
-    assert sheerka.isinstance(evaluated.body, concept_a)
+    assert evaluated.body == Concept(
+        name="c",
+        body=Concept(
+            name="b",
+            body=Concept(
+                name="a").init_key()).init_key()).init_key()
+    assert sheerka.value(evaluated) == Concept(name="a").init_key()
+    assert evaluated.metadata.is_evaluated
 
 
 def test_i_can_evaluate_concept_when_properties_reference_others_concepts():
@@ -509,9 +527,9 @@ def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2():
     :return:
     """
     sheerka = get_sheerka()
-    concept_a = sheerka.add_in_cache(Concept(name="a").init_key())
+    concept_a = sheerka.add_in_cache(Concept(name="a"))
 
-    concept = Concept("foo", body="concept_a").set_prop("concept_a", "a").init_key()
+    concept = Concept("foo", body="concept_a").set_prop("concept_a", "a")
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
 
     assert evaluated.key == concept.key
@@ -520,10 +538,10 @@ def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2():
 
 def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body():
     sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="a", body="1").init_key())
-    sheerka.add_in_cache(Concept(name="b", body="2").init_key())
+    sheerka.add_in_cache(Concept(name="a", body="1"))
+    sheerka.add_in_cache(Concept(name="b", body="2"))
 
-    concept = Concept("foo", body="propA + propB").set_prop("propA", "a").set_prop("propB", "b").init_key()
+    concept = Concept("foo", body="propA + propB").set_prop("propA", "a").set_prop("propB", "b")
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
 
     assert evaluated.key == concept.key
@@ -542,33 +560,33 @@ def test_i_can_reference_sheerka():
 
 def test_properties_values_takes_precedence_over_the_outside_world():
     sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="a", body="'concept_a'").init_key())
-    sheerka.add_in_cache(Concept(name="b", body="'concept_b'").init_key())
+    sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
+    sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
 
-    concept = Concept("foo", body="a").init_key()
+    concept = Concept("foo", body="a")
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
     assert evaluated.key == concept.key
-    assert evaluated.body == 'concept_a'  # this test was already done
+    assert evaluated.body == Concept(name="a", body="concept_a").init_key()  # this test was already done
 
     # so check this one.
-    concept = Concept("foo", body="a").set_prop("a", "'property_a'").init_key()
+    concept = Concept("foo", body="a").set_prop("a", "'property_a'")
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
     assert evaluated.key == concept.key
     assert evaluated.body == 'property_a'
 
     # or this one.
-    concept = Concept("foo", body="a").set_prop("a", "b").init_key()
+    concept = Concept("foo", body="a").set_prop("a", "b")
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
     assert evaluated.key == concept.key
-    assert evaluated.body == 'concept_b'
+    assert evaluated.body == Concept(name="b", body="concept_b").init_key()
 
 
 def test_properties_values_takes_precedence():
     sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="a", body="'concept_a'").init_key())
-    sheerka.add_in_cache(Concept(name="b", body="'concept_b'").init_key())
+    sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
+    sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
 
-    concept = Concept("foo", body="a + b").set_prop("a", "'prop_a'").init_key()
+    concept = Concept("foo", body="a + b").set_prop("a", "'prop_a'")
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
     assert evaluated.key == concept.key
     assert evaluated.body == 'prop_aconcept_b'
@@ -576,9 +594,9 @@ def test_properties_values_takes_precedence():
 
 def test_i_can_reference_sub_property_of_a_property():
     sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="concept_a").set_prop("subProp", "'sub_a'").init_key())
+    sheerka.add_in_cache(Concept(name="concept_a").set_prop("subProp", "'sub_a'"))
 
-    concept = Concept("foo", body="a.props['subProp'].value").set_prop("a", "concept_a").init_key()
+    concept = Concept("foo", body="a.props['subProp'].value").set_prop("a", "concept_a")
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
     assert evaluated.key == concept.key
     assert evaluated.body == 'sub_a'
@@ -587,7 +605,7 @@ def test_i_can_reference_sub_property_of_a_property():
 def test_i_cannot_evaluate_concept_if_property_is_in_error():
     sheerka = get_sheerka()
 
-    concept = Concept(name="concept_a").set_prop("subProp", "undef_concept").init_key()
+    concept = Concept(name="concept_a").set_prop("subProp", "undef_concept")
     evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
     assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
 

tests/test_sheerka_call_evaluators.py

@@ -1,5 +1,5 @@
 # Make sure that the evaluators works as expected
-from core.builtin_concepts import BuiltinConcepts, SuccessConcept
+from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
 from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator
@@ -157,7 +157,7 @@ class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority):
 
     def eval(self, context, return_values):
         super().eval(context, return_values)
-        ret = get_ret_val(context.sheerka, SuccessConcept())
+        ret = get_ret_val(context.sheerka, context.sheerka.new(BuiltinConcepts.SUCCESS))
         ret.parents = return_values
         return ret
 

tests/test_sheerka_non_reg.py

@@ -240,13 +240,17 @@ def test_i_can_eval_concept_with_variable():
 
 def test_i_can_eval_concept_with_variable_and_python_as_body():
     sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a"))
+    hello_a = sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a"))
     sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
 
     res = sheerka.evaluate_user_input("hello foo")
     assert len(res) == 1
     assert res[0].status
-    assert res[0].value, "hello foo"
+    assert sheerka.isinstance(res[0].value, hello_a)
+    assert res[0].value.body == "hello foo"
+    assert res[0].value.metadata.is_evaluated
+    assert res[0].value.props["a"].value == Concept(name="foo", body="foo").init_key()
+    assert res[0].value.props["a"].value.metadata.is_evaluated
 
 
 def test_i_can_eval_duplicate_concepts_with_same_value():
@@ -259,7 +263,7 @@ def test_i_can_eval_duplicate_concepts_with_same_value():
     res = sheerka.evaluate_user_input("hello foo")
     assert len(res) == 1
     assert res[0].status
-    assert res[0].value, "hello foo"
+    assert res[0].value.body == "hello foo"
     assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
 
 
@@ -362,7 +366,11 @@ def test_i_can_eval_bnf_definitions_with_variables():
     return_value = res[0].value
 
     assert sheerka.isinstance(return_value, concept_b)
-    assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one"))
+    assert return_value.body == "one three"
+    assert return_value.metadata.is_evaluated
+
+    assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one").init_key())
+    assert return_value.props["a"].value.metadata.is_evaluated
 
 
 def test_i_can_eval_bnf_definitions_from_separate_instances():
@@ -390,6 +398,8 @@ def test_i_can_eval_bnf_definitions_from_separate_instances():
     assert len(res) == 1
     assert res[0].status
     assert sheerka.isinstance(res[0].value, concept_b)
+    assert res[0].value.body == "one two three"
+    assert res[0].value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key())
 
 
 def test_i_can_say_that_a_concept_isa_another_concept():
@@ -398,6 +408,7 @@ def test_i_can_say_that_a_concept_isa_another_concept():
     sheerka.evaluate_user_input("def concept bar")
 
     res = sheerka.evaluate_user_input("foo isa bar")
+    assert len(res) == 1
     assert res[0].status
     assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
 
@@ -436,3 +447,40 @@ def test_i_can_manage_tokenizer_error(text):
     assert len(res) > 1
     for r in [r for r in res if r.who.startswith("parsers.")]:
         assert not r.status
+
+
+def test_i_can_recognize_concept_from_string():
+    sheerka = get_sheerka()
+    sheerka.add_in_cache(Concept("one", body="1"))
+
+    res = sheerka.evaluate_user_input("'one'")
+
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == "one"
+
+    res = sheerka.evaluate_user_input("eval 'one'")
+
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == "one"
+
+
+@pytest.mark.parametrize("expression", [
+    "def concept twenties from bnf 'twenty' (one | two)=unit as 20 + unit",
+    "def concept twenties from bnf 'twenty' (one | two)=unit as twenty + unit",
+    "def concept twenties from bnf twenty (one | two)=unit as 20 + unit",
+    "def concept twenties from bnf twenty (one | two)=unit as twenty + unit",
+])
+def test_i_can_evaluate_bnf_concepts(expression):
+    sheerka = get_sheerka()
+
+    sheerka.evaluate_user_input("def concept one as 1")
+    sheerka.evaluate_user_input("def concept two as 2")
+    sheerka.evaluate_user_input("def concept twenty as 20")
+    sheerka.evaluate_user_input(expression)
+    res = sheerka.evaluate_user_input("eval twenty one")
+
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == 21

tests/test_utils.py

@@ -125,6 +125,11 @@ def test_i_can_strip_eof():
     assert actual == expected
 
 
+def test_i_can_escape():
+    actual = core.utils.escape_char("hello 'world' my friend", "'")
+    assert actual == "hello \\'world\\' my friend"
+
+
 def get_tokens(lst):
     res = []
     for e in lst: