Refactored to use a single implementation for concept evaluation

core/builtin_concepts.py

@@ -41,10 +41,11 @@ class BuiltinConcepts(Enum):
     INVALID_RETURN_VALUE = "invalid return value"  # the return value of an evaluator is not correct
     CONCEPT_ALREADY_DEFINED = "concept already defined"  # when you try to add the same concept twice
     NOP = "no operation"  # no operation concept. Does nothing
-    PROPERTY_EVAL_ERROR = "property evaluation error"  # cannot evaluate a property of a concept
+    CONCEPT_EVAL_ERROR = "concept evaluation error"  # cannot evaluate a property or metadata of a concept
     ENUMERATION = "enum"  # represents a list or a set
     LIST = "list"  # represents a list
     CANNOT_RESOLVE_VALUE_ERROR = "value cannot be resolved"  # don't know how to find concept value
+    CONCEPT_NOT_INITIALIZED = "concept not evaluated"  # Try to work on a concept that is not evaluated
 
     NODE = "node"
     GENERIC_NODE = "generic node"
@@ -56,6 +57,21 @@ class BuiltinConcepts(Enum):
     def __str__(self):
         return "__" + self.name
 
+
+BuiltinErrors = [str(e) for e in {
+    BuiltinConcepts.ERROR,
+    BuiltinConcepts.UNKNOWN_CONCEPT,
+    BuiltinConcepts.CONCEPT_TOO_LONG,
+    BuiltinConcepts.UNKNOWN_PROPERTY,
+    BuiltinConcepts.TOO_MANY_SUCCESS,
+    BuiltinConcepts.TOO_MANY_ERRORS,
+    BuiltinConcepts.INVALID_RETURN_VALUE,
+    BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
+    BuiltinConcepts.CONCEPT_EVAL_ERROR,
+    BuiltinConcepts.CANNOT_RESOLVE_VALUE_ERROR,
+    BuiltinConcepts.CONCEPT_NOT_INITIALIZED
+}]
+
 """
 Some concepts have a specific implementation
 It's mainly to a have proper __repr__ implementation, or because they are singleton (is_unique=True)
@@ -251,31 +267,52 @@ class AfterEvaluationConcept(Concept):
         super().__init__(BuiltinConcepts.AFTER_EVALUATION, True, True, BuiltinConcepts.AFTER_EVALUATION)
 
 
-class PropertyEvalError(Concept):
+class ConceptEvalError(Concept):
-    def __init__(self, property_name=None, concept=None, error=None):
+    def __init__(self, error=None, concept=None, property_name=None):
-        super().__init__(BuiltinConcepts.PROPERTY_EVAL_ERROR,
+        super().__init__(BuiltinConcepts.CONCEPT_EVAL_ERROR,
                          True,
                          False,
-                         BuiltinConcepts.PROPERTY_EVAL_ERROR,
+                         BuiltinConcepts.CONCEPT_EVAL_ERROR,
-                         property_name)
+                         error)
         self.set_prop("concept", concept)
-        self.set_prop("error", error)
+        self.set_prop("property_name", property_name)
 
     def __repr__(self):
-        return f"PropertyEvalError(property={self.property_name}, concept={self.concept}), error={self.error})"
+        return f"ConceptEvalError(error={self.error}, concept={self.concept}, property={self.property_name})"
+
+    @property
+    def error(self):
+        return self.body
 
     @property
     def concept(self):
         return self.props["concept"].value
 
     @property
-    def error(self):
+    def property_name(self):
-        return self.props["error"].value
+        return self.props["property_name"].value
+
+
+class ConceptNotInitialized(Concept):
+    def __init__(self, error=None, concept=None, property_name=None):
+        super().__init__(BuiltinConcepts.CONCEPT_NOT_INITIALIZED,
+                         True,
+                         False,
+                         BuiltinConcepts.CONCEPT_NOT_INITIALIZED,
+                         error)
+        self.set_prop("concept", concept)
+
+    def __repr__(self):
+        return f"ConceptNotInitialized(error={self.error}, concept={self.concept})"
 
     @property
-    def property_name(self):
+    def error(self):
         return self.body
 
+    @property
+    def concept(self):
+        return self.props["concept"].value
+
 
 class EnumerationConcept(Concept):
     def __init__(self, iteration=None):

core/builtin_helpers.py

@@ -1,4 +1,6 @@
 import ast
+import logging
+
 import core.ast.nodes
 from core.ast.nodes import CallNodeConcept, GenericNodeConcept
 from core.ast.visitors import UnreferencedNamesVisitor
@@ -31,12 +33,13 @@ def is_same_success(sheerka, return_values):
     return True
 
 
-def expect_one(context, return_values):
+def expect_one(context, return_values, logger=None):
     """
     Checks if there is at least one success return value
     If there is more than one, check if it's the same value
     :param context:
     :param return_values:
+    :param logger:
     :return:
     """
 
@@ -58,7 +61,6 @@ def expect_one(context, return_values):
 
     # remove errors when a winner is found
     if number_of_successful == 1:
-        # log.debug(f"1 / {total_items} good item found.")
         return sheerka.ret(
             context.who,
             True,
@@ -74,6 +76,10 @@ def expect_one(context, return_values):
                 successful_results[0].value,
                 parents=return_values)
         else:
+            if logger and logger.isEnabledFor(logging.DEBUG):
+                context.log(logger, f"Too many successful results found by expect_one()", context.who)
+                for s in successful_results:
+                    context.log(logger, f"-> {s}", context.who)
             return sheerka.ret(
                 context.who,
                 False,
@@ -81,11 +87,23 @@ def expect_one(context, return_values):
                 parents=return_values)
 
     # only errors, i cannot help you
-    return sheerka.ret(
+    if logger and logger.isEnabledFor(logging.DEBUG):
-        context.who,
+        context.log(logger, f"Too many errors found by expect_one()", context.who)
-        False,
+        for s in successful_results:
-        sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=return_values),
+            context.log(logger, f"-> {s}", context.who)
-        parents=return_values)
+
+    if len(return_values) == 1:
+        return sheerka.ret(
+            context.who,
+            False,
+            return_values[0],
+            parents=return_values)
+    else:
+        return sheerka.ret(
+            context.who,
+            False,
+            sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=return_values),
+            parents=return_values)
 
 
 def get_names(sheerka, concept_node):

core/concept.py

@@ -6,7 +6,6 @@ from core.sheerka_logger import get_logger
 import core.utils
 from core.tokenizer import Tokenizer, TokenKind
 
-
 PROPERTIES_FOR_DIGEST = ("name", "key",
                          "definition", "definition_type",
                          "is_builtin", "is_unique",
@@ -44,6 +43,7 @@ class ConceptMetadata:
     definition_type: str  # definition can be done with something else than regex
     desc: str  # possible description for the concept
     id: str  # unique identifier for a concept. The id will never be modified (but the key can)
+    is_evaluated: bool = False  # True is the concept is evaluated by sheerka.eval_concept()
 
 
 class Concept:

core/sheerka.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 
-from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept
+from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_DIGEST
 from parsers.BaseParser import BaseParser
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderDuplicateKeyError
@@ -11,7 +11,10 @@ from core.sheerka_logger import console_handler, get_logger
 
 import logging
 
-concept_evaluation_steps = [BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION]
+CONCEPT_EVALUATION_STEPS = [
+    BuiltinConcepts.BEFORE_EVALUATION,
+    BuiltinConcepts.EVALUATION,
+    BuiltinConcepts.AFTER_EVALUATION]
 CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
 DEBUG_TAB_SIZE = 4
 
@@ -416,17 +419,21 @@ class Sheerka(Concept):
             source = getattr(concept.metadata, part_key.value)
             if source is None or not isinstance(source, str) or source == "":
                 # the only sources that I am sure to parse are strings
-                # I refuse empty strings for performance, I don't want to handle useless NOPConcepts
+                # I refuse empty strings for performance matters, I don't want to handle useless NOPConcepts
                 continue
             else:
                 to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=source))
                 concept.cached_asts[part_key] = self.execute(context, to_parse, steps, logger)
 
         for prop in concept.props:
-            to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=concept.props[prop].value))
+            if concept.props[prop].value:
-            concept.cached_asts[prop] = self.execute(context, to_parse, steps)
+                to_parse = self.ret(
+                    context.who,
+                    True,
+                    self.new(BuiltinConcepts.USER_INPUT, body=concept.props[prop].value))
+                concept.cached_asts[prop] = self.execute(context, to_parse, steps)
 
-        # updates the code of the reference when possible
+        # Updates the cache of concepts when possible
         if concept.key in self.concepts_cache:
             entry = self.concepts_cache[concept.key]
             if isinstance(entry, list):
@@ -435,32 +442,69 @@ class Sheerka(Concept):
             else:
                 self.concepts_cache[concept.key].cached_asts = concept.cached_asts
 
-    def eval_concept(self, context, concept: Concept, properties_to_eval=None, logger=None):
+    def evaluate_concept(self, context, concept: Concept, logger=None):
         """
         Evaluation a concept
         It means that if the where clause is True, will evaluate the body
         :param context:
         :param concept:
-        :param properties_to_eval:
         :param logger:
-        :return:
+        :return: value of the evaluation or error
         """
+
+        if concept.metadata.is_evaluated:
+            return concept
+
+        def _resolve(resolve_context, return_value):
+            r = self.execute(resolve_context, return_value, CONCEPT_EVALUATION_STEPS, logger or self.log)
+            return core.builtin_helpers.expect_one(context, r)
+
+        # WHERE condition should already be validated by the parser.
+        # It's a mandatory condition for the concept before it can be recognized
+
+        #
+        # TODO : Validate the PRE condition
+        #
+
         if len(concept.cached_asts) == 0:
             self.initialize_concept_asts(context, concept, logger)
 
-        if properties_to_eval is None:
+        # to make sure of the order, it don't use ConceptParts.get_parts()
-            properties_to_eval = ["where", "pre", "post", "body", "props"]
+        # props must be evaluated first
+        properties_to_eval = ["props", "where", "pre", "post", "body"]
 
-        for prop in properties_to_eval:
+        for prop_to_eval in properties_to_eval:
-            if prop == "props":
+            if prop_to_eval == "props":
-                pass
+                for prop_name in (p for p in concept.props if p in concept.cached_asts):
+                    sub_context = context.push(desc=f"Evaluating property '{prop_name}'")
+                    res = _resolve(sub_context, concept.cached_asts[prop_name])
+                    if res.status:
+                        concept.set_prop(prop_name, res.value)
+                    else:
+                        return self.new(BuiltinConcepts.CONCEPT_EVAL_ERROR,
+                                        body=res.value,
+                                        concept=concept,
+                                        property_name=prop_name)
             else:
-                part_key = ConceptParts(prop)
+                part_key = ConceptParts(prop_to_eval)
-                if concept.cached_asts[part_key] is None:
+                if part_key in concept.cached_asts and concept.cached_asts[part_key] is not None:
-                    continue
+                    sub_context = context.push(desc=f"Evaluating '{part_key}'", obj=concept)
-                res = self.execute(context, concept.cached_asts[part_key], concept_evaluation_steps, logger)
+                    res = _resolve(sub_context, concept.cached_asts[part_key])
-                res = core.builtin_helpers.expect_one(context, res)
+                    if res.status:
-                setattr(concept.metadata, prop, res.value)
+                        setattr(concept.metadata, prop_to_eval, res.value)
+                    else:
+                        return self.new(BuiltinConcepts.CONCEPT_EVAL_ERROR,
+                                        body=res.value,
+                                        concept=concept,
+                                        property_name=part_key)
+
+        #
+        # TODO : Validate the POST condition
+        #
+
+        concept.init_key()  # only does it if needed
+        concept.metadata.is_evaluated = True
+        return concept
 
     def add_in_cache(self, concept: Concept):
         """
@@ -601,16 +645,16 @@ class Sheerka(Concept):
         return (self.value(obj) for obj in objs)
 
     def is_success(self, obj):
-        if isinstance(obj, bool):
+        if isinstance(obj, bool):  # quick win
             return obj
 
-        if self.isinstance(obj, BuiltinConcepts.RETURN_VALUE):
+        if isinstance(obj, ReturnValueConcept):
             return obj.status
 
-        if self.isinstance(obj, BuiltinConcepts.ERROR):
+        if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
             return False
 
-        return False
+        return obj
 
     def isinstance(self, a, b):
         """
@@ -724,10 +768,6 @@ class Sheerka(Concept):
             log_format = "%(message)s"
             log_level = logging.INFO
 
-        # logging.root.setLevel(log_level)
-        # fmt = logging.Formatter(log_format, None, "%")
-        # console_handler.setFormatter(fmt)
-
         logging.basicConfig(format=log_format, level=log_level, handlers=[console_handler])
 
 

core/tokenizer.py

@@ -195,6 +195,14 @@ class Tokenizer:
                 yield Token(TokenKind.AMPER, "&", self.i, self.line, self.column)
                 self.i += 1
                 self.column += 1
+            elif c == "<":
+                yield Token(TokenKind.LESS, "<", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == ">":
+                yield Token(TokenKind.GREATER, ">", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
             elif c == "\n" or c == "\r":
                 newline = self.eat_newline(self.i)
                 yield Token(TokenKind.NEWLINE, newline, self.i, self.line, self.column)

evaluators/AddConceptEvaluator.py

@@ -3,6 +3,7 @@ from core.builtin_concepts import ParserResultConcept, ReturnValueConcept
 from core.builtin_helpers import get_names
 from core.concept import Concept
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
+from parsers.BaseParser import NotInitializedNode
 from parsers.ConceptLexerParser import ParsingExpression, ParsingExpressionVisitor
 from parsers.DefaultParser import DefConceptNode
 
@@ -58,7 +59,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
             # put back the sources
             part_ret_val = getattr(def_concept_node, prop)
             if not isinstance(part_ret_val, ReturnValueConcept) or not part_ret_val.status:
-                continue  # not quite sure that it's possible
+                continue  # Nothing to do is not initialized
 
             # update the parts
             source = self.get_source(part_ret_val)
@@ -82,7 +83,8 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         # finish initialisation
         concept.init_key(def_concept_node.name.tokens)
         concept.add_codes(def_concept_node.get_asts())
-        if sheerka.is_success(def_concept_node.definition):
+        if not isinstance(def_concept_node.definition, NotInitializedNode) and \
+            sheerka.is_success(def_concept_node.definition):
             concept.bnf = def_concept_node.definition.value.value
 
         ret = sheerka.create_new_concept(context, concept)

evaluators/ConceptEvaluator.py

@@ -1,5 +1,4 @@
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
-import core.builtin_helpers
 from core.concept import Concept, ConceptParts
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 
@@ -31,42 +30,25 @@ class ConceptEvaluator(OneReturnValueEvaluator):
         concept = return_value.value.value
         context.log(self.verbose_log, f"Evaluating concept {concept}.", self.name)
 
-        # pre condition should already be validated by the parser.
+        # If the concept that is requested is in the context(at least its name), drop the call.
-        # It's a mandatory condition for the concept before it can be recognized
+        # Why ?
+        # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
+        # The body should be 'property_a', and not a concept called a in our universe
+        if context.obj and concept.name in context.obj.props:
+            return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME), parents=[return_value])
 
-        if len(concept.cached_asts) == 0:
+        evaluated = sheerka.evaluate_concept(context, concept, self.verbose_log)
-            sheerka.initialize_concept_asts(context, concept, self.verbose_log)
 
-        # TODO; check pre
+        if evaluated.key != concept.key:
-        # if pre is not true, return Concept with a false value
+            # evaluated.key != concept.key means that we have transformed the concept
+            # When you successfully evaluate an error, the status should not be false
+            return sheerka.ret(
+                self.name,
+                False,
+                evaluated,
+                parents=[return_value])
 
-        # Evaluate the properties
+        if ConceptParts.BODY not in evaluated.cached_asts:
-        for prop in concept.props:
+            return sheerka.ret(self.name, True, evaluated, parents=[return_value])
-            sub_context = context.push(self.name, desc=f"Evaluating property '{prop}'", obj=concept)
+        else:
-            res = self.evaluate_parsing(sheerka, sub_context, concept.cached_asts[prop])
+            return sheerka.ret(self.name, True, evaluated.body, parents=[return_value])
-            if res.status:
-                concept.set_prop(prop, res.value)
-            else:
-                return sheerka.ret(
-                    self.name,
-                    False,
-                    sheerka.new(BuiltinConcepts.PROPERTY_EVAL_ERROR, body=prop, concept=concept, error=res.value),
-                    parents=[return_value])
-
-        # Returns the concept when no body
-        if ConceptParts.BODY not in concept.cached_asts:
-            return sheerka.ret(self.name, True, concept, parents=[return_value])
-
-        # Evaluate the body otherwise
-        body = concept.cached_asts[ConceptParts.BODY]
-        if body is None:
-            raise NotImplementedError("Seems weird !")
-
-        sub_context = context.push(self.name, desc="Evaluating body", obj=concept)
-        res = self.evaluate_parsing(sheerka, sub_context, body)
-        return sheerka.ret(self.name, res.status, res.value, parents=[return_value])
-
-    def evaluate_parsing(self, sheerka, context, parsing_result):
-        res = sheerka.execute(context, parsing_result, self.evaluation_steps, self.log)
-        res = core.builtin_helpers.expect_one(context, res)
-        return res

evaluators/ConceptNodeEvaluator.py

@@ -54,7 +54,8 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
 
     def update_concept(self, sheerka, concept, underlying, init_empty_body=True):
         """
-        Updates the property of the concept
+        Updates the properties of the concept
+        Goes in recursion if the property is a concept
         """
 
         def _add_prop(c, prop_name, value):

evaluators/PythonEvaluator.py

@@ -28,6 +28,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
         node = return_value.value.value
         try:
             context.log(self.verbose_log, f"Evaluating python node {node}.", self.name)
+
             my_locals = self.get_locals(context, node.ast_)
             context.log(self.verbose_log, f"locals={my_locals}", self.name)
 
@@ -51,6 +52,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
         if context.obj:
             context.log(self.verbose_log,
                         f"Concept '{context.obj}' is in context. Adding its properties to locals if any.", self.name)
+
             for prop_name, prop_value in context.obj.props.items():
                 my_locals[prop_name] = prop_value.value
 
@@ -60,18 +62,23 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
         for name in unreferenced_names_visitor.names:
             context.log(self.verbose_log, f"Resolving '{name}'.", self.name)
+
+            if name in my_locals:
+                context.log(self.verbose_log, f"Using value from property.", self.name)
+                continue
+
             concept = context.sheerka.new(name)
             if context.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
                 context.log(self.verbose_log, f"'{name}' is not a concept. Skipping.", self.name)
                 continue
 
-            context.log(self.verbose_log, f"'{name}' is a concept. Evaluating body.", self.name)
+            context.log(self.verbose_log, f"'{name}' is a concept. Evaluating.", self.name)
-            sub_context = context.push(self.name, desc=f"Evaluating {concept}'s body", obj=concept)
+            sub_context = context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept)
             sub_context.log_new(self.verbose_log)
-            context.sheerka.eval_concept(sub_context, concept, ["body"], self.verbose_log)
+            evaluated = context.sheerka.evaluate_concept(sub_context, concept, self.verbose_log)
 
-            if not context.sheerka.isa(concept.body, BuiltinConcepts.ERROR):
+            if evaluated.key == concept.key:
-                my_locals[name] = concept.body
+                my_locals[name] = evaluated.body or evaluated
 
         return my_locals
 

parsers/DefaultParser.py

@@ -371,7 +371,7 @@ class DefaultParser(BaseParser):
                 self.sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens))
             steps = [BuiltinConcepts.PARSING]
             parsed = self.sheerka.execute(new_context, to_parse, steps, self.verbose_log)
-            parsing_result = core.builtin_helpers.expect_one(new_context, parsed)
+            parsing_result = core.builtin_helpers.expect_one(new_context, parsed, self.verbose_log)
             if not parsing_result.status:
                 self.add_error(parsing_result.value)
                 continue
@@ -379,75 +379,3 @@ class DefaultParser(BaseParser):
             asts_found_by_parts[keyword] = parsing_result
 
         return asts_found_by_parts
-
-    # def parse_expression(self):
-    #     return self.parse_addition()
-    #
-    # def parse_addition(self):
-    #     left = self.parse_multiply()
-    #     token = self.get_token()
-    #     if token is None or token.type == TokenKind.EOF:
-    #         return left
-    #
-    #     if token.type == TokenKind.NUMBER:  # example 15 +5 or 15 -5
-    #         right = self.parse_addition()
-    #         return BinaryNode(self.collect_tokens(left, token, right), TokenKind.PLUS, left, right)
-    #
-    #     if token.type not in (TokenKind.PLUS, TokenKind.MINUS):
-    #         return left
-    #
-    #     self.next_token()
-    #     right = self.parse_addition()
-    #     return BinaryNode(self.collect_tokens(left, token, right), token.type, left, right)
-    #
-    # def parse_multiply(self):
-    #     left = self.parse_atom()
-    #     token = self.get_token()
-    #     if token is None or token.type == TokenKind.EOF:
-    #         return left
-    #
-    #     if token.type not in (TokenKind.STAR, TokenKind.SLASH):
-    #         return left
-    #
-    #     self.next_token()
-    #     right = self.parse_multiply()
-    #     return BinaryNode(self.collect_tokens(left, token, right), token.type, left, right)
-    #
-    # def parse_atom(self):
-    #     token = self.get_token()
-    #     if token.type == TokenKind.NUMBER:
-    #         self.next_token()
-    #         return NumberNode([token], float(token.value) if '.' in token.value else int(token.value))
-    #     elif token.type == TokenKind.STRING:
-    #         self.next_token()
-    #         return StringNode([token], token.value[1:-1], token.value[0])
-    #     elif token.type == TokenKind.IDENTIFIER:
-    #         if token.value == "true":
-    #             self.next_token()
-    #             return TrueNode([token])
-    #         elif token.value == "false":
-    #             self.next_token()
-    #             return FalseNode([token])
-    #         elif token.value == "null":
-    #             self.next_token()
-    #             return NullNode([token])
-    #         else:
-    #             self.next_token()
-    #             return VariableNode([token], token.value)
-    #     elif token.type == TokenKind.LPAR:
-    #         self.next_token()
-    #         exp = self.parse_expression()
-    #         token = self.get_token()
-    #         self.next_token()
-    #
-    #         if token.type != TokenKind.RPAR:
-    #             error = UnexpectedTokenErrorNode([token], "Right parenthesis not found.", [TokenKind.RPAR])
-    #             self.add_error(error)
-    #             return error
-    #
-    #         return exp
-    #     else:
-    #         error = UnexpectedTokenErrorNode([token], "Unexpected token",
-    #                                          [TokenKind.NUMBER, TokenKind.STRING, TokenKind.IDENTIFIER, "true", "false",
-    #                                           "null", TokenKind.LPAR])
-    #         return self.add_error(error)

tests/test_ConceptEvaluator.py

@@ -35,9 +35,12 @@ def test_i_can_match(ret_val, expected):
     assert ConceptEvaluator().matches(context, ret_val) == expected
 
 
-def test_concept_is_returned_when_no_body():
+def test_i_can_evaluate_concept():
     context = get_context()
-    concept = Concept(name="one").init_key()
+    concept = Concept(name="foo",
+                      where="1",
+                      pre="2",
+                      post="3").set_prop("a", "4").set_prop("b", "5")
 
     evaluator = ConceptEvaluator()
     item = get_return_value(concept)
@@ -45,13 +48,20 @@ def test_concept_is_returned_when_no_body():
 
     assert result.who == evaluator.name
     assert result.status
-    assert result.value == concept
+    assert result.value == Concept(name="foo",
+                                   where=1,
+                                   pre=2,
+                                   post=3).set_prop("a", 4).set_prop("b", 5).init_key()
     assert result.parents == [item]
 
 
-def test_body_is_evaluated_when_python_body():
+def test_body_is_returned_when_defined():
     context = get_context()
-    concept = Concept(name="one", body="1").init_key()
+    concept = Concept(name="foo",
+                      body="'I have a value'",
+                      where="1",
+                      pre="2",
+                      post="3").set_prop("a", "4").set_prop("b", "5")
 
     evaluator = ConceptEvaluator()
     item = get_return_value(concept)
@@ -59,136 +69,23 @@ def test_body_is_evaluated_when_python_body():
 
     assert result.who == evaluator.name
     assert result.status
-    assert result.value == 1
+    assert result.value == "I have a value"
     assert result.parents == [item]
 
 
-def test_body_is_evaluated_when_concept_body():
+def test_i_cannot_eval_if_with_the_same_name_is_defined_in_the_context():
+    # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
+    # ConceptEvaluator will be called to resolve 'a' while we know that 'a' refers to the string 'property_a'
+
     context = get_context()
-    concept_one = Concept(name="one").init_key()
+    context.obj = Concept("other").set_prop("foo", "'some_other_value'")
-    context.sheerka.add_in_cache(concept_one)
+    concept = Concept(name="foo")
-    concept_un = Concept(name="un", body="one").init_key()
 
-    evaluator = ConceptEvaluator()
+    item = get_return_value(concept)
-    item = get_return_value(concept_un)
+    result = ConceptEvaluator().eval(context, item)
-    result = evaluator.eval(context, item)
 
-    assert result.who == evaluator.name
+    assert not result.status
-    assert result.status
+    assert context.sheerka.isinstance(result.value, BuiltinConcepts.NOT_FOR_ME)
-    assert result.value == concept_one
-    assert result.parents == [item]
-
-
-def test_body_is_evaluated_when_concept_body_with_a_body():
-    """
-    The concept refers to another concept which has a body
-    :return:
-    """
-    context = get_context()
-    concept_one = Concept(name="one", body="1").init_key()
-    context.sheerka.add_in_cache(concept_one)
-    concept_un = Concept(name="un", body="one").init_key()
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept_un)
-    result = evaluator.eval(context, item)
-
-    assert result.who == evaluator.name
-    assert result.status
-    assert result.value == 1
-    assert result.parents == [item]
-
-
-def test_i_can_evaluate_longer_chains():
-    context = get_context()
-    context.sheerka.add_in_cache(Concept(name="a", body="'a'").init_key())
-    context.sheerka.add_in_cache(Concept(name="b", body="a").init_key())
-    context.sheerka.add_in_cache(Concept(name="c", body="b").init_key())
-    concept_d = context.sheerka.add_in_cache(Concept(name="d", body="c").init_key())
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept_d)
-    result = evaluator.eval(context, item)
-
-    assert result.status
-    assert result.value == 'a'
-
-
-def test_i_can_evaluate_longer_chains_2():
-    context = get_context()
-    concept_a = context.sheerka.add_in_cache(Concept(name="a").init_key())
-    context.sheerka.add_in_cache(Concept(name="b", body="a").init_key())
-    context.sheerka.add_in_cache(Concept(name="c", body="b").init_key())
-    concept_d = context.sheerka.add_in_cache(Concept(name="d", body="c").init_key())
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept_d)
-    result = evaluator.eval(context, item)
-
-    assert result.status
-    assert result.value == concept_a
-
-
-def test_i_can_recognize_concept_properties():
-    """
-    The concept 'plus' has some properties.
-    Let's check if they are recognized as concepts
-    :return:
-    """
-    context = get_context()
-    concept_one = context.sheerka.add_in_cache(Concept(name="one").init_key())
-    concept_two = context.sheerka.add_in_cache(Concept(name="two").init_key())
-    concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b")
-                                                .set_prop("a", "one")
-                                                .set_prop("b", "two").init_key())
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept_plus)
-    result = evaluator.eval(context, item)
-
-    assert result.status
-    assert context.sheerka.isinstance(result.value, concept_plus)
-    assert result.value.props["a"].value == concept_one
-    assert result.value.props["b"].value == concept_two
-
-
-def test_i_can_recognize_concept_properties_with_body():
-    """
-    The concept 'plus' has some properties.
-    Let's check if they are recognized as concepts with Python code
-    :return:
-    """
-    context = get_context()
-    context.sheerka.add_in_cache(Concept(name="one", body="1").init_key())
-    context.sheerka.add_in_cache(Concept(name="two", body="2").init_key())
-    concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b")
-                                                .set_prop("a", "one")
-                                                .set_prop("b", "two").init_key())
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept_plus)
-    result = evaluator.eval(context, item)
-
-    assert result.status
-    assert context.sheerka.isinstance(result.value, concept_plus)
-    assert result.value.props["a"].value == 1
-    assert result.value.props["b"].value == 2
-
-
-def test_i_can_recognize_concept_properties_with_body_when_concept_has_a_body():
-    context = get_context()
-    context.sheerka.add_in_cache(Concept(name="one", body="1").init_key())
-    context.sheerka.add_in_cache(Concept(name="two", body="2").init_key())
-    concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b", body="a + b")
-                                                .set_prop("a", "one")
-                                                .set_prop("b", "two").init_key())
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept_plus)
-    result = evaluator.eval(context, item)
-
-    assert result.status
-    assert result.value == 3
 
 
 def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown():
@@ -203,7 +100,19 @@ def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown():
     result = evaluator.eval(context, item)
 
     assert not result.status
-    assert context.sheerka.isinstance(result.value, BuiltinConcepts.PROPERTY_EVAL_ERROR)
+    assert context.sheerka.isinstance(result.value, BuiltinConcepts.CONCEPT_EVAL_ERROR)
     assert result.value.property_name == "b"
     assert context.sheerka.isinstance(result.value.error, BuiltinConcepts.TOO_MANY_ERRORS)
     assert result.value.concept == concept_plus
+
+
+def test_that_error_concepts_are_transformed_into_errors_only_if_the_key_is_different():
+    context = get_context()
+
+    error_concept = context.sheerka.new(BuiltinConcepts.ERROR)
+    item = get_return_value(error_concept)
+    result = ConceptEvaluator().eval(context, item)
+
+    assert not context.sheerka.is_success(error_concept)  # it's indeed an error
+    assert result.status
+    assert result.value == error_concept

tests/test_PythonEvaluator.py

@@ -1,6 +1,6 @@
 import pytest
 
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.sheerka import Sheerka, ExecutionContext
 from core.concept import Concept
 from evaluators.PythonEvaluator import PythonEvaluator
@@ -39,31 +39,79 @@ def test_i_can_eval(text, expected):
     assert evaluated.value == expected
 
 
-def test_i_can_eval_expression_with_variables():
+def test_i_can_eval_expression_that_references_concepts():
+    context = get_context()
+    context.sheerka.add_in_cache(Concept("foo"))
+
+    parsed = PythonParser().parse(context, "foo")
+    evaluated = PythonEvaluator().eval(context, parsed)
+
+    assert evaluated.status
+    assert evaluated.value == Concept("foo").init_key()
+
+
+def test_i_can_eval_expression_that_references_concepts_with_body():
     """
     I can test expression with variables
     :return:
     """
     context = get_context()
     context.sheerka.add_in_cache(Concept("foo", body="2"))
-    parsed = PythonParser().parse(context, "foo + 2")
 
+    parsed = PythonParser().parse(context, "foo")
     evaluated = PythonEvaluator().eval(context, parsed)
 
     assert evaluated.status
-    assert evaluated.value == 4
+    assert evaluated.value == 2
 
 
-def test_i_can_eval_module_with_variables():
+def test_i_can_eval_module_with_that_references_concepts():
+    """
+    I can test modules with variables
+    :return:
+    """
+    context = get_context()
+    context.sheerka.add_in_cache(Concept("foo"))
+
+    parsed = PythonParser().parse(context, "def a(b):\n    return b\na(foo)")
+    evaluated = PythonEvaluator().eval(context, parsed)
+
+    assert evaluated.status
+    assert evaluated.value == Concept("foo").init_key()
+
+
+def test_i_can_eval_module_with_that_references_concepts_with_body():
     """
     I can test modules with variables
     :return:
     """
     context = get_context()
     context.sheerka.add_in_cache(Concept("foo", body="2"))
-    parsed = PythonParser().parse(context, "def a(b):\n    return b\na(foo)")
 
+    parsed = PythonParser().parse(context, "def a(b):\n    return b\na(foo)")
     evaluated = PythonEvaluator().eval(context, parsed)
 
     assert evaluated.status
     assert evaluated.value == 2
+
+
+def test_i_can_eval_concept_with_props():
+    context = get_context()
+    context.sheerka.add_in_cache(Concept("foo").set_prop("prop", "'a'"))
+
+    parsed = PythonParser().parse(context, "foo")
+    evaluated = PythonEvaluator().eval(context, parsed)
+
+    assert evaluated.status
+    assert evaluated.value == Concept("foo").set_prop("prop", "a").init_key()  # evaluated version of foo
+
+
+def test_i_cannot_eval_when_body_references_unknown_concept():
+    context = get_context()
+    context.sheerka.add_in_cache(Concept("foo", body="bar"))
+
+    parsed = PythonParser().parse(context, "foo")
+    evaluated = PythonEvaluator().eval(context, parsed)
+
+    assert not evaluated.status
+    assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.ERROR)

tests/test_builtin_helpers.py

@@ -46,12 +46,11 @@ def test_i_can_use_expect_when_only_errors_1():
     sheerka = get_sheerka()
 
     items = [
-        ReturnValueConcept("who", False, None),
+        ReturnValueConcept("who", False, sheerka.new(BuiltinConcepts.ERROR)),
     ]
     res = core.builtin_helpers.expect_one(get_context(sheerka), items)
     assert not res.status
-    assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
+    assert res.value, items[0]
-    assert res.value.body == items
     assert res.parents == items
 
 

tests/test_sheerka.py

@@ -30,6 +30,32 @@ def init_test():
     os.chdir(current_pwd)
 
 
+def get_sheerka(use_dict=True, skip_builtins_in_db=True):
+    root = "mem://" if use_dict else root_folder
+    sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db)
+    sheerka.initialize(root)
+
+    return sheerka
+
+
+def get_context(sheerka):
+    return ExecutionContext("test", "xxx", sheerka)
+
+
+def get_default_concept():
+    concept = Concept(
+        name="a + b",
+        where="isinstance(a, int) and isinstance(b, int)",
+        pre="isinstance(a, int) and isinstance(b, int)",
+        post="isinstance(res, int)",
+        body="def func(x,y):\n    return x+y\nfunc(a,b)",
+        desc="specific description")
+    concept.set_prop("a", "value1")
+    concept.set_prop("b", "value2")
+
+    return concept
+
+
 class ConceptWithGetValue(Concept):
     def get_value(self):
         return self.get_prop("my_prop")
@@ -260,7 +286,7 @@ def test_instances_are_different_when_asking_for_new():
 
 def test_i_get_the_same_instance_when_is_unique_is_true():
     sheerka = get_sheerka()
-    concept = get_unique_concept()
+    concept = Concept(name="unique", is_unique=True)
     sheerka.create_new_concept(get_context(sheerka), concept)
 
     new1 = sheerka.new(concept.key, a=10, b="value")
@@ -320,389 +346,250 @@ def test_list_of_concept_is_sorted_by_id():
     assert concepts[0].id < concepts[-1].id
 
 
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+@pytest.mark.parametrize("body, expected", [
-#
+    (None, None),
-#               E V A L U A T I O N S
+    ("", ""),
-#
+    ("1", 1),
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    ("1+1", 2),
-
+    ("'one'", "one"),
-@pytest.mark.parametrize("text, expected", [
+    ("'one' + 'two'", "onetwo"),
-    ("1 + 1", 2),
+    ("True", True),
-    ("sheerka.test()", 'I have access to Sheerka !')
+    ("1 > 2", False),
 ])
-def test_i_can_eval_python_expressions_with_no_variable(text, expected):
+def test_i_can_evaluate_a_concept_with_simple_body(body, expected):
     sheerka = get_sheerka()
 
-    res = sheerka.evaluate_user_input(text)
+    concept = Concept("foo", body=body).init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
 
-    assert len(res) == 1
+    assert evaluated.key == concept.key
-    assert res[0].status
+    assert evaluated.body == expected
-    assert res[0].value == expected
+    assert evaluated.metadata.pre is None
+    assert evaluated.metadata.post is None
+    assert evaluated.metadata.where is None
+    assert evaluated.props == {}
+    assert evaluated.metadata.is_evaluated
 
 
-def test_i_can_eval_concept_with_python_body():
+@pytest.mark.parametrize("expr, expected", [
-    sheerka = get_sheerka()
+    ("", ""),
-    concept = Concept(name="one", body="1")
+    ("1", 1),
-    sheerka.add_in_cache(concept)
+    ("1+1", 2),
-
+    ("'one'", "one"),
-    text = "one"
+    ("'one' + 'two'", "onetwo"),
-    res = sheerka.evaluate_user_input(text)
+    ("True", True),
-    assert len(res) == 1
+    ("1 > 2", False),
-    assert res[0].status
+])
-    assert res[0].value == 1
+def test_i_can_evaluate_the_other_metadata(expr, expected):
-
+    """
-
+    I only test WHERE, it's the same for the others
-def test_i_can_eval_concept_with_concept_body():
+    :param expr:
-    sheerka = get_sheerka()
+    :param expected:
-    concept_one = Concept(name="one")
+    :return:
-    concept_un = Concept(name="un", body="one")
-    sheerka.add_in_cache(concept_one)
-    sheerka.add_in_cache(concept_un)
-
-    res = sheerka.evaluate_user_input("un")
-    return_value = res[0].value
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(return_value, concept_one)
-
-
-def test_i_can_eval_concept_with_no_body():
-    sheerka = get_sheerka()
-    concept = Concept(name="one")
-    sheerka.add_in_cache(concept)
-
-    text = "one"
-    res = sheerka.evaluate_user_input(text)
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value == concept
-    assert id(res[0].value) != id(concept)
-
-
-def test_is_unique_property_is_used_when_evaluating():
-    sheerka = get_sheerka()
-    concept = Concept(name="one", is_unique=True)
-    sheerka.add_in_cache(concept)
-
-    text = "one"
-    res = sheerka.evaluate_user_input(text)
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value == concept
-    assert id(res[0].value) == id(concept)
-
-
-def test_i_can_eval_def_concept_request():
-    text = """
-def concept a + b
-where isinstance(a, int) and isinstance(b, int)
-pre isinstance(a, int) and isinstance(b, int)
-post isinstance(res, int)
-as:
-    def func(x,y):
-        return x+y
-    func(a,b)
     """
 
-    expected = get_default_concept()
-    expected.metadata.id = "1001"
-    expected.metadata.desc = None
-    expected.init_key()
-
     sheerka = get_sheerka()
-    res = sheerka.evaluate_user_input(text)
 
-    assert len(res) == 1
+    concept = Concept("foo", where=expr).init_key()
-    assert res[0].status
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
 
-    concept_saved = res[0].value.body
+    assert evaluated.key == concept.key
-
+    assert evaluated.body is None
-    for prop in PROPERTIES_TO_SERIALIZE:
+    assert evaluated.metadata.pre is None
-        assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
+    assert evaluated.metadata.post is None
-
+    assert evaluated.metadata.where == expected
-    assert concept_saved.key in sheerka.concepts_cache
+    assert evaluated.props == {}
-    assert sheerka.sdp.io.exists(
+    assert evaluated.metadata.is_evaluated
-        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
 
 
-def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept():
+@pytest.mark.parametrize("expr, expected", [
+    (None, None),
+    ("", ""),
+    ("1", 1),
+    ("1+1", 2),
+    ("'one'", "one"),
+    ("'one' + 'two'", "onetwo"),
+    ("True", True),
+    ("1 > 2", False),
+])
+def test_i_can_evaluate_a_concept_with_prop(expr, expected):
+    sheerka = get_sheerka()
+
+    concept = Concept("foo").set_prop("a", expr).init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
+
+    assert evaluated.key == concept.key
+    assert evaluated.body is None
+    assert evaluated.metadata.pre is None
+    assert evaluated.metadata.post is None
+    assert evaluated.metadata.where is None
+    assert evaluated.props == {"a": Property("a", expected)}
+    assert evaluated.metadata.is_evaluated
+
+
+def test_props_are_evaluated_before_body():
+    sheerka = get_sheerka()
+
+    concept = Concept("foo", body="a+1").set_prop("a", "10").init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
+
+    assert evaluated.key == concept.key
+    assert evaluated.body == 11
+
+
+def test_i_can_evaluate_when_another_concept_is_referenced():
+    sheerka = get_sheerka()
+    concept_a = Concept("a")
+    sheerka.add_in_cache(concept_a)
+
+    concept = Concept("foo", body="a").init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
+
+    assert evaluated.key == concept.key
+    assert sheerka.isinstance(evaluated.body, concept_a)
+    assert id(evaluated.body) != id(concept_a)
+    assert evaluated.metadata.is_evaluated
+
+
+def test_i_can_evaluate_when_the_referenced_concept_has_a_body():
+    sheerka = get_sheerka()
+    concept_a = Concept("a", body="1")
+    sheerka.add_in_cache(concept_a)
+
+    concept = Concept("foo", body="a").init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
+
+    assert evaluated.key == concept.key
+    assert evaluated.body == 1
+    assert not concept_a.metadata.is_evaluated  #
+
+
+def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body():
+    sheerka = get_sheerka()
+    sheerka.add_in_cache(Concept(name="a", body="'a'").init_key())
+    sheerka.add_in_cache(Concept(name="b", body="a").init_key())
+    sheerka.add_in_cache(Concept(name="c", body="b").init_key())
+    concept_d = sheerka.add_in_cache(Concept(name="d", body="c").init_key())
+
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d)
+
+    assert evaluated.key == concept_d.key
+    assert evaluated.body == 'a'
+
+
+def test_i_can_evaluate_concept_of_concept_does_not_have_a_body():
+    sheerka = get_sheerka()
+    concept_a = sheerka.add_in_cache(Concept(name="a").init_key())
+    sheerka.add_in_cache(Concept(name="b", body="a").init_key())
+    sheerka.add_in_cache(Concept(name="c", body="b").init_key())
+    concept_d = sheerka.add_in_cache(Concept(name="d", body="c").init_key())
+
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d)
+
+    assert evaluated.key == concept_d.key
+    assert sheerka.isinstance(evaluated.body, concept_a)
+
+
+def test_i_can_evaluate_concept_when_properties_reference_others_concepts():
+    sheerka = get_sheerka()
+    concept_a = sheerka.add_in_cache(Concept(name="a").init_key())
+
+    concept = Concept("foo", body="a").set_prop("a", "a").init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
+
+    assert evaluated.key == concept.key
+    assert evaluated.body == concept_a
+
+
+def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2():
     """
-    In this test, we test that the properties of 'concept a xx b' (which are 'a' and 'b')
+    Same test,
-    are correctly detected, thanks to the source code 'a plus b' in its body
+    but the name of the property and the name of the concept are different
     :return:
     """
     sheerka = get_sheerka()
+    concept_a = sheerka.add_in_cache(Concept(name="a").init_key())
 
-    # concept 'a plus b' is known
+    concept = Concept("foo", body="concept_a").set_prop("concept_a", "a").init_key()
-    concept_a_plus_b = Concept(name="a plus b").set_prop("a").set_prop("b")
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    sheerka.add_in_cache(concept_a_plus_b)
 
-    res = sheerka.evaluate_user_input("def concept a xx b as a plus b")
+    assert evaluated.key == concept.key
-    expected = Concept(name="a xx b", body="a plus b").set_prop("a").set_prop("b").init_key()
+    assert evaluated.body == concept_a
-    expected.metadata.id = "1001"
-
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
-
-    concept_saved = res[0].value.body
-
-    for prop in PROPERTIES_TO_SERIALIZE:
-        assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
-
-    assert concept_saved.key in sheerka.concepts_cache
-    assert sheerka.sdp.io.exists(
-        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
 
 
-def test_i_cannot_eval_the_same_def_concept_twice():
+def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body():
-    text = """
-def concept a + b
-where isinstance(a, int) and isinstance(b, int)
-pre isinstance(a, int) and isinstance(b, int)
-post isinstance(res, int)
-as:
-    def func(x,y):
-        return x+y
-    func(a,b)
-        """
-
     sheerka = get_sheerka()
-    sheerka.evaluate_user_input(text)
+    sheerka.add_in_cache(Concept(name="a", body="1").init_key())
-    res = sheerka.evaluate_user_input(text)
+    sheerka.add_in_cache(Concept(name="b", body="2").init_key())
 
-    assert len(res) == 1
+    concept = Concept("foo", body="propA + propB").set_prop("propA", "a").set_prop("propB", "b").init_key()
-    assert not res[0].status
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+
+    assert evaluated.key == concept.key
+    assert evaluated.body == 3
 
 
-def test_i_can_disable_an_evaluator():
+def test_i_can_reference_sheerka():
-    sheerka = get_sheerka()
-    concept = Concept(name="one", body="1")
-    sheerka.add_in_cache(concept)
-
-    text = "one"
-    p = next(e for e in sheerka.evaluators if e.__name__ == "PythonEvaluator")
-    p.enabled = False  # not that you disable the class, not the instance
-
-    res = sheerka.evaluate_user_input(text)
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
-
-    p.enabled = True  # put back for the remaining unit tests
-
-
-@pytest.mark.parametrize("text", [
-    "",
-    "   ",
-    "\n",
-])
-def test_i_can_eval_a_empty_input(text):
     sheerka = get_sheerka()
 
-    res = sheerka.evaluate_user_input(text)
+    concept = Concept("foo", body="sheerka.test()").init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
 
-    assert len(res) == 1
+    assert evaluated.key == concept.key
-    assert res[0].status
+    assert evaluated.body == sheerka.test()
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NOP)
 
 
-def test_i_can_eval_concept_with_variable():
+def test_properties_values_takes_precedence_over_the_outside_world():
     sheerka = get_sheerka()
-    concept_hello = Concept(name="hello a").set_prop("a")
+    sheerka.add_in_cache(Concept(name="a", body="'concept_a'").init_key())
-    concept_foo = Concept(name="foo")
+    sheerka.add_in_cache(Concept(name="b", body="'concept_b'").init_key())
-    sheerka.add_in_cache(concept_hello)
-    sheerka.add_in_cache(concept_foo)
 
-    res = sheerka.evaluate_user_input("hello foo")
+    concept = Concept("foo", body="a").init_key()
-    return_value = res[0].value
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert len(res) == 1
+    assert evaluated.key == concept.key
-    assert res[0].status
+    assert evaluated.body == 'concept_a'  # this test was already done
-    assert sheerka.isinstance(return_value, concept_hello)
+
-    assert return_value.props["a"].value == concept_foo
+    # so check this one.
+    concept = Concept("foo", body="a").set_prop("a", "'property_a'").init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
+    assert evaluated.key == concept.key
+    assert evaluated.body == 'property_a'
+
+    # or this one.
+    concept = Concept("foo", body="a").set_prop("a", "b").init_key()
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
+    assert evaluated.key == concept.key
+    assert evaluated.body == 'concept_b'
 
 
-def test_i_can_eval_concept_with_variable_and_python_as_body():
+def test_i_can_reference_sub_property_of_a_property():
     sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a"))
+    sheerka.add_in_cache(Concept(name="concept_a").set_prop("subProp", "'sub_a'").init_key())
-    sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
 
-    res = sheerka.evaluate_user_input("hello foo")
+    concept = Concept("foo", body="a.props['subProp'].value").set_prop("a", "concept_a").init_key()
-    assert len(res) == 1
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert res[0].status
+    assert evaluated.key == concept.key
-    assert res[0].value, "hello foo"
+    assert evaluated.body == 'sub_a'
 
 
-def test_i_can_eval_duplicate_concepts_with_same_value():
+def test_i_cannot_evaluate_concept_if_property_is_in_error():
     sheerka = get_sheerka()
 
-    sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a"))
+    concept = Concept(name="concept_a").set_prop("subProp", "undef_concept").init_key()
-    sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
+    assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
-
-    res = sheerka.evaluate_user_input("hello foo")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value, "hello foo"
-    assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
 
 
-def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different():
+def test_key_is_initialized_by_evaluation():
     sheerka = get_sheerka()
 
-    sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a"))
+    concept = Concept("foo")
-    sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
+    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    sheerka.add_in_cache(Concept(name="foo", body="'another value'"))
 
-    res = sheerka.evaluate_user_input("hello foo")
+    assert evaluated.key == concept.init_key().key
-    assert len(res) == 1
-    assert not res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS)
-
-    concepts = res[0].value.body
-    assert len(concepts) == 2
-    sorted_values = sorted(concepts, key=lambda x: x.value)
-    assert sorted_values[0].value == "hello another value"
-    assert sorted_values[1].value == "hello foo"
 
 
-def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same():
+def test_builtin_error_concept_are_errors():
+    # only test a random one, it will be the same for the others
     sheerka = get_sheerka()
-    context = get_context(sheerka)
+    assert not sheerka.is_success(sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS))
-
-    sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").set_prop("a"))
-    sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").set_prop("b"))
-
-    res = sheerka.evaluate_user_input("hello 'foo'")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value == "hello foo"
-    assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
-
-
-def test_i_can_create_concepts_with_python_code_as_body():
-    sheerka = get_sheerka()
-    context = get_context(sheerka)
-
-    sheerka.create_new_concept(context, Concept(name="concepts", body="sheerka.concepts()"))
-    res = sheerka.evaluate_user_input("concepts")
-
-    assert len(res) == 1
-    assert res[0].status
-    assert isinstance(res[0].value, list)
-
-
-def test_i_can_create_concept_with_bnf_definition():
-    sheerka = get_sheerka(False, False)
-    a = Concept("a")
-    sheerka.add_in_cache(a)
-    sheerka.concepts_definition_cache = {a: OrderedChoice("one", "two")}
-
-    res = sheerka.evaluate_user_input("def concept plus from bnf a ('plus' plus)?")
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
-
-    saved_concept = sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, "plus")
-    assert saved_concept.key == "plus"
-    assert saved_concept.metadata.definition == "a ('plus' plus)?"
-    assert "a" in saved_concept.props
-    assert "plus" in saved_concept.props
-
-    saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY)
-    expected_bnf = Sequence(
-        ConceptMatch("a", rule_name="a"),
-        Optional(Sequence(StrMatch("plus"), ConceptMatch("plus", rule_name="plus"))))
-    assert saved_definitions[saved_concept] == expected_bnf
-
-    new_concept = res[0].value.body
-    assert new_concept.metadata.name == "plus"
-    assert new_concept.metadata.definition == "a ('plus' plus)?"
-    assert new_concept.bnf == expected_bnf
-    assert "a" in new_concept.props
-    assert "plus" in new_concept.props
-
-
-def test_i_can_eval_bnf_definitions():
-    sheerka = get_sheerka()
-    concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
-
-    res = sheerka.evaluate_user_input("one")
-
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, concept_a)
-
-
-def test_i_can_eval_bnf_definitions_with_variables():
-    sheerka = get_sheerka()
-    concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
-    concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
-
-    res = sheerka.evaluate_user_input("one three")
-
-    assert len(res) == 1
-    assert res[0].status
-    return_value = res[0].value
-
-    assert sheerka.isinstance(return_value, concept_b)
-    assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one"))
-
-
-def test_i_can_eval_bnf_definitions_from_separate_instances():
-    """
-    Same test then before,
-    but make sure that the BNF are correctly persisted and loaded
-    """
-    sheerka = get_sheerka(False)
-    concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' 'two'")[0].body.body
-
-    res = get_sheerka(False).evaluate_user_input("one two")
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, concept_a)
-
-    # add another bnf definition
-    concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
-
-    res = get_sheerka(False).evaluate_user_input("one two")  # previous one still works
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, concept_a)
-
-    res = get_sheerka(False).evaluate_user_input("one two three")  # new one works
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, concept_b)
-
-
-def get_sheerka(use_dict=True, skip_builtins_in_db=True):
-    root = "mem://" if use_dict else root_folder
-    sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db)
-    sheerka.initialize(root)
-
-    return sheerka
-
-
-def get_context(sheerka):
-    return ExecutionContext("test", "xxx", sheerka)
-
-
-def get_default_concept():
-    concept = Concept(
-        name="a + b",
-        where="isinstance(a, int) and isinstance(b, int)",
-        pre="isinstance(a, int) and isinstance(b, int)",
-        post="isinstance(res, int)",
-        body="def func(x,y):\n    return x+y\nfunc(a,b)",
-        desc="specific description")
-    concept.set_prop("a", "value1")
-    concept.set_prop("b", "value2")
-
-    return concept
-
-
-def get_unique_concept():
-    return Concept(name="unique", is_unique=True)

tests/test_sheerka_non_reg.py

@@ -0,0 +1,408 @@
+import pytest
+import os
+from os import path
+import shutil
+
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, Property
+from core.sheerka import Sheerka, ExecutionContext
+from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
+from parsers.ConceptLexerParser import Sequence, ZeroOrMore, StrMatch, OrderedChoice, Optional, ConceptMatch, \
+    ConceptLexerParser
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+
+tests_root = path.abspath("../build/tests")
+root_folder = "init_folder"
+
+
+@pytest.fixture(autouse=True)
+def init_test():
+    if path.exists(tests_root):
+        shutil.rmtree(tests_root)
+
+    if not path.exists(tests_root):
+        os.makedirs(tests_root)
+    current_pwd = os.getcwd()
+    os.chdir(tests_root)
+
+    yield None
+
+    os.chdir(current_pwd)
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("1 + 1", 2),
+    ("sheerka.test()", 'I have access to Sheerka !')
+])
+def test_i_can_eval_python_expressions_with_no_variable(text, expected):
+    sheerka = get_sheerka()
+
+    res = sheerka.evaluate_user_input(text)
+
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].value == expected
+
+
+def test_i_can_eval_concept_with_python_body():
+    sheerka = get_sheerka()
+    concept = Concept(name="one", body="1")
+    sheerka.add_in_cache(concept)
+
+    text = "one"
+    res = sheerka.evaluate_user_input(text)
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].value == 1
+
+
+def test_i_can_eval_concept_with_concept_body():
+    sheerka = get_sheerka()
+    concept_one = Concept(name="one")
+    concept_un = Concept(name="un", body="one")
+    sheerka.add_in_cache(concept_one)
+    sheerka.add_in_cache(concept_un)
+
+    res = sheerka.evaluate_user_input("un")
+    return_value = res[0].value
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(return_value, concept_one)
+
+
+def test_i_can_eval_concept_with_no_body():
+    sheerka = get_sheerka()
+    concept = Concept(name="one")
+    sheerka.add_in_cache(concept)
+
+    text = "one"
+    res = sheerka.evaluate_user_input(text)
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].value == concept
+    assert id(res[0].value) != id(concept)
+
+
+def test_is_unique_property_is_used_when_evaluating():
+    sheerka = get_sheerka()
+    concept = Concept(name="one", is_unique=True)
+    sheerka.add_in_cache(concept)
+
+    text = "one"
+    res = sheerka.evaluate_user_input(text)
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].value == concept
+    assert id(res[0].value) == id(concept)
+
+
+def test_i_can_eval_def_concept_request():
+    text = """
+def concept a + b
+where isinstance(a, int) and isinstance(b, int)
+pre isinstance(a, int) and isinstance(b, int)
+post isinstance(res, int)
+as:
+    def func(x,y):
+        return x+y
+    func(a,b)
+    """
+
+    expected = get_default_concept()
+    expected.metadata.id = "1001"
+    expected.metadata.desc = None
+    expected.init_key()
+
+    sheerka = get_sheerka()
+    res = sheerka.evaluate_user_input(text)
+
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
+
+    concept_saved = res[0].value.body
+
+    for prop in PROPERTIES_TO_SERIALIZE:
+        assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
+
+    assert concept_saved.key in sheerka.concepts_cache
+    assert sheerka.sdp.io.exists(
+        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+
+
+def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept():
+    """
+    In this test, we test that the properties of 'concept a xx b' (which are 'a' and 'b')
+    are correctly detected, thanks to the source code 'a plus b' in its body
+    :return:
+    """
+    sheerka = get_sheerka()
+
+    # concept 'a plus b' is known
+    concept_a_plus_b = Concept(name="a plus b").set_prop("a").set_prop("b")
+    sheerka.add_in_cache(concept_a_plus_b)
+
+    res = sheerka.evaluate_user_input("def concept a xx b as a plus b")
+    expected = Concept(name="a xx b", body="a plus b").set_prop("a").set_prop("b").init_key()
+    expected.metadata.id = "1001"
+
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
+
+    concept_saved = res[0].value.body
+
+    for prop in PROPERTIES_TO_SERIALIZE:
+        assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
+
+    assert concept_saved.key in sheerka.concepts_cache
+    assert sheerka.sdp.io.exists(
+        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+
+
+def test_i_cannot_eval_the_same_def_concept_twice():
+    text = """
+def concept a + b
+where isinstance(a, int) and isinstance(b, int)
+pre isinstance(a, int) and isinstance(b, int)
+post isinstance(res, int)
+as:
+    def func(x,y):
+        return x+y
+    func(a,b)
+        """
+
+    sheerka = get_sheerka()
+    sheerka.evaluate_user_input(text)
+    res = sheerka.evaluate_user_input(text)
+
+    assert len(res) == 1
+    assert not res[0].status
+    assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+
+
+def test_i_can_disable_an_evaluator():
+    sheerka = get_sheerka()
+    concept = Concept(name="one", body="1")
+    sheerka.add_in_cache(concept)
+
+    text = "one"
+    p = next(e for e in sheerka.evaluators if e.__name__ == "PythonEvaluator")
+    p.enabled = False  # not that you disable the class, not the instance
+
+    res = sheerka.evaluate_user_input(text)
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
+
+    p.enabled = True  # put back for the remaining unit tests
+
+
+@pytest.mark.parametrize("text", [
+    "",
+    "   ",
+    "\n",
+])
+def test_i_can_eval_a_empty_input(text):
+    sheerka = get_sheerka()
+
+    res = sheerka.evaluate_user_input(text)
+
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NOP)
+
+
+def test_i_can_eval_concept_with_variable():
+    sheerka = get_sheerka()
+    concept_hello = Concept(name="hello a").set_prop("a")
+    concept_foo = Concept(name="foo")
+    sheerka.add_in_cache(concept_hello)
+    sheerka.add_in_cache(concept_foo)
+
+    res = sheerka.evaluate_user_input("hello foo")
+    return_value = res[0].value
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(return_value, concept_hello)
+    assert return_value.props["a"].value == concept_foo
+
+
+def test_i_can_eval_concept_with_variable_and_python_as_body():
+    sheerka = get_sheerka()
+    sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a"))
+    sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
+
+    res = sheerka.evaluate_user_input("hello foo")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].value, "hello foo"
+
+
+def test_i_can_eval_duplicate_concepts_with_same_value():
+    sheerka = get_sheerka()
+
+    sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a"))
+    sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
+    sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
+
+    res = sheerka.evaluate_user_input("hello foo")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].value, "hello foo"
+    assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
+
+
+def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different():
+    sheerka = get_sheerka()
+
+    sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a"))
+    sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
+    sheerka.add_in_cache(Concept(name="foo", body="'another value'"))
+
+    res = sheerka.evaluate_user_input("hello foo")
+    assert len(res) == 1
+    assert not res[0].status
+    assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS)
+
+    concepts = res[0].value.body
+    assert len(concepts) == 2
+    sorted_values = sorted(concepts, key=lambda x: x.value)
+    assert sorted_values[0].value == "hello another value"
+    assert sorted_values[1].value == "hello foo"
+
+
+def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same():
+    sheerka = get_sheerka()
+    context = get_context(sheerka)
+
+    sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").set_prop("a"))
+    sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").set_prop("b"))
+
+    res = sheerka.evaluate_user_input("hello 'foo'")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].value == "hello foo"
+    assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
+
+
+def test_i_can_create_concepts_with_python_code_as_body():
+    sheerka = get_sheerka()
+    context = get_context(sheerka)
+
+    sheerka.create_new_concept(context, Concept(name="concepts", body="sheerka.concepts()"))
+    res = sheerka.evaluate_user_input("concepts")
+
+    assert len(res) == 1
+    assert res[0].status
+    assert isinstance(res[0].value, list)
+
+
+def test_i_can_create_concept_with_bnf_definition():
+    sheerka = get_sheerka(False, False)
+    a = Concept("a")
+    sheerka.add_in_cache(a)
+    sheerka.concepts_definition_cache = {a: OrderedChoice("one", "two")}
+
+    res = sheerka.evaluate_user_input("def concept plus from bnf a ('plus' plus)?")
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
+
+    saved_concept = sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, "plus")
+    assert saved_concept.key == "plus"
+    assert saved_concept.metadata.definition == "a ('plus' plus)?"
+    assert "a" in saved_concept.props
+    assert "plus" in saved_concept.props
+
+    saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY)
+    expected_bnf = Sequence(
+        ConceptMatch("a", rule_name="a"),
+        Optional(Sequence(StrMatch("plus"), ConceptMatch("plus", rule_name="plus"))))
+    assert saved_definitions[saved_concept] == expected_bnf
+
+    new_concept = res[0].value.body
+    assert new_concept.metadata.name == "plus"
+    assert new_concept.metadata.definition == "a ('plus' plus)?"
+    assert new_concept.bnf == expected_bnf
+    assert "a" in new_concept.props
+    assert "plus" in new_concept.props
+
+
+def test_i_can_eval_bnf_definitions():
+    sheerka = get_sheerka()
+    concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
+
+    res = sheerka.evaluate_user_input("one")
+
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(res[0].value, concept_a)
+
+
+def test_i_can_eval_bnf_definitions_with_variables():
+    sheerka = get_sheerka()
+    concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
+    concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
+
+    res = sheerka.evaluate_user_input("one three")
+
+    assert len(res) == 1
+    assert res[0].status
+    return_value = res[0].value
+
+    assert sheerka.isinstance(return_value, concept_b)
+    assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one"))
+
+
+def test_i_can_eval_bnf_definitions_from_separate_instances():
+    """
+    Same test then before,
+    but make sure that the BNF are correctly persisted and loaded
+    """
+    sheerka = get_sheerka(False)
+    concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' 'two'")[0].body.body
+
+    res = get_sheerka(False).evaluate_user_input("one two")
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(res[0].value, concept_a)
+
+    # add another bnf definition
+    concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
+
+    res = get_sheerka(False).evaluate_user_input("one two")  # previous one still works
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(res[0].value, concept_a)
+
+    res = get_sheerka(False).evaluate_user_input("one two three")  # new one works
+    assert len(res) == 1
+    assert res[0].status
+    assert sheerka.isinstance(res[0].value, concept_b)
+
+
+def get_sheerka(use_dict=True, skip_builtins_in_db=True):
+    root = "mem://" if use_dict else root_folder
+    sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db)
+    sheerka.initialize(root)
+
+    return sheerka
+
+
+def get_context(sheerka):
+    return ExecutionContext("test", "xxx", sheerka)
+
+
+def get_default_concept():
+    concept = Concept(
+        name="a + b",
+        where="isinstance(a, int) and isinstance(b, int)",
+        pre="isinstance(a, int) and isinstance(b, int)",
+        post="isinstance(res, int)",
+        body="def func(x,y):\n    return x+y\nfunc(a,b)",
+        desc="specific description")
+    concept.set_prop("a", "value1")
+    concept.set_prop("b", "value2")
+
+    return concept

tests/test_tokenizer.py

@@ -3,7 +3,7 @@ from core.tokenizer import Tokenizer, Token, TokenKind, LexerError, Keywords
 
 
 def test_i_can_tokenize():
-    source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&"
+    source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&<>"
     tokens = list(Tokenizer(source))
     assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
     assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
@@ -37,6 +37,8 @@ def test_i_can_tokenize():
     assert tokens[29] == Token(TokenKind.EQUALS, '=', 76, 6, 18)
     assert tokens[30] == Token(TokenKind.VBAR, '|', 77, 6, 19)
     assert tokens[31] == Token(TokenKind.AMPER, '&', 78, 6, 20)
+    assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 21)
+    assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 22)
 
 
 @pytest.mark.parametrize("text, expected", [