Refactored sheerka class: splitted to use sub handlers. Refactored unit tests to use classes.

tests/BaseTest.py

@@ -0,0 +1,58 @@
+import ast
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept
+from core.concept import Concept
+from core.sheerka.ExecutionContext import ExecutionContext
+from sdp.sheerkaDataProvider import Event
+
+
+class BaseTest:
+    def get_sheerka(self):
+        pass
+
+    def get_context(self, sheerka=None):
+        return ExecutionContext("test", Event(), sheerka or self.get_sheerka())
+
+    def get_default_concept(self):
+        concept = Concept(
+            name="a + b",
+            where="isinstance(a, int) and isinstance(b, int)",
+            pre="isinstance(a, int) and isinstance(b, int)",
+            post="isinstance(res, int)",
+            body="def func(x,y):\n    return x+y\nfunc(a,b)",
+            desc="specific description")
+        concept.def_prop("a", "value1")
+        concept.def_prop("b", "value2")
+
+        return concept
+
+    def dump_ast(self, node):
+        dump = ast.dump(node)
+        for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
+            dump = dump.replace(to_remove, "")
+        return dump
+
+    @staticmethod
+    def retval(obj, who="who", status=True):
+        """ret_val"""
+        return ReturnValueConcept.ret(who, status, obj)
+
+    @staticmethod
+    def tretval(sheerka, obj, who="who"):
+        """True ret_val + add concept in cache"""
+        if isinstance(obj, Concept):
+            obj.init_key()
+            if obj.key not in sheerka.cache_by_key:
+                sheerka.cache_by_key[obj.key] = obj
+        return sheerka.ret(who, True, obj)
+
+    @staticmethod
+    def pretval(concept, source=None, parser="parsers.name"):
+        """ParserResult ret_val"""
+        return ReturnValueConcept(
+            "some_name",
+            True,
+            ParserResultConcept(parser=parser,
+                                source=source or concept.name,
+                                value=concept,
+                                try_parsed=concept))

tests/TestUsingFileBasedSheerka.py

@@ -0,0 +1,34 @@
+import os
+import shutil
+from os import path
+
+import pytest
+from core.sheerka.Sheerka import Sheerka
+from tests.BaseTest import BaseTest
+
+
+class TestUsingFileBasedSheerka(BaseTest):
+
+    tests_root = path.abspath("../../build/tests")
+    root_folder = "init_folder"
+
+    @pytest.fixture(autouse=True)
+    def init_test(self):
+        if path.exists(self.tests_root):
+            shutil.rmtree(self.tests_root)
+
+        if not path.exists(self.tests_root):
+            os.makedirs(self.tests_root)
+        current_pwd = os.getcwd()
+        os.chdir(self.tests_root)
+
+        yield None
+
+        os.chdir(current_pwd)
+
+    def get_sheerka(self, use_dict=True, skip_builtins_in_db=True):
+        root = "mem://" if use_dict else self.root_folder
+        sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db)
+        sheerka.initialize(root)
+
+        return sheerka

tests/TestUsingMemoryBasedSheerka.py

@@ -0,0 +1,10 @@
+from core.sheerka.Sheerka import Sheerka
+from tests.BaseTest import BaseTest
+
+
+class TestUsingMemoryBasedSheerka(BaseTest):
+
+    def get_sheerka(self, skip_builtins_in_db=True):
+        sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db)
+        sheerka.initialize("mem://")
+        return sheerka

tests/core/test_ExecutionContext.py

@@ -2,7 +2,7 @@ import pytest
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.sheerka import ExecutionContext
+from core.sheerka.Sheerka import ExecutionContext
 from sdp.sheerkaDataProvider import Event
 
 

tests/core/test_SheerkaCreateNewConcept.py

@@ -0,0 +1,180 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import PROPERTIES_TO_SERIALIZE
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
+    def test_i_can_add_a_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_found = res.value.body
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop)
+
+        assert concept_found.key == "__var__0 + __var__1"
+        assert concept_found.id == "1001"
+
+        assert concept.key in sheerka.cache_by_key
+        assert concept.id in sheerka.cache_by_id
+        assert sheerka.sdp.io.exists(
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest()))
+
+    def test_i_cannot_add_the_same_concept_twice(self):
+        """
+        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+        assert res.value.body == concept
+
+    def test_i_can_get_a_newly_created_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        from_cache = sheerka.get(concept.key)
+        assert from_cache is not None
+        assert from_cache == concept
+
+        from_cache = sheerka.get_by_id(concept.id)
+        assert from_cache is not None
+        assert from_cache == concept
+
+    def test_i_first_look_in_local_cache(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+        sheerka.cache_by_key[concept.key].pre = "I have modified the concept in cache"
+
+        from_cache = sheerka.get(concept.key)
+        assert from_cache is not None
+        assert from_cache.key == concept.key
+        assert from_cache.pre == "I have modified the concept in cache"
+
+    def test_i_can_get_a_known_concept_when_not_in_cache(self):
+        """
+        When not in cache, uses sdp
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        sheerka.cache_by_key = {}  # reset the cache
+        loaded = sheerka.get(concept.key)
+
+        assert loaded is not None
+        assert loaded == concept
+
+        # I can also get it by its id
+        loaded = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert loaded is not None
+        assert loaded == concept
+
+    def test_i_can_get_a_concept_by_its_id(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        sheerka.cache_by_key = {}  # reset the cache
+        loaded = sheerka.get_by_id(concept.id)
+
+        assert loaded is not None
+        assert loaded == concept
+
+    def test_i_can_get_list_of_concept_when_same_key_when_no_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        sheerka.cache_by_key = {}  # reset the cache
+
+        result = sheerka.get(concept1.key)
+        assert len(result) == 2
+        assert result[0] == concept1
+        assert result[1] == concept2
+
+    def test_i_can_get_list_of_concept_when_same_key_when_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        # sheerka.cache_by_key = {}  # Do not reset the cache
+
+        result = sheerka.get(concept1.key)
+        assert len(result) == 2
+        assert result[0] == concept1
+        assert result[1] == concept2
+
+    def test_i_can_get_the_correct_concept_using_the_id_when_same_key_when_no_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get(concept1.key, res2.body.body.id)
+        assert result.name == "a + b"
+        assert result.metadata.body == "a+b"
+
+    def test_i_can_get_the_correct_concept_using_the_id__when_same_key_when_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get(concept1.key, res2.body.body.id)
+        assert result.name == "a + b"
+        assert result.metadata.body == "a+b"
+
+    def test_i_cannot_get_the_correct_concept_id_the_id_is_wrong(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get(concept1.key, "wrong id")
+        assert sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT)

tests/core/test_SheerkaEvaluateConcept.py

@@ -0,0 +1,332 @@
+import pytest
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
+from core.concept import Concept, simplec, DoNotResolve, ConceptParts, Property
+from parsers.PythonParser import PythonNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("body, expected", [
+        # (None, None),
+        # ("", ""),
+        ("1", 1),
+        ("1+1", 2),
+        ("'one'", "one"),
+        ("'one' + 'two'", "onetwo"),
+        ("True", True),
+        ("1 > 2", False),
+    ])
+    def test_i_can_evaluate_a_concept_with_simple_body(self,body, expected):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body=body)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == expected
+        assert evaluated.metadata.body == body
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.post is None
+        assert evaluated.metadata.where is None
+        assert evaluated.props == {}
+        assert evaluated.metadata.is_evaluated
+        assert len(evaluated.values) == 0 if body is None else 1
+
+    @pytest.mark.parametrize("expr, expected", [
+        ("", ""),
+        ("1", 1),
+        ("1+1", 2),
+        ("'one'", "one"),
+        ("'one' + 'two'", "onetwo"),
+        ("True", True),
+        ("1 > 2", False),
+    ])
+    def test_i_can_evaluate_the_other_metadata(self,expr, expected):
+        """
+        I only test WHERE, it's the same for the others
+        :param expr:
+        :param expected:
+        :return:
+        """
+
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", where=expr)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.metadata.body is None
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.post is None
+        assert evaluated.metadata.where == expr
+        assert evaluated.get_metadata_value(ConceptParts.WHERE) == expected
+        assert evaluated.props == {}
+        assert evaluated.metadata.is_evaluated
+        assert len(evaluated.values) == 0 if expr is None else 1
+
+    @pytest.mark.parametrize("expr, expected", [
+        (None, None),
+        ("", ""),
+        ("1", 1),
+        ("1+1", 2),
+        ("'one'", "one"),
+        ("'one' + 'two'", "onetwo"),
+        ("True", True),
+        ("1 > 2", False),
+    ])
+    def test_i_can_evaluate_a_concept_with_prop(self, expr, expected):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo").def_prop("a", expr)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.post is None
+        assert evaluated.metadata.where is None
+        assert evaluated.props == {"a": Property("a", expected)}
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_metadata_using_do_not_resolve(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo")
+        concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.body == "do not resolve"
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_property_using_do_not_resolve(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo").def_prop("a")
+        concept.compiled["a"] = DoNotResolve("do not resolve")
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.get_prop("a") == "do not resolve"
+        assert evaluated.metadata.is_evaluated
+
+    def test_original_value_is_overridden_when_using_do_no_resolve(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo", body="original value").set_prop("a", "original value")
+        concept.compiled["a"] = DoNotResolve("do not resolve")
+        concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.body == "do not resolve"
+        assert evaluated.get_prop("a") == "do not resolve"
+        assert evaluated.metadata.is_evaluated
+
+    def test_props_are_evaluated_before_body(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body="a+1").def_prop("a", "10").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == 11
+
+    def test_i_can_evaluate_when_another_concept_is_referenced(self):
+        sheerka = self.get_sheerka()
+        concept_a = Concept("a")
+        sheerka.add_in_cache(concept_a)
+
+        concept = Concept("foo", body="a").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated == simplec("foo", simplec("a", None))
+        assert id(evaluated.body) != id(concept_a)
+        assert evaluated.metadata.is_evaluated
+        assert evaluated.body.metadata.is_evaluated
+
+    def test_i_can_evaluate_when_the_referenced_concept_has_a_body(self):
+        sheerka = self.get_sheerka()
+        concept_a = Concept("a", body="1")
+        sheerka.add_in_cache(concept_a)
+
+        concept = Concept("foo", body="a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == simplec("a", 1)
+        assert not concept_a.metadata.is_evaluated
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="'a'"))
+        sheerka.add_in_cache(Concept(name="b", body="a"))
+        sheerka.add_in_cache(Concept(name="c", body="b"))
+        concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept_d)
+
+        assert evaluated.key == concept_d.key
+        expected = simplec("c", simplec("b", simplec("a", "a")))
+        assert evaluated.body == expected
+        assert sheerka.value(evaluated) == 'a'
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a"))
+        sheerka.add_in_cache(Concept(name="b", body="a"))
+        sheerka.add_in_cache(Concept(name="c", body="b"))
+        concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept_d)
+
+        assert evaluated.key == concept_d.key
+        expected = simplec("c", simplec("b", simplec("a", None)))
+        assert evaluated.body == expected
+        assert sheerka.value(evaluated) == Concept(name="a").init_key()
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_concept_when_properties_reference_others_concepts(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.add_in_cache(Concept(name="a").init_key())
+
+        concept = Concept("foo", body="a").def_prop("a", "a").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        # first prop a is evaluated to concept_a
+        # then body is evaluated to prop a -> concept_a
+        assert evaluated.key == concept.key
+        assert evaluated.body == concept_a
+
+    def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2(self):
+        """
+        Same test,
+        but the name of the property and the name of the concept are different
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.add_in_cache(Concept(name="a"))
+
+        concept = Concept("foo", body="concept_a").def_prop("concept_a", "a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == concept_a
+
+    def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="1"))
+        sheerka.add_in_cache(Concept(name="b", body="2"))
+
+        concept = Concept("foo", body="propA + propB").def_prop("propA", "a").def_prop("propB", "b")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == 3
+
+    def test_i_can_evaluate_concept_when_properties_is_a_concept(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.add_in_cache(Concept(name="a", body="'a'").init_key())
+
+        concept = Concept("foo").def_prop("a")
+        concept.compiled["a"] = concept_a
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.get_prop("a") == simplec("a", "a")
+
+    def test_i_can_evaluate_when_property_asts_is_a_list(self):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo", body="1")
+
+        concept = Concept("to_eval").def_prop("prop")
+        concept.compiled["prop"] = [foo, DoNotResolve("1")]
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        props = evaluated.get_prop("prop")
+        assert len(props) == 2
+        assert props[0] == simplec("foo", 1)
+        assert props[1] == "1"
+
+    def test_i_can_evaluate_when_compiled_is_set_up_with_return_value(self):
+        sheerka = self.get_sheerka()
+
+        python_node = PythonNode("1 +1 ")
+        parser_result = ParserResultConcept(parser="who", value=python_node)
+
+        concept = Concept("to_eval").def_prop("prop")
+        concept.compiled["prop"] = [ReturnValueConcept("who", True, parser_result)]
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.get_prop("prop") == 2
+
+        # also works when only one return value
+        concept = Concept("to_eval").def_prop("prop")
+        concept.compiled["prop"] = ReturnValueConcept("who", True, parser_result)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.get_prop("prop") == 2
+
+    def test_i_can_reference_sheerka(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body="sheerka.test()").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == sheerka.test()
+
+    def test_properties_values_takes_precedence_over_the_outside_world(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
+        sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
+
+        concept = Concept("foo", body="a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == simplec("a", "concept_a")  # this test was already done
+
+        # so check this one.
+        concept = Concept("foo", body="a").def_prop("a", "'property_a'")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == 'property_a'
+
+        # or this one.
+        concept = Concept("foo", body="a").def_prop("a", "b")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == simplec(name="b", body="concept_b")
+
+    def test_properties_values_takes_precedence(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
+        sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
+
+        concept = Concept("foo", body="a + b").def_prop("a", "'prop_a'")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == 'prop_aconcept_b'
+
+    def test_i_can_reference_sub_property_of_a_property(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="concept_a").def_prop("subProp", "'sub_a'"))
+
+        concept = Concept("foo", body="a.props['subProp'].value").def_prop("a", "concept_a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated == simplec(concept.key, "sub_a")
+
+    def test_i_cannot_evaluate_concept_if_property_is_in_error(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept(name="concept_a").def_prop("subProp", "undef_concept")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
+
+    def test_key_is_initialized_by_evaluation(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.init_key().key
+

tests/core/test_SheerkaSetsManager.py

@@ -0,0 +1,124 @@
+import os
+import shutil
+from os import path
+
+import pytest
+from core.builtin_concepts import ConceptAlreadyInSet, BuiltinConcepts
+from core.concept import Concept
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
+
+
+
+    def test_i_can_add_concept_to_set(self):
+        sheerka = self.get_sheerka(False, False)
+
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+
+        all_foos = Concept("all_foos")
+        sheerka.set_id_if_needed(all_foos, False)
+
+        context = self.get_context(sheerka)
+        res = sheerka.add_concept_to_set(context, foo, all_foos)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 1
+        assert foo.id in all_entries
+
+    def test_i_can_add_several_concepts_to_set(self):
+        sheerka = self.get_sheerka(False, False)
+
+        foo1 = Concept("foo1")
+        sheerka.set_id_if_needed(foo1, False)
+
+        foo2 = Concept("foo1")
+        sheerka.set_id_if_needed(foo2, False)
+
+        all_foos = Concept("all_foos")
+        sheerka.set_id_if_needed(all_foos, False)
+
+        context = self.get_context(sheerka)
+        sheerka.add_concept_to_set(context, foo1, all_foos)
+        res = sheerka.add_concept_to_set(context, foo2, all_foos)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 2
+        assert foo1.id in all_entries
+        assert foo2.id in all_entries
+
+    def test_i_cannot_add_the_same_concept_twice_in_a_set(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+
+        all_foos = Concept("all_foos")
+        sheerka.set_id_if_needed(all_foos, False)
+
+        context = self.get_context(sheerka)
+        sheerka.add_concept_to_set(context, foo, all_foos)
+        res = sheerka.add_concept_to_set(context, foo, all_foos)
+
+        assert not res.status
+        assert res.body == ConceptAlreadyInSet(foo, all_foos)
+
+        all_entries = sheerka.sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 1
+        assert foo.id in all_entries
+
+    def test_i_get_elements_from_a_set(self):
+        sheerka = self.get_sheerka()
+
+        one = Concept("one")
+        two = Concept("two")
+        three = Concept("three")
+        number = Concept("number")
+
+        for c in [one, two, three, number]:
+            sheerka.set_id_if_needed(c, False)
+            sheerka.add_in_cache(c)
+
+        context = self.get_context(sheerka)
+        for c in [one, two, three]:
+            sheerka.add_concept_to_set(context, c, number)
+
+        elements = sheerka.get_set_elements(number)
+
+        assert set(elements) == {one, two, three}
+
+    def test_i_cannot_get_elements_if_not_a_set(self):
+        sheerka = self.get_sheerka()
+        one = Concept("one")
+        sheerka.set_id_if_needed(one, False)
+        sheerka.add_in_cache(one)
+
+        error = sheerka.get_set_elements(one)
+
+        assert sheerka.isinstance(error, BuiltinConcepts.NOT_A_SET)
+        assert error.body == one
+
+    def test_isa_and_isa_group(self):
+        sheerka = self.get_sheerka()
+
+        group = Concept("group").init_key()
+        group.metadata.id = "1001"
+        assert not sheerka.isagroup(group)
+
+        foo = Concept("foo").init_key()
+        foo.metadata.id = "1002"
+        assert not sheerka.isa(foo, group)
+
+        context = self.get_context(sheerka)
+        sheerka.add_concept_to_set(context, foo, group)
+        assert sheerka.isagroup(group)
+        assert sheerka.isa(foo, group)

tests/core/test_ast.py

@@ -6,7 +6,7 @@ from core.ast.nodes import NodeParent, GenericNodeConcept
 import core.ast.nodes
 from core.ast.visitors import ConceptNodeVisitor, UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
-from core.sheerka import Sheerka
+from core.sheerka.Sheerka import Sheerka
 
 
 def get_sheerka():

tests/core/test_builtin_helpers.py

@@ -0,0 +1,145 @@
+import ast
+import pytest
+import core.builtin_helpers
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_use_expect_one_when_empty(self):
+        sheerka = self.get_sheerka()
+
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), [])
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY)
+
+    def test_i_can_use_expect_one_when_too_many_success(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", True, "value1"),
+            ReturnValueConcept("who", True, "value2"),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS)
+        assert res.value.body == items
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_same_success(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", True, "value"),
+            ReturnValueConcept("who", True, "value"),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert res.status
+        assert res.value == items[0].value
+        assert res.parents == items
+
+    def test_i_can_use_expect_when_only_errors_1(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", False, sheerka.new(BuiltinConcepts.ERROR)),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert not res.status
+        assert res.value, items[0]
+        assert res.parents == items
+
+    def test_i_can_use_expect_when_only_errors_2(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", False, None),
+            ReturnValueConcept("who", False, None),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
+        assert res.value.body == items
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_one_success_1(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", True, None),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert res.status
+        assert res.body == items[0].body
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_one_success_2(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", False, None),
+            ReturnValueConcept("who", True, None),
+            ReturnValueConcept("who", False, None),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert res.status
+        assert res.body == items[1].body
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_not_a_list_true(self):
+        sheerka = self.get_sheerka()
+
+        item = ReturnValueConcept("who", True, None)
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), item)
+        assert res.status
+        assert res == item
+
+    def test_i_can_use_expect_one_when_not_a_list_false(self):
+        sheerka = self.get_sheerka()
+
+        item = ReturnValueConcept("who", False, None)
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), item)
+
+        assert not res.status
+        assert res == item
+
+    @pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [
+        ("a == 1", [], [], []),
+        ("a == 1", ["a"], [], ["a == 1"]),
+        ("a == 1", [], ["a"], []),
+        ("predicate(a)", [], [], []),
+        ("predicate(a)", ["a"], [], ["predicate(a)"]),
+        ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]),
+        ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]),
+        ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]),
+        ("predicate(a, b)", ["a"], ["b"], []),
+        ("a + b == 1", [], [], []),
+        ("a + b == 1", ["a"], [], ["a + b == 1"]),
+        ("a + b == 1", ["a"], ["b"], []),
+        ("a + b == 1", ["b"], [], ["a + b == 1"]),
+        ("a + b == 1", ["a", "b"], [], ["a + b == 1"]),
+        ("a == 1 and b == 2", [], [], []),
+        ("a == 1 and b == 2", ["a"], [], ["a == 1"]),
+        ("a == 1 and b == 2", ["b"], [], ["b == 2"]),
+        ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]),
+        ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]),
+        ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]),
+        ("not(a == 1)", [], [], []),
+        ("not(a == 1)", ["a"], [], ["not(a==1)"]),
+        ("a == 1 or b == 2", [], [], []),
+        ("a == 1 or b == 2", ["a"], [], ["a == 1"]),
+        ("a == 1 or b == 2", ["b"], [], ["b == 2"]),
+        ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]),
+        ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]),
+    ])
+    def test_i_can_extract_predicates(self, expression, vars_to_include, vars_to_exclude, expected_expr):
+        sheerka = self.get_sheerka()
+        expected = [ast.parse(expr, mode="eval") for expr in expected_expr]
+
+        actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude)
+        assert len(actual) == len(expected)
+        for i in range(len(actual)):
+            assert self.dump_ast(actual[i]) == self.dump_ast(expected[i])

tests/core/test_sheerka.py

@@ -0,0 +1,244 @@
+import pytest
+import os
+
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE
+from core.sheerka.Sheerka import Sheerka
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class ConceptWithGetValue(Concept):
+    def get_value(self):
+        return self.get_prop("my_prop")
+
+
+class TestSheerka(TestUsingFileBasedSheerka):
+
+    def test_root_folder_is_created_after_initialization(self):
+        return_value = Sheerka().initialize(self.root_folder)
+        assert return_value.status, "initialisation should be successful"
+        assert os.path.exists(self.root_folder), "init folder should be created"
+
+    def test_i_can_list_builtin_concepts(self):
+        sheerka = self.get_sheerka()
+        builtins = list(sheerka.get_builtins_classes_as_dict())
+
+        assert str(BuiltinConcepts.ERROR) in builtins
+        assert str(BuiltinConcepts.RETURN_VALUE) in builtins
+
+    def test_builtin_concepts_are_initialized(self):
+        sheerka = self.get_sheerka(skip_builtins_in_db=False)
+        assert len(sheerka.cache_by_key) == len(BuiltinConcepts)
+        for concept_name in BuiltinConcepts:
+            assert str(concept_name) in sheerka.cache_by_key
+            assert sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, str(concept_name)) is not None
+
+        for key, concept_class in sheerka.get_builtins_classes_as_dict().items():
+            assert isinstance(sheerka.cache_by_key[key], concept_class)
+
+    def test_builtin_concepts_can_be_updated(self):
+        sheerka = self.get_sheerka(False, False)
+        loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
+        loaded_sheerka.metadata.desc = "I have a description"
+        sheerka.sdp.modify("Test", sheerka.CONCEPTS_ENTRY, loaded_sheerka.key, loaded_sheerka)
+
+        sheerka = self.get_sheerka(False, False)
+        loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
+
+        assert loaded_sheerka.metadata.desc == "I have a description"
+
+    def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string(self):
+        """
+        Checks that a concept can be found its name
+        even when there are variables in the name (ex 'hello + a' or 'a + b' )
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        for key in sheerka.get_builtins_classes_as_dict():
+            assert sheerka.get(key) is not None
+            assert sheerka.get(str(key)) is not None
+
+    def test_i_cannot_get_when_key_is_none(self):
+        sheerka = self.get_sheerka()
+
+        res = sheerka.get(None)
+        assert sheerka.isinstance(res, BuiltinConcepts.ERROR)
+        assert res.body == "Concept key is undefined."
+
+    def test_unknown_concept_is_return_when_the_concept_key_is_not_found(self):
+        sheerka = self.get_sheerka()
+
+        loaded = sheerka.get("key_that_does_not_exist")
+
+        assert loaded is not None
+        assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert loaded.body == ("key", "key_that_does_not_exist")
+        assert loaded.metadata.is_evaluated
+
+    def test_unknown_concept_is_return_when_the_concept_id_is_not_found(self):
+        sheerka = self.get_sheerka()
+
+        loaded = sheerka.get_by_id("id_that_does_not_exist")
+
+        assert loaded is not None
+        assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert loaded.body == ("id", "id_that_does_not_exist")
+        assert loaded.metadata.is_evaluated
+
+    def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self):
+        sheerka = self.get_sheerka()
+        ret = sheerka.new(BuiltinConcepts.RETURN_VALUE, who="who", status="status", value="value", message="message")
+
+        assert isinstance(ret, ReturnValueConcept)
+        assert ret.key == str(BuiltinConcepts.RETURN_VALUE)
+        assert ret.who == "who"
+        assert ret.status == "status"
+        assert ret.value == "value"
+        assert ret.message == "message"
+
+    def test_i_can_instantiate_a_builtin_concept_when_no_specific_class(self):
+        sheerka = self.get_sheerka()
+        ret = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body="fake_concept")
+
+        assert isinstance(ret, Concept)
+        assert ret.key == str(BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert ret.body == "fake_concept"
+
+    def test_i_can_instantiate_a_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new = sheerka.new(concept.key, a=10, b="value")
+
+        assert sheerka.isinstance(new, concept)
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(new.metadata, prop) == getattr(concept.metadata, prop)
+
+        assert new.props["a"].value == 10
+        assert new.props["b"].value == "value"
+
+    def test_i_can_instantiate_with_the_name_and_the_id(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
+
+        concepts = sheerka.new("foo")
+        assert len(concepts) == 2
+
+        foo1 = sheerka.new(("foo", "1001"))
+        assert foo1.metadata.body == "foo1"
+
+        foo2 = sheerka.new(("foo", "1002"))
+        assert foo2.metadata.body == "foo2"
+
+    def test_instances_are_different_when_asking_for_new(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new1 = sheerka.new(concept.key, a=10, b="value")
+        new2 = sheerka.new(concept.key, a=10, b="value")
+
+        assert new1 == new2
+        assert id(new1) != id(new2)
+
+    def test_i_get_the_same_instance_when_is_unique_is_true(self):
+        sheerka = self.get_sheerka()
+        concept = Concept(name="unique", is_unique=True)
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new1 = sheerka.new(concept.key, a=10, b="value")
+        new2 = sheerka.new(concept.key, a=10, b="value")
+
+        assert new1 == new2
+        assert id(new1) == id(new2)
+
+    def test_i_cannot_instantiate_an_unknown_concept(self):
+        sheerka = self.get_sheerka()
+
+        new = sheerka.new("fake_concept")
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert new.body == ('key', 'fake_concept')
+
+    def test_i_cannot_instantiate_with_invalid_id(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
+
+        new = sheerka.new(("foo", "invalid_id"))
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert new.body == [('key', 'foo'), ('id', 'invalid_id')]
+
+    def test_i_cannot_instantiate_with_invalid_key(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
+
+        new = sheerka.new(("invalid_key", "1001"))
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert new.body == [('key', 'invalid_key'), ('id', '1001')]
+
+    def test_concept_id_is_irrelevant_when_only_one_concept(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+
+        new = sheerka.new(("foo", "invalid_id"))
+
+        assert sheerka.isinstance(new, "foo")
+        assert new.metadata.body == "foo1"
+
+    def test_i_cannot_instantiate_when_properties_are_not_recognized(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new = sheerka.new(concept.key, a=10, c="value")
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_PROPERTY)
+        assert new.property_name == "c"
+        assert sheerka.isinstance(new.concept, concept)
+
+    @pytest.mark.parametrize("concept, reduce_simple_list, expected", [
+        (None, False, None),
+        (3.14, False, 3.14),
+        ("foo", False, "foo"),
+        (True, False, True),
+        (Concept("name", body="foo"), False, "foo"),
+        (Concept("name"), False, Concept("name")),
+        (ConceptWithGetValue("name").set_prop("my_prop", "my_value"), False, "my_value"),
+        (ReturnValueConcept(value="return_value"), False, "return_value"),
+        (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
+        (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
+        (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]),
+        (Concept("name", body=["foo"]), True, "foo"),
+        (Concept("name", body=Concept("foo")), False, Concept("foo")),
+        (Concept("name", body=Concept("foo", body="value")), False, "value"),
+        (Concept("name", body=Concept("foo", body=Concept("bar", body="value"))), False, "value"),
+        (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"),
+    ])
+    def test_i_can_get_value(self, concept, reduce_simple_list, expected):
+        sheerka = self.get_sheerka()
+
+        # I use auto_init() instead of evaluate_concept() to be quicker
+        c = concept
+        while isinstance(c, Concept):
+            c.auto_init()
+            c = c.body
+
+        assert sheerka.value(concept, reduce_simple_list) == expected
+
+    def test_list_of_concept_is_sorted_by_id(self):
+        sheerka = self.get_sheerka(False, False)
+        concepts = sheerka.concepts()
+
+        assert concepts[0].id < concepts[-1].id
+
+    def test_builtin_error_concept_are_errors(self):
+        # only test a random one, it will be the same for the others
+        sheerka = self.get_sheerka()
+        assert not sheerka.is_success(sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS))

tests/core/test_sheerka_call_evaluators.py

@@ -0,0 +1,375 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator
+
+from tests.BaseTest import BaseTest
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class Out:
+    debug_out = []
+
+    def out(self, method, name, context, return_value):
+        name = name[len(BaseEvaluator.PREFIX):]
+        if isinstance(return_value, list):
+            target = [str(r.body.key) for r in return_value]
+        else:
+            target = str(return_value.body.key)
+        step = str(context.step)
+        text = f"{step} [{context.iteration}] "
+        text += f"{name} - {method} - target={target}"
+        self.debug_out.append(text)
+
+    def out_all(self, method, name, context, return_values):
+        name = name[len(BaseEvaluator.PREFIX):]
+        target = [str(r.body.key) for r in return_values]
+        step = str(context.step)
+        text = f"{step} [{context.iteration}] "
+        text += f"{name} - {method} - target={target}"
+        self.debug_out.append(text)
+
+
+class OneReturnValueEvaluatorForTestingPurpose(OneReturnValueEvaluator, Out):
+    def __init__(self, name, steps, priority):
+        super().__init__(name, steps, priority)
+
+    def matches(self, context, return_value):
+        self.out("matches", self.name, context, return_value)
+        return True
+
+    def eval(self, context, return_value):
+        self.out("eval", self.name, context, return_value)
+
+
+class AllReturnValueEvaluatorForTestingPurpose(AllReturnValuesEvaluator, Out):
+    def __init__(self, name, steps, priority):
+        super().__init__(name, steps, priority)
+
+    def matches(self, context, return_values):
+        self.out("matches", self.name, context, return_values)
+        return True
+
+    def eval(self, context, return_values):
+        self.out("eval", self.name, context, return_values)
+
+
+class EvaluatorOneWithPriority(OneReturnValueEvaluatorForTestingPurpose):
+    def __init__(self, name, priority):
+        super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
+
+
+class EvaluatorOneWithPriority10(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("priority10", 10)
+
+
+class EvaluatorOneWithPriority15(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("priority15", 15)
+
+
+class EvaluatorOneWithPriority20(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("priority20", 20)
+
+
+class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose):
+    def __init__(self, name, priority):
+        super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
+
+
+class EvaluatorAllWithPriority10(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_priority10", 10)
+
+
+class EvaluatorAllWithPriority15(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_priority15", 15)
+
+
+class EvaluatorAllWithPriority20(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_priority20", 20)
+
+
+class EvaluatorOneModifyFoo(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("modifyFoo", 10)
+
+    def matches(self, context, return_value):
+        super().matches(context, return_value)
+        return context.sheerka.isinstance(return_value.body, "foo")
+
+    def eval(self, context, return_value):
+        super().eval(context, return_value)
+        return BaseTest.tretval(context.sheerka, Concept("bar"))
+
+
+class EvaluatorOneModifyBar(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("modifyBar", 10)
+
+    def matches(self, context, return_value):
+        super().matches(context, return_value)
+        return context.sheerka.isinstance(return_value.body, "bar")
+
+    def eval(self, context, return_value):
+        super().eval(context, return_value)
+        return BaseTest.tretval(context.sheerka, Concept("baz"))
+
+
+class EvaluatorOnePreEvaluation(OneReturnValueEvaluatorForTestingPurpose):
+    def __init__(self):
+        super().__init__("preEval", [BuiltinConcepts.BEFORE_EVALUATION], 10)
+
+
+class EvaluatorOneMultiSteps(OneReturnValueEvaluatorForTestingPurpose):
+    def __init__(self):
+        super().__init__("multiStep", [BuiltinConcepts.EVALUATION, BuiltinConcepts.BEFORE_EVALUATION], 10)
+
+
+class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_reduce_foobar", 10)
+
+    def matches(self, context, return_values):
+        super().matches(context, return_values)
+        keys = [c.body.key for c in return_values]
+        return "foo" in keys and "bar" in keys
+
+    def eval(self, context, return_values):
+        super().eval(context, return_values)
+        ret = BaseTest.tretval(context.sheerka, context.sheerka.new(BuiltinConcepts.SUCCESS))
+        ret.parents = return_values
+        return ret
+
+
+class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority):
+
+    def __init__(self):
+        super().__init__("suppress", 100)
+
+    def matches(self, context, return_values):
+        super().matches(context, return_values)
+        return True
+
+    def eval(self, context, return_values):
+        super().eval(context, return_values)
+        foo = None
+        for ret in return_values:
+            if ret.body.name == "foo":
+                foo = ret
+
+        if foo:
+            return context.sheerka.ret(self.name, False, Concept("does not matter"), parents=[foo])
+        else:
+            return None
+
+
+class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
+
+    def test_that_return_values_is_unchanged_when_no_evaluator(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = []
+
+        entries = self.tretval(sheerka, Concept("foo"))
+        return_values = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert return_values == [entries]
+
+    def test_i_can_use_a_list_as_input(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = []
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        return_values = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert return_values == entries
+
+    def test_step_concept_is_removed_after_processing_if_not_reduced(self):
+        """
+        No evaluator
+        """
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = []
+
+        entry = self.tretval(sheerka, Concept("foo"))
+        return_values = sheerka.execute(self.get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
+
+        assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
+
+    def test_step_concept_is_removed_after_processing_if_not_reduced_2(self):
+        """
+        This time the entry is modified by an evaluator,
+        nevertheless, step concept is removed
+        """
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+        entry = self.tretval(sheerka, Concept("foo"))
+        return_values = sheerka.execute(self.get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
+
+        assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
+
+    def test_that_higher_priority_evaluators_are_evaluated_first(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [
+            EvaluatorAllWithPriority10,
+            EvaluatorOneWithPriority20,
+            EvaluatorAllWithPriority15,
+            EvaluatorOneWithPriority10,
+            EvaluatorOneWithPriority15,
+            EvaluatorAllWithPriority20, ]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__EVALUATION [0] priority20 - matches - target=foo',
+            '__EVALUATION [0] priority20 - eval - target=foo',
+            '__EVALUATION [0] priority20 - matches - target=__EVALUATION',
+            '__EVALUATION [0] priority20 - eval - target=__EVALUATION',
+            "__EVALUATION [0] all_priority20 - matches - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority20 - eval - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority15 - matches - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority15 - eval - target=['foo', '__EVALUATION']",
+            '__EVALUATION [0] priority15 - matches - target=foo',
+            '__EVALUATION [0] priority15 - eval - target=foo',
+            '__EVALUATION [0] priority15 - matches - target=__EVALUATION',
+            '__EVALUATION [0] priority15 - eval - target=__EVALUATION',
+            "__EVALUATION [0] all_priority10 - matches - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority10 - eval - target=['foo', '__EVALUATION']",
+            '__EVALUATION [0] priority10 - matches - target=foo',
+            '__EVALUATION [0] priority10 - eval - target=foo',
+            '__EVALUATION [0] priority10 - matches - target=__EVALUATION',
+            '__EVALUATION [0] priority10 - eval - target=__EVALUATION'
+        ]
+
+    def test_that_predicate_is_checked_before_evaluation_for_one_return(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+        entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__EVALUATION [0] modifyFoo - matches - target=foo',
+            '__EVALUATION [0] modifyFoo - eval - target=foo',
+            '__EVALUATION [0] modifyFoo - matches - target=baz',
+            '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyFoo - matches - target=bar',
+            '__EVALUATION [1] modifyFoo - matches - target=baz',
+            '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
+        ]
+
+    def test_that_predicate_is_checked_before_evaluation_for_all_return(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorAllReduceFooBar]
+
+        entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', 'bar', '__EVALUATION']",
+            "__EVALUATION [0] all_reduce_foobar - eval - target=['foo', 'bar', '__EVALUATION']",
+            "__EVALUATION [1] all_reduce_foobar - matches - target=['__SUCCESS']"
+        ]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', '__EVALUATION']"
+        ]
+
+    def test_evaluation_continue_until_no_more_modification(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar]
+
+        entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__EVALUATION [0] modifyFoo - matches - target=foo',
+            '__EVALUATION [0] modifyFoo - eval - target=foo',
+            '__EVALUATION [0] modifyFoo - matches - target=baz',
+            '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [0] modifyBar - matches - target=foo',
+            '__EVALUATION [0] modifyBar - matches - target=baz',
+            '__EVALUATION [0] modifyBar - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyFoo - matches - target=bar',
+            '__EVALUATION [1] modifyFoo - matches - target=baz',
+            '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyBar - matches - target=bar',
+            '__EVALUATION [1] modifyBar - eval - target=bar',
+            '__EVALUATION [1] modifyBar - matches - target=baz',
+            '__EVALUATION [1] modifyBar - matches - target=__EVALUATION',
+            '__EVALUATION [2] modifyFoo - matches - target=baz',
+            '__EVALUATION [2] modifyFoo - matches - target=baz',
+            '__EVALUATION [2] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [2] modifyBar - matches - target=baz',
+            '__EVALUATION [2] modifyBar - matches - target=baz',
+            '__EVALUATION [2] modifyBar - matches - target=__EVALUATION'
+        ]
+
+    def test_evaluation_steps_are_respected(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION])
+
+        assert Out.debug_out == [
+            '__BEFORE_EVALUATION [0] preEval - matches - target=foo',
+            '__BEFORE_EVALUATION [0] preEval - eval - target=foo',
+            '__BEFORE_EVALUATION [0] preEval - matches - target=__BEFORE_EVALUATION',
+            '__BEFORE_EVALUATION [0] preEval - eval - target=__BEFORE_EVALUATION']
+
+    def test_evaluation_multi_steps_are_respected(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneMultiSteps]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries,
+                        [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__BEFORE_EVALUATION [0] multiStep - matches - target=foo',
+            '__BEFORE_EVALUATION [0] multiStep - eval - target=foo',
+            '__BEFORE_EVALUATION [0] multiStep - matches - target=__BEFORE_EVALUATION',
+            '__BEFORE_EVALUATION [0] multiStep - eval - target=__BEFORE_EVALUATION',
+            '__EVALUATION [0] multiStep - matches - target=foo',
+            '__EVALUATION [0] multiStep - eval - target=foo',
+            '__EVALUATION [0] multiStep - matches - target=__EVALUATION',
+            '__EVALUATION [0] multiStep - eval - target=__EVALUATION'
+        ]
+
+    def test_evaluators_can_be_pre_processed(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+
+        # disable evaluator
+        context = self.get_context(sheerka)
+        context.add_preprocess(EvaluatorOneModifyFoo().name, enabled=False)  # disabled  for this exec context
+        Out.debug_out = []
+        sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == []
+
+        # other contextes are not impacted
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            '__EVALUATION [0] modifyFoo - matches - target=foo',
+            '__EVALUATION [0] modifyFoo - eval - target=foo',
+            '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyFoo - matches - target=bar',
+            '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
+        ]

tests/core/test_sheerka_call_parsers.py

@@ -0,0 +1,363 @@
+from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept
+from parsers.BaseParser import BaseParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_ret_val(text, who="who"):
+    return ReturnValueConcept(who, True, UserInputConcept(text, "user_name"))
+
+
+class BaseTestParser(BaseParser):
+    debug_out = []
+
+    def __init__(self, name, priority, status=None, parser_result=None):
+        super().__init__(name, priority)
+        self.status = status
+        self.parser_result = parser_result
+
+    @staticmethod
+    def _get_name(name):
+        return name[8:] if name.startswith("parsers.") else name
+
+    @staticmethod
+    def _get_source(text_):
+        return text_ if isinstance(text_, str) else text_.body
+
+    def _out(self, name, priority, status, source):
+        debug = f"name={name}"
+        debug += f", priority={priority}"
+        debug += f", status={status}"
+        debug += f", source={source}"
+        self.debug_out.append(debug)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        parser_result = ParserResultConcept(parser=self, value=value)
+        return ReturnValueConcept(self, self.status, self.parser_result or parser_result)
+
+
+class Enabled90FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled90False", 90, False)
+
+
+class Enabled80FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled80False", 80, False)
+
+
+class Enabled80MultipleFalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled80MultipleFalse", 80, False)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
+        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        return [
+            ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)),
+            ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)),
+        ]
+
+
+class Enabled80MultipleTrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled80MultipleTrue", 80)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
+        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        return [
+            ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)),
+            ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)),
+        ]
+
+
+class Enabled70FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled70False", 70, False, "Not a ParserResult")
+
+
+class Enabled50TrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled50True", 50, True)
+
+    def parse(self, context, text):
+        source = self._get_source(text)
+        status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world"
+        self._out(self._get_name(self.name), self.priority, status, source)
+
+        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        return_value = ParserResultConcept(parser=self, value=value)
+        return ReturnValueConcept(self, status, return_value)
+
+
+class Enabled50bisTrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled50BisTrue", 50, True)
+
+
+class Enabled50FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled50False", 50, False)
+
+
+class Enabled10TrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled10True", 10, True)
+
+
+class DisabledParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Disabled", 90, True)
+        self.enabled = False
+
+
+class NoneParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("None", 90, True, None)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        return None
+
+
+class ListOfNoneParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("ListOfNone", 90, True, None)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        return [None, None]
+
+
+class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
+
+    def test_disabled_parsers_are_not_executed(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled": Enabled10TrueParser,
+            "Disabled": DisabledParser
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == ['name=Enabled10True, priority=10, status=True, source=hello world']
+
+    def test_parser_are_executed_by_priority(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled90False, priority=90, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
+        ]
+
+    def test_parsing_stop_at_the_first_success(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50bisTrue": Enabled50bisTrueParser,
+            "Enabled10True": Enabled10TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
+        ]
+
+    def test_parsing_stop_at_the_first_success_2(self):
+        """
+        Same test than before, but Enabled50True takes more time to find a match
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+            "Enabled10True": Enabled10TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled90False, priority=90, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
+        ]
+
+    def test_all_parsers_of_a_given_priority_are_executed(self):
+        """
+        Make sure that all parsers with priority 50 are executed
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+            "Enabled50bisTrue": Enabled50bisTrueParser,
+            "Enabled50False": Enabled50FalseParser,
+            "Enabled10True": Enabled10TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled90False, priority=90, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
+            'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
+            'name=Enabled50False, priority=50, status=False, source=hello world',
+            'name=Enabled50False, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50False, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50False, priority=50, status=False, source=Enabled80False:Enabled90False:hello world',
+        ]
+
+    def test_a_parser_has_access_to_the_output_of_its_predecessors(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+        res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
+        assert res_as_tuple == [
+            ('Enabled90False', False, 'Enabled90False:hello world'),
+            ('Enabled80False', False, 'Enabled80False:hello world'),
+            ('Enabled80False', False, 'Enabled80False:Enabled90False:hello world'),
+            ('Enabled50True', False, 'Enabled50True:hello world'),
+            ('Enabled50True', False, 'Enabled50True:Enabled90False:hello world'),
+            ('Enabled50True', False, 'Enabled50True:Enabled80False:hello world'),
+            ('Enabled50True', True, 'Enabled50True:Enabled80False:Enabled90False:hello world'),
+        ]
+
+    def test_none_return_values_are_discarded(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "NoneParser": NoneParser,
+            "ListOfNone": ListOfNoneParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+
+        BaseTestParser.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+        assert res == []
+        assert BaseTestParser.debug_out == [
+            'name=None, priority=90, status=True, source=hello world',
+            'name=ListOfNone, priority=90, status=True, source=hello world'
+        ]
+
+    def test_following_priorities_can_only_see_parser_result_return_values(self):
+        """
+        Normally, lower priority parsers can see the result of the higher priority parsers
+        This is true only if the higher priority parser return a ParserResultConcept
+        :return:
+        """
+
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled70False": Enabled70FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled70False, priority=70, status=False, source=hello world',
+            'name=Enabled70False, priority=70, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+        ]
+
+    def test_i_can_manage_parser_with_multiple_results(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80MultipleFalse": Enabled80MultipleFalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80MultipleFalse, priority=80, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_1',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_2',
+        ]
+
+        res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
+        assert res_as_tuple == [
+            ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_1'),
+            ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_2'),
+            ('Enabled50True', False, 'Enabled50True:hello world'),
+            ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_1'),
+            ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'),
+        ]
+
+    def test_i_can_manage_parser_with_multiple_results_and_a_sucess(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80MultipleTrue": Enabled80MultipleTrueParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80MultipleTrue, priority=80, status=None, source=hello world',
+        ]
+
+        res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
+        assert res_as_tuple == [
+            ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'),
+            ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'),
+        ]

tests/core/test_sheerka_transform.py

@@ -0,0 +1,318 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY
+from sdp.sheerkaDataProvider import Event
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaTransform(TestUsingMemoryBasedSheerka):
+    def test_i_can_transform_an_unknown_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo", body="body")
+        concept_with_sub = Concept("concept_with_sub", body=foo)
+
+        concept = Concept(
+            name="concept_name",
+            is_builtin=True,
+            is_unique=True,
+            key="concept_key",
+            body=concept_with_sub,
+            where=[foo, 1, "1", True, 1.0],
+            pre=foo,
+            post=None,  # will not appear
+            definition="it is a definition",
+            definition_type="def type",
+            desc="this this the desc"
+        ).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub")
+
+        # add values and props
+        concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init()
+        concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0]
+        concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init()
+        concept.values[ConceptParts.POST] = "a value for POST"
+        concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub)
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(concept)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            'meta.name': 'concept_name',
+            'meta.key': 'concept_key',
+            'meta.is_builtin': True,
+            'meta.is_unique': True,
+            'meta.definition': 'it is a definition',
+            'meta.definition_type': 'def type',
+            'meta.desc': 'this this the desc',
+            'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                            OBJ_ID_KEY: 1,
+                            'meta.body': 'body',
+                            'meta.name': 'foo'}, 1, '1', True, 1.0],
+            'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1},
+            'meta.body': {
+                '__type__': SheerkaTransformType.Concept,
+                '__id__': 2,
+                'meta.name': 'concept_with_sub',
+                'meta.body': {
+                    '__type__': SheerkaTransformType.Reference,
+                    '__id__': 1}},
+            'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']],
+            'pre': {'__type__': SheerkaTransformType.Concept,
+                    '__id__': 4,
+                    'meta.body': 'body',
+                    'meta.name': 'foo',
+                    'body': 'body'},
+            'post': "a value for POST",
+            'body': {'__type__': SheerkaTransformType.Concept,
+                     '__id__': 3,
+                     'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference},
+                     'meta.name': 'concept_with_sub',
+                     'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}},
+            'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0],
+            'props': [('a', 10),
+                      ('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}),
+                      ('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})],
+        }
+
+    def test_i_can_transform_unknown_concept_with_almost_same_value(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo")
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(concept)
+
+        assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'}
+
+    def test_i_can_transform_known_concept_when_the_values_are_the_same(self):
+        """
+        Values are the same means that we are serializing a concept which has kept all its default values
+        There is not diff between the concept to serialize and the one which was registered with create_new_concept()
+        We serialize only the id of the concept
+        :return:
+        """
+        sheerka = self.get_sheerka()
+
+        concept = Concept(
+            name="concept_name",
+            is_builtin=True,
+            is_unique=False,
+            key="concept_key",
+            body="body definition",
+            where="where definition",
+            pre="pre definition",
+            post="post definition",
+            definition="it is a definition",
+            definition_type="def type",
+            desc="this this the desc"
+        ).def_prop("a").def_prop("b")
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new_concept = sheerka.new(concept.key)
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(new_concept)
+
+        assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"}
+
+    def test_i_can_transform_known_concept_when_the_values_are_different(self):
+        """
+        Values are the different means the concept was modified.
+        It's different from the one which was registered with create_new_concept()
+        We serialize only the differences
+        :return:
+        """
+        sheerka = self.get_sheerka()
+
+        concept = Concept(
+            name="concept_name",
+            is_builtin=True,
+            is_unique=False,
+            key="concept_key",
+            body="body definition",
+            where="where definition",
+            pre="pre definition",
+            post="post definition",
+            definition="it is a definition",
+            definition_type="def type",
+            desc="this this the desc"
+        ).def_prop("a").def_prop("b")
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre")
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(new_concept)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            "id": "1001",
+            'pre': 'another pre',
+            "body": "another",
+            'props': [('a', 10)]
+        }
+
+    def test_i_can_transform_concept_with_circular_reference(self):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo")
+        bar = Concept("bar", body=foo)
+        foo.metadata.body = bar
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(foo)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            'meta.name': 'foo',
+            'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                          OBJ_ID_KEY: 1,
+                          'meta.name': 'bar',
+                          'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                                        OBJ_ID_KEY: 0},
+                          },
+        }
+
+    def test_i_can_transform_concept_with_circular_reference_2(self):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo")
+        bar = Concept("foo", body=foo)
+        foo.metadata.body = bar
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(foo)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            'meta.name': 'foo',
+            'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                          OBJ_ID_KEY: 1,
+                          'meta.name': 'foo',
+                          'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                                        OBJ_ID_KEY: 0},
+                          },
+        }
+
+    def test_i_can_transform_the_unknown_concept(self):
+        sheerka = self.get_sheerka(False)
+
+        unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(unknown)
+
+        assert len(to_dict) == 3
+        assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+        assert to_dict[OBJ_ID_KEY] == 0
+        assert "id" in to_dict
+
+    def test_i_can_transform_simple_execution_context(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(context)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
+            OBJ_ID_KEY: 0,
+            '_parent': None,
+            '_id': 0,
+            '_tab': '',
+            '_bag': {},
+            '_start': 0,
+            '_stop': 0,
+            'who': 'requester',
+            'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'},
+            'desc': 'this is the desc',
+            'children': [],
+            'preprocess': None,
+            'inputs': {},
+            'values': {},
+            'obj': None,
+            'concepts': {}
+        }
+
+    def test_i_can_transform_list(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict([context])
+
+        assert len(to_dict) == 1
+        assert isinstance(to_dict, list)
+        assert to_dict[0]["who"] == "requester"
+        assert to_dict[0]["desc"] == "this is the desc"
+
+    def test_i_can_transform_set(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict({context})
+
+        assert len(to_dict) == 1
+        assert isinstance(to_dict, list)
+        assert to_dict[0]["who"] == "requester"
+        assert to_dict[0]["desc"] == "this is the desc"
+
+    def test_i_can_transform_dict(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+        known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key()
+        sheerka.create_new_concept(self.get_context(sheerka), known_concept)
+        unknown_concept = Concept("bar")
+        known = sheerka.new("foo")
+
+        bag = {
+            "context": context,
+            "known_concept": known_concept,
+            "unknown_concept": unknown_concept,
+            "True": True,
+            "Number": 1.1,
+            "String": "a string value",
+            "None": None,
+            unknown_concept: "hello",
+            known: "world"
+        }
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(bag)
+
+        assert isinstance(to_dict, dict)
+        assert to_dict['Number'] == 1.1
+        assert to_dict['String'] == 'a string value'
+        assert to_dict['True']
+        assert to_dict['None'] is None
+        assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+        assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+        assert to_dict["known_concept"]["id"] == '1001'
+        assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+        assert to_dict["(None)bar"] == "hello"
+        assert to_dict["(1001)foo"] == "world"
+
+    def test_i_can_transform_when_circular_references(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+        context.push("another requester", "another desc")
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(context)
+
+        assert isinstance(to_dict, dict)
+        assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+        assert len(to_dict["children"]) == 1
+        assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+        assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
+        assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0
+        assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
+        assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1

tests/evaluators/test_AddConceptEvaluator.py

@@ -0,0 +1,155 @@
+import ast
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import VARIABLE_PREFIX, Concept
+from core.tokenizer import Tokenizer
+from evaluators.AddConceptEvaluator import AddConceptEvaluator
+from parsers.BaseParser import BaseParser
+from parsers.ConceptLexerParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
+from parsers.BnfParser import BnfParser
+from parsers.DefaultParser import DefConceptNode, NameNode
+from parsers.PythonParser import PythonNode, PythonParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
+
+    @staticmethod
+    def get_concept_part(part):
+        if isinstance(part, str):
+            node = PythonNode(part, ast.parse(part, mode="eval"))
+            return ReturnValueConcept(
+                who="parsers.Default",
+                status=True,
+                value=ParserResultConcept(
+                    source=part,
+                    parser=PythonParser(),
+                    value=node))
+
+        if isinstance(part, PythonNode):
+            return ReturnValueConcept(
+                who="parsers.Default",
+                status=True,
+                value=ParserResultConcept(
+                    source=part.source,
+                    parser=PythonParser(),
+                    value=part))
+
+        if isinstance(part, ReturnValueConcept):
+            return part
+
+    @staticmethod
+    def get_return_value(source, parsing_expression):
+        return ReturnValueConcept(
+            who="Parsers:RegexParser",
+            status=True,
+            value=ParserResultConcept(
+                source=source,
+                parser=BnfParser(),
+                value=parsing_expression
+            )
+        )
+
+    def get_def_concept(self, name, where=None, pre=None, post=None, body=None, definition=None):
+        concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
+
+        if body:
+            concept.body = self.get_concept_part(body)
+        if where:
+            concept.where = self.get_concept_part(where)
+        if pre:
+            concept.pre = self.get_concept_part(pre)
+        if post:
+            concept.post = self.get_concept_part(post)
+        if definition:
+            concept.definition = definition
+
+        return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=concept))
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))),
+            True),
+        (ReturnValueConcept(BaseParser.PREFIX + "some_name", False, ParserResultConcept(value=DefConceptNode([]))),
+         False),
+        (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not a ParserResultConcept"), False),
+        (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept()), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert AddConceptEvaluator().matches(context, ret_val) == expected
+
+    def test_that_the_source_is_correctly_set(self):
+        context = self.get_context()
+        def_concept_return_value = self.get_def_concept(
+            name="hello a",
+            definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
+            where="isinstance(a, str )",
+            pre="a is not None",
+            body="print('hello' + a)")
+
+        evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
+
+        assert evaluated.status
+        assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
+
+        created_concept = evaluated.body.body
+        assert created_concept.metadata.name == "hello a"
+        assert created_concept.metadata.where == "isinstance(a, str )"
+        assert created_concept.metadata.pre == "a is not None"
+        assert created_concept.metadata.post is None
+        assert created_concept.metadata.body == "print('hello' + a)"
+        assert created_concept.metadata.definition == "hello a"
+
+    def test_that_the_new_concept_is_correctly_saved_in_db(self):
+        context = self.get_context()
+        def_concept_return_value = self.get_def_concept(
+            name="hello a",
+            definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
+            where="isinstance(a, str )",
+            pre="a is not None",
+            body="print('hello' + a)")
+
+        # sanity. Make sure that the concept does not already exist
+        from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0")
+        assert context.sheerka.isinstance(from_db, BuiltinConcepts.UNKNOWN_CONCEPT)
+
+        AddConceptEvaluator().eval(context, def_concept_return_value)
+        context.sheerka.concepts_cache = {}  # reset cache
+        from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0")
+
+        assert from_db.metadata.key == f"hello {VARIABLE_PREFIX}0"
+        assert from_db.metadata.name == "hello a"
+        assert from_db.metadata.where == "isinstance(a, str )"
+        assert from_db.metadata.pre == "a is not None"
+        assert from_db.metadata.post is None
+        assert from_db.metadata.body == "print('hello' + a)"
+        assert from_db.metadata.definition == "hello a"
+        assert len(from_db.metadata.props) == 1
+        assert from_db.metadata.props[0] == ("a", None)
+        assert "a" in from_db.props
+
+        assert from_db.compiled == {}  # ast is not saved in db
+
+    def test_i_can_get_props_from_python_node(self):
+        ret_val = self.get_concept_part("isinstance(a, str)")
+        context = self.get_context()
+
+        assert AddConceptEvaluator.get_props(context.sheerka, ret_val, ["a"]) == ["a"]
+
+    def test_i_can_get_props_from_another_concept(self):
+        concept = Concept("hello").def_prop("a").def_prop("b")
+        ret_val = ReturnValueConcept(who="some_parser",
+                                     status=True,
+                                     value=ParserResultConcept(value=concept))
+
+        assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["a", "b"]
+
+    def test_i_can_get_props_from_definition(self):
+        parsing_expression = Sequence(ConceptExpression('mult'),
+                                      ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
+        ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression)
+
+        assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["add", "mult"]

tests/evaluators/test_AddConceptInSetEvaluator.py

@@ -0,0 +1,102 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Tokenizer
+from evaluators.AddConceptInSetEvaluator import AddConceptInSetEvaluator
+from parsers.DefaultParser import IsaConceptNode, NameNode
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_ret_val(concept_name, concept_set_name):
+    n1 = NameNode(list(Tokenizer(concept_name)))
+    n2 = NameNode(list(Tokenizer(concept_set_name)))
+
+    return ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([], n1, n2)))
+
+
+class TestAssConceptInSetEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([]))), True),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=IsaConceptNode([]))), False),
+        (ReturnValueConcept("some_name", True, "not a ParserResultConcept"), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept()), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert AddConceptInSetEvaluator().matches(context, ret_val) == expected
+
+    def test_i_cannot_add_if_the_concept_does_not_exists(self):
+        context = self.get_context()
+
+        ret_val = get_ret_val("foo", "bar")
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == "foo"
+
+    def test_i_cannot_add_if_the_set_does_not_exists(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        context.sheerka.set_id_if_needed(foo, False)
+        context.sheerka.add_in_cache(foo)
+
+        ret_val = get_ret_val("foo", "bar")
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == "bar"
+
+    def test_i_can_add_concept_to_a_set_of_concept(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        context.sheerka.set_id_if_needed(foo, False)
+        context.sheerka.add_in_cache(foo)
+
+        bar = Concept("bar")
+        context.sheerka.set_id_if_needed(bar, False)
+        context.sheerka.add_in_cache(bar)
+
+        ret_val = get_ret_val("foo", "bar")
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
+
+    def test_i_can_add_concept_with_a_body_to_a_set_of_concept(self):
+        context = self.get_context()
+        foo = Concept("foo", body="1")
+        context.sheerka.set_id_if_needed(foo, False)
+        context.sheerka.add_in_cache(foo)
+
+        bar = Concept("bar")
+        context.sheerka.set_id_if_needed(bar, False)
+        context.sheerka.add_in_cache(bar)
+
+        ret_val = get_ret_val("foo", "bar")
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
+
+    def test_i_cannot_add_the_same_concept_twice(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        context.sheerka.set_id_if_needed(foo, False)
+        context.sheerka.add_in_cache(foo)
+
+        bar = Concept("bar")
+        context.sheerka.set_id_if_needed(bar, False)
+        context.sheerka.add_in_cache(bar)
+
+        ret_val = get_ret_val("foo", "bar")
+        AddConceptInSetEvaluator().eval(context, ret_val)
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_IN_SET)
+        assert res.value.concept == foo
+        assert res.value.concept_set == bar

tests/evaluators/test_ConceptEvaluator.py

@@ -0,0 +1,117 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from evaluators.ConceptEvaluator import ConceptEvaluator
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=Concept())), True),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=Concept())), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept")), False),
+        (ReturnValueConcept("some_name", True, Concept()), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert ConceptEvaluator().matches(context, ret_val) == expected
+
+    def test_i_can_evaluate_concept(self):
+        context = self.get_context()
+        concept = Concept(name="foo",
+                          where="1",
+                          pre="2",
+                          post="3").def_prop("a", "4").def_prop("b", "5")
+
+        evaluator = ConceptEvaluator()
+        item = self.pretval(concept)
+        result = evaluator.eval(context, item)
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert result.value.name == "foo"
+        assert result.value.get_metadata_value(ConceptParts.WHERE) == 1
+        assert result.value.get_metadata_value(ConceptParts.PRE) == 2
+        assert result.value.get_metadata_value(ConceptParts.POST) == 3
+        assert result.value.get_prop("a") == 4
+        assert result.value.get_prop("b") == 5
+        assert result.value.key == "foo"
+        assert result.parents == [item]
+
+    def test_body_is_returned_when_defined_and_requested(self):
+        context = self.get_context()
+        concept = Concept(name="foo",
+                          body="'I have a value'",
+                          where="1",
+                          pre="2",
+                          post="3").set_prop("a", "4").set_prop("b", "5")
+
+        evaluator = ConceptEvaluator(return_body=True)
+        item = self.pretval(concept)
+        result = evaluator.eval(context, item)
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert result.value == "I have a value"
+        assert result.parents == [item]
+
+    def test_body_is_not_returned_if_not_requested(self):
+        context = self.get_context()
+        concept = Concept(name="foo",
+                          body="'I have a value'",
+                          where="1",
+                          pre="2",
+                          post="3").set_prop("a", "4").set_prop("b", "5")
+
+        evaluator = ConceptEvaluator(return_body=False)  # which is the default behaviour
+        item = self.pretval(concept)
+        result = evaluator.eval(context, item)
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert result.value == concept
+        assert result.parents == [item]
+
+    def test_i_can_eval_if_with_the_same_name_is_defined_in_the_context(self):
+        # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
+        # ConceptEvaluator will be called to resolve 'a' while we know that 'a' refers to the string 'property_a'
+
+        context = self.get_context()
+        context.obj = Concept("other").set_prop("foo", "'some_other_value'")
+        concept = Concept(name="foo")
+
+        item = self.pretval(concept)
+        result = ConceptEvaluator().eval(context, item)
+
+        assert result.status
+        assert result.value == "'some_other_value'"
+
+    def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(self):
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept(name="one").init_key())
+        concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b")
+                                                    .def_prop("a", "one")
+                                                    .def_prop("b", "two").init_key())
+
+        evaluator = ConceptEvaluator()
+        item = self.pretval(concept_plus)
+        result = evaluator.eval(context, item)
+
+        assert not result.status
+        assert context.sheerka.isinstance(result.value, BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        assert result.value.concept == concept_plus
+        assert result.value.property_name == "b"
+        assert context.sheerka.isinstance(result.value.error, BuiltinConcepts.TOO_MANY_ERRORS)
+
+    def test_that_error_concepts_are_transformed_into_errors_only_if_the_key_is_different(self):
+        context = self.get_context()
+
+        error_concept = context.sheerka.new(BuiltinConcepts.ERROR)
+        item = self.pretval(error_concept)
+        result = ConceptEvaluator().eval(context, item)
+
+        assert not context.sheerka.is_success(error_concept)  # it's indeed an error
+        assert result.status
+        assert result.value == error_concept

tests/evaluators/test_EvalEvaluator.py

@@ -0,0 +1,72 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.EvalEvaluator import EvalEvaluator
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+eval_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.CONCEPT_EVAL_REQUESTED))
+
+
+def retval(obj, who="who", status=True):
+    """ret_val"""
+    return ReturnValueConcept(who, status, obj)
+
+
+class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
+    def test_i_can_match_and_eval(self):
+        context = self.get_context()
+
+        to_eval1 = ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init())
+        to_eval2 = ReturnValueConcept("some_name", True, Concept(name="3", body="also to eval").auto_init())
+
+        return_values = [
+            ReturnValueConcept("some_name", True, "not to eval"),
+            ReturnValueConcept("some_name", True, Concept(name="not to eval")),
+            ReturnValueConcept("some_name", False, Concept(name="1", body="'not to eval'")),
+            to_eval1,
+            to_eval2,
+            eval_requested
+        ]
+
+        evaluator = EvalEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert len(evaluated) == 2
+        assert evaluated[0].value == to_eval1.body.body
+        assert evaluated[0].parents == [to_eval1, eval_requested]
+
+        assert evaluated[1].value == to_eval2.body.body
+        assert evaluated[1].parents == [to_eval2, eval_requested]
+
+    @pytest.mark.parametrize("return_values, expected", [
+        ([retval(Concept("foo", body="bar")), eval_requested], True),
+        ([retval(Concept("status is false", body="bar"), status=False), eval_requested], True),
+        ([retval("string_value"), eval_requested], True),
+        ([retval(Concept("no body")), eval_requested], True),
+        ([retval(Concept("eval requested missing", body="bar"))], False),
+    ])
+    def test_i_cannot_match_if_eval_request_is_not_present(self, return_values, expected):
+        context = self.get_context()
+        assert EvalEvaluator().matches(context, return_values) == expected
+
+    def test_concept_eval_requested_is_reduced_when_nothing_to_reduce(self):
+        context = self.get_context()
+
+        return_values = [
+            ReturnValueConcept("some_name", True, "not to eval"),
+            ReturnValueConcept("some_name", True, Concept(name="not to eval")),
+            ReturnValueConcept("some_name", False, Concept(name="1", body="not to eval")),
+            eval_requested
+        ]
+
+        evaluator = EvalEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated == ReturnValueConcept(
+            "evaluators.Eval",
+            False,
+            context.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_REQUESTED))
+        assert evaluated.parents == [eval_requested]

tests/evaluators/test_LexerNodeEvaluator.py

@@ -0,0 +1,104 @@
+import ast
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import Concept, ConceptParts, DoNotResolve
+from evaluators.LexerNodeEvaluator import LexerNodeEvaluator
+from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, StrMatch, UnrecognizedTokensNode, SourceCodeNode
+from parsers.PythonParser import PythonNode
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka):
+
+    def from_parsing(self, context, grammar, expression):
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+
+        ret_val = parser.parse(context, expression)
+        assert ret_val.status
+        return ret_val
+
+    def from_fragments(self, *fragments):
+        nodes = []
+        for fragment in fragments:
+            if isinstance(fragment, str):
+                node = PythonNode(fragment, ast.parse(fragment.strip(), mode="eval"))
+                nodes.append(SourceCodeNode(node, 0, 0, [], fragment))
+            else:
+                nodes.append(ConceptNode(fragment, 0, 0, [], fragment.name))
+
+        return ReturnValueConcept("somme_name", True, ParserResultConcept(value=nodes))
+
+    def init(self, concept, grammar, text):
+        context = self.get_context()
+        if isinstance(concept, list):
+            for c in concept:
+                context.sheerka.add_in_cache(c)
+        else:
+            context.sheerka.add_in_cache(concept)
+        ret_val = self.from_parsing(context, grammar, text)
+        node = ret_val.value.value[0]
+
+        return context, node
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=SourceCodeNode(0, 0, []))), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=[UnrecognizedTokensNode(0, 0, [])])), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=UnrecognizedTokensNode(0, 0, []))), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=SourceCodeNode(0, 0, []))), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept node")), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=["Not a concept node"])), False),
+        (ReturnValueConcept("some_name", True, [ConceptNode(Concept(), 0, 0)]), False),
+        (ReturnValueConcept("some_name", True, ConceptNode(Concept(), 0, 0)), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert LexerNodeEvaluator().matches(context, ret_val) == expected
+
+    def test_concept_is_returned_when_only_one_in_the_list(self):
+        foo = Concept("foo")
+        context = self.get_context()
+        context.sheerka.add_in_cache(foo)
+
+        ret_val = self.from_parsing(context, {foo: StrMatch("foo")}, "foo")
+
+        evaluator = LexerNodeEvaluator()
+        result = evaluator.eval(context, ret_val)
+        wrapper = result.body
+        return_value = result.body.body
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.parser == evaluator
+        assert wrapper.source == "foo"
+        assert return_value == Concept("foo").init_key()
+        assert return_value.compiled[ConceptParts.BODY] == DoNotResolve("foo")
+        assert result.parents == [ret_val]
+
+    def test_concept_python_node_is_returned_when_source_code(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        ret_val = self.from_fragments(foo, " + 1")
+
+        evaluator = LexerNodeEvaluator()
+        result = evaluator.eval(context, ret_val)
+        wrapper = result.body
+        return_value = result.body.body
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.parser == evaluator
+        assert wrapper.source == "foo + 1"
+
+        assert return_value == PythonNode('foo + 1', ast.parse("__C__foo__C__ + 1", mode="eval"))
+        assert return_value.concepts == {"__C__foo__C__": foo}
+        assert result.parents == [ret_val]

tests/evaluators/test_MultipleSameSuccessEvaluator.py

@@ -0,0 +1,200 @@
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.BaseEvaluator import BaseEvaluator
+from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
+from parsers.BaseParser import BaseParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestMultipleSameSuccessEvaluator(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_match_and_eval(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == Concept(name="1", body="value").auto_init()  # the first concept is returned
+
+    def test_i_can_match_and_eval_when_no_body(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == Concept(name="1").auto_init()
+
+    def test_i_can_match_and_eval_when_value_is_not_a_concept(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == "value"
+
+    def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_concept_evaluator_first(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Concept", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="3", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == Concept(name="1", body="value").auto_init()  # the concept is returned, not the value
+
+    def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_python_evaluator_first(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="3", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == Concept(name="1", body="value").auto_init()  # the concept is returned, not the value
+
+    def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value2").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+        assert evaluator.eval(context, return_values) is None
+
+    def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail_when_no_body(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+        assert evaluator.eval(context, return_values) is None
+
+    def test_i_can_match_if_no_parser(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        assert MultipleSameSuccessEvaluator().matches(context, return_values)
+
+    def test_i_cannot_match_if_not_reduced_requested(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, Concept(name="2", body="value")),
+            # ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        assert not MultipleSameSuccessEvaluator().matches(context, return_values)
+
+    def test_i_cannot_match_if_only_one_successful_evaluator(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            # ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, Concept(name="2", body="value")),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        assert not MultipleSameSuccessEvaluator().matches(context, return_values)
+
+    def test_i_cannot_match_if_at_least_one_parser_is_successful(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", True, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        assert not MultipleSameSuccessEvaluator().matches(context, return_values)

tests/evaluators/test_OneErrorEvaluator.py

@@ -0,0 +1,73 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.OneErrorEvaluator import OneErrorEvaluator
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def r(value, status=True):
+    return ReturnValueConcept(value, status, value)
+
+
+reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
+
+
+class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka):
+    @pytest.mark.parametrize("return_values, expected", [
+        ([r("evaluators.one error", False), reduce_requested], True),
+        ([r("evaluators.one error", False), r("failed", False), r("failed", False), reduce_requested], True),
+        ([r("evaluators.error", False), r("not a parser in success"), reduce_requested], True),
+        ([r("evaluators.no reduce required", False), r("failed", False), r("failed", False)], False),
+        ([r("evaluators.no reduce required", False)], False),
+        ([r("evaluators.error", False), r("evaluators.success"), reduce_requested], False),
+        ([r("evaluators.error", False), r("parsers.success"), reduce_requested], False),
+        ([r("evaluators.success"), r("not an evaluator in error", False), reduce_requested], False),
+        ([r("evaluators.error", False), r("evaluators.another error", False), reduce_requested], False),
+    ])
+    def test_i_can_match(self, return_values, expected):
+        context = self.get_context()
+        assert OneErrorEvaluator().matches(context, return_values) == expected
+
+    def test_i_can_eval(self):
+        context = self.get_context()
+
+        return_values = [
+            r("evaluators.one error", False),
+            r("parsers.failed", False),
+            r("parsers.failed", False),
+            reduce_requested
+        ]
+
+        evaluator = OneErrorEvaluator()
+        evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert not res.status
+        assert res.body == "evaluators.one error"
+        assert len(res.parents) == 4
+
+    def test_unwanted_return_values_are_not_eaten(self):
+        context = self.get_context()
+
+        a_successful_concept = r("successful concept")
+        a_concept_in_error = r("concept in error", False)
+        return_values = [
+            r("evaluators.one error", False),
+            r("parsers.failed", False),
+            r("parsers.failed", False),
+            a_successful_concept,
+            a_concept_in_error,
+            reduce_requested
+        ]
+
+        evaluator = OneErrorEvaluator()
+        evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert not res.status
+        assert res.body == "evaluators.one error"
+        assert len(res.parents) == 4
+
+        assert a_successful_concept not in res.parents

tests/evaluators/test_OneSuccessEvaluator.py

@@ -0,0 +1,72 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def r(value, status=True):
+    return ReturnValueConcept(value, status, value)
+
+
+reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
+
+
+class TestOneSuccessEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("return_values, expected", [
+        ([r("evaluators.one success"), reduce_requested], True),
+        ([r("evaluators.one success"), r("failed", False), r("failed", False), reduce_requested], True),
+        ([r("evaluators.no reduce required"), r("failed", False), r("failed", False)], False),
+        ([r("evaluators.no reduce required")], False),
+        ([r("evaluators.failed", False), r("failed", False), r("failed", False), reduce_requested], False),
+        ([r("evaluators.failed", False), r("not evaluator success"), reduce_requested], False),
+        ([r("evaluators.success"), r("evaluators.another success"), reduce_requested], False),
+        ([r("evaluators.one success"), r("other success"), r("failed", False), reduce_requested], True),
+        ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False),
+    ])
+    def test_i_can_match(self, return_values, expected):
+        context = self.get_context()
+        assert OneSuccessEvaluator().matches(context, return_values) == expected
+
+    def test_i_can_eval(self):
+        context = self.get_context()
+
+        return_values = [
+            r("evaluators.one success"),
+            r("failed", False),
+            r("failed", False),
+            reduce_requested
+        ]
+
+        evaluator = OneSuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert res.status
+        assert res.body == "evaluators.one success"
+        assert len(res.parents) == 4
+
+    def test_i_do_not_eat_the_other_success(self):
+        context = self.get_context()
+
+        not_a_parser_success = r("other success")
+        return_values = [
+            r("evaluators.one success"),
+            not_a_parser_success,
+            r("failed", False),
+            reduce_requested
+        ]
+
+        evaluator = OneSuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert res.status
+        assert res.body == "evaluators.one success"
+        assert len(res.parents) == 3
+
+        assert not_a_parser_success not in res.parents

tests/evaluators/test_PrepareEvalEvaluator.py

@@ -0,0 +1,49 @@
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
+from core.concept import Concept
+from evaluators.PrepareEvalEvaluator import PrepareEvalEvaluator
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+r = ReturnValueConcept
+
+
+class TestPrepareEvalEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (r("name", True, UserInputConcept("eval 1 + 1")), True),
+        (r("name", True, UserInputConcept("  eval 1 + 1")), True),
+        (r("name", True, UserInputConcept("eval")), False),
+        (r("name", True, UserInputConcept("1+1")), False),
+        (r("name", True, UserInputConcept("")), False),
+        (r("name", True, UserInputConcept("eval   ")), False),
+        (r("name", True, UserInputConcept([])), False),
+        (r("name", True, Concept("foo")), False),
+        (r("name", True, "not a concept"), False),
+        (r("name", False, UserInputConcept("eval 1 + 1")), False),
+        (r("name", False, UserInputConcept("  eval 1 + 1")), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert PrepareEvalEvaluator().matches(context, ret_val) == expected
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (r("name", True, UserInputConcept("eval 1 + 1")), "1 + 1"),
+        (r("name", True, UserInputConcept("   eval 1 + 1")), "1 + 1"),
+    ])
+    def test_i_can_eval(self, ret_val, expected):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        prepare_evaluator = PrepareEvalEvaluator()
+        prepare_evaluator.matches(context, ret_val)
+        res = prepare_evaluator.eval(context, ret_val)
+
+        assert len(res) == 2
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.USER_INPUT)
+        assert res[0].body.body == expected
+
+        assert res[1].status
+        assert sheerka.isinstance(res[1].body, BuiltinConcepts.CONCEPT_EVAL_REQUESTED)

tests/evaluators/test_PythonEvaluator.py

@@ -0,0 +1,138 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.PythonEvaluator import PythonEvaluator
+from parsers.PythonParser import PythonNode, PythonParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_context_name(context):
+    return context.name
+
+
+class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=PythonNode("", None))), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False),
+        (ReturnValueConcept("some_name", False, "not relevant"), False),
+        (ReturnValueConcept("some_name", True, Concept()), False)
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert PythonEvaluator().matches(context, ret_val) == expected
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1 + 1", 2),
+        ("sheerka.test()", "I have access to Sheerka !"),
+        ("a=10\na", 10),
+    ])
+    def test_i_can_eval(self, text, expected):
+        context = self.get_context()
+        parsed = PythonParser().parse(context, text)
+
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == expected
+
+    @pytest.mark.parametrize("concept", [
+        Concept("foo"),
+        Concept("foo", body="2"),
+        Concept("foo").set_prop("prop", "'a'"),
+        Concept("foo", body="bar")
+    ])
+    def test_i_cannot_eval_simple_concept(self, concept):
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo"))
+
+        parsed = PythonParser().parse(context, "foo")
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert not evaluated.status
+        assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.NOT_FOR_ME)
+
+    def test_i_can_eval_expression_with_that_references_concepts(self):
+        """
+        I can test modules with variables
+        :return:
+        """
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo", body="1"))
+
+        parsed = PythonParser().parse(context, "foo + 2")
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == 3
+
+    def test_i_can_eval_module_with_that_references_concepts(self):
+        """
+        I can test modules with variables
+        :return:
+        """
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo"))
+
+        parsed = PythonParser().parse(context, "def a(b):\n    return b\na(c:foo:)")
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == Concept("foo").init_key()
+
+    def test_i_can_eval_module_with_that_references_concepts_with_body(self):
+        """
+        I can test modules with variables
+        :return:
+        """
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo", body="2"))
+
+        parsed = PythonParser().parse(context, "def a(b):\n    return b\na(foo)")
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == 2
+
+    def test_i_can_eval_concept_token(self):
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo", body="2"))
+
+        parsed = PythonParser().parse(context, "get_context_name(c:foo:)")
+        python_evaluator = PythonEvaluator()
+        python_evaluator.locals["get_context_name"] = get_context_name
+        evaluated = python_evaluator.eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == "foo"
+
+        # sanity, does not work otherwise
+        parsed = PythonParser().parse(context, "get_context_name(foo)")
+        python_evaluator = PythonEvaluator()
+        python_evaluator.locals["get_context_name"] = get_context_name
+        evaluated = python_evaluator.eval(context, parsed)
+
+        assert not evaluated.status
+        assert evaluated.body.body.args[0] == "'int' object has no attribute 'name'"
+
+    @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [
+        ("__C__key__C__", "key", None, False),
+        ("__C__key__id__C__", "key", "id", False),
+        ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+        ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+    ])
+    def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept):
+        context = self.get_context()
+        assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept)
+
+    @pytest.mark.parametrize("text", [
+        "__C__",
+        "__C__key",
+        "__C__key____",
+        "__C____",
+        "__C__USE_CONCEPT__",
+    ])
+    def test_i_cannot_resolve_name(self, text):
+        context = self.get_context()
+        assert PythonEvaluator().resolve_name(context, text) is None

tests/evaluators/test_TooManySucessEvaluator.py

@@ -0,0 +1,105 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.TooManySuccessEvaluator import TooManySuccessEvaluator
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def r(name, status=True, value=None):
+    value = value or name
+    return ReturnValueConcept(name, status, value)
+
+
+reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
+
+
+class TestTooManySuccessEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("return_values, expected", [
+        ([r("evaluators.success1"), r("evaluators.success2"), reduce_requested], True),
+        ([r("evaluators.success1"), r("evaluators.success2"), r("other"), reduce_requested], True),
+        ([r("evaluators.success1"), r("evaluators.success2"), r("other", False), reduce_requested], True),
+        ([r("evaluators.a", value=Concept("c1", body="1")),
+          r("evaluators.a", value=Concept("c2", body="1")),
+          r("other"),
+          reduce_requested], True),
+        ([r("evaluators.a", value=Concept("c1", body="1")),
+          r("evaluators.a", value=Concept("c2", body="1")),
+          r("parsers.other", False),
+          reduce_requested], True),
+        ([r("evaluators.a", value=Concept("c1", body="1")),
+          r("evaluators.a", value=Concept("c2", body="1")),
+          r("parsers.other"),
+          reduce_requested], False),
+        ([r("evaluators.success1"), reduce_requested], False),
+        ([reduce_requested], False),
+        ([r("evaluators.success1"), r("evaluators.success2")], False),
+    ])
+    def test_i_can_match(self, return_values, expected):
+        context = self.get_context()
+        assert TooManySuccessEvaluator().matches(context, return_values) == expected
+
+    def test_i_can_eval(self):
+        context = self.get_context()
+
+        value1 = r("evaluators.a", value=Concept("c1", body="1"))
+        value2 = r("evaluators.a", value=Concept("c2", body="2"))
+        return_values = [
+            value1,
+            value2,
+            r("other", False),
+            reduce_requested
+        ]
+
+        evaluator = TooManySuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert not res.status
+        assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS)
+        assert res.body.body == [value1, value2]
+        assert len(res.parents) == 4
+
+    def test_i_can_eval_when_same_success(self):
+        context = self.get_context()
+
+        return_values = [
+            r("evaluators.a", value=Concept("c1", body="1").auto_init()),
+            r("evaluators.a", value=Concept("c2", body="1").auto_init()),
+            r("other", False),
+            reduce_requested
+        ]
+
+        evaluator = TooManySuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert res is None
+
+    def test_other_success_are_not_reduced(self):
+        context = self.get_context()
+
+        value1 = r("evaluators.a", value=Concept("c1", body="1").auto_init())
+        value2 = r("evaluators.a", value=Concept("c2", body="2").auto_init())
+        other_success = r("other")
+        return_values = [
+            value1,
+            value2,
+            other_success,
+            reduce_requested
+        ]
+
+        evaluator = TooManySuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert not res.status
+        assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS)
+        assert res.body.body == [value1, value2]
+        assert len(res.parents) == 3
+        assert other_success not in res.parents

tests/non_reg/test_sheerka_non_reg.py

@@ -0,0 +1,625 @@
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, Property, simplec
+from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
+from parsers.ConceptLexerParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class TestSheerkaNonReg(TestUsingFileBasedSheerka):
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1 + 1", 2),
+        ("sheerka.test()", 'I have access to Sheerka !')
+    ])
+    def test_i_can_eval_python_expressions_with_no_variable(self, text, expected):
+        sheerka = self.get_sheerka()
+
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == expected
+
+    def test_i_can_eval_concept_with_python_body(self):
+        sheerka = self.get_sheerka()
+        concept = Concept(name="one", body="1")
+        sheerka.add_in_cache(concept)
+
+        text = "one"
+        res = sheerka.evaluate_user_input(text)
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == simplec("one", 1)  # by default, the concept is returned
+
+    def test_i_can_eval_concept_with_concept_body(self):
+        sheerka = self.get_sheerka()
+        concept_one = Concept(name="one")
+        concept_un = Concept(name="un", body="one")
+        sheerka.add_in_cache(concept_one)
+        sheerka.add_in_cache(concept_un)
+
+        res = sheerka.evaluate_user_input("un")
+        return_value = res[0].value
+        assert len(res) == 1
+        assert res[0].status
+        assert return_value == simplec("un", simplec("one", None))
+
+    def test_i_can_eval_concept_with_no_body(self):
+        sheerka = self.get_sheerka()
+        concept = Concept(name="one")
+        sheerka.add_in_cache(concept)
+
+        text = "one"
+        res = sheerka.evaluate_user_input(text)
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == concept
+        assert id(res[0].value) != id(concept)
+
+    def test_is_unique_property_is_used_when_evaluating(self):
+        sheerka = self.get_sheerka()
+        concept = Concept(name="one", is_unique=True)
+        sheerka.add_in_cache(concept)
+
+        text = "one"
+        res = sheerka.evaluate_user_input(text)
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == concept
+        assert id(res[0].value) == id(concept)
+
+    def test_i_can_eval_def_concept_request(self):
+        text = """
+def concept a + b
+where isinstance(a, int) and isinstance(b, int)
+pre isinstance(a, int) and isinstance(b, int)
+post isinstance(res, int)
+as:
+    def func(x,y):
+        return x+y
+    func(a,b)
+        """
+
+        expected = self.get_default_concept()
+        expected.metadata.id = "1001"
+        expected.metadata.desc = None
+        expected.metadata.props = [("a", None), ("b", None)]
+        expected.init_key()
+
+        sheerka = self.get_sheerka()
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_saved = res[0].value.body
+
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
+
+        assert concept_saved.key in sheerka.cache_by_key
+        assert concept_saved.id in sheerka.cache_by_id
+        assert sheerka.sdp.io.exists(
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+
+    def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept(self):
+        """
+        In this test, we test that the properties of 'concept a xx b' (which are 'a' and 'b')
+        are correctly detected, thanks to the source code 'a plus b' in its body
+        :return:
+        """
+        sheerka = self.get_sheerka()
+
+        # concept 'a plus b' is known
+        concept_a_plus_b = Concept(name="a plus b").def_prop("a").def_prop("b")
+        sheerka.add_in_cache(concept_a_plus_b)
+
+        res = sheerka.evaluate_user_input("def concept a xx b as a plus b")
+        expected = Concept(name="a xx b", body="a plus b").def_prop("a").def_prop("b").init_key()
+        expected.metadata.id = "1001"
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_saved = res[0].value.body
+
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
+
+        assert concept_saved.key in sheerka.cache_by_key
+        assert concept_saved.id in sheerka.cache_by_id
+        assert sheerka.sdp.io.exists(
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+
+    def test_i_cannot_eval_the_same_def_concept_twice(self):
+        text = """
+def concept a + b
+where isinstance(a, int) and isinstance(b, int)
+pre isinstance(a, int) and isinstance(b, int)
+post isinstance(res, int)
+as:
+    def func(x,y):
+        return x+y
+    func(a,b)
+            """
+
+        sheerka = self.get_sheerka()
+        sheerka.evaluate_user_input(text)
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) == 1
+        assert not res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+
+    @pytest.mark.parametrize("text", [
+        "",
+        "   ",
+        "\n",
+    ])
+    def test_i_can_eval_a_empty_input(self, text):
+        sheerka = self.get_sheerka()
+
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.NOP)
+
+    def test_i_can_eval_concept_with_variable(self):
+        sheerka = self.get_sheerka()
+        concept_hello = Concept(name="hello a").def_prop("a")
+        concept_foo = Concept(name="foo")
+        sheerka.add_in_cache(concept_hello)
+        sheerka.add_in_cache(concept_foo)
+
+        res = sheerka.evaluate_user_input("hello foo")
+        return_value = res[0].value
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(return_value, concept_hello)
+        assert return_value.props["a"].value == concept_foo
+
+    def test_i_can_eval_concept_with_variable_and_python_as_body(self):
+        sheerka = self.get_sheerka()
+        hello_a = sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a"))
+        sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
+
+        res = sheerka.evaluate_user_input("hello foo")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, hello_a)
+        assert res[0].value.body == "hello foo"
+        assert res[0].value.metadata.is_evaluated
+        assert res[0].value.props["a"].value == simplec("foo", "foo")
+        assert res[0].value.props["a"].value.metadata.is_evaluated
+
+    def test_i_can_eval_duplicate_concepts_with_same_value(self):
+        sheerka = self.get_sheerka()
+
+        sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a"))
+        sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
+        sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
+
+        res = sheerka.evaluate_user_input("hello foo")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value.body == "hello foo"
+        assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
+
+    def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different(self):
+        sheerka = self.get_sheerka()
+
+        sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a"))
+        sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
+        sheerka.add_in_cache(Concept(name="foo", body="'another value'"))
+
+        res = sheerka.evaluate_user_input("hello foo")
+        assert len(res) == 1
+        assert not res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS)
+
+        concepts = res[0].value.body
+        assert len(concepts) == 2
+        sorted_values = sorted(concepts, key=lambda x: x.value.body)
+        assert sorted_values[0].value.body == "hello another value"
+        assert sorted_values[1].value.body == "hello foo"
+
+    def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+
+        sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").def_prop("a"))
+        sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").def_prop("b"))
+
+        res = sheerka.evaluate_user_input("hello 'foo'")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value.body == "hello foo"  # I don't know yet the one to choose
+        assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
+
+    def test_i_can_create_concepts_with_python_code_as_body(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+
+        sheerka.create_new_concept(context, Concept(name="concepts", body="sheerka.concepts()"))
+        res = sheerka.evaluate_user_input("concepts")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert isinstance(res[0].value.body, list)
+
+    def test_i_can_create_concept_with_bnf_definition(self):
+        sheerka = self.get_sheerka(False, False)
+        a = Concept("a")
+        sheerka.add_in_cache(a)
+        sheerka.concepts_definition_cache = {a: OrderedChoice("one", "two")}
+
+        res = sheerka.evaluate_user_input("def concept plus from bnf a ('plus' plus)?")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
+
+        saved_concept = sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, "plus")
+        assert saved_concept.key == "plus"
+        assert saved_concept.metadata.definition == "a ('plus' plus)?"
+        assert "a" in saved_concept.props
+        assert "plus" in saved_concept.props
+
+        saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY)
+        expected_bnf = Sequence(
+            a, Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus"))))
+        assert saved_definitions[saved_concept] == expected_bnf
+
+        new_concept = res[0].value.body
+        assert new_concept.metadata.name == "plus"
+        assert new_concept.metadata.definition == "a ('plus' plus)?"
+        assert new_concept.bnf == expected_bnf
+        assert "a" in new_concept.props
+        assert "plus" in new_concept.props
+
+    def test_i_can_eval_bnf_definitions(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
+
+        res = sheerka.evaluate_user_input("one")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, concept_a)
+
+    def test_i_can_eval_bnf_definitions_with_variables(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
+        concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
+
+        res = sheerka.evaluate_user_input("one three")
+
+        assert len(res) == 1
+        assert res[0].status
+        return_value = res[0].value
+
+        assert sheerka.isinstance(return_value, concept_b)
+        assert return_value.body == "one three"
+        assert return_value.metadata.is_evaluated
+
+        assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one").init_key())
+        assert return_value.props["a"].value.metadata.is_evaluated
+
+    def test_i_can_eval_bnf_definitions_from_separate_instances(self):
+        """
+        Same test then before,
+        but make sure that the BNF are correctly persisted and loaded
+        """
+        sheerka = self.get_sheerka(False)
+        concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' 'two'")[0].body.body
+
+        res = self.get_sheerka(False).evaluate_user_input("one two")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, concept_a)
+
+        # add another bnf definition
+        concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
+
+        res = self.get_sheerka(False).evaluate_user_input("one two")  # previous one still works
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, concept_a)
+
+        res = self.get_sheerka(False).evaluate_user_input("one two three")  # new one works
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, concept_b)
+        assert res[0].value.body == "one two three"
+        assert res[0].value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key())
+
+    @pytest.mark.parametrize("desc, definitions", [
+        ("Simple form", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit"
+        ]),
+        ("When twenty is a concept", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenty as 20",
+            "def concept twenties from bnf twenty (one|two)=unit as twenty + unit"
+        ]),
+        ("When digit is a concept", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenty as 20",
+            "def concept digit from bnf one|two",
+            "def concept twenties from bnf twenty digit as twenty + digit"
+        ]),
+        ("When using isa and concept twenty", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept number",
+            "one isa number",
+            "two isa number",
+            "def concept twenties from bnf 'twenty' number as 20 + number"
+        ]),
+        ("When using isa and concept twenty", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenty as 20",
+            "def concept number",
+            "one isa number",
+            "two isa number",
+            "def concept twenties from bnf twenty number as 20 + number"
+        ]),
+    ])
+    def test_i_can_mix_concept_with_python_to_define_numbers(self, desc, definitions):
+        sheerka = self.get_sheerka()
+
+        for definition in definitions:
+            sheerka.evaluate_user_input(definition)
+
+        res = sheerka.evaluate_user_input("twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, "twenties")
+        assert res[0].body.body == 21
+
+        res = sheerka.evaluate_user_input("twenty one + 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("twenty one + one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("twenty one + twenty two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 43
+
+        res = sheerka.evaluate_user_input("1 + twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("1 + 1 + twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 23
+
+    def test_i_can_mix_bnf_and_isa(self):
+        """
+        if 'one' isa 'number, twenty number should be recognized
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        sheerka.evaluate_user_input("def concept one as 1")
+        sheerka.evaluate_user_input("def concept two as 2")
+        sheerka.evaluate_user_input("def concept number")
+        sheerka.evaluate_user_input("one isa number")
+        sheerka.evaluate_user_input("two isa number")
+        sheerka.evaluate_user_input("def concept twenties from bnf 'twenty' number as 20 + number")
+
+        res = sheerka.evaluate_user_input("twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == simplec("twenties", 21)
+
+        res = sheerka.evaluate_user_input("twenty one + 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("twenty one + one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("twenty one + twenty two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 43
+
+        res = sheerka.evaluate_user_input("1 + twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("1 + 1 + twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 23
+
+    def test_i_can_mix_concept_of_concept(self):
+        sheerka = self.get_sheerka()
+
+        definitions = [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit",
+            "def concept a plus b as a + b"
+        ]
+
+        for definition in definitions:
+            sheerka.evaluate_user_input(definition)
+
+        res = sheerka.evaluate_user_input("1 plus 2")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 3
+
+        res = sheerka.evaluate_user_input("1 plus one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 2
+
+        res = sheerka.evaluate_user_input("1 + 1 plus 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 3
+
+        res = sheerka.evaluate_user_input("1 plus twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 22
+
+        res = sheerka.evaluate_user_input("one plus 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 2
+
+        res = sheerka.evaluate_user_input("one plus two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 3
+
+        res = sheerka.evaluate_user_input("one plus twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 22
+
+        res = sheerka.evaluate_user_input("twenty one plus 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 22
+
+        res = sheerka.evaluate_user_input("twenty one plus one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 22
+
+        res = sheerka.evaluate_user_input("twenty one plus twenty two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 43
+
+    @pytest.mark.xfail
+    def test_i_can_evaluate_concept_of_concept_when_multiple_choices(self):
+        sheerka = self.get_sheerka()
+
+        definitions = [
+            "def concept little a where a",
+            "def concept blue a where a",
+            "def concept little blue a where a",
+            "def concept house"
+        ]
+
+        for definition in definitions:
+            sheerka.evaluate_user_input(definition)
+
+        ### CAUTION ####
+        # this test cannot work !!
+        # it is just to hint the result that I would like to achieve
+
+        res = sheerka.evaluate_user_input("little blue house")
+        assert len(res) == 2
+        assert res[0].status
+        assert res[0].body == "little(blue(house))"
+
+        assert res[1].status
+        assert res[1].body == "little blue(house)"
+
+    def test_i_can_say_that_a_concept_isa_another_concept(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluate_user_input("def concept foo")
+        sheerka.evaluate_user_input("def concept bar")
+
+        res = sheerka.evaluate_user_input("foo isa bar")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
+
+    def test_eval_does_not_break_valid_result(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluate_user_input("def concept one as 1")
+        sheerka.evaluate_user_input("def concept two as 2")
+
+        res = sheerka.evaluate_user_input("one + two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 3
+
+        res = sheerka.evaluate_user_input("eval one + two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 3
+
+    @pytest.mark.parametrize("text", [
+        "'hello",
+        '"foo" + "string',
+        "c::",
+        "c:foo\nbar:",
+        "c:foo",
+        "def concept 'name",
+        "def concept name from bnf 'name"
+    ])
+    def test_i_can_manage_tokenizer_error(self, text):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept("foo"))
+
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) > 1
+        for r in [r for r in res if r.who.startswith("parsers.")]:
+            assert not r.status
+
+    def test_i_can_recognize_concept_from_string(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept("one", body="1"))
+
+        res = sheerka.evaluate_user_input("'one'")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == "one"
+
+        res = sheerka.evaluate_user_input("eval 'one'")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == "one"
+
+    @pytest.mark.parametrize("expression", [
+        "def concept twenties from bnf 'twenty' (one | two)=unit as 20 + unit",
+        "def concept twenties from bnf 'twenty' (one | two)=unit as twenty + unit",
+        "def concept twenties from bnf twenty (one | two)=unit as 20 + unit",
+        "def concept twenties from bnf twenty (one | two)=unit as twenty + unit",
+    ])
+    def test_i_can_evaluate_bnf_concepts(self, expression):
+        sheerka = self.get_sheerka()
+
+        sheerka.evaluate_user_input("def concept one as 1")
+        sheerka.evaluate_user_input("def concept two as 2")
+        sheerka.evaluate_user_input("def concept twenty as 20")
+        sheerka.evaluate_user_input(expression)
+        res = sheerka.evaluate_user_input("eval twenty one")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 21

tests/parsers/test_BaseParser.py

@@ -1,6 +1,6 @@
 import pytest
 
-from core.tokenizer import Tokenizer, Token, TokenKind
+from core.tokenizer import Tokenizer, TokenKind
 from parsers.BaseParser import BaseParser
 
 

tests/parsers/test_BnfParser.py

@@ -0,0 +1,176 @@
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Tokenizer, TokenKind, LexerError
+from parsers.BaseParser import UnexpectedTokenErrorNode
+from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
+from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
+    ConceptLexerParser, ConceptExpression, cnode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class ClassWithName:
+    def __init__(self, name):
+        self.name = name
+
+
+class TestBnfParser(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("expression, expected", [
+        ("'str'", StrMatch("str")),
+        ("1", StrMatch("1")),
+        ("  1", StrMatch("1")),
+        (",", StrMatch(",")),
+        ("'foo'?", Optional(StrMatch("foo"))),
+        ("'foo'*", ZeroOrMore(StrMatch("foo"))),
+        ("'foo'+", OneOrMore(StrMatch("foo"))),
+        ("1 | 2 | 3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))),
+        ("1|2|3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))),
+        ("1'|' 2 '|' 3", Sequence(StrMatch("1"), StrMatch("|"), StrMatch("2"), StrMatch("|"), StrMatch("3"))),
+        ("1 2 'foo'", Sequence(StrMatch("1"), StrMatch("2"), StrMatch("foo"))),
+        ("1 2 | 3 4+", OrderedChoice(
+            Sequence(StrMatch("1"), StrMatch("2")),
+            Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))),
+        (
+            "1 (2 | 3) 4+",
+            Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))),
+        ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))),
+        ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))),
+        ("1 *", Sequence(StrMatch("1"), StrMatch("*"))),
+        ("1 ?", Sequence(StrMatch("1"), StrMatch("?"))),
+        ("1 +", Sequence(StrMatch("1"), StrMatch("+"))),
+        ("(1|*) +", Sequence(OrderedChoice(StrMatch("1"), StrMatch("*")), StrMatch("+"))),
+        ("1, :&", Sequence(StrMatch("1"), StrMatch(","), StrMatch(":"), StrMatch("&"))),
+        ("(1 )", StrMatch("1")),
+        ("'str'=var", StrMatch("str", rule_name="var")),
+        ("'foo'?=var", Optional(StrMatch("foo"), rule_name="var")),
+        ("('foo'?)=var", Optional(StrMatch("foo"), rule_name="var")),
+        ("'foo'*=var", ZeroOrMore(StrMatch("foo"), rule_name="var")),
+        ("('foo'*)=var", ZeroOrMore(StrMatch("foo"), rule_name="var")),
+        ("'foo'+=var", OneOrMore(StrMatch("foo"), rule_name="var")),
+        ("('foo'+)=var", OneOrMore(StrMatch("foo"), rule_name="var")),
+        ("'foo'=var?", Optional(StrMatch("foo", rule_name="var"))),
+        ("('foo'=var)?", Optional(StrMatch("foo", rule_name="var"))),
+        ("'foo'=var*", ZeroOrMore(StrMatch("foo", rule_name="var"))),
+        ("('foo'=var)*", ZeroOrMore(StrMatch("foo", rule_name="var"))),
+        ("'foo'=var+", OneOrMore(StrMatch("foo", rule_name="var"))),
+        ("('foo'=var)+", OneOrMore(StrMatch("foo", rule_name="var"))),
+        ("(1 | 2 | 3)=var", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"), rule_name="var")),
+        ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")),
+        ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")),
+        ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))),
+    ])
+    def test_i_can_parse_regex(self, expression, expected):
+        parser = BnfParser()
+        res = parser.parse(self.get_context(), Tokenizer(expression))
+
+        assert not parser.has_error
+        assert res.status
+        assert res.value.value == expected
+        assert res.value.source == expression
+
+    @pytest.mark.parametrize("expression, expected", [
+        ("foo", Concept("foo").init_key()),
+        ("foo*", ZeroOrMore(Concept("foo").init_key())),
+        ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))),
+        ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))),
+        ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+        ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+    ])
+    def test_i_can_parse_regex_with_concept(self, expression, expected):
+        foo = Concept("foo")
+        bar = Concept("bar")
+        var = Concept("var")
+        context = self.get_context()
+
+        for c in (foo, bar, var):
+            context.sheerka.add_in_cache(c)
+        parser = BnfParser()
+        res = parser.parse(context, Tokenizer(expression))
+
+        assert not parser.has_error
+        assert res.status
+        assert res.value.value == expected
+        assert res.value.source == expression
+
+    def test_i_can_parse_regex_with_concept_when_the_concept_is_still_under_definition(self):
+        expression = "foo"
+        expected = ConceptExpression("foo")
+
+        context = self.get_context()
+        context.obj = ClassWithName("foo")
+
+        parser = BnfParser()
+        res = parser.parse(context, Tokenizer(expression))
+
+        assert not parser.has_error
+        assert res.status
+        assert res.value.value == expected
+        assert res.value.source == expression
+
+    @pytest.mark.parametrize("expression, error", [
+        ("1 ", UnexpectedEndOfFileError()),
+        ("1|", UnexpectedEndOfFileError()),
+        ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.RPAR])),
+        ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.IDENTIFIER])),
+        ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
+    ])
+    def test_i_can_detect_errors(self, expression, error):
+        parser = BnfParser()
+        res = parser.parse(self.get_context(), Tokenizer(expression))
+        ret_value = res.value.value
+        assert parser.has_error
+        assert not res.status
+        assert ret_value[0] == error
+
+    def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        context = self.get_context()
+        context.sheerka.add_in_cache(foo)
+        context.sheerka.add_in_cache(bar)
+
+        regex_parser = BnfParser()
+        foo_definition = regex_parser.parse(context, "'twenty' | 'thirty'").value.value
+        bar_definition = regex_parser.parse(context, "foo ('one' | 'two')").value.value
+
+        concepts = {bar: bar_definition, foo: foo_definition}
+        concept_parser = ConceptLexerParser()
+        concept_parser.initialize(context, concepts)
+
+        res = concept_parser.parse(context, "twenty two")
+        assert res.status
+        assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
+
+        res = concept_parser.parse(context, "thirty one")
+        assert res.status
+        assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
+
+        res = concept_parser.parse(context, "twenty")
+        assert res.status
+        assert res.value.body == [cnode("foo", 0, 0, "twenty")]
+
+    def test_i_cannot_parse_when_too_many_concepts(self):
+        foo1 = Concept(name="foo", body="1")
+        foo2 = Concept(name="foo", body="2")
+        context = self.get_context()
+        context.sheerka.cache_by_key["foo"] = [foo1, foo2]
+
+        regex_parser = BnfParser()
+        res = regex_parser.parse(context, "foo")
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
+        assert res.value.body == ('key', 'foo')
+
+    def test_i_cannot_parse_when_unknown_concept(self):
+        context = self.get_context()
+
+        regex_parser = BnfParser()
+        res = regex_parser.parse(self.get_context(), "foo")
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == ('key', 'foo')

tests/parsers/test_ConceptsWithConceptsParser.py

@@ -0,0 +1,193 @@
+import ast
+
+import pytest
+
+from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Token, TokenKind, Tokenizer
+from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode
+from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser
+from parsers.MultipleConceptsParser import MultipleConceptsParser
+from parsers.PythonParser import PythonNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+multiple_concepts_parser = MultipleConceptsParser()
+
+
+def ret_val(*args):
+    result = []
+    index = 0
+    source = ""
+    for item in args:
+        if isinstance(item, Concept):
+            tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
+            result.append(ConceptNode(item, index, index, tokens, item.name))
+            index += 1
+            source += item.name
+        elif isinstance(item, PythonNode):
+            tokens = list(Tokenizer(item.source))[:-1]  # strip trailing EOF
+            result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source))
+            index += len(tokens)
+            source += item.source
+        else:
+            tokens = list(Tokenizer(item))[:-1]  # strip trailing EOF
+            result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
+            index += len(tokens)
+            source += item
+
+    return ReturnValueConcept(
+        "who",
+        False,
+        ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source))
+
+
+class TestConceptsWithConceptsParser(TestUsingMemoryBasedSheerka):
+
+    def init(self, concepts, inputs):
+        context = self.get_context()
+        for concept in concepts:
+            context.sheerka.create_new_concept(context, concept)
+
+        return context, ret_val(*inputs)
+
+    def execute(self, concepts, inputs):
+        context, input_return_values = self.init(concepts, inputs)
+
+        parser = ConceptsWithConceptsParser()
+        result = parser.parse(context, input_return_values.body)
+
+        wrapper = result.body
+        return_value = result.body.body
+
+        return context, parser, result, wrapper, return_value
+
+    @pytest.mark.parametrize("text, interested", [
+        ("not parser result", False),
+        (ParserResultConcept(parser="not multiple_concepts_parser"), False),
+        (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True),
+    ])
+    def test_not_interested(self, text, interested):
+        context = self.get_context()
+
+        res = ConceptsWithConceptsParser().parse(context, text)
+        if interested:
+            assert res is not None
+        else:
+            assert res is None
+
+    def test_i_can_parse_composition_of_concepts(self):
+        foo = Concept("foo")
+        bar = Concept("bar")
+        plus = Concept("a plus b").def_prop("a").def_prop("b")
+
+        context, parser, result, wrapper, return_value = self.execute([foo, bar, plus], [foo, " plus ", bar])
+
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert result.who == wrapper.parser.name
+        assert wrapper.source == "foo plus bar"
+        assert context.sheerka.isinstance(return_value, plus)
+
+        assert return_value.compiled["a"] == foo
+        assert return_value.compiled["b"] == bar
+
+        # sanity check, I can evaluate the result
+        evaluated = context.sheerka.evaluate_concept(context, return_value)
+        assert evaluated.key == return_value.key
+        assert evaluated.get_prop("a") == foo.init_key()
+        assert evaluated.get_prop("b") == bar.init_key()
+
+    def test_i_can_parse_when_composition_of_source_code(self):
+        plus = Concept("a plus b", body="a + b").def_prop("a").def_prop("b")
+        left = PythonNode("1+1", ast.parse("1+1", mode="eval"))
+        right = PythonNode("2+2", ast.parse("2+2", mode="eval"))
+        context, parser, result, wrapper, return_value = self.execute([plus], [left, " plus ", right])
+
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert result.who == wrapper.parser.name
+        assert wrapper.source == "1+1 plus 2+2"
+        assert context.sheerka.isinstance(return_value, plus)
+
+        left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left)
+        right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right)
+        assert return_value.compiled["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)]
+        assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)]
+
+        # sanity check, I can evaluate the result
+        evaluated = context.sheerka.evaluate_concept(context, return_value)
+        assert evaluated.key == return_value.key
+        assert evaluated.get_prop("a") == 2
+        assert evaluated.get_prop("b") == 4
+        assert evaluated.body == 6
+
+    def test_i_can_parse_when_mix_of_concept_and_code(self):
+        plus = Concept("a plus b").def_prop("a").def_prop("b")
+        code = PythonNode("1+1", ast.parse("1+1", mode="eval"))
+        foo = Concept("foo")
+        context, parser, result, wrapper, return_value = self.execute([plus, foo], [foo, " plus ", code])
+
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert result.who == wrapper.parser.name
+        assert wrapper.source == "foo plus 1+1"
+        assert context.sheerka.isinstance(return_value, plus)
+
+        code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code)
+        assert return_value.compiled["a"] == foo
+        assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)]
+
+        # sanity check, I can evaluate the result
+        evaluated = context.sheerka.evaluate_concept(context, return_value)
+        assert evaluated.key == return_value.key
+        assert evaluated.get_prop("a") == foo.init_key()
+        assert evaluated.get_prop("b") == 2
+
+    def test_i_can_parse_when_multiple_concepts_are_recognized(self):
+        foo = Concept("foo")
+        bar = Concept("bar")
+        plus_1 = Concept("a plus b", body="body1").def_prop("a").def_prop("b")
+        plus_2 = Concept("a plus b", body="body2").def_prop("a").def_prop("b")
+
+        context, input_return_values = self.init([foo, bar, plus_1, plus_2], [foo, " plus ", bar])
+        parser = ConceptsWithConceptsParser()
+        result = parser.parse(context, input_return_values.body)
+
+        assert len(result) == 2
+
+        res = result[0]
+        wrapper = res.value
+        return_value = res.value.value
+        assert res.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert res.who == wrapper.parser.name
+        assert wrapper.source == "foo plus bar"
+        assert context.sheerka.isinstance(return_value, plus_1)
+        assert return_value.compiled["a"] == foo
+        assert return_value.compiled["b"] == bar
+
+        res = result[1]
+        wrapper = res.value
+        return_value = res.value.value
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert res.who == wrapper.parser.name
+        assert wrapper.source == "foo plus bar"
+        assert context.sheerka.isinstance(return_value, plus_2)
+        assert return_value.compiled["a"] == foo
+        assert return_value.compiled["b"] == bar
+
+    def test_i_cannot_parse_when_unknown_concept(self):
+        foo = Concept("foo")
+        bar = Concept("bar")
+
+        context, input_return_values = self.init([foo, bar], [foo, " plus ", bar])
+        parser = ConceptsWithConceptsParser()
+        result = parser.parse(context, input_return_values.body)
+        wrapper = result.body
+        return_value = result.body.body
+
+        assert not result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME)
+        assert result.who == parser.name
+        assert return_value == input_return_values.body.body

tests/parsers/test_DefaultParser.py

@@ -0,0 +1,345 @@
+import pytest
+import ast
+
+from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
+from core.concept import Concept
+from parsers.ConceptLexerParser import OrderedChoice, StrMatch
+from parsers.PythonParser import PythonParser, PythonNode
+from core.tokenizer import Keywords, Tokenizer, LexerError
+from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode
+from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode
+from parsers.BnfParser import BnfParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None):
+    def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
+
+    if body:
+        def_concept.body = get_concept_part(body)
+    if where:
+        def_concept.where = get_concept_part(where)
+    if pre:
+        def_concept.pre = get_concept_part(pre)
+    if post:
+        def_concept.post = get_concept_part(post)
+    if definition:
+        def_concept.definition = ReturnValueConcept(
+            "parsers.Bnf",
+            True,
+            definition)
+
+    return def_concept
+
+
+def get_concept_part(part):
+    if isinstance(part, str):
+        node = PythonNode(part, ast.parse(part, mode="eval"))
+        return ReturnValueConcept(
+            who="parsers.Default",
+            status=True,
+            value=ParserResultConcept(
+                source=part,
+                parser=PythonParser(),
+                value=node))
+
+    if isinstance(part, PythonNode):
+        return ReturnValueConcept(
+            who="parsers.Default",
+            status=True,
+            value=ParserResultConcept(
+                source=part.source,
+                parser=PythonParser(),
+                value=part))
+
+    if isinstance(part, ReturnValueConcept):
+        return part
+
+
+class TestDefaultParser(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("text, expected", [
+        ("def concept hello", get_def_concept(name="hello")),
+        ("def concept hello  ", get_def_concept(name="hello")),
+        ("def concept a + b", get_def_concept(name="a + b")),
+        ("def concept a+b", get_def_concept(name="a + b")),
+        ("def concept 'a+b'+c", get_def_concept(name="'a+b' + c")),
+        ("def concept 'as if'", get_def_concept(name="'as if'")),
+        ("def concept 'as' if", get_def_concept(name="'as if'")),
+        ("def concept hello as 'hello'", get_def_concept(name="hello", body="'hello'")),
+        ("def concept hello as 1", get_def_concept(name="hello", body="1")),
+        ("def concept hello as 1 + 1", get_def_concept(name="hello", body="1 + 1")),
+    ])
+    def test_i_can_parse_def_concept(self, text, expected):
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        node = res.value.value
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
+    def test_i_can_parse_complex_def_concept_statement(self):
+        text = """def concept a plus b
+where a,b
+pre isinstance(a, int) and isinstance(b, float)
+post isinstance(res, int)
+as res = a + b
+    """
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+        expected_concept = get_def_concept(
+            name="a plus b",
+            where="a,b",
+            pre="isinstance(a, int) and isinstance(b, float)",
+            post="isinstance(res, int)",
+            body=PythonNode("res = a + b", ast.parse("res = a + b", mode="exec"))
+        )
+
+        assert res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert return_value.value == expected_concept
+
+    def test_i_can_have_mutilines_declarations(self):
+        text = """
+def concept add one to a as
+def func(x):
+    return x+1
+func(a)
+        """
+
+        expected_concept = get_def_concept(
+            name="add one to a ",
+            body=PythonNode(
+                "def func(x):\n    return x+1\nfunc(a)",
+                ast.parse("def func(x):\n    return x+1\nfunc(a)", mode="exec"))
+        )
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert return_value.value == expected_concept
+
+    def test_i_can_use_colon_to_use_indentation(self):
+        text = """
+def concept add one to a as:
+    def func(x):
+        return x+1
+    func(a)
+    """
+
+        expected_concept = get_def_concept(
+            name="add one to a ",
+            body=PythonNode(
+                "def func(x):\n    return x+1\nfunc(a)",
+                ast.parse("def func(x):\n    return x+1\nfunc(a)", mode="exec"))
+        )
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert return_value.value == expected_concept
+
+    def test_indentation_is_mandatory_after_a_colon(self):
+        text = """
+def concept add one to a as:
+def func(x):
+    return x+1
+func(a)
+    """
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert not res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert isinstance(return_value.value[0], SyntaxErrorNode)
+        assert return_value.value[0].message == "Indentation not found."
+
+    def test_indentation_is_not_allowed_if_the_colon_is_missing(self):
+        text = """
+def concept add one to a as
+    def func(x):
+        return x+1
+    func(a)
+        """
+        context = self.get_context()
+
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+        return_value = res.value
+
+        assert not res.status
+        assert context.sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
+
+    def test_name_is_mandatory(self):
+        text = "def concept as 'hello'"
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert not res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert isinstance(return_value.value[0], SyntaxErrorNode)
+        assert return_value.value[0].message == "Name is mandatory"
+
+    def test_concept_keyword_is_mandatory_but_the_concept_is_recognized(self):
+        text = "def hello as a where b pre c post d"
+
+        expected_concept = get_def_concept(name="hello", body="a", where="b", pre="c", post="d")
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert not res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert isinstance(return_value.value[0], UnexpectedTokenErrorNode)
+        assert return_value.value[0].message == "Syntax error."
+        assert return_value.value[0].expected_tokens == [Keywords.CONCEPT]
+        assert return_value.try_parsed == expected_concept
+
+    @pytest.mark.parametrize("text", [
+        "def concept hello where 1+",
+        "def concept hello pre 1+",
+        "def concept hello post 1+",
+        "def concept hello as 1+"
+    ])
+    def test_i_can_detect_error_in_declaration(self, text):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+        return_value = res.value
+
+        assert not res.status
+        assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
+
+    def test_new_line_is_not_allowed_in_the_name(self):
+        text = "def concept hello \n my friend as 'hello'"
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert not res.status
+        assert return_value.value == [SyntaxErrorNode([], "Newline are not allowed in name.")]
+
+    def test_i_can_parse_def_concept_from_regex(self):
+        context = self.get_context()
+        a_concept = Concept("a_concept")
+        context.sheerka.add_in_cache(a_concept)
+
+        text = "def concept name from bnf a_concept | 'a_string' as __definition[0]"
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+        node = res.value.value
+        definition = OrderedChoice(a_concept, StrMatch("a_string"))
+        parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition)
+        expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result)
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
+    def test_i_can_parse_def_concept_where_bnf_references_itself(self):
+        context = self.get_context()
+        a_concept = Concept("a_concept")
+        context.sheerka.add_in_cache(a_concept)
+
+        text = "def concept name from bnf 'a' + name?"
+        parser = DefaultParser()
+        parser.parse(context, text)
+
+        assert not parser.has_error
+
+    def test_i_can_detect_empty_bnf_declaration(self):
+        text = "def concept name from bnf as __definition[0]"
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration")
+
+    def test_i_can_detect_not_for_me(self):
+        text = "hello world"
+        context = self.get_context()
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
+        assert isinstance(res.value.body[0], CannotHandleErrorNode)
+
+    def test_i_can_parse_is_a(self):
+        parser = DefaultParser()
+        text = "the name of my 'concept' isa the name of the set"
+        res = parser.parse(self.get_context(), text)
+        expected = IsaConceptNode([],
+                                  concept=NameNode(list(Tokenizer("the name of my 'concept'"))),
+                                  set=NameNode(list(Tokenizer("the name of the set"))))
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert res.value.value == expected
+
+    @pytest.mark.parametrize("text", [
+        "concept",
+        "isa number",
+        "name isa",
+        "def",
+        "def concept_name"
+    ])
+    def test_i_cannot_parse_invalid_entries(self, text):
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert isinstance(res.body, ParserResultConcept)
+        assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
+
+    @pytest.mark.parametrize("text, error_msg, error_text", [
+        ("'name", "Missing Trailing quote", "'name"),
+        ("foo isa 'name", "Missing Trailing quote", "'name"),
+        ("def concept 'name", "Missing Trailing quote", "'name"),
+        ("def concept name as 'body", "Missing Trailing quote", "'body"),
+        ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"),
+        ("def concept c::", "Concept name not found", ""),
+    ])
+    def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text):
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert isinstance(res.body, ParserResultConcept)
+        assert isinstance(res.body.body[0], LexerError)
+        assert res.body.body[0].message == error_msg
+        assert res.body.body[0].text == error_text
+
+    def test_i_cannot_parse_bnf_definition_referencing_unknown_concept(self):
+        context = self.get_context()
+        text = "def concept name from bnf unknown"
+
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == ("key", "unknown")

tests/parsers/test_ExactConceptParser.py

@@ -0,0 +1,138 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Tokenizer
+from parsers.ExactConceptParser import ExactConceptParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def metadata_prop(concept, prop_name):
+    for name, value in concept.metadata.props:
+        if name == prop_name:
+            return value
+
+    return None
+
+
+def get_concept(name, variables):
+    c = Concept(name=name)
+    if variables:
+        for v in variables:
+            c.def_prop(v)
+    c.init_key()
+    return c
+
+
+class TestExactConceptParser(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_compute_combinations(self):
+        parser = ExactConceptParser()
+        res = parser.combinations(["foo", "bar", "baz"])
+
+        assert res == {('foo', 'bar', 'baz'),
+                       ('__var__0', 'bar', 'baz'),
+                       ('foo', '__var__0', 'baz'),
+                       ('foo', 'bar', '__var__0'),
+                       ('__var__0', '__var__1', 'baz'),
+                       ('__var__0', 'bar', '__var__1'),
+                       ('foo', '__var__0', '__var__1'),
+                       ('__var__0', '__var__1', '__var__2')}
+
+    def test_i_can_compute_combinations_with_duplicates(self):
+        parser = ExactConceptParser()
+        res = parser.combinations(["foo", "bar", "foo"])
+
+        assert res == {('foo', 'bar', 'foo'),
+                       ('__var__0', 'bar', '__var__0'),
+                       ('foo', '__var__0', 'foo'),
+                       ('__var__0', '__var__1', '__var__0'),
+                       ('__var__1', '__var__0', '__var__1')}
+        # TODO: the last tuple is not possible, so the algo can be improved
+
+    def test_i_can_recognize_a_simple_concept(self):
+        context = self.get_context()
+        concept = get_concept("hello world", [])
+        context.sheerka.add_in_cache(concept)
+
+        source = "hello world"
+        results = ExactConceptParser().parse(context, source)
+
+        assert len(results) == 1
+        assert results[0].status
+        assert results[0].value.value == concept
+
+    def test_i_can_recognize_concepts_defined_several_times(self):
+        context = self.get_context()
+        context.sheerka.add_in_cache(get_concept("hello world", []))
+        context.sheerka.add_in_cache(get_concept("hello a", ["a"]))
+
+        source = "hello world"
+        results = ExactConceptParser().parse(context, source)
+
+        assert len(results) == 2
+        results = sorted(results, key=lambda x: x.value.value.name)  # because of the usage of sets
+
+        assert results[0].status
+        assert results[0].value.value.name == "hello a"
+        assert metadata_prop(results[0].value.value, "a") == "world"
+
+        assert results[1].status
+        assert results[1].value.value.name == "hello world"
+
+    def test_i_can_recognize_a_concept_with_variables(self):
+        context = self.get_context()
+        concept = get_concept("a + b", ["a", "b"])
+        context.sheerka.add_in_cache(concept)
+        source = "10 + 5"
+        results = ExactConceptParser().parse(context, source)
+
+        assert len(results) == 1
+        assert results[0].status
+        concept_found = results[0].value.value
+        assert concept_found.key == concept.key
+        assert metadata_prop(concept_found, "a") == "10"
+        assert metadata_prop(concept_found, "b") == "5"
+
+    def test_i_can_recognize_a_concept_with_duplicate_variables(self):
+        context = self.get_context()
+        concept = get_concept("a + b + a", ["a", "b"])
+        context.sheerka.cache_by_key[concept.key] = concept
+        source = "10 + 5 + 10"
+        results = ExactConceptParser().parse(context, source)
+
+        assert len(results) == 1
+        assert results[0].status
+        concept_found = results[0].value.value
+        assert concept_found.key == concept.key
+        assert metadata_prop(concept_found, "a") == "10"
+        assert metadata_prop(concept_found, "b") == "5"
+
+    def test_i_can_manage_unknown_concept(self):
+        context = self.get_context()
+        source = "def concept hello world"  # this is not a concept by itself
+        res = ExactConceptParser().parse(context, source)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == "def concept hello world"
+
+    def test_i_can_detect_concepts_too_long(self):
+        context = self.get_context()
+        source = "a very very long concept that cannot be an unique one"
+        res = ExactConceptParser().parse(context, source)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_TOO_LONG)
+        assert res.value.body == "a very very long concept that cannot be an unique one"
+
+    def test_i_can_detect_concept_from_tokens(self):
+        context = self.get_context()
+        concept = get_concept("hello world", [])
+        context.sheerka.add_in_cache(concept)
+
+        source = "hello world"
+        results = ExactConceptParser().parse(context, list(Tokenizer(source)))
+
+        assert len(results) == 1
+        assert results[0].status
+        assert results[0].value.value == concept

tests/parsers/test_MultipleConceptsParser.py

@@ -0,0 +1,215 @@
+import pytest
+
+from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Tokenizer, TokenKind, Token
+from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, cnode, utnode, scnode, SourceCodeNode
+from parsers.MultipleConceptsParser import MultipleConceptsParser
+from parsers.PythonParser import PythonNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_return_value(context, grammar, expression):
+    parser = ConceptLexerParser()
+    parser.initialize(context, grammar)
+
+    ret_val = parser.parse(context, expression)
+    assert not ret_val.status
+    return ret_val
+
+
+class TestMultipleConceptsParser(TestUsingMemoryBasedSheerka):
+
+    def init(self, concepts, grammar, expression):
+        context = self.get_context()
+        for c in concepts:
+            context.sheerka.create_new_concept(context, c)
+        return_value = get_return_value(context, grammar, expression)
+
+        return context, return_value
+
+    def test_not_interested_if_not_parser_result(self):
+        context = self.get_context()
+        text = "not parser result"
+
+        res = MultipleConceptsParser().parse(context, text)
+        assert res is None
+
+    def test_not_interested_if_not_from_concept_lexer_parser(self):
+        context = self.get_context()
+        text = ParserResultConcept(parser="not concept lexer", value="some value")
+
+        res = MultipleConceptsParser().parse(context, text)
+        assert res is None
+
+    def test_i_can_parse_exact_concepts(self):
+        foo = Concept("foo", body="'foo'")
+        bar = Concept("bar", body="'bar'")
+        baz = Concept("baz", body="'baz'")
+        grammar = {}
+        context, return_value = self.init([foo, bar, baz], grammar, "bar foo    baz")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [
+            ConceptNode(bar, 0, 0, source="bar"),
+            ConceptNode(foo, 2, 2, source="foo"),
+            ConceptNode(baz, 4, 4, source="baz")]
+        assert ret_val.value.source == "bar foo    baz"
+
+    def test_i_can_parse_when_ending_with_bnf(self):
+        foo = Concept("foo", body="'foo'")
+        bar = Concept("bar", body="'bar'")
+        grammar = {foo: Sequence("foo1", "foo2", "foo3")}
+        context, return_value = self.init([foo, bar], grammar, "bar foo1 foo2 foo3")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [cnode("bar", 0, 0, "bar"), cnode("foo", 2, 6, "foo1 foo2 foo3")]
+        assert ret_val.value.source == "bar foo1 foo2 foo3"
+
+    def test_i_can_parse_when_starting_with_bnf(self):
+        foo = Concept("foo", body="'foo'")
+        bar = Concept("bar", body="'bar'")
+        grammar = {foo: Sequence("foo1", "foo2", "foo3")}
+        context, return_value = self.init([foo, bar], grammar, "foo1 foo2 foo3 bar")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [cnode("foo", 0, 4, "foo1 foo2 foo3"), cnode("bar", 6, 6, "bar")]
+        assert ret_val.value.source == "foo1 foo2 foo3 bar"
+
+    def test_i_can_parse_when_concept_are_mixed(self):
+        foo = Concept("foo")
+        bar = Concept("bar")
+        baz = Concept("baz")
+        grammar = {foo: Sequence("foo1", "foo2", "foo3")}
+        context, return_value = self.init([foo, bar, baz], grammar, "baz foo1 foo2 foo3 bar")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [
+            cnode("baz", 0, 0, "baz"),
+            cnode("foo", 2, 6, "foo1 foo2 foo3"),
+            cnode("bar", 8, 8, "bar")]
+        assert ret_val.value.source == "baz foo1 foo2 foo3 bar"
+
+    def test_i_can_parse_when_multiple_concepts_are_matching(self):
+        foo = Concept("foo")
+        bar = Concept("bar", body="bar1")
+        baz = Concept("bar", body="bar2")
+        grammar = {foo: "foo"}
+        context, return_value = self.init([foo, bar, baz], grammar, "foo bar")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert len(ret_val) == 2
+        assert ret_val[0].status
+        assert ret_val[0].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
+        assert ret_val[0].value.source == "foo bar"
+        assert ret_val[0].value.value[1].concept.metadata.body == "bar1"
+
+        assert ret_val[1].status
+        assert ret_val[1].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
+        assert ret_val[1].value.source == "foo bar"
+        assert ret_val[1].value.value[1].concept.metadata.body == "bar2"
+
+    def test_i_can_parse_when_source_code(self):
+        foo = Concept("foo")
+        grammar = {foo: "foo"}
+        context, return_value = self.init([foo], grammar, "1 foo")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+        wrapper = ret_val.value
+        value = ret_val.value.value
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.source == "1 foo"
+        assert value == [
+            scnode(0, 1, "1 "),
+            cnode("foo", 2, 2, "foo")]
+
+    def test_i_cannot_parse_when_unrecognized_token(self):
+        twenty_two = Concept("twenty two")
+        one = Concept("one")
+        grammar = {twenty_two: Sequence("twenty", "two")}
+        context, return_value = self.init([twenty_two, one], grammar, "twenty two + one")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert not ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [
+            cnode("twenty two", 0, 2, "twenty two"),
+            utnode(3, 5, " + "),
+            cnode("one", 6, 6, "one")
+        ]
+        assert ret_val.value.source == "twenty two + one"
+
+    def test_i_cannot_parse_when_unknown_concepts(self):
+        twenty_two = Concept("twenty two")
+        one = Concept("one")
+        grammar = {twenty_two: Sequence("twenty", "two")}
+        context, return_value = self.init([twenty_two, one], grammar, "twenty two plus one")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert not ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [
+            cnode("twenty two", 0, 2, "twenty two"),
+            utnode(3, 5, " plus "),
+            cnode("one", 6, 6, "one")
+        ]
+        assert ret_val.value.source == "twenty two plus one"
+
+    @pytest.mark.parametrize("text, expected_source, expected_end", [
+        ("True", "True", 0),
+        ("1 == 1", "1 == 1", 5),
+        ("1!xdf", "1", 0),
+        ("1", "1", 0),
+    ])
+    def test_i_can_get_source_code_node(self, text, expected_source, expected_end):
+        tokens = list(Tokenizer(text))[:-1]  # strip trailing EOF
+
+        start_index = 5  # a random number different of zero
+        res = MultipleConceptsParser().get_source_code_node(self.get_context(), start_index, tokens)
+
+        assert isinstance(res, SourceCodeNode)
+        assert isinstance(res.node, PythonNode)
+        assert res.source == expected_source
+        assert res.start == start_index
+        assert res.end == start_index + expected_end
+
+    def test_i_cannot_parse_null_text(self):
+        res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, [])
+        assert res is None
+
+        eof = Token(TokenKind.EOF, "", 0, 0, 0)
+        res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, [eof])
+        assert res is None

tests/parsers/test_PythonParser.py

@@ -0,0 +1,77 @@
+import ast
+import pytest
+from core.builtin_concepts import ParserResultConcept
+from core.tokenizer import Tokenizer, LexerError
+from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestPythonParser(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))),
+        ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))),
+    ])
+    def test_i_can_parse_a_simple_expression(self, text, expected):
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert res.status
+        assert res.who == parser.name
+        assert isinstance(res.value, ParserResultConcept)
+        assert res.value.value == expected
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))),
+        ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))),
+    ])
+    def test_i_can_parse_from_tokens(self, text, expected):
+        parser = PythonParser()
+        tokens = list(Tokenizer(text))
+        res = parser.parse(self.get_context(), tokens)
+
+        assert res.status
+        assert res.who == parser.name
+        assert isinstance(res.value, ParserResultConcept)
+        assert res.value.value == expected
+
+    @pytest.mark.parametrize("text", [
+        "1+",
+        "'name",
+        "foo = 'name"
+    ])
+    def test_i_can_detect_error(self, text):
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert res.who == parser.name
+        assert isinstance(res.value, ParserResultConcept)
+        assert isinstance(res.value.value[0], PythonErrorNode)
+        assert isinstance(res.value.value[0].exception, SyntaxError)
+
+    @pytest.mark.parametrize("text, error_msg, error_text", [
+        ("c::", "Concept name not found", ""),
+        ("c:: + 1", "Concept name not found", ""),
+    ])
+    def test_i_can_detect_lexer_errors(self, text, error_msg, error_text):
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert isinstance(res.body, ParserResultConcept)
+        assert isinstance(res.body.body[0], LexerError)
+        assert res.body.body[0].message == error_msg
+        assert res.body.body[0].text == error_text
+
+    def test_i_can_parse_a_concept(self):
+        text = "c:concept_name: + 1"
+
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert res
+        assert res.value.value == PythonNode(
+            "c:concept_name: + 1",
+            ast.parse("__C__USE_CONCEPT__concept_name__C__+1", mode="eval"))

tests/parsers/test_PythonWithConceptsParser.py

@@ -0,0 +1,138 @@
+import ast
+
+import pytest
+
+from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
+from core.concept import Concept
+from core.tokenizer import Token, TokenKind, Tokenizer
+from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode
+from parsers.MultipleConceptsParser import MultipleConceptsParser
+from parsers.PythonParser import PythonNode, PythonErrorNode
+from parsers.PythonWithConceptsParser import PythonWithConceptsParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+multiple_concepts_parser = MultipleConceptsParser()
+
+
+def ret_val(*args):
+    result = []
+    index = 0
+    for item in args:
+        if isinstance(item, Concept):
+            tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
+            result.append(ConceptNode(item, index, index, tokens, item.name))
+            index += 1
+        else:
+            tokens = list(Tokenizer(item))
+            result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
+            index += len(tokens)
+
+    return ReturnValueConcept("who", False, ParserResultConcept(parser=multiple_concepts_parser, value=result))
+
+
+def to_str_ast(expression):
+    return PythonNode.get_dump(ast.parse(expression, mode="eval"))
+
+
+class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("text, interested", [
+        ("not parser result", False),
+        (ParserResultConcept(parser="not multiple_concepts_parser"), False),
+        (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True),
+    ])
+    def test_not_interested(self, text, interested):
+        context = self.get_context()
+
+        res = PythonWithConceptsParser().parse(context, text)
+        if interested:
+            assert res is not None
+        else:
+            assert res is None
+
+    def test_i_can_parse_concepts_and_python(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        input_return_value = ret_val(foo, " + 1")
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        wrapper = result.value
+        return_value = result.value.value
+
+        assert result.status
+        assert result.who == parser.name
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.source == "foo + 1"
+        assert isinstance(return_value, PythonNode)
+        assert return_value.source == "foo + 1"
+        assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1")
+        assert return_value.concepts["__C__foo__C__"] == foo
+
+    def test_i_can_parse_concepts_and_python_when_concept_is_known(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        foo = context.sheerka.create_new_concept(context, foo).body.body
+        input_return_value = ret_val(foo, " + 1")
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        wrapper = result.value
+        return_value = result.value.value
+
+        assert result.status
+        assert result.who == parser.name
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.source == "foo + 1"
+        assert isinstance(return_value, PythonNode)
+        assert return_value.source == "foo + 1"
+        assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__1001__C__ + 1")
+        assert return_value.concepts["__C__foo__1001__C__"] == foo
+
+    def test_i_can_parse_when_concept_name_has_invalid_characters(self):
+        context = self.get_context()
+        foo = Concept("foo et > (,")
+        foo = context.sheerka.create_new_concept(context, foo).body.body
+        input_return_value = ret_val(foo, " + 1")
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        return_value = result.value.value
+
+        assert result.status
+        assert return_value.concepts["__C__foo0et000000__1001__C__"] == foo
+
+    def test_python_ids_mappings_are_correct_when_concepts_with_the_same_name(self):
+        context = self.get_context()
+        foo1 = Concept("foo")
+        foo2 = Concept("foo")
+        foo3 = context.sheerka.create_new_concept(context, Concept("foo", body="foo3")).body.body
+        foo4 = context.sheerka.create_new_concept(context, Concept("foo", body="foo4")).body.body
+
+        input_return_value = ret_val(foo1, "+", foo2, "+", foo3, "+", foo4)
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        return_value = result.value.value
+
+        assert result.status
+        assert return_value.concepts["__C__foo__C__"] == foo1
+        assert return_value.concepts["__C__foo_1__C__"] == foo2
+        assert return_value.concepts["__C__foo__1001__C__"] == foo3
+        assert return_value.concepts["__C__foo__1002__C__"] == foo4
+
+    def test_i_cannot_parse_if_syntax_error(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        foo = context.sheerka.create_new_concept(context, foo).body.body
+        input_return_value = ret_val(foo, " + ")
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        wrapper = result.value
+        return_value = result.value.value
+
+        assert not result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(return_value[0], PythonErrorNode)

tests/sdp/test_sheerkaDataProvider.py

@@ -12,7 +12,7 @@ import json
 from sdp.sheerkaSerializer import JsonSerializer, Serializer, PickleSerializer
 import core.utils
 
-tests_root = path.abspath("../build/tests")
+tests_root = path.abspath("../../build/tests")
 evt_digest = "3a571cb6034ef6fc8d7fe91948d0d29728eed74de02bac7968b0e9facca2c2d7"
 
 
@@ -821,17 +821,17 @@ def test_i_can_set_using_reference(root):
     entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"), use_ref=True)
 
     state = sdp.load_state(sdp.get_snapshot())
-    assert state.data == {"entry": {"2": '##REF##:95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268'}}
+    assert state.data == {"entry": {"2": '##REF##:43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9'}}
     assert entry == "entry"
     assert key == "2"
 
     assert sdp.io.exists(sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder,
-                                             "95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268"))
+                                             "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9"))
 
     # sanity check, make sure that I can load back
     loaded = sdp.get(entry, key)
     assert loaded == ObjWithKey(2, "foo")
-    assert getattr(loaded, Serializer.ORIGIN) == "95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268"
+    assert getattr(loaded, Serializer.ORIGIN) == "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9"
 
 
 @pytest.mark.parametrize("root", [
@@ -1430,7 +1430,7 @@ def test_i_can_modify_a_ref(root):
     state = sdp.load_state(sdp.get_snapshot())
     assert state.data == {"entry": {
         "key1": ObjWithKey("key1", "foo"),
-        "key2": "##REF##:d70b0247311645ed18d275337cbcf79ad186d995236cdc8ad4fcfc708085bd3d"}}
+        "key2": "##REF##:041d3cca905b51bc2c66251e73e56b836aae7b9435ee3d7eb05d44bb67ff575e"}}
     assert entry == "entry"
     assert key == "key2"
 

tests/sdp/test_sheerkaSerializer.py

@@ -37,7 +37,7 @@ def test_i_can_serialize_an_event():
 def test_i_can_serialize_an_object():
     obj = Obj("10", "value")
     serializer = Serializer()
-    serializer.register(JsonSerializer("tests.test_sheerkaSerializer.Obj"))
+    serializer.register(JsonSerializer("tests.sdp.test_sheerkaSerializer.Obj"))
     context = SerializerContext("kodjo", "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b")
 
     stream = serializer.serialize(obj, context)
@@ -51,7 +51,7 @@ def test_i_can_serialize_an_object():
 
 
 @pytest.mark.parametrize("obj, expected", [
-    (Obj("10", "value"), "tests.test_sheerkaSerializer.Obj")
+    (Obj("10", "value"), "tests.sdp.test_sheerkaSerializer.Obj")
 ])
 def test_get_full_qualified_name(obj, expected):
     assert expected == core.utils.get_full_qualified_name(obj)

tests/test_AddConceptEvaluator.py

@@ -1,194 +0,0 @@
-import ast
-
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import VARIABLE_PREFIX, ConceptParts, Concept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer
-from evaluators.AddConceptEvaluator import AddConceptEvaluator
-from parsers.BaseParser import BaseParser
-from parsers.ConceptLexerParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
-from parsers.BnfParser import BnfParser
-from parsers.DefaultParser import DefConceptNode, NameNode
-from parsers.PythonParser import PythonNode, PythonParser
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None):
-    concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
-
-    if body:
-        concept.body = get_concept_part(body)
-    if where:
-        concept.where = get_concept_part(where)
-    if pre:
-        concept.pre = get_concept_part(pre)
-    if post:
-        concept.post = get_concept_part(post)
-    if definition:
-        concept.definition = definition
-
-    return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=concept))
-
-
-def get_concept_part(part):
-    if isinstance(part, str):
-        node = PythonNode(part, ast.parse(part, mode="eval"))
-        return ReturnValueConcept(
-            who="parsers.Default",
-            status=True,
-            value=ParserResultConcept(
-                source=part,
-                parser=PythonParser(),
-                value=node))
-
-    if isinstance(part, PythonNode):
-        return ReturnValueConcept(
-            who="parsers.Default",
-            status=True,
-            value=ParserResultConcept(
-                source=part.source,
-                parser=PythonParser(),
-                value=part))
-
-    if isinstance(part, ReturnValueConcept):
-        return part
-
-
-def get_concept_definition(source, parsing_expression):
-    return ReturnValueConcept(
-        who="Parsers:RegexParser",
-        status=True,
-        value=ParserResultConcept(
-            source=source,
-            parser=BnfParser(),
-            value=parsing_expression
-        )
-    )
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))), True),
-    (ReturnValueConcept(BaseParser.PREFIX + "some_name", False, ParserResultConcept(value=DefConceptNode([]))), False),
-    (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not a ParserResultConcept"), False),
-    (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept()), False),
-])
-def test_i_can_match(ret_val, expected):
-    context = get_context()
-    assert AddConceptEvaluator().matches(context, ret_val) == expected
-
-
-def test_that_the_source_is_correctly_set():
-    context = get_context()
-    def_concept_return_value = get_def_concept(
-        name="hello a",
-        definition=get_concept_definition("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
-        where="isinstance(a, str )",
-        pre="a is not None",
-        body="print('hello' + a)")
-
-    evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
-
-    assert evaluated.status
-    assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
-
-    created_concept = evaluated.body.body
-    assert created_concept.metadata.name == "hello a"
-    assert created_concept.metadata.where == "isinstance(a, str )"
-    assert created_concept.metadata.pre == "a is not None"
-    assert created_concept.metadata.post is None
-    assert created_concept.metadata.body == "print('hello' + a)"
-    assert created_concept.metadata.definition == "hello a"
-
-
-# def test_that_the_ast_is_correctly_initialized():
-#     """
-#     When I parse the definition of a concept, I evaluate the metadata (like the body)
-#     I wanted to keep in cache these evaluation for further utilisation but I have
-#     a serialization issue.
-#     So I had to comment concept.add_codes(def_concept_node.get_asts()) around line 85
-#     So this test is now irrelevant
-#     :return:
-#     """
-#     context = get_context()
-#     def_concept_return_value = get_concept(
-#         name="hello a",
-#         definition=get_concept_definition("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
-#         where="isinstance(a, str )",
-#         pre="a is not None",
-#         body="print('hello' + a)")
-#
-#     evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
-#
-#     assert evaluated.status
-#     assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
-#
-#     created_concept = evaluated.body.body
-#
-#     assert ConceptParts.WHERE in created_concept.compiled
-#     assert ConceptParts.PRE in created_concept.compiled
-#     assert ConceptParts.BODY in created_concept.compiled
-#     assert ConceptParts.POST not in created_concept.compiled
-
-
-def test_that_the_new_concept_is_correctly_saved_in_db():
-    context = get_context()
-    def_concept_return_value = get_def_concept(
-        name="hello a",
-        definition=get_concept_definition("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
-        where="isinstance(a, str )",
-        pre="a is not None",
-        body="print('hello' + a)")
-
-    # sanity. Make sure that the concept does not already exist
-    from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0")
-    assert context.sheerka.isinstance(from_db, BuiltinConcepts.UNKNOWN_CONCEPT)
-
-    AddConceptEvaluator().eval(context, def_concept_return_value)
-    context.sheerka.concepts_cache = {}  # reset cache
-    from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0")
-
-    assert from_db.metadata.key == f"hello {VARIABLE_PREFIX}0"
-    assert from_db.metadata.name == "hello a"
-    assert from_db.metadata.where == "isinstance(a, str )"
-    assert from_db.metadata.pre == "a is not None"
-    assert from_db.metadata.post is None
-    assert from_db.metadata.body == "print('hello' + a)"
-    assert from_db.metadata.definition == "hello a"
-    assert len(from_db.metadata.props) == 1
-    assert from_db.metadata.props[0] == ("a", None)
-    assert "a" in from_db.props
-
-    assert from_db.compiled == {}  # ast is not saved in db
-
-
-def test_i_can_get_props_from_python_node():
-    ret_val = get_concept_part("isinstance(a, str)")
-    context = get_context()
-
-    assert AddConceptEvaluator.get_props(context.sheerka, ret_val, ["a"]) == ["a"]
-
-
-def test_i_can_get_props_from_another_concept():
-    concept = Concept("hello").def_prop("a").def_prop("b")
-    ret_val = ReturnValueConcept(who="some_parser",
-                                 status=True,
-                                 value=ParserResultConcept(value=concept))
-
-    assert AddConceptEvaluator.get_props(get_context(), ret_val, []) == ["a", "b"]
-
-
-def test_i_can_get_props_from_definition():
-    parsing_expression = Sequence(ConceptExpression('mult'), ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
-    ret_val = get_concept_definition("mult (('+'|'-') add)?", parsing_expression)
-
-    assert AddConceptEvaluator.get_props(get_context(), ret_val, []) == ["add", "mult"]
-
-

tests/test_AddConceptInSetEvaluator.py

@@ -1,112 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer
-from evaluators.AddConceptInSetEvaluator import AddConceptInSetEvaluator
-from parsers.DefaultParser import IsaConceptNode, NameNode
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_ret_val(concept_name, concept_set_name):
-    n1 = NameNode(list(Tokenizer(concept_name)))
-    n2 = NameNode(list(Tokenizer(concept_set_name)))
-
-    return ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([], n1, n2)))
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([]))), True),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=IsaConceptNode([]))), False),
-    (ReturnValueConcept("some_name", True, "not a ParserResultConcept"), False),
-    (ReturnValueConcept("some_name", True, ParserResultConcept()), False),
-])
-def test_i_can_match(ret_val, expected):
-    context = get_context()
-    assert AddConceptInSetEvaluator().matches(context, ret_val) == expected
-
-
-def test_i_cannot_add_if_the_concept_does_not_exists():
-    context = get_context()
-
-    ret_val = get_ret_val("foo", "bar")
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert res.value.body == "foo"
-
-
-def test_i_cannot_add_if_the_set_does_not_exists():
-    context = get_context()
-    foo = Concept("foo")
-    context.sheerka.set_id_if_needed(foo, False)
-    context.sheerka.add_in_cache(foo)
-
-    ret_val = get_ret_val("foo", "bar")
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert res.value.body == "bar"
-
-
-def test_i_can_add_concept_to_a_set_of_concept():
-    context = get_context()
-    foo = Concept("foo")
-    context.sheerka.set_id_if_needed(foo, False)
-    context.sheerka.add_in_cache(foo)
-
-    bar = Concept("bar")
-    context.sheerka.set_id_if_needed(bar, False)
-    context.sheerka.add_in_cache(bar)
-
-    ret_val = get_ret_val("foo", "bar")
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
-
-
-def test_i_can_add_concept_with_a_body_to_a_set_of_concept():
-    context = get_context()
-    foo = Concept("foo", body="1")
-    context.sheerka.set_id_if_needed(foo, False)
-    context.sheerka.add_in_cache(foo)
-
-    bar = Concept("bar")
-    context.sheerka.set_id_if_needed(bar, False)
-    context.sheerka.add_in_cache(bar)
-
-    ret_val = get_ret_val("foo", "bar")
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
-
-
-def test_i_cannot_add_the_same_concept_twice():
-    context = get_context()
-    foo = Concept("foo")
-    context.sheerka.set_id_if_needed(foo, False)
-    context.sheerka.add_in_cache(foo)
-
-    bar = Concept("bar")
-    context.sheerka.set_id_if_needed(bar, False)
-    context.sheerka.add_in_cache(bar)
-
-    ret_val = get_ret_val("foo", "bar")
-    AddConceptInSetEvaluator().eval(context, ret_val)
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_IN_SET)
-    assert res.value.concept == foo
-    assert res.value.concept_set == bar

tests/test_BnfParser.py

@@ -1,185 +0,0 @@
-import pytest
-
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer, TokenKind, LexerError
-from parsers.BaseParser import UnexpectedTokenErrorNode
-from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
-from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
-    ConceptLexerParser, ConceptNode, ConceptExpression, cnode
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-
-    return ExecutionContext("sheerka", Event(), sheerka)
-
-
-class ClassWithName():
-    def __init__(self, name):
-        self.name = name
-
-
-@pytest.mark.parametrize("expression, expected", [
-    ("'str'", StrMatch("str")),
-    ("1", StrMatch("1")),
-    ("  1", StrMatch("1")),
-    (",", StrMatch(",")),
-    ("'foo'?", Optional(StrMatch("foo"))),
-    ("'foo'*", ZeroOrMore(StrMatch("foo"))),
-    ("'foo'+", OneOrMore(StrMatch("foo"))),
-    ("1 | 2 | 3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))),
-    ("1|2|3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))),
-    ("1'|' 2 '|' 3", Sequence(StrMatch("1"), StrMatch("|"), StrMatch("2"), StrMatch("|"), StrMatch("3"))),
-    ("1 2 'foo'", Sequence(StrMatch("1"), StrMatch("2"), StrMatch("foo"))),
-    ("1 2 | 3 4+", OrderedChoice(
-        Sequence(StrMatch("1"), StrMatch("2")),
-        Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))),
-    ("1 (2 | 3) 4+", Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))),
-    ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))),
-    ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))),
-    ("1 *", Sequence(StrMatch("1"), StrMatch("*"))),
-    ("1 ?", Sequence(StrMatch("1"), StrMatch("?"))),
-    ("1 +", Sequence(StrMatch("1"), StrMatch("+"))),
-    ("(1|*) +", Sequence(OrderedChoice(StrMatch("1"), StrMatch("*")), StrMatch("+"))),
-    ("1, :&", Sequence(StrMatch("1"), StrMatch(","), StrMatch(":"), StrMatch("&"))),
-    ("(1 )", StrMatch("1")),
-    ("'str'=var", StrMatch("str", rule_name="var")),
-    ("'foo'?=var", Optional(StrMatch("foo"), rule_name="var")),
-    ("('foo'?)=var", Optional(StrMatch("foo"), rule_name="var")),
-    ("'foo'*=var", ZeroOrMore(StrMatch("foo"), rule_name="var")),
-    ("('foo'*)=var", ZeroOrMore(StrMatch("foo"), rule_name="var")),
-    ("'foo'+=var", OneOrMore(StrMatch("foo"), rule_name="var")),
-    ("('foo'+)=var", OneOrMore(StrMatch("foo"), rule_name="var")),
-    ("'foo'=var?", Optional(StrMatch("foo", rule_name="var"))),
-    ("('foo'=var)?", Optional(StrMatch("foo", rule_name="var"))),
-    ("'foo'=var*", ZeroOrMore(StrMatch("foo", rule_name="var"))),
-    ("('foo'=var)*", ZeroOrMore(StrMatch("foo", rule_name="var"))),
-    ("'foo'=var+", OneOrMore(StrMatch("foo", rule_name="var"))),
-    ("('foo'=var)+", OneOrMore(StrMatch("foo", rule_name="var"))),
-    ("(1 | 2 | 3)=var", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"), rule_name="var")),
-    ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")),
-    ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")),
-    ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))),
-])
-def test_i_can_parse_regex(expression, expected):
-    parser = BnfParser()
-    res = parser.parse(get_context(), Tokenizer(expression))
-
-    assert not parser.has_error
-    assert res.status
-    assert res.value.value == expected
-    assert res.value.source == expression
-
-
-@pytest.mark.parametrize("expression, expected", [
-    ("foo", Concept("foo").init_key()),
-    ("foo*", ZeroOrMore(Concept("foo").init_key())),
-    ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))),
-    ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))),
-    ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
-    ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
-])
-def test_i_can_parse_regex_with_concept(expression, expected):
-    foo = Concept("foo")
-    bar = Concept("bar")
-    var = Concept("var")
-    context = get_context()
-
-    for c in (foo, bar, var):
-        context.sheerka.add_in_cache(c)
-    parser = BnfParser()
-    res = parser.parse(context, Tokenizer(expression))
-
-    assert not parser.has_error
-    assert res.status
-    assert res.value.value == expected
-    assert res.value.source == expression
-
-
-def test_i_can_parse_regex_with_concept_when_the_concept_is_still_under_definition():
-    expression = "foo"
-    expected = ConceptExpression("foo")
-
-    context = get_context()
-    context.obj = ClassWithName("foo")
-
-    parser = BnfParser()
-    res = parser.parse(context, Tokenizer(expression))
-
-    assert not parser.has_error
-    assert res.status
-    assert res.value.value == expected
-    assert res.value.source == expression
-
-
-@pytest.mark.parametrize("expression, error", [
-    ("1 ", UnexpectedEndOfFileError()),
-    ("1|", UnexpectedEndOfFileError()),
-    ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.RPAR])),
-    ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.IDENTIFIER])),
-    ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
-])
-def test_i_can_detect_errors(expression, error):
-    parser = BnfParser()
-    res = parser.parse(get_context(), Tokenizer(expression))
-    ret_value = res.value.value
-    assert parser.has_error
-    assert not res.status
-    assert ret_value[0] == error
-
-
-def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text():
-    foo = Concept(name="foo")
-    bar = Concept(name="bar")
-    context = get_context()
-    context.sheerka.add_in_cache(foo)
-    context.sheerka.add_in_cache(bar)
-
-    regex_parser = BnfParser()
-    foo_definition = regex_parser.parse(context, "'twenty' | 'thirty'").value.value
-    bar_definition = regex_parser.parse(context, "foo ('one' | 'two')").value.value
-
-    concepts = {bar: bar_definition, foo: foo_definition}
-    concept_parser = ConceptLexerParser()
-    concept_parser.initialize(context, concepts)
-
-    res = concept_parser.parse(context, "twenty two")
-    assert res.status
-    assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
-
-    res = concept_parser.parse(context, "thirty one")
-    assert res.status
-    assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
-
-    res = concept_parser.parse(context, "twenty")
-    assert res.status
-    assert res.value.body == [cnode("foo", 0, 0, "twenty")]
-
-
-def test_i_cannot_parse_when_too_many_concepts():
-    foo1 = Concept(name="foo", body="1")
-    foo2 = Concept(name="foo", body="2")
-    context = get_context()
-    context.sheerka.cache_by_key["foo"] = [foo1, foo2]
-
-    regex_parser = BnfParser()
-    res = regex_parser.parse(context, "foo")
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
-    assert res.value.body == ('key', 'foo')
-
-
-def test_i_cannot_parse_when_unknown_concept():
-    context = get_context()
-
-    regex_parser = BnfParser()
-    res = regex_parser.parse(get_context(), "foo")
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert res.value.body == ('key', 'foo')

tests/test_ConceptEvaluator.py

@@ -1,139 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import Concept, ConceptParts
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.ConceptEvaluator import ConceptEvaluator
-from parsers.ExactConceptParser import ExactConceptParser
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_return_value(concept, source=None):
-    return ReturnValueConcept(
-        "some_name",
-        True,
-        ParserResultConcept(parser=ExactConceptParser(),
-                            source=source or concept.name,
-                            value=concept,
-                            try_parsed=concept))
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=Concept())), True),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=Concept())), False),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept")), False),
-    (ReturnValueConcept("some_name", True, Concept()), False),
-])
-def test_i_can_match(ret_val, expected):
-    context = get_context()
-    assert ConceptEvaluator().matches(context, ret_val) == expected
-
-
-def test_i_can_evaluate_concept():
-    context = get_context()
-    concept = Concept(name="foo",
-                      where="1",
-                      pre="2",
-                      post="3").def_prop("a", "4").def_prop("b", "5")
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept)
-    result = evaluator.eval(context, item)
-
-    assert result.who == evaluator.name
-    assert result.status
-    assert result.value.name == "foo"
-    assert result.value.get_metadata_value(ConceptParts.WHERE) == 1
-    assert result.value.get_metadata_value(ConceptParts.PRE) == 2
-    assert result.value.get_metadata_value(ConceptParts.POST) == 3
-    assert result.value.get_prop("a") == 4
-    assert result.value.get_prop("b") == 5
-    assert result.value.key == "foo"
-    assert result.parents == [item]
-
-
-def test_body_is_returned_when_defined_and_requested():
-    context = get_context()
-    concept = Concept(name="foo",
-                      body="'I have a value'",
-                      where="1",
-                      pre="2",
-                      post="3").set_prop("a", "4").set_prop("b", "5")
-
-    evaluator = ConceptEvaluator(return_body=True)
-    item = get_return_value(concept)
-    result = evaluator.eval(context, item)
-
-    assert result.who == evaluator.name
-    assert result.status
-    assert result.value == "I have a value"
-    assert result.parents == [item]
-
-
-def test_body_is_not_returned_if_not_requested():
-    context = get_context()
-    concept = Concept(name="foo",
-                      body="'I have a value'",
-                      where="1",
-                      pre="2",
-                      post="3").set_prop("a", "4").set_prop("b", "5")
-
-    evaluator = ConceptEvaluator(return_body=False)  # which is the default behaviour
-    item = get_return_value(concept)
-    result = evaluator.eval(context, item)
-
-    assert result.who == evaluator.name
-    assert result.status
-    assert result.value == concept
-    assert result.parents == [item]
-
-
-def test_i_can_eval_if_with_the_same_name_is_defined_in_the_context():
-    # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
-    # ConceptEvaluator will be called to resolve 'a' while we know that 'a' refers to the string 'property_a'
-
-    context = get_context()
-    context.obj = Concept("other").set_prop("foo", "'some_other_value'")
-    concept = Concept(name="foo")
-
-    item = get_return_value(concept)
-    result = ConceptEvaluator().eval(context, item)
-
-    assert result.status
-    assert result.value == "'some_other_value'"
-
-
-def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown():
-    context = get_context()
-    context.sheerka.add_in_cache(Concept(name="one").init_key())
-    concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b")
-                                                .def_prop("a", "one")
-                                                .def_prop("b", "two").init_key())
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept_plus)
-    result = evaluator.eval(context, item)
-
-    assert not result.status
-    assert context.sheerka.isinstance(result.value, BuiltinConcepts.CONCEPT_EVAL_ERROR)
-    assert result.value.property_name == "b"
-    assert context.sheerka.isinstance(result.value.error, BuiltinConcepts.TOO_MANY_ERRORS)
-    assert result.value.concept == concept_plus
-
-
-def test_that_error_concepts_are_transformed_into_errors_only_if_the_key_is_different():
-    context = get_context()
-
-    error_concept = context.sheerka.new(BuiltinConcepts.ERROR)
-    item = get_return_value(error_concept)
-    result = ConceptEvaluator().eval(context, item)
-
-    assert not context.sheerka.is_success(error_concept)  # it's indeed an error
-    assert result.status
-    assert result.value == error_concept

tests/test_ConceptsWithConceptsParser.py

@@ -1,204 +0,0 @@
-import ast
-
-import pytest
-
-from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Token, TokenKind, Tokenizer
-from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode
-from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser
-from parsers.MultipleConceptsParser import MultipleConceptsParser
-from parsers.PythonParser import PythonNode
-from sdp.sheerkaDataProvider import Event
-
-multiple_concepts_parser = MultipleConceptsParser()
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_ret_from(*args):
-    result = []
-    index = 0
-    source = ""
-    for item in args:
-        if isinstance(item, Concept):
-            tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
-            result.append(ConceptNode(item, index, index, tokens, item.name))
-            index += 1
-            source += item.name
-        elif isinstance(item, PythonNode):
-            tokens = list(Tokenizer(item.source))[:-1]  # strip trailing EOF
-            result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source))
-            index += len(tokens)
-            source += item.source
-        else:
-            tokens = list(Tokenizer(item))[:-1]  # strip trailing EOF
-            result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
-            index += len(tokens)
-            source += item
-
-    return ReturnValueConcept(
-        "who",
-        False,
-        ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source))
-
-
-def init(concepts, inputs):
-    context = get_context()
-    for concept in concepts:
-        context.sheerka.create_new_concept(context, concept)
-
-    return context, get_ret_from(*inputs)
-
-
-def execute(concepts, inputs):
-    context, input_return_values = init(concepts, inputs)
-
-    parser = ConceptsWithConceptsParser()
-    result = parser.parse(context, input_return_values.body)
-
-    wrapper = result.body
-    return_value = result.body.body
-
-    return context, parser, result, wrapper, return_value
-
-
-@pytest.mark.parametrize("text, interested", [
-    ("not parser result", False),
-    (ParserResultConcept(parser="not multiple_concepts_parser"), False),
-    (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True),
-])
-def test_not_interested(text, interested):
-    context = get_context()
-
-    res = ConceptsWithConceptsParser().parse(context, text)
-    if interested:
-        assert res is not None
-    else:
-        assert res is None
-
-
-def test_i_can_parse_composition_of_concepts():
-    foo = Concept("foo")
-    bar = Concept("bar")
-    plus = Concept("a plus b").def_prop("a").def_prop("b")
-
-    context, parser, result, wrapper, return_value = execute([foo, bar, plus], [foo, " plus ", bar])
-
-    assert result.status
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert result.who == wrapper.parser.name
-    assert wrapper.source == "foo plus bar"
-    assert context.sheerka.isinstance(return_value, plus)
-
-    assert return_value.compiled["a"] == foo
-    assert return_value.compiled["b"] == bar
-
-    # sanity check, I can evaluate the result
-    evaluated = context.sheerka.evaluate_concept(context, return_value)
-    assert evaluated.key == return_value.key
-    assert evaluated.get_prop("a") == foo.init_key()
-    assert evaluated.get_prop("b") == bar.init_key()
-
-
-def test_i_can_parse_when_composition_of_source_code():
-    plus = Concept("a plus b", body="a + b").def_prop("a").def_prop("b")
-    left = PythonNode("1+1", ast.parse("1+1", mode="eval"))
-    right = PythonNode("2+2", ast.parse("2+2", mode="eval"))
-    context, parser, result, wrapper, return_value = execute([plus], [left, " plus ", right])
-
-    assert result.status
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert result.who == wrapper.parser.name
-    assert wrapper.source == "1+1 plus 2+2"
-    assert context.sheerka.isinstance(return_value, plus)
-
-    left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left)
-    right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right)
-    assert return_value.compiled["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)]
-    assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)]
-
-    # sanity check, I can evaluate the result
-    evaluated = context.sheerka.evaluate_concept(context, return_value)
-    assert evaluated.key == return_value.key
-    assert evaluated.get_prop("a") == 2
-    assert evaluated.get_prop("b") == 4
-    assert evaluated.body == 6
-
-
-def test_i_can_parse_when_mix_of_concept_and_code():
-    plus = Concept("a plus b").def_prop("a").def_prop("b")
-    code = PythonNode("1+1", ast.parse("1+1", mode="eval"))
-    foo = Concept("foo")
-    context, parser, result, wrapper, return_value = execute([plus, foo], [foo, " plus ", code])
-
-    assert result.status
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert result.who == wrapper.parser.name
-    assert wrapper.source == "foo plus 1+1"
-    assert context.sheerka.isinstance(return_value, plus)
-
-    code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code)
-    assert return_value.compiled["a"] == foo
-    assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)]
-
-    # sanity check, I can evaluate the result
-    evaluated = context.sheerka.evaluate_concept(context, return_value)
-    assert evaluated.key == return_value.key
-    assert evaluated.get_prop("a") == foo.init_key()
-    assert evaluated.get_prop("b") == 2
-
-
-def test_i_can_parse_when_multiple_concepts_are_recognized():
-    foo = Concept("foo")
-    bar = Concept("bar")
-    plus_1 = Concept("a plus b", body="body1").def_prop("a").def_prop("b")
-    plus_2 = Concept("a plus b", body="body2").def_prop("a").def_prop("b")
-
-    context, input_return_values = init([foo, bar, plus_1, plus_2], [foo, " plus ", bar])
-    parser = ConceptsWithConceptsParser()
-    result = parser.parse(context, input_return_values.body)
-
-    assert len(result) == 2
-
-    res = result[0]
-    wrapper = res.value
-    return_value = res.value.value
-    assert res.status
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert res.who == wrapper.parser.name
-    assert wrapper.source == "foo plus bar"
-    assert context.sheerka.isinstance(return_value, plus_1)
-    assert return_value.compiled["a"] == foo
-    assert return_value.compiled["b"] == bar
-
-    res = result[1]
-    wrapper = res.value
-    return_value = res.value.value
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert res.who == wrapper.parser.name
-    assert wrapper.source == "foo plus bar"
-    assert context.sheerka.isinstance(return_value, plus_2)
-    assert return_value.compiled["a"] == foo
-    assert return_value.compiled["b"] == bar
-
-
-def test_i_cannot_parse_when_unknown_concept():
-    foo = Concept("foo")
-    bar = Concept("bar")
-
-    context, input_return_values = init([foo, bar], [foo, " plus ", bar])
-    parser = ConceptsWithConceptsParser()
-    result = parser.parse(context, input_return_values.body)
-    wrapper = result.body
-    return_value = result.body.body
-
-    assert not result.status
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME)
-    assert result.who == parser.name
-    assert return_value == input_return_values.body.body

tests/test_DefaultParser.py

@@ -1,368 +0,0 @@
-import pytest
-import ast
-
-from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptExpression
-from parsers.PythonParser import PythonParser, PythonNode
-from core.tokenizer import Keywords, Tokenizer, LexerError
-from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode
-from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode
-from parsers.BnfParser import BnfParser
-
-from sdp.sheerkaDataProvider import Event
-
-
-def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None):
-    def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
-
-    if body:
-        def_concept.body = get_concept_part(body)
-    if where:
-        def_concept.where = get_concept_part(where)
-    if pre:
-        def_concept.pre = get_concept_part(pre)
-    if post:
-        def_concept.post = get_concept_part(post)
-    if definition:
-        def_concept.definition = ReturnValueConcept(
-            "parsers.Bnf",
-            True,
-            definition)
-
-    return def_concept
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_concept_part(part):
-    if isinstance(part, str):
-        node = PythonNode(part, ast.parse(part, mode="eval"))
-        return ReturnValueConcept(
-            who="parsers.Default",
-            status=True,
-            value=ParserResultConcept(
-                source=part,
-                parser=PythonParser(),
-                value=node))
-
-    if isinstance(part, PythonNode):
-        return ReturnValueConcept(
-            who="parsers.Default",
-            status=True,
-            value=ParserResultConcept(
-                source=part.source,
-                parser=PythonParser(),
-                value=part))
-
-    if isinstance(part, ReturnValueConcept):
-        return part
-
-
-@pytest.mark.parametrize("text, expected", [
-    ("def concept hello", get_def_concept(name="hello")),
-    ("def concept hello  ", get_def_concept(name="hello")),
-    ("def concept a + b", get_def_concept(name="a + b")),
-    ("def concept a+b", get_def_concept(name="a + b")),
-    ("def concept 'a+b'+c", get_def_concept(name="'a+b' + c")),
-    ("def concept 'as if'", get_def_concept(name="'as if'")),
-    ("def concept 'as' if", get_def_concept(name="'as if'")),
-    ("def concept hello as 'hello'", get_def_concept(name="hello", body="'hello'")),
-    ("def concept hello as 1", get_def_concept(name="hello", body="1")),
-    ("def concept hello as 1 + 1", get_def_concept(name="hello", body="1 + 1")),
-])
-def test_i_can_parse_def_concept(text, expected):
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-    node = res.value.value
-
-    assert res.status
-    assert res.who == parser.name
-    assert res.value.source == text
-    assert isinstance(res.value, ParserResultConcept)
-    assert node == expected
-
-
-def test_i_can_parse_complex_def_concept_statement():
-    text = """def concept a plus b
-where a,b
-pre isinstance(a, int) and isinstance(b, float)
-post isinstance(res, int)
-as res = a + b
-"""
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-    return_value = res.value
-    expected_concept = get_def_concept(
-        name="a plus b",
-        where="a,b",
-        pre="isinstance(a, int) and isinstance(b, float)",
-        post="isinstance(res, int)",
-        body=PythonNode("res = a + b", ast.parse("res = a + b", mode="exec"))
-    )
-
-    assert res.status
-    assert isinstance(return_value, ParserResultConcept)
-    assert return_value.value == expected_concept
-
-
-def test_i_can_have_mutilines_declarations():
-    text = """
-def concept add one to a as
-def func(x):
-    return x+1
-func(a)
-    """
-
-    expected_concept = get_def_concept(
-        name="add one to a ",
-        body=PythonNode(
-            "def func(x):\n    return x+1\nfunc(a)",
-            ast.parse("def func(x):\n    return x+1\nfunc(a)", mode="exec"))
-    )
-
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-    return_value = res.value
-
-    assert res.status
-    assert isinstance(return_value, ParserResultConcept)
-    assert return_value.value == expected_concept
-
-
-def test_i_can_use_colon_to_use_indentation():
-    text = """
-def concept add one to a as:
-    def func(x):
-        return x+1
-    func(a)
-"""
-
-    expected_concept = get_def_concept(
-        name="add one to a ",
-        body=PythonNode(
-            "def func(x):\n    return x+1\nfunc(a)",
-            ast.parse("def func(x):\n    return x+1\nfunc(a)", mode="exec"))
-    )
-
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-    return_value = res.value
-
-    assert res.status
-    assert isinstance(return_value, ParserResultConcept)
-    assert return_value.value == expected_concept
-
-
-def test_indentation_is_mandatory_after_a_colon():
-    text = """
-def concept add one to a as:
-def func(x):
-    return x+1
-func(a)
-"""
-
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-    return_value = res.value
-
-    assert not res.status
-    assert isinstance(return_value, ParserResultConcept)
-    assert isinstance(return_value.value[0], SyntaxErrorNode)
-    assert return_value.value[0].message == "Indentation not found."
-
-
-def test_indentation_is_not_allowed_if_the_colon_is_missing():
-    text = """
-def concept add one to a as
-    def func(x):
-        return x+1
-    func(a)
-    """
-    context = get_context()
-    sheerka = context.sheerka
-
-    parser = DefaultParser()
-    res = parser.parse(context, text)
-    return_value = res.value
-
-    assert not res.status
-    assert context.sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
-
-
-def test_name_is_mandatory():
-    text = "def concept as 'hello'"
-
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-    return_value = res.value
-
-    assert not res.status
-    assert isinstance(return_value, ParserResultConcept)
-    assert isinstance(return_value.value[0], SyntaxErrorNode)
-    assert return_value.value[0].message == "Name is mandatory"
-
-
-def test_concept_keyword_is_mandatory_but_the_concept_is_recognized():
-    text = "def hello as a where b pre c post d"
-
-    expected_concept = get_def_concept(name="hello", body="a", where="b", pre="c", post="d")
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-    return_value = res.value
-
-    assert not res.status
-    assert isinstance(return_value, ParserResultConcept)
-    assert isinstance(return_value.value[0], UnexpectedTokenErrorNode)
-    assert return_value.value[0].message == "Syntax error."
-    assert return_value.value[0].expected_tokens == [Keywords.CONCEPT]
-    assert return_value.try_parsed == expected_concept
-
-
-@pytest.mark.parametrize("text", [
-    "def concept hello where 1+",
-    "def concept hello pre 1+",
-    "def concept hello post 1+",
-    "def concept hello as 1+"
-])
-def test_i_can_detect_error_in_declaration(text):
-    context = get_context()
-    sheerka = context.sheerka
-
-    parser = DefaultParser()
-    res = parser.parse(context, text)
-    return_value = res.value
-
-    assert not res.status
-    assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
-
-
-def test_new_line_is_not_allowed_in_the_name():
-    text = "def concept hello \n my friend as 'hello'"
-
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-    return_value = res.value
-
-    assert not res.status
-    assert return_value.value == [SyntaxErrorNode([], "Newline are not allowed in name.")]
-
-
-def test_i_can_parse_def_concept_from_regex():
-    context = get_context()
-    a_concept = Concept("a_concept")
-    context.sheerka.add_in_cache(a_concept)
-
-    text = "def concept name from bnf a_concept | 'a_string' as __definition[0]"
-    parser = DefaultParser()
-    res = parser.parse(context, text)
-    node = res.value.value
-    definition = OrderedChoice(a_concept, StrMatch("a_string"))
-    parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition)
-    expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result)
-
-    assert res.status
-    assert res.who == parser.name
-    assert res.value.source == text
-    assert isinstance(res.value, ParserResultConcept)
-    assert node == expected
-
-
-def test_i_can_parse_def_concept_where_bnf_references_itself():
-    context = get_context()
-    a_concept = Concept("a_concept")
-    context.sheerka.add_in_cache(a_concept)
-
-    text = "def concept name from bnf 'a' + name?"
-    parser = DefaultParser()
-    parser.parse(context, text)
-
-    assert not parser.has_error
-
-
-def test_i_can_detect_empty_bnf_declaration():
-    text = "def concept name from bnf as __definition[0]"
-
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-
-    assert not res.status
-    assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration")
-
-
-def test_i_can_detect_not_for_me():
-    text = "hello world"
-    context = get_context()
-    parser = DefaultParser()
-    res = parser.parse(context, text)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
-    assert isinstance(res.value.body[0], CannotHandleErrorNode)
-
-
-def test_i_can_parse_is_a():
-    parser = DefaultParser()
-    text = "the name of my 'concept' isa the name of the set"
-    res = parser.parse(get_context(), text)
-    expected = IsaConceptNode([],
-                              concept=NameNode(list(Tokenizer("the name of my 'concept'"))),
-                              set=NameNode(list(Tokenizer("the name of the set"))))
-
-    assert res.status
-    assert res.who == parser.name
-    assert res.value.source == text
-    assert isinstance(res.value, ParserResultConcept)
-    assert res.value.value == expected
-
-
-@pytest.mark.parametrize("text", [
-    "concept",
-    "isa number",
-    "name isa",
-    "def",
-    "def concept_name"
-])
-def test_i_cannot_parse_invalid_entries(text):
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-
-    assert not res.status
-    assert isinstance(res.body, ParserResultConcept)
-    assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
-
-
-@pytest.mark.parametrize("text, error_msg, error_text", [
-    ("'name", "Missing Trailing quote", "'name"),
-    ("foo isa 'name", "Missing Trailing quote", "'name"),
-    ("def concept 'name", "Missing Trailing quote", "'name"),
-    ("def concept name as 'body", "Missing Trailing quote", "'body"),
-    ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"),
-    ("def concept c::", "Concept name not found", ""),
-])
-def test_i_cannot_parse_when_tokenizer_fails(text, error_msg, error_text):
-    parser = DefaultParser()
-    res = parser.parse(get_context(), text)
-
-    assert not res.status
-    assert isinstance(res.body, ParserResultConcept)
-    assert isinstance(res.body.body[0], LexerError)
-    assert res.body.body[0].message == error_msg
-    assert res.body.body[0].text == error_text
-
-
-def test_i_cannot_parse_bnf_definition_referencing_unknown_concept():
-    context = get_context()
-    text = "def concept name from bnf unknown"
-
-    parser = DefaultParser()
-    res = parser.parse(context, text)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert res.value.body == ("key", "unknown")

tests/test_EvalEvaluator.py

@@ -1,80 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.EvalEvaluator import EvalEvaluator
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def r(value, status=True):
-    return ReturnValueConcept("some_name", status, value)
-
-
-eval_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.CONCEPT_EVAL_REQUESTED))
-
-
-def test_i_can_match_and_eval():
-    context = get_context()
-
-    to_eval1 = ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init())
-    to_eval2 = ReturnValueConcept("some_name", True, Concept(name="3", body="also to eval").auto_init())
-
-    return_values = [
-        ReturnValueConcept("some_name", True, "not to eval"),
-        ReturnValueConcept("some_name", True, Concept(name="not to eval")),
-        ReturnValueConcept("some_name", False, Concept(name="1", body="'not to eval'")),
-        to_eval1,
-        to_eval2,
-        eval_requested
-    ]
-
-    evaluator = EvalEvaluator()
-    assert evaluator.matches(context, return_values)
-
-    evaluated = evaluator.eval(context, return_values)
-    assert len(evaluated) == 2
-    assert evaluated[0].value == to_eval1.body.body
-    assert evaluated[0].parents == [to_eval1, eval_requested]
-
-    assert evaluated[1].value == to_eval2.body.body
-    assert evaluated[1].parents == [to_eval2, eval_requested]
-
-
-@pytest.mark.parametrize("return_values, expected", [
-    ([r(Concept("foo", body="bar")), eval_requested], True),
-    ([r(Concept("status is false", body="bar"), False), eval_requested], True),
-    ([r("string_value"), eval_requested], True),
-    ([r(Concept("no body")), eval_requested], True),
-    ([r(Concept("eval requested missing", body="bar"))], False),
-])
-def test_i_cannot_match_if_eval_request_is_not_present(return_values, expected):
-    context = get_context()
-    assert EvalEvaluator().matches(context, return_values) == expected
-
-
-def test_concept_eval_requested_is_reduced_when_nothing_to_reduce():
-    context = get_context()
-
-    return_values = [
-        ReturnValueConcept("some_name", True, "not to eval"),
-        ReturnValueConcept("some_name", True, Concept(name="not to eval")),
-        ReturnValueConcept("some_name", False, Concept(name="1", body="not to eval")),
-        eval_requested
-    ]
-
-    evaluator = EvalEvaluator()
-    assert evaluator.matches(context, return_values)
-
-    evaluated = evaluator.eval(context, return_values)
-    assert evaluated == ReturnValueConcept(
-        "evaluators.Eval",
-        False,
-        context.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_REQUESTED))
-    assert evaluated.parents == [eval_requested]

tests/test_ExactConceptParser.py

@@ -1,151 +0,0 @@
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, Property
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer
-from parsers.ExactConceptParser import ExactConceptParser
-from sdp.sheerkaDataProvider import Event
-
-
-def metadata_prop(concept, prop_name):
-    for name, value in concept.metadata.props:
-        if name == prop_name:
-            return value
-
-    return None
-
-
-def test_i_can_compute_combinations():
-    parser = ExactConceptParser()
-    res = parser.combinations(["foo", "bar", "baz"])
-
-    assert res == {('foo', 'bar', 'baz'),
-                   ('__var__0', 'bar', 'baz'),
-                   ('foo', '__var__0', 'baz'),
-                   ('foo', 'bar', '__var__0'),
-                   ('__var__0', '__var__1', 'baz'),
-                   ('__var__0', 'bar', '__var__1'),
-                   ('foo', '__var__0', '__var__1'),
-                   ('__var__0', '__var__1', '__var__2')}
-
-
-def test_i_can_compute_combinations_with_duplicates():
-    parser = ExactConceptParser()
-    res = parser.combinations(["foo", "bar", "foo"])
-
-    assert res == {('foo', 'bar', 'foo'),
-                   ('__var__0', 'bar', '__var__0'),
-                   ('foo', '__var__0', 'foo'),
-                   ('__var__0', '__var__1', '__var__0'),
-                   ('__var__1', '__var__0', '__var__1')}
-    # TODO: the last tuple is not possible, so the algo can be improved
-
-
-def test_i_can_recognize_a_simple_concept():
-    context = get_context()
-    concept = get_concept("hello world", [])
-    context.sheerka.add_in_cache(concept)
-
-    source = "hello world"
-    results = ExactConceptParser().parse(context, source)
-
-    assert len(results) == 1
-    assert results[0].status
-    assert results[0].value.value == concept
-
-
-def test_i_can_recognize_concepts_defined_several_times():
-    context = get_context()
-    context.sheerka.add_in_cache(get_concept("hello world", []))
-    context.sheerka.add_in_cache(get_concept("hello a", ["a"]))
-
-    source = "hello world"
-    results = ExactConceptParser().parse(context, source)
-
-    assert len(results) == 2
-    results = sorted(results, key=lambda x: x.value.value.name)  # because of the usage of sets
-
-    assert results[0].status
-    assert results[0].value.value.name == "hello a"
-    assert metadata_prop(results[0].value.value, "a") == "world"
-
-    assert results[1].status
-    assert results[1].value.value.name == "hello world"
-
-
-def test_i_can_recognize_a_concept_with_variables():
-    context = get_context()
-    concept = get_concept("a + b", ["a", "b"])
-    context.sheerka.add_in_cache(concept)
-    source = "10 + 5"
-    results = ExactConceptParser().parse(context, source)
-
-    assert len(results) == 1
-    assert results[0].status
-    concept_found = results[0].value.value
-    assert concept_found.key == concept.key
-    assert metadata_prop(concept_found, "a") == "10"
-    assert metadata_prop(concept_found, "b") == "5"
-
-
-def test_i_can_recognize_a_concept_with_duplicate_variables():
-    context = get_context()
-    concept = get_concept("a + b + a", ["a", "b"])
-    context.sheerka.cache_by_key[concept.key] = concept
-    source = "10 + 5 + 10"
-    results = ExactConceptParser().parse(context, source)
-
-    assert len(results) == 1
-    assert results[0].status
-    concept_found = results[0].value.value
-    assert concept_found.key == concept.key
-    assert metadata_prop(concept_found, "a") == "10"
-    assert metadata_prop(concept_found, "b") == "5"
-
-
-def test_i_can_manage_unknown_concept():
-    context = get_context()
-    source = "def concept hello world"  # this is not a concept by itself
-    res = ExactConceptParser().parse(context, source)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert res.value.body == "def concept hello world"
-
-
-def test_i_can_detect_concepts_too_long():
-    context = get_context()
-    source = "a very very long concept that cannot be an unique one"
-    res = ExactConceptParser().parse(context, source)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_TOO_LONG)
-    assert res.value.body == "a very very long concept that cannot be an unique one"
-
-
-def test_i_can_detect_concept_from_tokens():
-    context = get_context()
-    concept = get_concept("hello world", [])
-    context.sheerka.add_in_cache(concept)
-
-    source = "hello world"
-    results = ExactConceptParser().parse(context, list(Tokenizer(source)))
-
-    assert len(results) == 1
-    assert results[0].status
-    assert results[0].value.value == concept
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-
-    return ExecutionContext("sheerka", Event(), sheerka)
-
-
-def get_concept(name, variables):
-    c = Concept(name=name)
-    if variables:
-        for v in variables:
-            c.def_prop(v)
-    c.init_key()
-    return c

tests/test_LexerNodeEvaluator.py

@@ -1,115 +0,0 @@
-import ast
-
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import Concept, ConceptParts, DoNotResolve
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.LexerNodeEvaluator import LexerNodeEvaluator
-from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, StrMatch, UnrecognizedTokensNode, SourceCodeNode
-from parsers.PythonParser import PythonNode
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def from_parsing(context, grammar, expression):
-    parser = ConceptLexerParser()
-    parser.initialize(context, grammar)
-
-    ret_val = parser.parse(context, expression)
-    assert ret_val.status
-    return ret_val
-
-
-def from_fragments(*fragments):
-    nodes = []
-    for fragment in fragments:
-        if isinstance(fragment, str):
-            node = PythonNode(fragment, ast.parse(fragment.strip(), mode="eval"))
-            nodes.append(SourceCodeNode(node, 0, 0, [], fragment))
-        else:
-            nodes.append(ConceptNode(fragment, 0, 0, [], fragment.name))
-
-    return ReturnValueConcept("somme_name", True, ParserResultConcept(value=nodes))
-
-
-def init(concept, grammar, text):
-    context = get_context()
-    if isinstance(concept, list):
-        for c in concept:
-            context.sheerka.add_in_cache(c)
-    else:
-        context.sheerka.add_in_cache(concept)
-    ret_val = from_parsing(context, grammar, text)
-    node = ret_val.value.value[0]
-
-    return context, node
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), True),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), True),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), True),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=SourceCodeNode(0, 0, []))), True),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=[UnrecognizedTokensNode(0, 0, [])])), False),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=UnrecognizedTokensNode(0, 0, []))), False),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), False),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), False),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), False),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=SourceCodeNode(0, 0, []))), False),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept node")), False),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=["Not a concept node"])), False),
-    (ReturnValueConcept("some_name", True, [ConceptNode(Concept(), 0, 0)]), False),
-    (ReturnValueConcept("some_name", True, ConceptNode(Concept(), 0, 0)), False),
-])
-def test_i_can_match(ret_val, expected):
-    context = get_context()
-    assert LexerNodeEvaluator().matches(context, ret_val) == expected
-
-
-def test_concept_is_returned_when_only_one_in_the_list():
-    foo = Concept("foo")
-    context = get_context()
-    context.sheerka.add_in_cache(foo)
-
-    ret_val = from_parsing(context, {foo: StrMatch("foo")}, "foo")
-
-    evaluator = LexerNodeEvaluator()
-    result = evaluator.eval(context, ret_val)
-    wrapper = result.body
-    return_value = result.body.body
-
-    assert result.who == evaluator.name
-    assert result.status
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert wrapper.parser == evaluator
-    assert wrapper.source == "foo"
-    assert return_value == Concept("foo").init_key()
-    assert return_value.compiled[ConceptParts.BODY] == DoNotResolve("foo")
-    assert result.parents == [ret_val]
-
-
-def test_concept_python_node_is_returned_when_source_code():
-    context = get_context()
-    foo = Concept("foo")
-    ret_val = from_fragments(foo, " + 1")
-
-    evaluator = LexerNodeEvaluator()
-    result = evaluator.eval(context, ret_val)
-    wrapper = result.body
-    return_value = result.body.body
-
-    assert result.who == evaluator.name
-    assert result.status
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert wrapper.parser == evaluator
-    assert wrapper.source == "foo + 1"
-
-    assert return_value == PythonNode('foo + 1', ast.parse("__C__foo__C__ + 1", mode="eval"))
-    assert return_value.concepts == {"__C__foo__C__": foo}
-    assert result.parents == [ret_val]

tests/test_MultipleConceptsParser.py

@@ -1,231 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer, TokenKind, Token
-from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, cnode, utnode, scnode, SourceCodeNode
-from parsers.MultipleConceptsParser import MultipleConceptsParser
-from parsers.PythonParser import PythonNode
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_return_value(context, grammar, expression):
-    parser = ConceptLexerParser()
-    parser.initialize(context, grammar)
-
-    ret_val = parser.parse(context, expression)
-    assert not ret_val.status
-    return ret_val
-
-
-def init(concepts, grammar, expression):
-    context = get_context()
-    for c in concepts:
-        context.sheerka.create_new_concept(context, c)
-    return_value = get_return_value(context, grammar, expression)
-
-    return context, return_value
-
-
-def test_not_interested_if_not_parser_result():
-    context = get_context()
-    text = "not parser result"
-
-    res = MultipleConceptsParser().parse(context, text)
-    assert res is None
-
-
-def test_not_interested_if_not_from_concept_lexer_parser():
-    context = get_context()
-    text = ParserResultConcept(parser="not concept lexer", value="some value")
-
-    res = MultipleConceptsParser().parse(context, text)
-    assert res is None
-
-
-def test_i_can_parse_exact_concepts():
-    foo = Concept("foo", body="'foo'")
-    bar = Concept("bar", body="'bar'")
-    baz = Concept("baz", body="'baz'")
-    grammar = {}
-    context, return_value = init([foo, bar, baz], grammar, "bar foo    baz")
-
-    parser = MultipleConceptsParser()
-    ret_val = parser.parse(context, return_value.body)
-
-    assert ret_val.status
-    assert ret_val.who == parser.name
-    assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [
-        ConceptNode(bar, 0, 0, source="bar"),
-        ConceptNode(foo, 2, 2, source="foo"),
-        ConceptNode(baz, 4, 4, source="baz")]
-    assert ret_val.value.source == "bar foo    baz"
-
-
-def test_i_can_parse_when_ending_with_bnf():
-    foo = Concept("foo", body="'foo'")
-    bar = Concept("bar", body="'bar'")
-    grammar = {foo: Sequence("foo1", "foo2", "foo3")}
-    context, return_value = init([foo, bar], grammar, "bar foo1 foo2 foo3")
-
-    parser = MultipleConceptsParser()
-    ret_val = parser.parse(context, return_value.body)
-
-    assert ret_val.status
-    assert ret_val.who == parser.name
-    assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [cnode("bar", 0, 0, "bar"), cnode("foo", 2, 6, "foo1 foo2 foo3")]
-    assert ret_val.value.source == "bar foo1 foo2 foo3"
-
-
-def test_i_can_parse_when_starting_with_bnf():
-    foo = Concept("foo", body="'foo'")
-    bar = Concept("bar", body="'bar'")
-    grammar = {foo: Sequence("foo1", "foo2", "foo3")}
-    context, return_value = init([foo, bar], grammar, "foo1 foo2 foo3 bar")
-
-    parser = MultipleConceptsParser()
-    ret_val = parser.parse(context, return_value.body)
-
-    assert ret_val.status
-    assert ret_val.who == parser.name
-    assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [cnode("foo", 0, 4, "foo1 foo2 foo3"), cnode("bar", 6, 6, "bar")]
-    assert ret_val.value.source == "foo1 foo2 foo3 bar"
-
-
-def test_i_can_parse_when_concept_are_mixed():
-    foo = Concept("foo")
-    bar = Concept("bar")
-    baz = Concept("baz")
-    grammar = {foo: Sequence("foo1", "foo2", "foo3")}
-    context, return_value = init([foo, bar, baz], grammar, "baz foo1 foo2 foo3 bar")
-
-    parser = MultipleConceptsParser()
-    ret_val = parser.parse(context, return_value.body)
-
-    assert ret_val.status
-    assert ret_val.who == parser.name
-    assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [
-        cnode("baz", 0, 0, "baz"),
-        cnode("foo", 2, 6, "foo1 foo2 foo3"),
-        cnode("bar", 8, 8, "bar")]
-    assert ret_val.value.source == "baz foo1 foo2 foo3 bar"
-
-
-def test_i_can_parse_when_multiple_concepts_are_matching():
-    foo = Concept("foo")
-    bar = Concept("bar", body="bar1")
-    baz = Concept("bar", body="bar2")
-    grammar = {foo: "foo"}
-    context, return_value = init([foo, bar, baz], grammar, "foo bar")
-
-    parser = MultipleConceptsParser()
-    ret_val = parser.parse(context, return_value.body)
-
-    assert len(ret_val) == 2
-    assert ret_val[0].status
-    assert ret_val[0].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
-    assert ret_val[0].value.source == "foo bar"
-    assert ret_val[0].value.value[1].concept.metadata.body == "bar1"
-
-    assert ret_val[1].status
-    assert ret_val[1].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
-    assert ret_val[1].value.source == "foo bar"
-    assert ret_val[1].value.value[1].concept.metadata.body == "bar2"
-
-
-def test_i_can_parse_when_source_code():
-    foo = Concept("foo")
-    grammar = {foo: "foo"}
-    context, return_value = init([foo], grammar, "1 foo")
-
-    parser = MultipleConceptsParser()
-    ret_val = parser.parse(context, return_value.body)
-    wrapper = ret_val.value
-    value = ret_val.value.value
-
-    assert ret_val.status
-    assert ret_val.who == parser.name
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert wrapper.source == "1 foo"
-    assert value == [
-        scnode(0, 1, "1 "),
-        cnode("foo", 2, 2, "foo")]
-
-
-def test_i_cannot_parse_when_unrecognized_token():
-    twenty_two = Concept("twenty two")
-    one = Concept("one")
-    grammar = {twenty_two: Sequence("twenty", "two")}
-    context, return_value = init([twenty_two, one], grammar, "twenty two + one")
-
-    parser = MultipleConceptsParser()
-    ret_val = parser.parse(context, return_value.body)
-
-    assert not ret_val.status
-    assert ret_val.who == parser.name
-    assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [
-        cnode("twenty two", 0, 2, "twenty two"),
-        utnode(3, 5, " + "),
-        cnode("one", 6, 6, "one")
-    ]
-    assert ret_val.value.source == "twenty two + one"
-
-
-def test_i_cannot_parse_when_unknown_concepts():
-    twenty_two = Concept("twenty two")
-    one = Concept("one")
-    grammar = {twenty_two: Sequence("twenty", "two")}
-    context, return_value = init([twenty_two, one], grammar, "twenty two plus one")
-
-    parser = MultipleConceptsParser()
-    ret_val = parser.parse(context, return_value.body)
-
-    assert not ret_val.status
-    assert ret_val.who == parser.name
-    assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [
-        cnode("twenty two", 0, 2, "twenty two"),
-        utnode(3, 5, " plus "),
-        cnode("one", 6, 6, "one")
-    ]
-    assert ret_val.value.source == "twenty two plus one"
-
-
-@pytest.mark.parametrize("text, expected_source, expected_end", [
-    ("True", "True", 0),
-    ("1 == 1", "1 == 1", 5),
-    ("1!xdf", "1", 0),
-    ("1", "1", 0),
-])
-def test_i_can_get_source_code_node(text, expected_source, expected_end):
-    tokens = list(Tokenizer(text))[:-1]  # strip trailing EOF
-
-    start_index = 5  # a random number different of zero
-    res = MultipleConceptsParser().get_source_code_node(get_context(), start_index, tokens)
-
-    assert isinstance(res, SourceCodeNode)
-    assert isinstance(res.node, PythonNode)
-    assert res.source == expected_source
-    assert res.start == start_index
-    assert res.end == start_index + expected_end
-
-
-def test_i_cannot_parse_null_text():
-    res = MultipleConceptsParser().get_source_code_node(get_context(), 0, [])
-    assert res is None
-
-    eof = Token(TokenKind.EOF, "", 0, 0, 0)
-    res = MultipleConceptsParser().get_source_code_node(get_context(), 0, [eof])
-    assert res is None

tests/test_MultipleSameSuccessEvaluator.py

@@ -1,216 +0,0 @@
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.BaseEvaluator import BaseEvaluator
-from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
-from parsers.BaseParser import BaseParser
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def test_i_can_match_and_eval():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    evaluator = MultipleSameSuccessEvaluator()
-    assert evaluator.matches(context, return_values)
-
-    evaluated = evaluator.eval(context, return_values)
-    assert evaluated.status
-    assert evaluated.value == Concept(name="1", body="value").auto_init()  # the first concept is returned
-
-
-def test_i_can_match_and_eval_when_no_body():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    evaluator = MultipleSameSuccessEvaluator()
-    assert evaluator.matches(context, return_values)
-
-    evaluated = evaluator.eval(context, return_values)
-    assert evaluated.status
-    assert evaluated.value == Concept(name="1").auto_init()
-
-
-def test_i_can_match_and_eval_when_value_is_not_a_concept():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    evaluator = MultipleSameSuccessEvaluator()
-    assert evaluator.matches(context, return_values)
-
-    evaluated = evaluator.eval(context, return_values)
-    assert evaluated.status
-    assert evaluated.value == "value"
-
-
-def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_concept_evaluator_first():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="2", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "Concept", True, Concept(name="1", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="3", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    evaluator = MultipleSameSuccessEvaluator()
-    assert evaluator.matches(context, return_values)
-
-    evaluated = evaluator.eval(context, return_values)
-    assert evaluated.status
-    assert evaluated.value == Concept(name="1", body="value").auto_init()  # the concept is returned, not the value
-
-
-def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_python_evaluator_first():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="2", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="1", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="3", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    evaluator = MultipleSameSuccessEvaluator()
-    assert evaluator.matches(context, return_values)
-
-    evaluated = evaluator.eval(context, return_values)
-    assert evaluated.status
-    assert evaluated.value == Concept(name="1", body="value").auto_init()  # the concept is returned, not the value
-
-
-def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value2").auto_init()),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    evaluator = MultipleSameSuccessEvaluator()
-    assert evaluator.matches(context, return_values)
-    assert evaluator.eval(context, return_values) is None
-
-
-def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail_when_no_body():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2").auto_init()),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    evaluator = MultipleSameSuccessEvaluator()
-    assert evaluator.matches(context, return_values)
-    assert evaluator.eval(context, return_values) is None
-
-
-def test_i_can_match_if_no_parser():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    assert MultipleSameSuccessEvaluator().matches(context, return_values)
-
-
-def test_i_cannot_match_if_not_reduced_requested():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
-        ReturnValueConcept("some_name", True, Concept(name="2", body="value")),
-        # ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    assert not MultipleSameSuccessEvaluator().matches(context, return_values)
-
-
-def test_i_cannot_match_if_only_one_successful_evaluator():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
-        # ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
-        ReturnValueConcept("some_name", True, Concept(name="2", body="value")),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    assert not MultipleSameSuccessEvaluator().matches(context, return_values)
-
-
-def test_i_cannot_match_if_at_least_one_parser_is_successful():
-    context = get_context()
-    sheerka = context.sheerka
-
-    return_values = [
-        ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
-        ReturnValueConcept(BaseParser.PREFIX + "some_name2", True, "Not relevant"),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
-        ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
-        ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
-    ]
-
-    assert not MultipleSameSuccessEvaluator().matches(context, return_values)
-

tests/test_OneErrorEvaluator.py

@@ -1,80 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.OneErrorEvaluator import OneErrorEvaluator
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def r(value, status=True):
-    return ReturnValueConcept(value, status, value)
-
-
-reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
-
-
-@pytest.mark.parametrize("return_values, expected", [
-    ([r("evaluators.one error", False), reduce_requested], True),
-    ([r("evaluators.one error", False), r("failed", False), r("failed", False), reduce_requested], True),
-    ([r("evaluators.error", False), r("not a parser in success"), reduce_requested], True),
-    ([r("evaluators.no reduce required", False), r("failed", False), r("failed", False)], False),
-    ([r("evaluators.no reduce required", False)], False),
-    ([r("evaluators.error", False), r("evaluators.success"), reduce_requested], False),
-    ([r("evaluators.error", False), r("parsers.success"), reduce_requested], False),
-    ([r("evaluators.success"), r("not an evaluator in error", False), reduce_requested], False),
-    ([r("evaluators.error", False), r("evaluators.another error", False), reduce_requested], False),
-])
-def test_i_can_match(return_values, expected):
-    context = get_context()
-    assert OneErrorEvaluator().matches(context, return_values) == expected
-
-
-def test_i_can_eval():
-    context = get_context()
-
-    return_values = [
-        r("evaluators.one error", False),
-        r("parsers.failed", False),
-        r("parsers.failed", False),
-        reduce_requested
-    ]
-
-    evaluator = OneErrorEvaluator()
-    evaluator.matches(context, return_values)
-    res = evaluator.eval(context, return_values)
-
-    assert not res.status
-    assert res.body == "evaluators.one error"
-    assert len(res.parents) == 4
-
-
-def test_unwanted_return_values_are_not_eaten():
-    context = get_context()
-
-    a_successful_concept = r("successful concept")
-    a_concept_in_error = r("concept in error", False)
-    return_values = [
-        r("evaluators.one error", False),
-        r("parsers.failed", False),
-        r("parsers.failed", False),
-        a_successful_concept,
-        a_concept_in_error,
-        reduce_requested
-    ]
-
-    evaluator = OneErrorEvaluator()
-    evaluator.matches(context, return_values)
-    res = evaluator.eval(context, return_values)
-
-    assert not res.status
-    assert res.body == "evaluators.one error"
-    assert len(res.parents) == 4
-
-    assert a_successful_concept not in res.parents

tests/test_OneSuccessEvaluator.py

@@ -1,79 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def r(value, status=True):
-    return ReturnValueConcept(value, status, value)
-
-
-reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
-
-
-@pytest.mark.parametrize("return_values, expected", [
-    ([r("evaluators.one success"), reduce_requested], True),
-    ([r("evaluators.one success"), r("failed", False), r("failed", False), reduce_requested], True),
-    ([r("evaluators.no reduce required"), r("failed", False), r("failed", False)], False),
-    ([r("evaluators.no reduce required")], False),
-    ([r("evaluators.failed", False), r("failed", False), r("failed", False), reduce_requested], False),
-    ([r("evaluators.failed", False), r("not evaluator success"), reduce_requested], False),
-    ([r("evaluators.success"), r("evaluators.another success"), reduce_requested], False),
-    ([r("evaluators.one success"), r("other success"), r("failed", False), reduce_requested], True),
-    ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False),
-])
-def test_i_can_match(return_values, expected):
-    context = get_context()
-    assert OneSuccessEvaluator().matches(context, return_values) == expected
-
-
-def test_i_can_eval():
-    context = get_context()
-
-    return_values = [
-        r("evaluators.one success"),
-        r("failed", False),
-        r("failed", False),
-        reduce_requested
-    ]
-
-    evaluator = OneSuccessEvaluator()
-    matches = evaluator.matches(context, return_values)
-    res = evaluator.eval(context, return_values)
-
-    assert matches
-    assert res.status
-    assert res.body == "evaluators.one success"
-    assert len(res.parents) == 4
-
-
-def test_i_do_not_eat_the_other_success():
-    context = get_context()
-
-    not_a_parser_success = r("other success")
-    return_values = [
-        r("evaluators.one success"),
-        not_a_parser_success,
-        r("failed", False),
-        reduce_requested
-    ]
-
-    evaluator = OneSuccessEvaluator()
-    matches = evaluator.matches(context, return_values)
-    res = evaluator.eval(context, return_values)
-
-    assert matches
-    assert res.status
-    assert res.body == "evaluators.one success"
-    assert len(res.parents) == 3
-
-    assert not_a_parser_success not in res.parents

tests/test_PrepareEvalEvaluator.py

@@ -1,55 +0,0 @@
-import pytest
-
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.PrepareEvalEvaluator import PrepareEvalEvaluator
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-r = ReturnValueConcept
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (r("name", True, UserInputConcept("eval 1 + 1")), True),
-    (r("name", True, UserInputConcept("  eval 1 + 1")), True),
-    (r("name", True, UserInputConcept("eval")), False),
-    (r("name", True, UserInputConcept("1+1")), False),
-    (r("name", True, UserInputConcept("")), False),
-    (r("name", True, UserInputConcept("eval   ")), False),
-    (r("name", True, UserInputConcept([])), False),
-    (r("name", True, Concept("foo")), False),
-    (r("name", True, "not a concept"), False),
-    (r("name", False, UserInputConcept("eval 1 + 1")), False),
-    (r("name", False, UserInputConcept("  eval 1 + 1")), False),
-])
-def test_i_can_match(ret_val, expected):
-    context = get_context()
-    assert PrepareEvalEvaluator().matches(context, ret_val) == expected
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (r("name", True, UserInputConcept("eval 1 + 1")), "1 + 1"),
-    (r("name", True, UserInputConcept("   eval 1 + 1")), "1 + 1"),
-])
-def test_i_can_eval(ret_val, expected):
-    context = get_context()
-    sheerka = context.sheerka
-
-    prepare_evaluator = PrepareEvalEvaluator()
-    prepare_evaluator.matches(context, ret_val)
-    res = prepare_evaluator.eval(context, ret_val)
-
-    assert len(res) == 2
-    assert res[0].status
-    assert sheerka.isinstance(res[0].body, BuiltinConcepts.USER_INPUT)
-    assert res[0].body.body == expected
-
-    assert res[1].status
-    assert sheerka.isinstance(res[1].body, BuiltinConcepts.CONCEPT_EVAL_REQUESTED)

tests/test_PythonEvaluator.py

@@ -1,151 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.sheerka import Sheerka, ExecutionContext
-from core.concept import Concept
-from evaluators.PythonEvaluator import PythonEvaluator
-from parsers.PythonParser import PythonNode, PythonParser
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_context_name(context):
-    return context.name
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=PythonNode("", None))), True),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False),
-    (ReturnValueConcept("some_name", False, "not relevant"), False),
-    (ReturnValueConcept("some_name", True, Concept()), False)
-])
-def test_i_can_match(ret_val, expected):
-    context = get_context()
-    assert PythonEvaluator().matches(context, ret_val) == expected
-
-
-@pytest.mark.parametrize("text, expected", [
-    ("1 + 1", 2),
-    ("sheerka.test()", "I have access to Sheerka !"),
-    ("a=10\na", 10),
-])
-def test_i_can_eval(text, expected):
-    context = get_context()
-    parsed = PythonParser().parse(context, text)
-
-    evaluated = PythonEvaluator().eval(context, parsed)
-
-    assert evaluated.status
-    assert evaluated.value == expected
-
-
-@pytest.mark.parametrize("concept", [
-    Concept("foo"),
-    Concept("foo", body="2"),
-    Concept("foo").set_prop("prop", "'a'"),
-    Concept("foo", body="bar")
-])
-def test_i_cannot_eval_simple_concept(concept):
-    context = get_context()
-    context.sheerka.add_in_cache(Concept("foo"))
-
-    parsed = PythonParser().parse(context, "foo")
-    evaluated = PythonEvaluator().eval(context, parsed)
-
-    assert not evaluated.status
-    assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.NOT_FOR_ME)
-
-
-def test_i_can_eval_expression_with_that_references_concepts():
-    """
-    I can test modules with variables
-    :return:
-    """
-    context = get_context()
-    context.sheerka.add_in_cache(Concept("foo", body="1"))
-
-    parsed = PythonParser().parse(context, "foo + 2")
-    evaluated = PythonEvaluator().eval(context, parsed)
-
-    assert evaluated.status
-    assert evaluated.value == 3
-
-
-def test_i_can_eval_module_with_that_references_concepts():
-    """
-    I can test modules with variables
-    :return:
-    """
-    context = get_context()
-    context.sheerka.add_in_cache(Concept("foo"))
-
-    parsed = PythonParser().parse(context, "def a(b):\n    return b\na(c:foo:)")
-    evaluated = PythonEvaluator().eval(context, parsed)
-
-    assert evaluated.status
-    assert evaluated.value == Concept("foo").init_key()
-
-
-def test_i_can_eval_module_with_that_references_concepts_with_body():
-    """
-    I can test modules with variables
-    :return:
-    """
-    context = get_context()
-    context.sheerka.add_in_cache(Concept("foo", body="2"))
-
-    parsed = PythonParser().parse(context, "def a(b):\n    return b\na(foo)")
-    evaluated = PythonEvaluator().eval(context, parsed)
-
-    assert evaluated.status
-    assert evaluated.value == 2
-
-
-def test_i_can_eval_concept_token():
-    context = get_context()
-    context.sheerka.add_in_cache(Concept("foo", body="2"))
-
-    parsed = PythonParser().parse(context, "get_context_name(c:foo:)")
-    python_evaluator = PythonEvaluator()
-    python_evaluator.locals["get_context_name"] = get_context_name
-    evaluated = python_evaluator.eval(context, parsed)
-
-    assert evaluated.status
-    assert evaluated.value == "foo"
-
-    # sanity, does not work otherwise
-    parsed = PythonParser().parse(context, "get_context_name(foo)")
-    python_evaluator = PythonEvaluator()
-    python_evaluator.locals["get_context_name"] = get_context_name
-    evaluated = python_evaluator.eval(context, parsed)
-
-    assert not evaluated.status
-    assert evaluated.body.body.args[0] == "'int' object has no attribute 'name'"
-
-
-@pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [
-    ("__C__key__C__", "key", None, False),
-    ("__C__key__id__C__", "key", "id", False),
-    ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
-    ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
-])
-def test_i_can_resolve_name(text, concept_key, concept_id, use_concept):
-    context = get_context()
-    assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept)
-
-
-@pytest.mark.parametrize("text", [
-    "__C__",
-    "__C__key",
-    "__C__key____",
-    "__C____",
-    "__C__USE_CONCEPT__",
-])
-def test_i_cannot_resolve_name(text):
-    context = get_context()
-    assert PythonEvaluator().resolve_name(context, text) is None

tests/test_PythonParser.py

@@ -1,87 +0,0 @@
-import ast
-
-import pytest
-
-from core.builtin_concepts import ParserResultConcept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer, LexerError
-from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-@pytest.mark.parametrize("text, expected", [
-    ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))),
-    ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))),
-])
-def test_i_can_parse_a_simple_expression(text, expected):
-    parser = PythonParser()
-    res = parser.parse(get_context(), text)
-
-    assert res.status
-    assert res.who == parser.name
-    assert isinstance(res.value, ParserResultConcept)
-    assert res.value.value == expected
-
-
-@pytest.mark.parametrize("text, expected", [
-    ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))),
-    ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))),
-])
-def test_i_can_parse_from_tokens(text, expected):
-    parser = PythonParser()
-    tokens = list(Tokenizer(text))
-    res = parser.parse(get_context(), tokens)
-
-    assert res.status
-    assert res.who == parser.name
-    assert isinstance(res.value, ParserResultConcept)
-    assert res.value.value == expected
-
-
-@pytest.mark.parametrize("text", [
-    "1+",
-    "'name",
-    "foo = 'name"
-])
-def test_i_can_detect_error(text):
-    parser = PythonParser()
-    res = parser.parse(get_context(), text)
-
-    assert not res.status
-    assert res.who == parser.name
-    assert isinstance(res.value, ParserResultConcept)
-    assert isinstance(res.value.value[0], PythonErrorNode)
-    assert isinstance(res.value.value[0].exception, SyntaxError)
-
-
-@pytest.mark.parametrize("text, error_msg, error_text", [
-    ("c::", "Concept name not found", ""),
-    ("c:: + 1", "Concept name not found", ""),
-])
-def test_i_can_detect_lexer_errors(text, error_msg, error_text):
-    parser = PythonParser()
-    res = parser.parse(get_context(), text)
-
-    assert not res.status
-    assert isinstance(res.body, ParserResultConcept)
-    assert isinstance(res.body.body[0], LexerError)
-    assert res.body.body[0].message == error_msg
-    assert res.body.body[0].text == error_text
-
-
-def test_i_can_parse_a_concept():
-    text = "c:concept_name: + 1"
-
-    parser = PythonParser()
-    res = parser.parse(get_context(), text)
-
-    assert res
-    assert res.value.value == PythonNode(
-        "c:concept_name: + 1",
-        ast.parse("__C__USE_CONCEPT__concept_name__C__+1", mode="eval"))

tests/test_PythonWithConceptsParser.py

@@ -1,147 +0,0 @@
-import ast
-
-import pytest
-
-from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Token, TokenKind, Tokenizer
-from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode
-from parsers.MultipleConceptsParser import MultipleConceptsParser
-from parsers.PythonParser import PythonNode, PythonErrorNode
-from parsers.PythonWithConceptsParser import PythonWithConceptsParser
-from sdp.sheerkaDataProvider import Event
-
-multiple_concepts_parser = MultipleConceptsParser()
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_ret_from(*args):
-    result = []
-    index = 0
-    for item in args:
-        if isinstance(item, Concept):
-            tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
-            result.append(ConceptNode(item, index, index, tokens, item.name))
-            index += 1
-        else:
-            tokens = list(Tokenizer(item))
-            result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
-            index += len(tokens)
-
-    return ReturnValueConcept("who", False, ParserResultConcept(parser=multiple_concepts_parser, value=result))
-
-
-def to_str_ast(expression):
-    return PythonNode.get_dump(ast.parse(expression, mode="eval"))
-
-
-@pytest.mark.parametrize("text, interested", [
-    ("not parser result", False),
-    (ParserResultConcept(parser="not multiple_concepts_parser"), False),
-    (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True),
-])
-def test_not_interested(text, interested):
-    context = get_context()
-
-    res = PythonWithConceptsParser().parse(context, text)
-    if interested:
-        assert res is not None
-    else:
-        assert res is None
-
-
-def test_i_can_parse_concepts_and_python():
-    context = get_context()
-    foo = Concept("foo")
-    input_return_value = get_ret_from(foo, " + 1")
-
-    parser = PythonWithConceptsParser()
-    result = parser.parse(context, input_return_value.body)
-    wrapper = result.value
-    return_value = result.value.value
-
-    assert result.status
-    assert result.who == parser.name
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert wrapper.source == "foo + 1"
-    assert isinstance(return_value, PythonNode)
-    assert return_value.source == "foo + 1"
-    assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1")
-    assert return_value.concepts["__C__foo__C__"] == foo
-
-
-def test_i_can_parse_concepts_and_python_when_concept_is_known():
-    context = get_context()
-    foo = Concept("foo")
-    foo = context.sheerka.create_new_concept(context, foo).body.body
-    input_return_value = get_ret_from(foo, " + 1")
-
-    parser = PythonWithConceptsParser()
-    result = parser.parse(context, input_return_value.body)
-    wrapper = result.value
-    return_value = result.value.value
-
-    assert result.status
-    assert result.who == parser.name
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert wrapper.source == "foo + 1"
-    assert isinstance(return_value, PythonNode)
-    assert return_value.source == "foo + 1"
-    assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__1001__C__ + 1")
-    assert return_value.concepts["__C__foo__1001__C__"] == foo
-
-
-def test_i_can_parse_when_concept_name_has_invalid_characters():
-    context = get_context()
-    foo = Concept("foo et > (,")
-    foo = context.sheerka.create_new_concept(context, foo).body.body
-    input_return_value = get_ret_from(foo, " + 1")
-
-    parser = PythonWithConceptsParser()
-    result = parser.parse(context, input_return_value.body)
-    return_value = result.value.value
-
-    assert result.status
-    assert return_value.concepts["__C__foo0et000000__1001__C__"] == foo
-
-
-def test_python_ids_mappings_are_correct_when_concepts_with_the_same_name():
-    context = get_context()
-    foo1 = Concept("foo")
-    foo2 = Concept("foo")
-    foo3 = context.sheerka.create_new_concept(context, Concept("foo", body="foo3")).body.body
-    foo4 = context.sheerka.create_new_concept(context, Concept("foo", body="foo4")).body.body
-
-    input_return_value = get_ret_from(foo1, "+", foo2, "+", foo3, "+", foo4)
-
-    parser = PythonWithConceptsParser()
-    result = parser.parse(context, input_return_value.body)
-    return_value = result.value.value
-
-    assert result.status
-    assert return_value.concepts["__C__foo__C__"] == foo1
-    assert return_value.concepts["__C__foo_1__C__"] == foo2
-    assert return_value.concepts["__C__foo__1001__C__"] == foo3
-    assert return_value.concepts["__C__foo__1002__C__"] == foo4
-
-
-def test_i_cannot_parse_if_syntax_error():
-    context = get_context()
-    foo = Concept("foo")
-    foo = context.sheerka.create_new_concept(context, foo).body.body
-    input_return_value = get_ret_from(foo, " + ")
-
-    parser = PythonWithConceptsParser()
-    result = parser.parse(context, input_return_value.body)
-    wrapper = result.value
-    return_value = result.value.value
-
-    assert not result.status
-    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert isinstance(return_value[0], PythonErrorNode)

tests/test_TooManySucessEvaluator.py

@@ -1,112 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.TooManySuccessEvaluator import TooManySuccessEvaluator
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def r(name, status=True, value=None):
-    value = value or name
-    return ReturnValueConcept(name, status, value)
-
-
-reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
-
-
-@pytest.mark.parametrize("return_values, expected", [
-    ([r("evaluators.success1"), r("evaluators.success2"), reduce_requested], True),
-    ([r("evaluators.success1"), r("evaluators.success2"), r("other"), reduce_requested], True),
-    ([r("evaluators.success1"), r("evaluators.success2"), r("other", False), reduce_requested], True),
-    ([r("evaluators.a", value=Concept("c1", body="1")),
-      r("evaluators.a", value=Concept("c2", body="1")),
-      r("other"),
-      reduce_requested], True),
-    ([r("evaluators.a", value=Concept("c1", body="1")),
-      r("evaluators.a", value=Concept("c2", body="1")),
-      r("parsers.other", False),
-      reduce_requested], True),
-    ([r("evaluators.a", value=Concept("c1", body="1")),
-      r("evaluators.a", value=Concept("c2", body="1")),
-      r("parsers.other"),
-      reduce_requested], False),
-    ([r("evaluators.success1"), reduce_requested], False),
-    ([reduce_requested], False),
-    ([r("evaluators.success1"), r("evaluators.success2")], False),
-])
-def test_i_can_match(return_values, expected):
-    context = get_context()
-    assert TooManySuccessEvaluator().matches(context, return_values) == expected
-
-
-def test_i_can_eval():
-    context = get_context()
-
-    value1 = r("evaluators.a", value=Concept("c1", body="1"))
-    value2 = r("evaluators.a", value=Concept("c2", body="2"))
-    return_values = [
-        value1,
-        value2,
-        r("other", False),
-        reduce_requested
-    ]
-
-    evaluator = TooManySuccessEvaluator()
-    matches = evaluator.matches(context, return_values)
-    res = evaluator.eval(context, return_values)
-
-    assert matches
-    assert not res.status
-    assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS)
-    assert res.body.body == [value1, value2]
-    assert len(res.parents) == 4
-
-
-def test_i_can_eval_when_same_success():
-    context = get_context()
-
-    return_values = [
-        r("evaluators.a", value=Concept("c1", body="1").auto_init()),
-        r("evaluators.a", value=Concept("c2", body="1").auto_init()),
-        r("other", False),
-        reduce_requested
-    ]
-
-    evaluator = TooManySuccessEvaluator()
-    matches = evaluator.matches(context, return_values)
-    res = evaluator.eval(context, return_values)
-
-    assert matches
-    assert res is None
-
-
-def test_other_success_are_not_reduced():
-    context = get_context()
-
-    value1 = r("evaluators.a", value=Concept("c1", body="1").auto_init())
-    value2 = r("evaluators.a", value=Concept("c2", body="2").auto_init())
-    other_success = r("other")
-    return_values = [
-        value1,
-        value2,
-        other_success,
-        reduce_requested
-    ]
-
-    evaluator = TooManySuccessEvaluator()
-    matches = evaluator.matches(context, return_values)
-    res = evaluator.eval(context, return_values)
-
-    assert matches
-    assert not res.status
-    assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS)
-    assert res.body.body == [value1, value2]
-    assert len(res.parents) == 3
-    assert other_success not in res.parents

tests/test_builtin_helpers.py

@@ -1,171 +0,0 @@
-import ast
-
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
-from core.sheerka import Sheerka, ExecutionContext
-import core.builtin_helpers
-from sdp.sheerkaDataProvider import Event
-
-
-def test_i_can_use_expect_one_when_empty():
-    sheerka = get_sheerka()
-
-    res = core.builtin_helpers.expect_one(get_context(sheerka), [])
-    assert not res.status
-    assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY)
-
-
-def test_i_can_use_expect_one_when_too_many_success():
-    sheerka = get_sheerka()
-
-    items = [
-        ReturnValueConcept("who", True, "value1"),
-        ReturnValueConcept("who", True, "value2"),
-    ]
-    res = core.builtin_helpers.expect_one(get_context(sheerka), items)
-    assert not res.status
-    assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS)
-    assert res.value.body == items
-    assert res.parents == items
-
-
-def test_i_can_use_expect_one_when_same_success():
-    sheerka = get_sheerka()
-
-    items = [
-        ReturnValueConcept("who", True, "value"),
-        ReturnValueConcept("who", True, "value"),
-    ]
-    res = core.builtin_helpers.expect_one(get_context(sheerka), items)
-    assert res.status
-    assert res.value == items[0].value
-    assert res.parents == items
-
-
-def test_i_can_use_expect_when_only_errors_1():
-    sheerka = get_sheerka()
-
-    items = [
-        ReturnValueConcept("who", False, sheerka.new(BuiltinConcepts.ERROR)),
-    ]
-    res = core.builtin_helpers.expect_one(get_context(sheerka), items)
-    assert not res.status
-    assert res.value, items[0]
-    assert res.parents == items
-
-
-def test_i_can_use_expect_when_only_errors_2():
-    sheerka = get_sheerka()
-
-    items = [
-        ReturnValueConcept("who", False, None),
-        ReturnValueConcept("who", False, None),
-    ]
-    res = core.builtin_helpers.expect_one(get_context(sheerka), items)
-    assert not res.status
-    assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
-    assert res.value.body == items
-    assert res.parents == items
-
-
-def test_i_can_use_expect_one_when_one_success_1():
-    sheerka = get_sheerka()
-
-    items = [
-        ReturnValueConcept("who", True, None),
-    ]
-    res = core.builtin_helpers.expect_one(get_context(sheerka), items)
-    assert res.status
-    assert res.body == items[0].body
-    assert res.parents == items
-
-
-def test_i_can_use_expect_one_when_one_success_2():
-    sheerka = get_sheerka()
-
-    items = [
-        ReturnValueConcept("who", False, None),
-        ReturnValueConcept("who", True, None),
-        ReturnValueConcept("who", False, None),
-    ]
-    res = core.builtin_helpers.expect_one(get_context(sheerka), items)
-    assert res.status
-    assert res.body == items[1].body
-    assert res.parents == items
-
-
-def test_i_can_use_expect_one_when_not_a_list_true():
-    sheerka = get_sheerka()
-
-    item = ReturnValueConcept("who", True, None)
-    res = core.builtin_helpers.expect_one(get_context(sheerka), item)
-    assert res.status
-    assert res == item
-
-
-def test_i_can_use_expect_one_when_not_a_list_false():
-    sheerka = get_sheerka()
-
-    item = ReturnValueConcept("who", False, None)
-    res = core.builtin_helpers.expect_one(get_context(sheerka), item)
-
-    assert not res.status
-    assert res == item
-
-
-@pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [
-    ("a == 1", [], [], []),
-    ("a == 1", ["a"], [], ["a == 1"]),
-    ("a == 1", [], ["a"], []),
-    ("predicate(a)", [], [], []),
-    ("predicate(a)", ["a"], [], ["predicate(a)"]),
-    ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]),
-    ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]),
-    ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]),
-    ("predicate(a, b)", ["a"], ["b"], []),
-    ("a + b == 1", [], [], []),
-    ("a + b == 1", ["a"], [], ["a + b == 1"]),
-    ("a + b == 1", ["a"], ["b"], []),
-    ("a + b == 1", ["b"], [], ["a + b == 1"]),
-    ("a + b == 1", ["a", "b"], [], ["a + b == 1"]),
-    ("a == 1 and b == 2", [], [], []),
-    ("a == 1 and b == 2", ["a"], [], ["a == 1"]),
-    ("a == 1 and b == 2", ["b"], [], ["b == 2"]),
-    ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]),
-    ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]),
-    ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]),
-    ("not(a == 1)", [], [], []),
-    ("not(a == 1)", ["a"], [], ["not(a==1)"]),
-    ("a == 1 or b == 2", [], [], []),
-    ("a == 1 or b == 2", ["a"], [], ["a == 1"]),
-    ("a == 1 or b == 2", ["b"], [], ["b == 2"]),
-    ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]),
-    ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]),
-])
-def test_i_can_extract_predicates(expression, vars_to_include, vars_to_exclude, expected_expr):
-    sheerka = get_sheerka()
-    expected = [ast.parse(expr, mode="eval") for expr in expected_expr]
-
-    actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude)
-    assert len(actual) == len(expected)
-    for i in range(len(actual)):
-        assert dump_ast(actual[i]) == dump_ast(expected[i])
-
-
-def get_sheerka():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-
-    return sheerka
-
-
-def get_context(sheerka):
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def dump_ast(node):
-    dump = ast.dump(node)
-    for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
-        dump = dump.replace(to_remove, "")
-    return dump

tests/test_sheerka.py

@@ -1,955 +0,0 @@
-import pytest
-import os
-from os import path
-import shutil
-
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept, ConceptAlreadyInSet, \
-    ParserResultConcept
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, Property, ConceptParts, DoNotResolve, simplec
-from core.sheerka import Sheerka, ExecutionContext
-from parsers.PythonParser import PythonNode
-from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
-
-tests_root = path.abspath("../build/tests")
-root_folder = "init_folder"
-
-
-@pytest.fixture(autouse=True)
-def init_test():
-    if path.exists(tests_root):
-        shutil.rmtree(tests_root)
-
-    if not path.exists(tests_root):
-        os.makedirs(tests_root)
-    current_pwd = os.getcwd()
-    os.chdir(tests_root)
-
-    yield None
-
-    os.chdir(current_pwd)
-
-
-def get_sheerka(use_dict=True, skip_builtins_in_db=True):
-    root = "mem://" if use_dict else root_folder
-    sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db)
-    sheerka.initialize(root)
-
-    return sheerka
-
-
-def get_context(sheerka):
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_default_concept():
-    concept = Concept(
-        name="a + b",
-        where="isinstance(a, int) and isinstance(b, int)",
-        pre="isinstance(a, int) and isinstance(b, int)",
-        post="isinstance(res, int)",
-        body="def func(x,y):\n    return x+y\nfunc(a,b)",
-        desc="specific description")
-    concept.def_prop("a", "value1")
-    concept.def_prop("b", "value2")
-
-    return concept
-
-
-class ConceptWithGetValue(Concept):
-    def get_value(self):
-        return self.get_prop("my_prop")
-
-
-def test_root_folder_is_created_after_initialization():
-    return_value = Sheerka().initialize(root_folder)
-    assert return_value.status, "initialisation should be successful"
-    assert os.path.exists(root_folder), "init folder should be created"
-
-
-def test_i_can_list_builtin_concepts():
-    sheerka = get_sheerka()
-    builtins = list(sheerka.get_builtins_classes_as_dict())
-
-    assert str(BuiltinConcepts.ERROR) in builtins
-    assert str(BuiltinConcepts.RETURN_VALUE) in builtins
-
-
-def test_builtin_concepts_are_initialized():
-    sheerka = get_sheerka(skip_builtins_in_db=False)
-    assert len(sheerka.cache_by_key) == len(BuiltinConcepts)
-    for concept_name in BuiltinConcepts:
-        assert str(concept_name) in sheerka.cache_by_key
-        assert sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, str(concept_name)) is not None
-
-    for key, concept_class in sheerka.get_builtins_classes_as_dict().items():
-        assert isinstance(sheerka.cache_by_key[key], concept_class)
-
-
-def test_builtin_concepts_can_be_updated():
-    sheerka = get_sheerka(False, False)
-    loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
-    loaded_sheerka.metadata.desc = "I have a description"
-    sheerka.sdp.modify("Test", sheerka.CONCEPTS_ENTRY, loaded_sheerka.key, loaded_sheerka)
-
-    sheerka = get_sheerka(False, False)
-    loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
-
-    assert loaded_sheerka.metadata.desc == "I have a description"
-
-
-def test_i_can_add_a_concept():
-    sheerka = get_sheerka()
-    concept = get_default_concept()
-
-    res = sheerka.create_new_concept(get_context(sheerka), concept)
-
-    assert res.status
-    assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
-
-    concept_found = res.value.body
-    for prop in PROPERTIES_TO_SERIALIZE:
-        assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop)
-
-    assert concept_found.key == "__var__0 + __var__1"
-    assert concept_found.id == "1001"
-
-    assert concept.key in sheerka.cache_by_key
-    assert concept.id in sheerka.cache_by_id
-    assert sheerka.sdp.io.exists(
-        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest()))
-
-
-def test_i_cannot_add_the_same_concept_twice():
-    """
-    Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
-    :return:
-    """
-    sheerka = get_sheerka()
-    concept = get_default_concept()
-
-    sheerka.create_new_concept(get_context(sheerka), concept)
-    res = sheerka.create_new_concept(get_context(sheerka), concept)
-
-    assert not res.status
-    assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
-    assert res.value.body == concept
-
-
-def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string():
-    """
-    Checks that a concept can be found its name
-    even when there are variables in the name (ex 'hello + a' or 'a + b' )
-    :return:
-    """
-    sheerka = get_sheerka()
-    for key in sheerka.get_builtins_classes_as_dict():
-        assert sheerka.get(key) is not None
-        assert sheerka.get(str(key)) is not None
-
-
-def test_i_can_get_a_newly_created_concept():
-    sheerka = get_sheerka()
-    concept = get_default_concept()
-
-    sheerka.create_new_concept(get_context(sheerka), concept)
-
-    from_cache = sheerka.get(concept.key)
-    assert from_cache is not None
-    assert from_cache == concept
-
-    from_cache = sheerka.get_by_id(concept.id)
-    assert from_cache is not None
-    assert from_cache == concept
-
-
-def test_i_first_look_in_local_cache():
-    sheerka = get_sheerka()
-    concept = get_default_concept()
-
-    sheerka.create_new_concept(get_context(sheerka), concept)
-    sheerka.cache_by_key[concept.key].pre = "I have modified the concept in cache"
-
-    from_cache = sheerka.get(concept.key)
-    assert from_cache is not None
-    assert from_cache.key == concept.key
-    assert from_cache.pre == "I have modified the concept in cache"
-
-
-def test_i_can_get_a_known_concept_when_not_in_cache():
-    """
-    When not in cache, uses sdp
-    :return:
-    """
-    sheerka = get_sheerka()
-    concept = get_default_concept()
-    sheerka.create_new_concept(get_context(sheerka), concept)
-
-    sheerka.cache_by_key = {}  # reset the cache
-    loaded = sheerka.get(concept.key)
-
-    assert loaded is not None
-    assert loaded == concept
-
-    # I can also get it by its id
-    loaded = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
-    assert loaded is not None
-    assert loaded == concept
-
-
-def test_i_can_get_a_concept_by_its_id():
-    sheerka = get_sheerka()
-    concept = get_default_concept()
-    sheerka.create_new_concept(get_context(sheerka), concept)
-
-    sheerka.cache_by_key = {}  # reset the cache
-    loaded = sheerka.get_by_id(concept.id)
-
-    assert loaded is not None
-    assert loaded == concept
-
-
-def test_i_can_get_list_of_concept_when_same_key_when_no_cache():
-    sheerka = get_sheerka()
-    concept1 = get_default_concept()
-    concept2 = get_default_concept()
-    concept2.metadata.body = "a+b"
-
-    res1 = sheerka.create_new_concept(get_context(sheerka), concept1)
-    res2 = sheerka.create_new_concept(get_context(sheerka), concept2)
-
-    assert res1.value.body.key == res2.value.body.key  # same key
-
-    sheerka.cache_by_key = {}  # reset the cache
-
-    result = sheerka.get(concept1.key)
-    assert len(result) == 2
-    assert result[0] == concept1
-    assert result[1] == concept2
-
-
-def test_i_can_get_list_of_concept_when_same_key_when_cache():
-    sheerka = get_sheerka()
-    concept1 = get_default_concept()
-    concept2 = get_default_concept()
-    concept2.metadata.body = "a+b"
-
-    res1 = sheerka.create_new_concept(get_context(sheerka), concept1)
-    res2 = sheerka.create_new_concept(get_context(sheerka), concept2)
-
-    assert res1.value.body.key == res2.value.body.key  # same key
-
-    # sheerka.cache_by_key = {}  # Do not reset the cache
-
-    result = sheerka.get(concept1.key)
-    assert len(result) == 2
-    assert result[0] == concept1
-    assert result[1] == concept2
-
-
-def test_i_can_get_the_correct_concept_using_the_id_when_same_key_when_no_cache():
-    sheerka = get_sheerka()
-    concept1 = get_default_concept()
-    concept2 = get_default_concept()
-    concept2.metadata.body = "a+b"
-
-    res1 = sheerka.create_new_concept(get_context(sheerka), concept1)
-    res2 = sheerka.create_new_concept(get_context(sheerka), concept2)
-
-    assert res1.value.body.key == res2.value.body.key  # same key
-
-    result = sheerka.get(concept1.key, res2.body.body.id)
-    assert result.name == "a + b"
-    assert result.metadata.body == "a+b"
-
-
-def test_i_can_get_the_correct_concept_using_the_id__when_same_key_when_cache():
-    sheerka = get_sheerka()
-    concept1 = get_default_concept()
-    concept2 = get_default_concept()
-    concept2.metadata.body = "a+b"
-
-    res1 = sheerka.create_new_concept(get_context(sheerka), concept1)
-    res2 = sheerka.create_new_concept(get_context(sheerka), concept2)
-
-    assert res1.value.body.key == res2.value.body.key  # same key
-
-    result = sheerka.get(concept1.key, res2.body.body.id)
-    assert result.name == "a + b"
-    assert result.metadata.body == "a+b"
-
-
-def test_i_cannot_get_the_correct_concept_id_the_id_is_wrong():
-    sheerka = get_sheerka()
-    concept1 = get_default_concept()
-    concept2 = get_default_concept()
-    concept2.metadata.body = "a+b"
-
-    res1 = sheerka.create_new_concept(get_context(sheerka), concept1)
-    res2 = sheerka.create_new_concept(get_context(sheerka), concept2)
-
-    assert res1.value.body.key == res2.value.body.key  # same key
-
-    result = sheerka.get(concept1.key, "wrong id")
-    assert sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT)
-
-
-def test_i_cannot_get_when_key_is_none():
-    sheerka = get_sheerka()
-
-    res = sheerka.get(None)
-    assert sheerka.isinstance(res, BuiltinConcepts.ERROR)
-    assert res.body == "Concept key is undefined."
-
-
-def test_unknown_concept_is_return_when_the_concept_key_is_not_found():
-    sheerka = get_sheerka()
-
-    loaded = sheerka.get("key_that_does_not_exist")
-
-    assert loaded is not None
-    assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert loaded.body == ("key", "key_that_does_not_exist")
-    assert loaded.metadata.is_evaluated
-
-
-def test_unknown_concept_is_return_when_the_concept_id_is_not_found():
-    sheerka = get_sheerka()
-
-    loaded = sheerka.get_by_id("id_that_does_not_exist")
-
-    assert loaded is not None
-    assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert loaded.body == ("id", "id_that_does_not_exist")
-    assert loaded.metadata.is_evaluated
-
-
-def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class():
-    sheerka = get_sheerka()
-    ret = sheerka.new(BuiltinConcepts.RETURN_VALUE, who="who", status="status", value="value", message="message")
-
-    assert isinstance(ret, ReturnValueConcept)
-    assert ret.key == str(BuiltinConcepts.RETURN_VALUE)
-    assert ret.who == "who"
-    assert ret.status == "status"
-    assert ret.value == "value"
-    assert ret.message == "message"
-
-
-def test_i_can_instantiate_a_builtin_concept_when_no_specific_class():
-    sheerka = get_sheerka()
-    ret = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body="fake_concept")
-
-    assert isinstance(ret, Concept)
-    assert ret.key == str(BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert ret.body == "fake_concept"
-
-
-def test_i_can_instantiate_a_concept():
-    sheerka = get_sheerka()
-    concept = get_default_concept()
-    sheerka.create_new_concept(get_context(sheerka), concept)
-
-    new = sheerka.new(concept.key, a=10, b="value")
-
-    assert sheerka.isinstance(new, concept)
-    for prop in PROPERTIES_TO_SERIALIZE:
-        assert getattr(new.metadata, prop) == getattr(concept.metadata, prop)
-
-    assert new.props["a"].value == 10
-    assert new.props["b"].value == "value"
-
-
-def test_i_can_instantiate_with_the_name_and_the_id():
-    sheerka = get_sheerka()
-    sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo1"))
-    sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo2"))
-
-    concepts = sheerka.new("foo")
-    assert len(concepts) == 2
-
-    foo1 = sheerka.new(("foo", "1001"))
-    assert foo1.metadata.body == "foo1"
-
-    foo2 = sheerka.new(("foo", "1002"))
-    assert foo2.metadata.body == "foo2"
-
-
-def test_instances_are_different_when_asking_for_new():
-    sheerka = get_sheerka()
-    concept = get_default_concept()
-    sheerka.create_new_concept(get_context(sheerka), concept)
-
-    new1 = sheerka.new(concept.key, a=10, b="value")
-    new2 = sheerka.new(concept.key, a=10, b="value")
-
-    assert new1 == new2
-    assert id(new1) != id(new2)
-
-
-def test_i_get_the_same_instance_when_is_unique_is_true():
-    sheerka = get_sheerka()
-    concept = Concept(name="unique", is_unique=True)
-    sheerka.create_new_concept(get_context(sheerka), concept)
-
-    new1 = sheerka.new(concept.key, a=10, b="value")
-    new2 = sheerka.new(concept.key, a=10, b="value")
-
-    assert new1 == new2
-    assert id(new1) == id(new2)
-
-
-def test_i_cannot_instantiate_an_unknown_concept():
-    sheerka = get_sheerka()
-
-    new = sheerka.new("fake_concept")
-
-    assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert new.body == ('key', 'fake_concept')
-
-
-def test_i_cannot_instantiate_with_invalid_id():
-    sheerka = get_sheerka()
-    sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo1"))
-    sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo2"))
-
-    new = sheerka.new(("foo", "invalid_id"))
-
-    assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert new.body == [('key', 'foo'), ('id', 'invalid_id')]
-
-
-def test_i_cannot_instantiate_with_invalid_key():
-    sheerka = get_sheerka()
-    sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo1"))
-    sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo2"))
-
-    new = sheerka.new(("invalid_key", "1001"))
-
-    assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert new.body == [('key', 'invalid_key'), ('id', '1001')]
-
-
-def test_concept_id_is_irrelevant_when_only_one_concept():
-    sheerka = get_sheerka()
-    sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo1"))
-
-    new = sheerka.new(("foo", "invalid_id"))
-
-    assert sheerka.isinstance(new, "foo")
-    assert new.metadata.body == "foo1"
-
-
-def test_i_cannot_instantiate_when_properties_are_not_recognized():
-    sheerka = get_sheerka()
-    concept = get_default_concept()
-    sheerka.create_new_concept(get_context(sheerka), concept)
-
-    new = sheerka.new(concept.key, a=10, c="value")
-
-    assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_PROPERTY)
-    assert new.property_name == "c"
-    assert sheerka.isinstance(new.concept, concept)
-
-
-@pytest.mark.parametrize("concept, reduce_simple_list, expected", [
-    (None, False, None),
-    (3.14, False, 3.14),
-    ("foo", False, "foo"),
-    (True, False, True),
-    (Concept("name", body="foo"), False, "foo"),
-    (Concept("name"), False, Concept("name")),
-    (ConceptWithGetValue("name").set_prop("my_prop", "my_value"), False, "my_value"),
-    (ReturnValueConcept(value="return_value"), False, "return_value"),
-    (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
-    (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
-    (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]),
-    (Concept("name", body=["foo"]), True, "foo"),
-    (Concept("name", body=Concept("foo")), False, Concept("foo")),
-    (Concept("name", body=Concept("foo", body="value")), False, "value"),
-    (Concept("name", body=Concept("foo", body=Concept("bar", body="value"))), False, "value"),
-    (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"),
-])
-def test_i_can_get_value(concept, reduce_simple_list, expected):
-    sheerka = get_sheerka()
-
-    # I use auto_init() instead of evaluate_concept() to be quicker
-    c = concept
-    while isinstance(c, Concept):
-        c.auto_init()
-        c = c.body
-
-    assert sheerka.value(concept, reduce_simple_list) == expected
-
-
-def test_list_of_concept_is_sorted_by_id():
-    sheerka = get_sheerka(False, False)
-    concepts = sheerka.concepts()
-
-    assert concepts[0].id < concepts[-1].id
-
-
-@pytest.mark.parametrize("body, expected", [
-    # (None, None),
-    # ("", ""),
-    ("1", 1),
-    ("1+1", 2),
-    ("'one'", "one"),
-    ("'one' + 'two'", "onetwo"),
-    ("True", True),
-    ("1 > 2", False),
-])
-def test_i_can_evaluate_a_concept_with_simple_body(body, expected):
-    sheerka = get_sheerka()
-
-    concept = Concept("foo", body=body)
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.key
-    assert evaluated.body == expected
-    assert evaluated.metadata.body == body
-    assert evaluated.metadata.pre is None
-    assert evaluated.metadata.post is None
-    assert evaluated.metadata.where is None
-    assert evaluated.props == {}
-    assert evaluated.metadata.is_evaluated
-    assert len(evaluated.values) == 0 if body is None else 1
-
-
-@pytest.mark.parametrize("expr, expected", [
-    ("", ""),
-    ("1", 1),
-    ("1+1", 2),
-    ("'one'", "one"),
-    ("'one' + 'two'", "onetwo"),
-    ("True", True),
-    ("1 > 2", False),
-])
-def test_i_can_evaluate_the_other_metadata(expr, expected):
-    """
-    I only test WHERE, it's the same for the others
-    :param expr:
-    :param expected:
-    :return:
-    """
-
-    sheerka = get_sheerka()
-
-    concept = Concept("foo", where=expr)
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.key
-    assert evaluated.metadata.body is None
-    assert evaluated.metadata.pre is None
-    assert evaluated.metadata.post is None
-    assert evaluated.metadata.where == expr
-    assert evaluated.get_metadata_value(ConceptParts.WHERE) == expected
-    assert evaluated.props == {}
-    assert evaluated.metadata.is_evaluated
-    assert len(evaluated.values) == 0 if expr is None else 1
-
-
-@pytest.mark.parametrize("expr, expected", [
-    (None, None),
-    ("", ""),
-    ("1", 1),
-    ("1+1", 2),
-    ("'one'", "one"),
-    ("'one' + 'two'", "onetwo"),
-    ("True", True),
-    ("1 > 2", False),
-])
-def test_i_can_evaluate_a_concept_with_prop(expr, expected):
-    sheerka = get_sheerka()
-
-    concept = Concept("foo").def_prop("a", expr)
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.key
-    assert evaluated.metadata.pre is None
-    assert evaluated.metadata.pre is None
-    assert evaluated.metadata.post is None
-    assert evaluated.metadata.where is None
-    assert evaluated.props == {"a": Property("a", expected)}
-    assert evaluated.metadata.is_evaluated
-
-
-def test_i_can_evaluate_metadata_using_do_not_resolve():
-    sheerka = get_sheerka()
-    concept = Concept("foo")
-    concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
-
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.body == "do not resolve"
-    assert evaluated.metadata.is_evaluated
-
-
-def test_i_can_evaluate_property_using_do_not_resolve():
-    sheerka = get_sheerka()
-    concept = Concept("foo").def_prop("a")
-    concept.compiled["a"] = DoNotResolve("do not resolve")
-
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.get_prop("a") == "do not resolve"
-    assert evaluated.metadata.is_evaluated
-
-
-def test_original_value_is_overridden_when_using_do_no_resolve():
-    sheerka = get_sheerka()
-    concept = Concept("foo", body="original value").set_prop("a", "original value")
-    concept.compiled["a"] = DoNotResolve("do not resolve")
-    concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
-
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.body == "do not resolve"
-    assert evaluated.get_prop("a") == "do not resolve"
-    assert evaluated.metadata.is_evaluated
-
-
-def test_props_are_evaluated_before_body():
-    sheerka = get_sheerka()
-
-    concept = Concept("foo", body="a+1").def_prop("a", "10").init_key()
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.key
-    assert evaluated.body == 11
-
-
-def test_i_can_evaluate_when_another_concept_is_referenced():
-    sheerka = get_sheerka()
-    concept_a = Concept("a")
-    sheerka.add_in_cache(concept_a)
-
-    concept = Concept("foo", body="a").init_key()
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated == simplec("foo", simplec("a", None))
-    assert id(evaluated.body) != id(concept_a)
-    assert evaluated.metadata.is_evaluated
-    assert evaluated.body.metadata.is_evaluated
-
-
-def test_i_can_evaluate_when_the_referenced_concept_has_a_body():
-    sheerka = get_sheerka()
-    concept_a = Concept("a", body="1")
-    sheerka.add_in_cache(concept_a)
-
-    concept = Concept("foo", body="a")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.key
-    assert evaluated.body == simplec("a", 1)
-    assert not concept_a.metadata.is_evaluated
-    assert evaluated.metadata.is_evaluated
-
-
-def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body():
-    sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="a", body="'a'"))
-    sheerka.add_in_cache(Concept(name="b", body="a"))
-    sheerka.add_in_cache(Concept(name="c", body="b"))
-    concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
-
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d)
-
-    assert evaluated.key == concept_d.key
-    expected = simplec("c", simplec("b", simplec("a", "a")))
-    assert evaluated.body == expected
-    assert sheerka.value(evaluated) == 'a'
-    assert evaluated.metadata.is_evaluated
-
-
-def test_i_can_evaluate_concept_of_concept_does_not_have_a_body():
-    sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="a"))
-    sheerka.add_in_cache(Concept(name="b", body="a"))
-    sheerka.add_in_cache(Concept(name="c", body="b"))
-    concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
-
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d)
-
-    assert evaluated.key == concept_d.key
-    expected = simplec("c", simplec("b", simplec("a", None)))
-    assert evaluated.body == expected
-    assert sheerka.value(evaluated) == Concept(name="a").init_key()
-    assert evaluated.metadata.is_evaluated
-
-
-def test_i_can_evaluate_concept_when_properties_reference_others_concepts():
-    sheerka = get_sheerka()
-    concept_a = sheerka.add_in_cache(Concept(name="a").init_key())
-
-    concept = Concept("foo", body="a").def_prop("a", "a").init_key()
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    # first prop a is evaluated to concept_a
-    # then body is evaluated to prop a -> concept_a
-    assert evaluated.key == concept.key
-    assert evaluated.body == concept_a
-
-
-def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2():
-    """
-    Same test,
-    but the name of the property and the name of the concept are different
-    :return:
-    """
-    sheerka = get_sheerka()
-    concept_a = sheerka.add_in_cache(Concept(name="a"))
-
-    concept = Concept("foo", body="concept_a").def_prop("concept_a", "a")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.key
-    assert evaluated.body == concept_a
-
-
-def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body():
-    sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="a", body="1"))
-    sheerka.add_in_cache(Concept(name="b", body="2"))
-
-    concept = Concept("foo", body="propA + propB").def_prop("propA", "a").def_prop("propB", "b")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.key
-    assert evaluated.body == 3
-
-
-def test_i_can_evaluate_concept_when_properties_is_a_concept():
-    sheerka = get_sheerka()
-    concept_a = sheerka.add_in_cache(Concept(name="a", body="'a'").init_key())
-
-    concept = Concept("foo").def_prop("a")
-    concept.compiled["a"] = concept_a
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.key
-    assert evaluated.get_prop("a") == simplec("a", "a")
-
-
-def test_i_can_evaluate_when_property_asts_is_a_list():
-    sheerka = get_sheerka()
-    foo = Concept("foo", body="1")
-
-    concept = Concept("to_eval").def_prop("prop")
-    concept.compiled["prop"] = [foo, DoNotResolve("1")]
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    props = evaluated.get_prop("prop")
-    assert len(props) == 2
-    assert props[0] == simplec("foo", 1)
-    assert props[1] == "1"
-
-
-def test_i_can_evaluate_when_compiled_is_set_up_with_return_value():
-    sheerka = get_sheerka()
-
-    python_node = PythonNode("1 +1 ")
-    parser_result = ParserResultConcept(parser="who", value=python_node)
-
-    concept = Concept("to_eval").def_prop("prop")
-    concept.compiled["prop"] = [ReturnValueConcept("who", True, parser_result)]
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert evaluated.get_prop("prop") == 2
-
-    # also works when only one return value
-    concept = Concept("to_eval").def_prop("prop")
-    concept.compiled["prop"] = ReturnValueConcept("who", True, parser_result)
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert evaluated.get_prop("prop") == 2
-
-
-def test_i_can_reference_sheerka():
-    sheerka = get_sheerka()
-
-    concept = Concept("foo", body="sheerka.test()").init_key()
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.key
-    assert evaluated.body == sheerka.test()
-
-
-def test_properties_values_takes_precedence_over_the_outside_world():
-    sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
-    sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
-
-    concept = Concept("foo", body="a")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert evaluated.key == concept.key
-    assert evaluated.body == simplec("a", "concept_a")  # this test was already done
-
-    # so check this one.
-    concept = Concept("foo", body="a").def_prop("a", "'property_a'")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert evaluated.key == concept.key
-    assert evaluated.body == 'property_a'
-
-    # or this one.
-    concept = Concept("foo", body="a").def_prop("a", "b")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert evaluated.key == concept.key
-    assert evaluated.body == simplec(name="b", body="concept_b")
-
-
-def test_properties_values_takes_precedence():
-    sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
-    sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
-
-    concept = Concept("foo", body="a + b").def_prop("a", "'prop_a'")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert evaluated.key == concept.key
-    assert evaluated.body == 'prop_aconcept_b'
-
-
-def test_i_can_reference_sub_property_of_a_property():
-    sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept(name="concept_a").def_prop("subProp", "'sub_a'"))
-
-    concept = Concept("foo", body="a.props['subProp'].value").def_prop("a", "concept_a")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert evaluated == simplec(concept.key, "sub_a")
-
-
-def test_i_cannot_evaluate_concept_if_property_is_in_error():
-    sheerka = get_sheerka()
-
-    concept = Concept(name="concept_a").def_prop("subProp", "undef_concept")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-    assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
-
-
-def test_key_is_initialized_by_evaluation():
-    sheerka = get_sheerka()
-
-    concept = Concept("foo")
-    evaluated = sheerka.evaluate_concept(get_context(sheerka), concept)
-
-    assert evaluated.key == concept.init_key().key
-
-
-def test_builtin_error_concept_are_errors():
-    # only test a random one, it will be the same for the others
-    sheerka = get_sheerka()
-    assert not sheerka.is_success(sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS))
-
-
-def test_i_can_add_concept_to_set():
-    sheerka = get_sheerka(False, False)
-
-    foo = Concept("foo")
-    sheerka.set_id_if_needed(foo, False)
-
-    all_foos = Concept("all_foos")
-    sheerka.set_id_if_needed(all_foos, False)
-
-    context = get_context(sheerka)
-    res = sheerka.add_concept_to_set(context, foo, all_foos)
-
-    assert res.status
-    assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
-
-    all_entries = get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
-    assert len(all_entries) == 1
-    assert foo.id in all_entries
-
-
-def test_i_can_add_several_concepts_to_set():
-    sheerka = get_sheerka(False, False)
-
-    foo1 = Concept("foo1")
-    sheerka.set_id_if_needed(foo1, False)
-
-    foo2 = Concept("foo1")
-    sheerka.set_id_if_needed(foo2, False)
-
-    all_foos = Concept("all_foos")
-    sheerka.set_id_if_needed(all_foos, False)
-
-    context = get_context(sheerka)
-    sheerka.add_concept_to_set(context, foo1, all_foos)
-    res = sheerka.add_concept_to_set(context, foo2, all_foos)
-
-    assert res.status
-    assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
-
-    all_entries = get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
-    assert len(all_entries) == 2
-    assert foo1.id in all_entries
-    assert foo2.id in all_entries
-
-
-def test_i_cannot_add_the_same_concept_twice_in_a_set():
-    sheerka = get_sheerka()
-
-    foo = Concept("foo")
-    sheerka.set_id_if_needed(foo, False)
-
-    all_foos = Concept("all_foos")
-    sheerka.set_id_if_needed(all_foos, False)
-
-    context = get_context(sheerka)
-    sheerka.add_concept_to_set(context, foo, all_foos)
-    res = sheerka.add_concept_to_set(context, foo, all_foos)
-
-    assert not res.status
-    assert res.body == ConceptAlreadyInSet(foo, all_foos)
-
-    all_entries = sheerka.sdp.get("All_" + all_foos.id, None, False)
-    assert len(all_entries) == 1
-    assert foo.id in all_entries
-
-
-def test_i_get_elements_from_a_set():
-    sheerka = get_sheerka(False, False)
-
-    one = Concept("one")
-    two = Concept("two")
-    three = Concept("three")
-    number = Concept("number")
-
-    for c in [one, two, three, number]:
-        sheerka.set_id_if_needed(c, False)
-        sheerka.add_in_cache(c)
-
-    context = get_context(sheerka)
-    for c in [one, two, three]:
-        sheerka.add_concept_to_set(context, c, number)
-
-    elements = sheerka.get_set_elements(number)
-
-    assert set(elements) == set([one, two, three])
-
-
-def test_i_cannot_get_elements_if_not_a_set():
-    sheerka = get_sheerka(False, False)
-    one = Concept("one")
-    sheerka.set_id_if_needed(one, False)
-    sheerka.add_in_cache(one)
-
-    error = sheerka.get_set_elements(one)
-
-    assert sheerka.isinstance(error, BuiltinConcepts.NOT_A_SET)
-    assert error.body == one
-
-
-def test_isa_and_isa_group():
-    sheerka = get_sheerka()
-
-    group = Concept("group").init_key()
-    group.metadata.id = "1001"
-    assert not sheerka.isagroup(group)
-
-    foo = Concept("foo").init_key()
-    foo.metadata.id = "1002"
-    assert not sheerka.isa(foo, group)
-
-    context = get_context(sheerka)
-    sheerka.add_concept_to_set(context, foo, group)
-    assert sheerka.isagroup(group)
-    assert sheerka.isa(foo, group)

tests/test_sheerka_call_evaluators.py

@@ -1,399 +0,0 @@
-# Make sure that the evaluators works as expected
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator
-from sdp.sheerkaDataProvider import Event
-
-
-def get_sheerka():
-    sheerka = Sheerka()
-    sheerka.initialize("mem://")
-    return sheerka
-
-
-def get_context(sheerka):
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_ret_val(sheerka, concept, who="who"):
-    concept.init_key()
-    if concept.key not in sheerka.cache_by_key:
-        sheerka.cache_by_key[concept.key] = concept
-    return sheerka.ret(who, True, sheerka.new(concept.key))
-
-
-class Out:
-    debug_out = []
-
-    def out(self, method, name, context, return_value):
-        name = name[len(BaseEvaluator.PREFIX):]
-        if isinstance(return_value, list):
-            target = [str(r.body.key) for r in return_value]
-        else:
-            target = str(return_value.body.key)
-        step = str(context.step)
-        text = f"{step} [{context.iteration}] "
-        text += f"{name} - {method} - target={target}"
-        self.debug_out.append(text)
-
-    def out_all(self, method, name, context, return_values):
-        name = name[len(BaseEvaluator.PREFIX):]
-        target = [str(r.body.key) for r in return_values]
-        step = str(context.step)
-        text = f"{step} [{context.iteration}] "
-        text += f"{name} - {method} - target={target}"
-        self.debug_out.append(text)
-
-
-class OneReturnValueEvaluatorForTestingPurpose(OneReturnValueEvaluator, Out):
-    def __init__(self, name, steps, priority):
-        super().__init__(name, steps, priority)
-
-    def matches(self, context, return_value):
-        self.out("matches", self.name, context, return_value)
-        return True
-
-    def eval(self, context, return_value):
-        self.out("eval", self.name, context, return_value)
-
-
-class AllReturnValueEvaluatorForTestingPurpose(AllReturnValuesEvaluator, Out):
-    def __init__(self, name, steps, priority):
-        super().__init__(name, steps, priority)
-
-    def matches(self, context, return_values):
-        self.out("matches", self.name, context, return_values)
-        return True
-
-    def eval(self, context, return_values):
-        self.out("eval", self.name, context, return_values)
-
-
-class EvaluatorOneWithPriority(OneReturnValueEvaluatorForTestingPurpose):
-    def __init__(self, name, priority):
-        super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
-
-
-class EvaluatorOneWithPriority10(EvaluatorOneWithPriority):
-    def __init__(self):
-        super().__init__("priority10", 10)
-
-
-class EvaluatorOneWithPriority15(EvaluatorOneWithPriority):
-    def __init__(self):
-        super().__init__("priority15", 15)
-
-
-class EvaluatorOneWithPriority20(EvaluatorOneWithPriority):
-    def __init__(self):
-        super().__init__("priority20", 20)
-
-
-class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose):
-    def __init__(self, name, priority):
-        super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
-
-
-class EvaluatorAllWithPriority10(EvaluatorAllWithPriority):
-    def __init__(self):
-        super().__init__("all_priority10", 10)
-
-
-class EvaluatorAllWithPriority15(EvaluatorAllWithPriority):
-    def __init__(self):
-        super().__init__("all_priority15", 15)
-
-
-class EvaluatorAllWithPriority20(EvaluatorAllWithPriority):
-    def __init__(self):
-        super().__init__("all_priority20", 20)
-
-
-class EvaluatorOneModifyFoo(EvaluatorOneWithPriority):
-    def __init__(self):
-        super().__init__("modifyFoo", 10)
-
-    def matches(self, context, return_value):
-        super().matches(context, return_value)
-        return context.sheerka.isinstance(return_value.body, "foo")
-
-    def eval(self, context, return_value):
-        super().eval(context, return_value)
-        return get_ret_val(context.sheerka, Concept("bar"))
-
-
-class EvaluatorOneModifyBar(EvaluatorOneWithPriority):
-    def __init__(self):
-        super().__init__("modifyBar", 10)
-
-    def matches(self, context, return_value):
-        super().matches(context, return_value)
-        return context.sheerka.isinstance(return_value.body, "bar")
-
-    def eval(self, context, return_value):
-        super().eval(context, return_value)
-        return get_ret_val(context.sheerka, Concept("baz"))
-
-
-class EvaluatorOnePreEvaluation(OneReturnValueEvaluatorForTestingPurpose):
-    def __init__(self):
-        super().__init__("preEval", [BuiltinConcepts.BEFORE_EVALUATION], 10)
-
-
-class EvaluatorOneMultiSteps(OneReturnValueEvaluatorForTestingPurpose):
-    def __init__(self):
-        super().__init__("multiStep", [BuiltinConcepts.EVALUATION, BuiltinConcepts.BEFORE_EVALUATION], 10)
-
-
-class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority):
-    def __init__(self):
-        super().__init__("all_reduce_foobar", 10)
-
-    def matches(self, context, return_values):
-        super().matches(context, return_values)
-        keys = [c.body.key for c in return_values]
-        return "foo" in keys and "bar" in keys
-
-    def eval(self, context, return_values):
-        super().eval(context, return_values)
-        ret = get_ret_val(context.sheerka, context.sheerka.new(BuiltinConcepts.SUCCESS))
-        ret.parents = return_values
-        return ret
-
-
-class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority):
-
-    def __init__(self):
-        super().__init__("suppress", 100)
-
-    def matches(self, context, return_values):
-        super().matches(context, return_values)
-        return True
-
-    def eval(self, context, return_values):
-        super().eval(context, return_values)
-        foo = None
-        for ret in return_values:
-            if ret.body.name == "foo":
-                foo = ret
-
-        if foo:
-            return context.sheerka.ret(self.name, False, Concept("does not matter"), parents=[foo])
-        else:
-            return None
-
-
-def test_that_return_values_is_unchanged_when_no_evaluator():
-    sheerka = get_sheerka()
-    sheerka.evaluators = []
-
-    entries = get_ret_val(sheerka, Concept("foo"))
-    return_values = sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
-
-    assert return_values == [entries]
-
-
-def test_i_can_use_a_list_as_input():
-    sheerka = get_sheerka()
-    sheerka.evaluators = []
-
-    entries = [get_ret_val(sheerka, Concept("foo"))]
-    return_values = sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
-
-    assert return_values == entries
-
-
-def test_step_concept_is_removed_after_processing_if_not_reduced():
-    """
-    No evaluator
-    """
-    sheerka = get_sheerka()
-    sheerka.evaluators = []
-
-    entry = get_ret_val(sheerka, Concept("foo"))
-    return_values = sheerka.execute(get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
-
-    assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
-
-
-def test_step_concept_is_removed_after_processing_if_not_reduced_2():
-    """
-    This time the entry is modified by an evaluator,
-    nevertheless, step concept is removed
-    """
-    sheerka = get_sheerka()
-    sheerka.evaluators = [EvaluatorOneModifyFoo]
-
-    entry = get_ret_val(sheerka, Concept("foo"))
-    return_values = sheerka.execute(get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
-
-    assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
-
-
-def test_that_higher_priority_evaluators_are_evaluated_first():
-    sheerka = get_sheerka()
-    sheerka.evaluators = [
-        EvaluatorAllWithPriority10,
-        EvaluatorOneWithPriority20,
-        EvaluatorAllWithPriority15,
-        EvaluatorOneWithPriority10,
-        EvaluatorOneWithPriority15,
-        EvaluatorAllWithPriority20, ]
-
-    entries = [get_ret_val(sheerka, Concept("foo"))]
-    Out.debug_out = []
-    sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
-
-    assert Out.debug_out == [
-        '__EVALUATION [0] priority20 - matches - target=foo',
-        '__EVALUATION [0] priority20 - eval - target=foo',
-        '__EVALUATION [0] priority20 - matches - target=__EVALUATION',
-        '__EVALUATION [0] priority20 - eval - target=__EVALUATION',
-        "__EVALUATION [0] all_priority20 - matches - target=['foo', '__EVALUATION']",
-        "__EVALUATION [0] all_priority20 - eval - target=['foo', '__EVALUATION']",
-        "__EVALUATION [0] all_priority15 - matches - target=['foo', '__EVALUATION']",
-        "__EVALUATION [0] all_priority15 - eval - target=['foo', '__EVALUATION']",
-        '__EVALUATION [0] priority15 - matches - target=foo',
-        '__EVALUATION [0] priority15 - eval - target=foo',
-        '__EVALUATION [0] priority15 - matches - target=__EVALUATION',
-        '__EVALUATION [0] priority15 - eval - target=__EVALUATION',
-        "__EVALUATION [0] all_priority10 - matches - target=['foo', '__EVALUATION']",
-        "__EVALUATION [0] all_priority10 - eval - target=['foo', '__EVALUATION']",
-        '__EVALUATION [0] priority10 - matches - target=foo',
-        '__EVALUATION [0] priority10 - eval - target=foo',
-        '__EVALUATION [0] priority10 - matches - target=__EVALUATION',
-        '__EVALUATION [0] priority10 - eval - target=__EVALUATION'
-    ]
-
-
-def test_that_predicate_is_checked_before_evaluation_for_one_return():
-    sheerka = get_sheerka()
-    sheerka.evaluators = [EvaluatorOneModifyFoo]
-
-    entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("baz"))]
-    Out.debug_out = []
-    sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
-
-    assert Out.debug_out == [
-        '__EVALUATION [0] modifyFoo - matches - target=foo',
-        '__EVALUATION [0] modifyFoo - eval - target=foo',
-        '__EVALUATION [0] modifyFoo - matches - target=baz',
-        '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
-        '__EVALUATION [1] modifyFoo - matches - target=bar',
-        '__EVALUATION [1] modifyFoo - matches - target=baz',
-        '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
-    ]
-
-
-def test_that_predicate_is_checked_before_evaluation_for_all_return():
-    sheerka = get_sheerka()
-    sheerka.evaluators = [EvaluatorAllReduceFooBar]
-
-    entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("bar"))]
-    Out.debug_out = []
-    sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
-    assert Out.debug_out == [
-        "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', 'bar', '__EVALUATION']",
-        "__EVALUATION [0] all_reduce_foobar - eval - target=['foo', 'bar', '__EVALUATION']",
-        "__EVALUATION [1] all_reduce_foobar - matches - target=['__SUCCESS']"
-    ]
-
-    entries = [get_ret_val(sheerka, Concept("foo"))]
-    Out.debug_out = []
-    sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
-    assert Out.debug_out == [
-        "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', '__EVALUATION']"
-    ]
-
-
-def test_evaluation_continue_until_no_more_modification():
-    sheerka = get_sheerka()
-    sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar]
-
-    entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("baz"))]
-    Out.debug_out = []
-    sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
-
-    assert Out.debug_out == [
-        '__EVALUATION [0] modifyFoo - matches - target=foo',
-        '__EVALUATION [0] modifyFoo - eval - target=foo',
-        '__EVALUATION [0] modifyFoo - matches - target=baz',
-        '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
-        '__EVALUATION [0] modifyBar - matches - target=foo',
-        '__EVALUATION [0] modifyBar - matches - target=baz',
-        '__EVALUATION [0] modifyBar - matches - target=__EVALUATION',
-        '__EVALUATION [1] modifyFoo - matches - target=bar',
-        '__EVALUATION [1] modifyFoo - matches - target=baz',
-        '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION',
-        '__EVALUATION [1] modifyBar - matches - target=bar',
-        '__EVALUATION [1] modifyBar - eval - target=bar',
-        '__EVALUATION [1] modifyBar - matches - target=baz',
-        '__EVALUATION [1] modifyBar - matches - target=__EVALUATION',
-        '__EVALUATION [2] modifyFoo - matches - target=baz',
-        '__EVALUATION [2] modifyFoo - matches - target=baz',
-        '__EVALUATION [2] modifyFoo - matches - target=__EVALUATION',
-        '__EVALUATION [2] modifyBar - matches - target=baz',
-        '__EVALUATION [2] modifyBar - matches - target=baz',
-        '__EVALUATION [2] modifyBar - matches - target=__EVALUATION'
-    ]
-
-
-def test_evaluation_steps_are_respected():
-    sheerka = get_sheerka()
-    sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation]
-
-    entries = [get_ret_val(sheerka, Concept("foo"))]
-    Out.debug_out = []
-    sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION])
-
-    assert Out.debug_out == [
-        '__BEFORE_EVALUATION [0] preEval - matches - target=foo',
-        '__BEFORE_EVALUATION [0] preEval - eval - target=foo',
-        '__BEFORE_EVALUATION [0] preEval - matches - target=__BEFORE_EVALUATION',
-        '__BEFORE_EVALUATION [0] preEval - eval - target=__BEFORE_EVALUATION']
-
-
-def test_evaluation_multi_steps_are_respected():
-    sheerka = get_sheerka()
-    sheerka.evaluators = [EvaluatorOneMultiSteps]
-
-    entries = [get_ret_val(sheerka, Concept("foo"))]
-    Out.debug_out = []
-    sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION])
-
-    assert Out.debug_out == [
-        '__BEFORE_EVALUATION [0] multiStep - matches - target=foo',
-        '__BEFORE_EVALUATION [0] multiStep - eval - target=foo',
-        '__BEFORE_EVALUATION [0] multiStep - matches - target=__BEFORE_EVALUATION',
-        '__BEFORE_EVALUATION [0] multiStep - eval - target=__BEFORE_EVALUATION',
-        '__EVALUATION [0] multiStep - matches - target=foo',
-        '__EVALUATION [0] multiStep - eval - target=foo',
-        '__EVALUATION [0] multiStep - matches - target=__EVALUATION',
-        '__EVALUATION [0] multiStep - eval - target=__EVALUATION'
-    ]
-
-
-def test_evaluators_can_be_pre_processed():
-    sheerka = get_sheerka()
-    sheerka.evaluators = [EvaluatorOneModifyFoo]
-
-    entries = [get_ret_val(sheerka, Concept("foo"))]
-
-    # disable evaluator
-    context = get_context(sheerka)
-    context.add_preprocess(EvaluatorOneModifyFoo().name, enabled=False)  # disabled  for this exec context
-    Out.debug_out = []
-    sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION])
-    assert Out.debug_out == []
-
-    # other contextes are not impacted
-    Out.debug_out = []
-    sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
-    assert Out.debug_out == [
-        '__EVALUATION [0] modifyFoo - matches - target=foo',
-        '__EVALUATION [0] modifyFoo - eval - target=foo',
-        '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
-        '__EVALUATION [1] modifyFoo - matches - target=bar',
-        '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
-    ]

tests/test_sheerka_call_parsers.py

@@ -1,380 +0,0 @@
-from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept
-from core.sheerka import Sheerka, ExecutionContext
-from parsers.BaseParser import BaseParser
-from sdp.sheerkaDataProvider import Event
-
-
-def get_sheerka():
-    sheerka = Sheerka()
-    sheerka.initialize("mem://")
-    return sheerka
-
-
-def get_context(sheerka):
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_ret_val(text, who="who"):
-    return ReturnValueConcept(who, True, UserInputConcept(text, "user_name"))
-
-
-class BaseTestParser(BaseParser):
-    debug_out = []
-
-    def __init__(self, name, priority, status=None, parser_result=None):
-        super().__init__(name, priority)
-        self.status = status
-        self.parser_result = parser_result
-
-    @staticmethod
-    def _get_name(name):
-        return name[8:] if name.startswith("parsers.") else name
-
-    @staticmethod
-    def _get_source(text_):
-        return text_ if isinstance(text_, str) else text_.body
-
-    def _out(self, name, priority, status, source):
-        debug = f"name={name}"
-        debug += f", priority={priority}"
-        debug += f", status={status}"
-        debug += f", source={source}"
-        self.debug_out.append(debug)
-
-    def parse(self, context, text):
-        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
-        parser_result = ParserResultConcept(parser=self, value=value)
-        return ReturnValueConcept(self, self.status, self.parser_result or parser_result)
-
-
-class Enabled90FalseParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Enabled90False", 90, False)
-
-
-class Enabled80FalseParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Enabled80False", 80, False)
-
-
-class Enabled80MultipleFalseParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Enabled80MultipleFalse", 80, False)
-
-    def parse(self, context, text):
-        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
-        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
-        return [
-            ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)),
-            ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)),
-        ]
-
-
-class Enabled80MultipleTrueParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Enabled80MultipleTrue", 80)
-
-    def parse(self, context, text):
-        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
-        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
-        return [
-            ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)),
-            ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)),
-        ]
-
-
-class Enabled70FalseParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Enabled70False", 70, False, "Not a ParserResult")
-
-
-class Enabled50TrueParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Enabled50True", 50, True)
-
-    def parse(self, context, text):
-        source = self._get_source(text)
-        status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world"
-        self._out(self._get_name(self.name), self.priority, status, source)
-
-        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
-        return_value = ParserResultConcept(parser=self, value=value)
-        return ReturnValueConcept(self, status, return_value)
-
-
-class Enabled50bisTrueParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Enabled50BisTrue", 50, True)
-
-
-class Enabled50FalseParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Enabled50False", 50, False)
-
-
-class Enabled10TrueParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Enabled10True", 10, True)
-
-
-class DisabledParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("Disabled", 90, True)
-        self.enabled = False
-
-
-class NoneParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("None", 90, True, None)
-
-    def parse(self, context, text):
-        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        return None
-
-
-class ListOfNoneParser(BaseTestParser):
-    def __init__(self, **kwargs):
-        super().__init__("ListOfNone", 90, True, None)
-
-    def parse(self, context, text):
-        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        return [None, None]
-
-
-def test_disabled_parsers_are_not_executed():
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "Enabled": Enabled10TrueParser,
-        "Disabled": DisabledParser
-    }
-
-    user_input = [get_ret_val("hello world")]
-    BaseTestParser.debug_out = []
-    sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-
-    assert BaseTestParser.debug_out == ['name=Enabled10True, priority=10, status=True, source=hello world']
-
-
-def test_parser_are_executed_by_priority():
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "Enabled90False": Enabled90FalseParser,
-        "Enabled80False": Enabled80FalseParser,
-        "Enabled50True": Enabled50TrueParser,
-    }
-
-    user_input = [get_ret_val("hello world")]
-    BaseTestParser.debug_out = []
-    sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-
-    assert BaseTestParser.debug_out == [
-        'name=Enabled90False, priority=90, status=False, source=hello world',
-        'name=Enabled80False, priority=80, status=False, source=hello world',
-        'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
-        'name=Enabled50True, priority=50, status=False, source=hello world',
-        'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
-        'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
-        'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
-    ]
-
-
-def test_parsing_stop_at_the_first_success():
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "Enabled80False": Enabled80FalseParser,
-        "Enabled50bisTrue": Enabled50bisTrueParser,
-        "Enabled10True": Enabled10TrueParser,
-    }
-
-    user_input = [get_ret_val("hello world")]
-    BaseTestParser.debug_out = []
-    sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-
-    assert BaseTestParser.debug_out == [
-        'name=Enabled80False, priority=80, status=False, source=hello world',
-        'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
-    ]
-
-
-def test_parsing_stop_at_the_first_success_2():
-    """
-    Same test than before, but Enabled50True takes more time to find a match
-    :return:
-    """
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "Enabled90False": Enabled90FalseParser,
-        "Enabled80False": Enabled80FalseParser,
-        "Enabled50True": Enabled50TrueParser,
-        "Enabled10True": Enabled10TrueParser,
-    }
-
-    user_input = [get_ret_val("hello world")]
-    BaseTestParser.debug_out = []
-    sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-
-    assert BaseTestParser.debug_out == [
-        'name=Enabled90False, priority=90, status=False, source=hello world',
-        'name=Enabled80False, priority=80, status=False, source=hello world',
-        'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
-        'name=Enabled50True, priority=50, status=False, source=hello world',
-        'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
-        'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
-        'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
-    ]
-
-
-def test_all_parsers_of_a_given_priority_are_executed():
-    """
-    Make sure that all parsers with priority 50 are executed
-    :return:
-    """
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "Enabled90False": Enabled90FalseParser,
-        "Enabled80False": Enabled80FalseParser,
-        "Enabled50True": Enabled50TrueParser,
-        "Enabled50bisTrue": Enabled50bisTrueParser,
-        "Enabled50False": Enabled50FalseParser,
-        "Enabled10True": Enabled10TrueParser,
-    }
-
-    user_input = [get_ret_val("hello world")]
-    BaseTestParser.debug_out = []
-    sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-
-    assert BaseTestParser.debug_out == [
-        'name=Enabled90False, priority=90, status=False, source=hello world',
-        'name=Enabled80False, priority=80, status=False, source=hello world',
-        'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
-        'name=Enabled50True, priority=50, status=False, source=hello world',
-        'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
-        'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
-        'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
-        'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
-        'name=Enabled50False, priority=50, status=False, source=hello world',
-        'name=Enabled50False, priority=50, status=False, source=Enabled90False:hello world',
-        'name=Enabled50False, priority=50, status=False, source=Enabled80False:hello world',
-        'name=Enabled50False, priority=50, status=False, source=Enabled80False:Enabled90False:hello world',
-    ]
-
-
-def test_a_parser_has_access_to_the_output_of_its_predecessors():
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "Enabled90False": Enabled90FalseParser,
-        "Enabled80False": Enabled80FalseParser,
-        "Enabled50True": Enabled50TrueParser,
-    }
-
-    user_input = [get_ret_val("hello world")]
-
-    res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-    res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
-    assert res_as_tuple == [
-        ('Enabled90False', False, 'Enabled90False:hello world'),
-        ('Enabled80False', False, 'Enabled80False:hello world'),
-        ('Enabled80False', False, 'Enabled80False:Enabled90False:hello world'),
-        ('Enabled50True', False, 'Enabled50True:hello world'),
-        ('Enabled50True', False, 'Enabled50True:Enabled90False:hello world'),
-        ('Enabled50True', False, 'Enabled50True:Enabled80False:hello world'),
-        ('Enabled50True', True, 'Enabled50True:Enabled80False:Enabled90False:hello world'),
-    ]
-
-
-def test_none_return_values_are_discarded():
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "NoneParser": NoneParser,
-        "ListOfNone": ListOfNoneParser,
-    }
-
-    user_input = [get_ret_val("hello world")]
-
-    BaseTestParser.debug_out = []
-    res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-    assert res == []
-    assert BaseTestParser.debug_out == [
-        'name=None, priority=90, status=True, source=hello world',
-        'name=ListOfNone, priority=90, status=True, source=hello world'
-    ]
-
-
-def test_following_priorities_can_only_see_parser_result_return_values():
-    """
-    Normally, lower priority parsers can see the result of the higher priority parsers
-    This is true only if the higher priority parser return a ParserResultConcept
-    :return:
-    """
-
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "Enabled80False": Enabled80FalseParser,
-        "Enabled70False": Enabled70FalseParser,
-        "Enabled50True": Enabled50TrueParser,
-    }
-
-    user_input = [get_ret_val("hello world")]
-    BaseTestParser.debug_out = []
-    sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-
-    assert BaseTestParser.debug_out == [
-        'name=Enabled80False, priority=80, status=False, source=hello world',
-        'name=Enabled70False, priority=70, status=False, source=hello world',
-        'name=Enabled70False, priority=70, status=False, source=Enabled80False:hello world',
-        'name=Enabled50True, priority=50, status=False, source=hello world',
-        'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
-    ]
-
-
-def test_i_can_manage_parser_with_multiple_results():
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "Enabled80MultipleFalse": Enabled80MultipleFalseParser,
-        "Enabled50True": Enabled50TrueParser,
-    }
-
-    user_input = [get_ret_val("hello world")]
-    BaseTestParser.debug_out = []
-    res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-
-    assert BaseTestParser.debug_out == [
-        'name=Enabled80MultipleFalse, priority=80, status=False, source=hello world',
-        'name=Enabled50True, priority=50, status=False, source=hello world',
-        'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_1',
-        'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_2',
-    ]
-
-    res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
-    assert res_as_tuple == [
-        ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_1'),
-        ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_2'),
-        ('Enabled50True', False, 'Enabled50True:hello world'),
-        ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_1'),
-        ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'),
-    ]
-
-
-def test_i_can_manage_parser_with_multiple_results_and_a_sucess():
-    sheerka = get_sheerka()
-    sheerka.parsers = {
-        "Enabled80MultipleTrue": Enabled80MultipleTrueParser,
-        "Enabled50True": Enabled50TrueParser,
-    }
-
-    user_input = [get_ret_val("hello world")]
-    BaseTestParser.debug_out = []
-    res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
-
-    assert BaseTestParser.debug_out == [
-        'name=Enabled80MultipleTrue, priority=80, status=None, source=hello world',
-    ]
-
-    res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
-    assert res_as_tuple == [
-        ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'),
-        ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'),
-    ]

tests/test_sheerka_non_reg.py

@@ -1,696 +0,0 @@
-import os
-import shutil
-from os import path
-
-import pytest
-
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, Property, simplec
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
-from parsers.ConceptLexerParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression
-from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
-
-tests_root = path.abspath("../build/tests")
-root_folder = "init_folder"
-
-
-@pytest.fixture(autouse=True)
-def init_test():
-    if path.exists(tests_root):
-        shutil.rmtree(tests_root)
-
-    if not path.exists(tests_root):
-        os.makedirs(tests_root)
-    current_pwd = os.getcwd()
-    os.chdir(tests_root)
-
-    yield None
-
-    os.chdir(current_pwd)
-
-
-def get_sheerka(use_dict=True, skip_builtins_in_db=True):
-    root = "mem://" if use_dict else root_folder
-    sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db)
-    sheerka.initialize(root)
-
-    return sheerka
-
-
-def get_context(sheerka):
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_default_concept():
-    concept = Concept(
-        name="a + b",
-        where="isinstance(a, int) and isinstance(b, int)",
-        pre="isinstance(a, int) and isinstance(b, int)",
-        post="isinstance(res, int)",
-        body="def func(x,y):\n    return x+y\nfunc(a,b)",
-        desc="specific description")
-    concept.def_prop("a", "value1")
-    concept.def_prop("b", "value2")
-
-    return concept
-
-
-@pytest.mark.parametrize("text, expected", [
-    ("1 + 1", 2),
-    ("sheerka.test()", 'I have access to Sheerka !')
-])
-def test_i_can_eval_python_expressions_with_no_variable(text, expected):
-    sheerka = get_sheerka()
-
-    res = sheerka.evaluate_user_input(text)
-
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value == expected
-
-
-def test_i_can_eval_concept_with_python_body():
-    sheerka = get_sheerka()
-    concept = Concept(name="one", body="1")
-    sheerka.add_in_cache(concept)
-
-    text = "one"
-    res = sheerka.evaluate_user_input(text)
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value == simplec("one", 1)  # by default, the concept is returned
-
-
-def test_i_can_eval_concept_with_concept_body():
-    sheerka = get_sheerka()
-    concept_one = Concept(name="one")
-    concept_un = Concept(name="un", body="one")
-    sheerka.add_in_cache(concept_one)
-    sheerka.add_in_cache(concept_un)
-
-    res = sheerka.evaluate_user_input("un")
-    return_value = res[0].value
-    assert len(res) == 1
-    assert res[0].status
-    assert return_value == simplec("un", simplec("one", None))
-
-
-def test_i_can_eval_concept_with_no_body():
-    sheerka = get_sheerka()
-    concept = Concept(name="one")
-    sheerka.add_in_cache(concept)
-
-    text = "one"
-    res = sheerka.evaluate_user_input(text)
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value == concept
-    assert id(res[0].value) != id(concept)
-
-
-def test_is_unique_property_is_used_when_evaluating():
-    sheerka = get_sheerka()
-    concept = Concept(name="one", is_unique=True)
-    sheerka.add_in_cache(concept)
-
-    text = "one"
-    res = sheerka.evaluate_user_input(text)
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value == concept
-    assert id(res[0].value) == id(concept)
-
-
-def test_i_can_eval_def_concept_request():
-    text = """
-def concept a + b
-where isinstance(a, int) and isinstance(b, int)
-pre isinstance(a, int) and isinstance(b, int)
-post isinstance(res, int)
-as:
-    def func(x,y):
-        return x+y
-    func(a,b)
-    """
-
-    expected = get_default_concept()
-    expected.metadata.id = "1001"
-    expected.metadata.desc = None
-    expected.metadata.props = [("a", None), ("b", None)]
-    expected.init_key()
-
-    sheerka = get_sheerka()
-    res = sheerka.evaluate_user_input(text)
-
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
-
-    concept_saved = res[0].value.body
-
-    for prop in PROPERTIES_TO_SERIALIZE:
-        assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
-
-    assert concept_saved.key in sheerka.cache_by_key
-    assert concept_saved.id in sheerka.cache_by_id
-    assert sheerka.sdp.io.exists(
-        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
-
-
-def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept():
-    """
-    In this test, we test that the properties of 'concept a xx b' (which are 'a' and 'b')
-    are correctly detected, thanks to the source code 'a plus b' in its body
-    :return:
-    """
-    sheerka = get_sheerka()
-
-    # concept 'a plus b' is known
-    concept_a_plus_b = Concept(name="a plus b").def_prop("a").def_prop("b")
-    sheerka.add_in_cache(concept_a_plus_b)
-
-    res = sheerka.evaluate_user_input("def concept a xx b as a plus b")
-    expected = Concept(name="a xx b", body="a plus b").def_prop("a").def_prop("b").init_key()
-    expected.metadata.id = "1001"
-
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
-
-    concept_saved = res[0].value.body
-
-    for prop in PROPERTIES_TO_SERIALIZE:
-        assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
-
-    assert concept_saved.key in sheerka.cache_by_key
-    assert concept_saved.id in sheerka.cache_by_id
-    assert sheerka.sdp.io.exists(
-        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
-
-
-def test_i_cannot_eval_the_same_def_concept_twice():
-    text = """
-def concept a + b
-where isinstance(a, int) and isinstance(b, int)
-pre isinstance(a, int) and isinstance(b, int)
-post isinstance(res, int)
-as:
-    def func(x,y):
-        return x+y
-    func(a,b)
-        """
-
-    sheerka = get_sheerka()
-    sheerka.evaluate_user_input(text)
-    res = sheerka.evaluate_user_input(text)
-
-    assert len(res) == 1
-    assert not res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
-
-
-@pytest.mark.parametrize("text", [
-    "",
-    "   ",
-    "\n",
-])
-def test_i_can_eval_a_empty_input(text):
-    sheerka = get_sheerka()
-
-    res = sheerka.evaluate_user_input(text)
-
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NOP)
-
-
-def test_i_can_eval_concept_with_variable():
-    sheerka = get_sheerka()
-    concept_hello = Concept(name="hello a").def_prop("a")
-    concept_foo = Concept(name="foo")
-    sheerka.add_in_cache(concept_hello)
-    sheerka.add_in_cache(concept_foo)
-
-    res = sheerka.evaluate_user_input("hello foo")
-    return_value = res[0].value
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(return_value, concept_hello)
-    assert return_value.props["a"].value == concept_foo
-
-
-def test_i_can_eval_concept_with_variable_and_python_as_body():
-    sheerka = get_sheerka()
-    hello_a = sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a"))
-    sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
-
-    res = sheerka.evaluate_user_input("hello foo")
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, hello_a)
-    assert res[0].value.body == "hello foo"
-    assert res[0].value.metadata.is_evaluated
-    assert res[0].value.props["a"].value == simplec("foo", "foo")
-    assert res[0].value.props["a"].value.metadata.is_evaluated
-
-
-def test_i_can_eval_duplicate_concepts_with_same_value():
-    sheerka = get_sheerka()
-
-    sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a"))
-    sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
-    sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
-
-    res = sheerka.evaluate_user_input("hello foo")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value.body == "hello foo"
-    assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
-
-
-def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different():
-    sheerka = get_sheerka()
-
-    sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a"))
-    sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
-    sheerka.add_in_cache(Concept(name="foo", body="'another value'"))
-
-    res = sheerka.evaluate_user_input("hello foo")
-    assert len(res) == 1
-    assert not res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS)
-
-    concepts = res[0].value.body
-    assert len(concepts) == 2
-    sorted_values = sorted(concepts, key=lambda x: x.value.body)
-    assert sorted_values[0].value.body == "hello another value"
-    assert sorted_values[1].value.body == "hello foo"
-
-
-def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same():
-    sheerka = get_sheerka()
-    context = get_context(sheerka)
-
-    sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").def_prop("a"))
-    sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").def_prop("b"))
-
-    res = sheerka.evaluate_user_input("hello 'foo'")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].value.body == "hello foo"  # I don't know yet the one to choose
-    assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
-
-
-def test_i_can_create_concepts_with_python_code_as_body():
-    sheerka = get_sheerka()
-    context = get_context(sheerka)
-
-    sheerka.create_new_concept(context, Concept(name="concepts", body="sheerka.concepts()"))
-    res = sheerka.evaluate_user_input("concepts")
-
-    assert len(res) == 1
-    assert res[0].status
-    assert isinstance(res[0].value.body, list)
-
-
-def test_i_can_create_concept_with_bnf_definition():
-    sheerka = get_sheerka(False, False)
-    a = Concept("a")
-    sheerka.add_in_cache(a)
-    sheerka.concepts_definition_cache = {a: OrderedChoice("one", "two")}
-
-    res = sheerka.evaluate_user_input("def concept plus from bnf a ('plus' plus)?")
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
-
-    saved_concept = sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, "plus")
-    assert saved_concept.key == "plus"
-    assert saved_concept.metadata.definition == "a ('plus' plus)?"
-    assert "a" in saved_concept.props
-    assert "plus" in saved_concept.props
-
-    saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY)
-    expected_bnf = Sequence(
-        a, Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus"))))
-    assert saved_definitions[saved_concept] == expected_bnf
-
-    new_concept = res[0].value.body
-    assert new_concept.metadata.name == "plus"
-    assert new_concept.metadata.definition == "a ('plus' plus)?"
-    assert new_concept.bnf == expected_bnf
-    assert "a" in new_concept.props
-    assert "plus" in new_concept.props
-
-
-def test_i_can_eval_bnf_definitions():
-    sheerka = get_sheerka()
-    concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
-
-    res = sheerka.evaluate_user_input("one")
-
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, concept_a)
-
-
-def test_i_can_eval_bnf_definitions_with_variables():
-    sheerka = get_sheerka()
-    concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
-    concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
-
-    res = sheerka.evaluate_user_input("one three")
-
-    assert len(res) == 1
-    assert res[0].status
-    return_value = res[0].value
-
-    assert sheerka.isinstance(return_value, concept_b)
-    assert return_value.body == "one three"
-    assert return_value.metadata.is_evaluated
-
-    assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one").init_key())
-    assert return_value.props["a"].value.metadata.is_evaluated
-
-
-def test_i_can_eval_bnf_definitions_from_separate_instances():
-    """
-    Same test then before,
-    but make sure that the BNF are correctly persisted and loaded
-    """
-    sheerka = get_sheerka(False)
-    concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' 'two'")[0].body.body
-
-    res = get_sheerka(False).evaluate_user_input("one two")
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, concept_a)
-
-    # add another bnf definition
-    concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
-
-    res = get_sheerka(False).evaluate_user_input("one two")  # previous one still works
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, concept_a)
-
-    res = get_sheerka(False).evaluate_user_input("one two three")  # new one works
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].value, concept_b)
-    assert res[0].value.body == "one two three"
-    assert res[0].value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key())
-
-
-@pytest.mark.parametrize("desc, definitions", [
-    ("Simple form", [
-        "def concept one as 1",
-        "def concept two as 2",
-        "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit"
-    ]),
-    ("When twenty is a concept", [
-        "def concept one as 1",
-        "def concept two as 2",
-        "def concept twenty as 20",
-        "def concept twenties from bnf twenty (one|two)=unit as twenty + unit"
-    ]),
-    ("When digit is a concept", [
-        "def concept one as 1",
-        "def concept two as 2",
-        "def concept twenty as 20",
-        "def concept digit from bnf one|two",
-        "def concept twenties from bnf twenty digit as twenty + digit"
-    ]),
-    ("When using isa and concept twenty", [
-        "def concept one as 1",
-        "def concept two as 2",
-        "def concept number",
-        "one isa number",
-        "two isa number",
-        "def concept twenties from bnf 'twenty' number as 20 + number"
-    ]),
-    ("When using isa and concept twenty", [
-        "def concept one as 1",
-        "def concept two as 2",
-        "def concept twenty as 20",
-        "def concept number",
-        "one isa number",
-        "two isa number",
-        "def concept twenties from bnf twenty number as 20 + number"
-    ]),
-])
-def test_i_can_mix_concept_with_python_to_define_numbers(desc, definitions):
-    sheerka = get_sheerka()
-
-    for definition in definitions:
-        sheerka.evaluate_user_input(definition)
-
-    res = sheerka.evaluate_user_input("twenty one")
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].body, "twenties")
-    assert res[0].body.body == 21
-
-    res = sheerka.evaluate_user_input("twenty one + 1")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 22
-
-    res = sheerka.evaluate_user_input("twenty one + one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 22
-
-    res = sheerka.evaluate_user_input("twenty one + twenty two")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 43
-
-    res = sheerka.evaluate_user_input("1 + twenty one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 22
-
-    res = sheerka.evaluate_user_input("1 + 1 + twenty one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 23
-
-
-def test_i_can_mix_bnf_and_isa():
-    """
-    if 'one' isa 'number, twenty number should be recognized
-    :return:
-    """
-    sheerka = get_sheerka()
-    sheerka.evaluate_user_input("def concept one as 1")
-    sheerka.evaluate_user_input("def concept two as 2")
-    sheerka.evaluate_user_input("def concept number")
-    sheerka.evaluate_user_input("one isa number")
-    sheerka.evaluate_user_input("two isa number")
-    sheerka.evaluate_user_input("def concept twenties from bnf 'twenty' number as 20 + number")
-
-    res = sheerka.evaluate_user_input("twenty one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == simplec("twenties", 21)
-
-    res = sheerka.evaluate_user_input("twenty one + 1")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 22
-
-    res = sheerka.evaluate_user_input("twenty one + one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 22
-
-    res = sheerka.evaluate_user_input("twenty one + twenty two")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 43
-
-    res = sheerka.evaluate_user_input("1 + twenty one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 22
-
-    res = sheerka.evaluate_user_input("1 + 1 + twenty one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 23
-
-
-def test_i_can_mix_concept_of_concept():
-    sheerka = get_sheerka()
-
-    definitions = [
-        "def concept one as 1",
-        "def concept two as 2",
-        "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit",
-        "def concept a plus b as a + b"
-    ]
-
-    for definition in definitions:
-        sheerka.evaluate_user_input(definition)
-
-    res = sheerka.evaluate_user_input("1 plus 2")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 3
-
-    res = sheerka.evaluate_user_input("1 plus one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 2
-
-    res = sheerka.evaluate_user_input("1 + 1 plus 1")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 3
-
-    res = sheerka.evaluate_user_input("1 plus twenty one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 22
-
-    res = sheerka.evaluate_user_input("one plus 1")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 2
-
-    res = sheerka.evaluate_user_input("one plus two")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 3
-
-    res = sheerka.evaluate_user_input("one plus twenty one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 22
-
-    res = sheerka.evaluate_user_input("twenty one plus 1")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 22
-
-    res = sheerka.evaluate_user_input("twenty one plus one")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 22
-
-    res = sheerka.evaluate_user_input("twenty one plus twenty two")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body.body == 43
-
-
-# def test_i_can_evaluate_concept_of_concept_when_multiple_choices():
-#     sheerka = get_sheerka()
-#
-#     definitions = [
-#         "def concept little a where a",
-#         "def concept blue a where a",
-#         "def concept little blue a where a",
-#         "def concept house"
-#     ]
-#
-#     for definition in definitions:
-#         sheerka.evaluate_user_input(definition)
-#
-#     ### CAUTION ####
-#     # this test cannot work !!
-#     # it is just to hint the result that I would like to achieve
-#
-#     res = sheerka.evaluate_user_input("little blue house")
-#     assert len(res) == 2
-#     assert res[0].status
-#     assert res[0].body == "little(blue(house))"
-#
-#     assert res[1].status
-#     assert res[1].body == "little blue(house)"
-
-
-def test_i_can_say_that_a_concept_isa_another_concept():
-    sheerka = get_sheerka()
-    sheerka.evaluate_user_input("def concept foo")
-    sheerka.evaluate_user_input("def concept bar")
-
-    res = sheerka.evaluate_user_input("foo isa bar")
-    assert len(res) == 1
-    assert res[0].status
-    assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
-
-
-def test_eval_does_not_break_valid_result():
-    sheerka = get_sheerka()
-    sheerka.evaluate_user_input("def concept one as 1")
-    sheerka.evaluate_user_input("def concept two as 2")
-
-    res = sheerka.evaluate_user_input("one + two")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 3
-
-    res = sheerka.evaluate_user_input("eval one + two")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 3
-
-
-@pytest.mark.parametrize("text", [
-    "'hello",
-    '"foo" + "string',
-    "c::",
-    "c:foo\nbar:",
-    "c:foo",
-    "def concept 'name",
-    "def concept name from bnf 'name"
-])
-def test_i_can_manage_tokenizer_error(text):
-    sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept("foo"))
-
-    res = sheerka.evaluate_user_input(text)
-
-    assert len(res) > 1
-    for r in [r for r in res if r.who.startswith("parsers.")]:
-        assert not r.status
-
-
-def test_i_can_recognize_concept_from_string():
-    sheerka = get_sheerka()
-    sheerka.add_in_cache(Concept("one", body="1"))
-
-    res = sheerka.evaluate_user_input("'one'")
-
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == "one"
-
-    res = sheerka.evaluate_user_input("eval 'one'")
-
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == "one"
-
-
-@pytest.mark.parametrize("expression", [
-    "def concept twenties from bnf 'twenty' (one | two)=unit as 20 + unit",
-    "def concept twenties from bnf 'twenty' (one | two)=unit as twenty + unit",
-    "def concept twenties from bnf twenty (one | two)=unit as 20 + unit",
-    "def concept twenties from bnf twenty (one | two)=unit as twenty + unit",
-])
-def test_i_can_evaluate_bnf_concepts(expression):
-    sheerka = get_sheerka()
-
-    sheerka.evaluate_user_input("def concept one as 1")
-    sheerka.evaluate_user_input("def concept two as 2")
-    sheerka.evaluate_user_input("def concept twenty as 20")
-    sheerka.evaluate_user_input(expression)
-    res = sheerka.evaluate_user_input("eval twenty one")
-
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 21

tests/test_sheerka_transform.py

@@ -1,336 +0,0 @@
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts
-from core.sheerka import Sheerka, ExecutionContext, ExecutionContextIdManager
-from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY
-from sdp.sheerkaDataProvider import Event
-
-
-def get_sheerka():
-    sheerka = Sheerka()
-    sheerka.initialize("mem://")
-    return sheerka
-
-
-def get_context(sheerka):
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def test_i_can_transform_an_unknown_concept():
-    sheerka = get_sheerka()
-
-    foo = Concept("foo", body="body")
-    concept_with_sub = Concept("concept_with_sub", body=foo)
-
-    concept = Concept(
-        name="concept_name",
-        is_builtin=True,
-        is_unique=True,
-        key="concept_key",
-        body=concept_with_sub,
-        where=[foo, 1, "1", True, 1.0],
-        pre=foo,
-        post=None,  # will not appear
-        definition="it is a definition",
-        definition_type="def type",
-        desc="this this the desc"
-    ).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub")
-
-    # add values and props
-    concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init()
-    concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0]
-    concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init()
-    concept.values[ConceptParts.POST] = "a value for POST"
-    concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub)
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(concept)
-
-    assert to_dict == {
-        OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-        OBJ_ID_KEY: 0,
-        'meta.name': 'concept_name',
-        'meta.key': 'concept_key',
-        'meta.is_builtin': True,
-        'meta.is_unique': True,
-        'meta.definition': 'it is a definition',
-        'meta.definition_type': 'def type',
-        'meta.desc': 'this this the desc',
-        'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-                        OBJ_ID_KEY: 1,
-                        'meta.body': 'body',
-                        'meta.name': 'foo'}, 1, '1', True, 1.0],
-        'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1},
-        'meta.body': {
-            '__type__': SheerkaTransformType.Concept,
-            '__id__': 2,
-            'meta.name': 'concept_with_sub',
-            'meta.body': {
-                '__type__': SheerkaTransformType.Reference,
-                '__id__': 1}},
-        'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']],
-        'pre': {'__type__': SheerkaTransformType.Concept,
-                '__id__': 4,
-                'meta.body': 'body',
-                'meta.name': 'foo',
-                'body': 'body'},
-        'post': "a value for POST",
-        'body': {'__type__': SheerkaTransformType.Concept,
-                 '__id__': 3,
-                 'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference},
-                 'meta.name': 'concept_with_sub',
-                 'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}},
-        'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0],
-        'props': [('a', 10),
-                  ('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}),
-                  ('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})],
-    }
-
-
-def test_i_can_transform_unknown_concept_with_almost_same_value():
-    sheerka = get_sheerka()
-    concept = Concept("foo")
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(concept)
-
-    assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'}
-
-
-def test_i_can_transform_known_concept_when_the_values_are_the_same():
-    """
-    Values are the same means that we are serializing a concept which has kept all its default values
-    There is not diff between the concept to serialize and the one which was registered with create_new_concept()
-    We serialize only the id of the concept
-    :return:
-    """
-    sheerka = get_sheerka()
-
-    concept = Concept(
-        name="concept_name",
-        is_builtin=True,
-        is_unique=False,
-        key="concept_key",
-        body="body definition",
-        where="where definition",
-        pre="pre definition",
-        post="post definition",
-        definition="it is a definition",
-        definition_type="def type",
-        desc="this this the desc"
-    ).def_prop("a").def_prop("b")
-    sheerka.create_new_concept(get_context(sheerka), concept)
-
-    new_concept = sheerka.new(concept.key)
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(new_concept)
-
-    assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"}
-
-
-def test_i_can_transform_known_concept_when_the_values_are_different():
-    """
-    Values are the different means the concept was modified.
-    It's different from the one which was registered with create_new_concept()
-    We serialize only the differences
-    :return:
-    """
-    sheerka = get_sheerka()
-
-    concept = Concept(
-        name="concept_name",
-        is_builtin=True,
-        is_unique=False,
-        key="concept_key",
-        body="body definition",
-        where="where definition",
-        pre="pre definition",
-        post="post definition",
-        definition="it is a definition",
-        definition_type="def type",
-        desc="this this the desc"
-    ).def_prop("a").def_prop("b")
-    sheerka.create_new_concept(get_context(sheerka), concept)
-
-    new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre")
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(new_concept)
-
-    assert to_dict == {
-        OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-        OBJ_ID_KEY: 0,
-        "id": "1001",
-        'pre': 'another pre',
-        "body": "another",
-        'props': [('a', 10)]
-    }
-
-
-def test_i_can_transform_concept_with_circular_reference():
-    sheerka = get_sheerka()
-    foo = Concept("foo")
-    bar = Concept("bar", body=foo)
-    foo.metadata.body = bar
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(foo)
-
-    assert to_dict == {
-        OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-        OBJ_ID_KEY: 0,
-        'meta.name': 'foo',
-        'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-                      OBJ_ID_KEY: 1,
-                      'meta.name': 'bar',
-                      'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
-                                    OBJ_ID_KEY: 0},
-                      },
-    }
-
-
-def test_i_can_transform_concept_with_circular_reference_2():
-    sheerka = get_sheerka()
-    foo = Concept("foo")
-    bar = Concept("foo", body=foo)
-    foo.metadata.body = bar
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(foo)
-
-    assert to_dict == {
-        OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-        OBJ_ID_KEY: 0,
-        'meta.name': 'foo',
-        'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-                      OBJ_ID_KEY: 1,
-                      'meta.name': 'foo',
-                      'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
-                                    OBJ_ID_KEY: 0},
-                      },
-    }
-
-
-def test_i_can_transform_the_unknown_concept():
-    sheerka = get_sheerka()
-
-    unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(unknown)
-
-    assert len(to_dict) == 3
-    assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept
-    assert to_dict[OBJ_ID_KEY] == 0
-    assert "id" in to_dict
-
-
-def test_i_can_transform_simple_execution_context():
-    sheerka = get_sheerka()
-    ExecutionContextIdManager.ids = {}
-    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(context)
-
-    assert to_dict == {
-        OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
-        OBJ_ID_KEY: 0,
-        '_parent': None,
-        '_id': 0,
-        '_tab': '',
-        '_bag': {},
-        '_start': 0,
-        '_stop': 0,
-        'who': 'requester',
-        'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'},
-        'desc': 'this is the desc',
-        'children': [],
-        'preprocess': None,
-        'inputs': {},
-        'values': {},
-        'obj': None,
-        'concepts': {}
-    }
-
-
-def test_i_can_transform_list():
-    sheerka = get_sheerka()
-    ExecutionContextIdManager.ids = {}
-    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict([context])
-
-    assert len(to_dict) == 1
-    assert isinstance(to_dict, list)
-    assert to_dict[0]["who"] == "requester"
-    assert to_dict[0]["desc"] == "this is the desc"
-
-
-def test_i_can_transform_set():
-    sheerka = get_sheerka()
-    ExecutionContextIdManager.ids = {}
-    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict({context})
-
-    assert len(to_dict) == 1
-    assert isinstance(to_dict, list)
-    assert to_dict[0]["who"] == "requester"
-    assert to_dict[0]["desc"] == "this is the desc"
-
-
-def test_i_can_transform_dict():
-    sheerka = get_sheerka()
-    ExecutionContextIdManager.ids = {}
-    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-    known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key()
-    sheerka.create_new_concept(get_context(sheerka), known_concept)
-    unknown_concept = Concept("bar")
-    known = sheerka.new("foo")
-
-    bag = {
-        "context": context,
-        "known_concept": known_concept,
-        "unknown_concept": unknown_concept,
-        "True": True,
-        "Number": 1.1,
-        "String": "a string value",
-        "None": None,
-        unknown_concept: "hello",
-        known: "world"
-    }
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(bag)
-
-    assert isinstance(to_dict, dict)
-    assert to_dict['Number'] == 1.1
-    assert to_dict['String'] == 'a string value'
-    assert to_dict['True']
-    assert to_dict['None'] is None
-    assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
-    assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
-    assert to_dict["known_concept"]["id"] == '1001'
-    assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
-    assert to_dict["(None)bar"] == "hello"
-    assert to_dict["(1001)foo"] == "world"
-
-
-def test_i_can_transform_when_circular_references():
-    sheerka = get_sheerka()
-    ExecutionContextIdManager.ids = {}
-    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-    context.push("another requester", "another desc")
-
-    st = SheerkaTransform(sheerka)
-    to_dict = st.to_dict(context)
-
-    assert isinstance(to_dict, dict)
-    assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
-    assert len(to_dict["children"]) == 1
-    assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
-    assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
-    assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0
-    assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
-    assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1