Refactored sheerka class: splitted to use sub handlers. Refactored unit tests to use classes.

tests/core/test_ExecutionContext.py

@@ -0,0 +1,72 @@
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.Sheerka import ExecutionContext
+from sdp.sheerkaDataProvider import Event
+
+
+def test_id_is_incremented_by_event_digest():
+    a = ExecutionContext("foo", Event("event_1"), None)
+    b = ExecutionContext("foo", Event("event_1"), None)
+    c = ExecutionContext("foo", Event("event_2"), None)
+    d = b.push()
+    e = c.push()
+
+    assert a.id == 0
+    assert b.id == 1
+    assert c.id == 0
+    assert d.id == 2
+    assert e.id == 1
+
+
+def test_i_can_use_with_statement():
+    with ExecutionContext("who_", Event("event"), "fake_sheerka") as e:
+        pass
+    assert e.elapsed > 0
+
+
+def test_i_can_push():
+    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", "some description",
+                         obj=Concept("foo"),
+                         step=BuiltinConcepts.EVALUATION,
+                         iteration=15,
+                         concepts={"bar": Concept("bar")})
+    a.preprocess = set()
+    a.preprocess.add("preprocess")
+
+    b = a.push()
+
+    assert b._parent == a
+    assert b.who == a.who
+    assert b.event == a.event
+    assert b.sheerka == a.sheerka
+    assert b.desc is None
+    assert b.obj == a.obj
+    assert b.step == a.step
+    assert b.iteration == a.iteration
+    assert b.concepts == a.concepts
+    assert b.id == a.id + 1
+    assert b._tab == a._tab + "    "
+    assert b.preprocess == a.preprocess
+
+
+def test_children_i_created_when_i_push():
+    e = ExecutionContext("who_", Event("event"), "fake_sheerka")
+    e.push("a", desc="I do something")
+    e.push("b", desc="oups! I did a again")
+    e.push("c", desc="I do something else")
+
+    assert len(e.children) == 3
+    assert e.children[0].who, e.children[0].who == ("a", "I do something")
+    assert e.children[1].who, e.children[1].who == ("b", "oups! I did a again")
+    assert e.children[2].who, e.children[2].who == ("c", "I do something else")
+
+
+def test_i_can_add_variable_when_i_push():
+    e = ExecutionContext("who_", Event("event"), "fake_sheerka")
+    sub_e = e.push("a", my_new_var="new var value")
+
+    assert sub_e.my_new_var == "new var value"
+    with pytest.raises(AttributeError):
+        assert e.my_new_var == ""  # my_new_var does not exist in parent

tests/core/test_SheerkaCreateNewConcept.py

@@ -0,0 +1,180 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import PROPERTIES_TO_SERIALIZE
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
+    def test_i_can_add_a_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_found = res.value.body
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop)
+
+        assert concept_found.key == "__var__0 + __var__1"
+        assert concept_found.id == "1001"
+
+        assert concept.key in sheerka.cache_by_key
+        assert concept.id in sheerka.cache_by_id
+        assert sheerka.sdp.io.exists(
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest()))
+
+    def test_i_cannot_add_the_same_concept_twice(self):
+        """
+        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+        assert res.value.body == concept
+
+    def test_i_can_get_a_newly_created_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        from_cache = sheerka.get(concept.key)
+        assert from_cache is not None
+        assert from_cache == concept
+
+        from_cache = sheerka.get_by_id(concept.id)
+        assert from_cache is not None
+        assert from_cache == concept
+
+    def test_i_first_look_in_local_cache(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+        sheerka.cache_by_key[concept.key].pre = "I have modified the concept in cache"
+
+        from_cache = sheerka.get(concept.key)
+        assert from_cache is not None
+        assert from_cache.key == concept.key
+        assert from_cache.pre == "I have modified the concept in cache"
+
+    def test_i_can_get_a_known_concept_when_not_in_cache(self):
+        """
+        When not in cache, uses sdp
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        sheerka.cache_by_key = {}  # reset the cache
+        loaded = sheerka.get(concept.key)
+
+        assert loaded is not None
+        assert loaded == concept
+
+        # I can also get it by its id
+        loaded = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert loaded is not None
+        assert loaded == concept
+
+    def test_i_can_get_a_concept_by_its_id(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        sheerka.cache_by_key = {}  # reset the cache
+        loaded = sheerka.get_by_id(concept.id)
+
+        assert loaded is not None
+        assert loaded == concept
+
+    def test_i_can_get_list_of_concept_when_same_key_when_no_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        sheerka.cache_by_key = {}  # reset the cache
+
+        result = sheerka.get(concept1.key)
+        assert len(result) == 2
+        assert result[0] == concept1
+        assert result[1] == concept2
+
+    def test_i_can_get_list_of_concept_when_same_key_when_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        # sheerka.cache_by_key = {}  # Do not reset the cache
+
+        result = sheerka.get(concept1.key)
+        assert len(result) == 2
+        assert result[0] == concept1
+        assert result[1] == concept2
+
+    def test_i_can_get_the_correct_concept_using_the_id_when_same_key_when_no_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get(concept1.key, res2.body.body.id)
+        assert result.name == "a + b"
+        assert result.metadata.body == "a+b"
+
+    def test_i_can_get_the_correct_concept_using_the_id__when_same_key_when_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get(concept1.key, res2.body.body.id)
+        assert result.name == "a + b"
+        assert result.metadata.body == "a+b"
+
+    def test_i_cannot_get_the_correct_concept_id_the_id_is_wrong(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get(concept1.key, "wrong id")
+        assert sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT)

tests/core/test_SheerkaEvaluateConcept.py

@@ -0,0 +1,332 @@
+import pytest
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
+from core.concept import Concept, simplec, DoNotResolve, ConceptParts, Property
+from parsers.PythonParser import PythonNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("body, expected", [
+        # (None, None),
+        # ("", ""),
+        ("1", 1),
+        ("1+1", 2),
+        ("'one'", "one"),
+        ("'one' + 'two'", "onetwo"),
+        ("True", True),
+        ("1 > 2", False),
+    ])
+    def test_i_can_evaluate_a_concept_with_simple_body(self,body, expected):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body=body)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == expected
+        assert evaluated.metadata.body == body
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.post is None
+        assert evaluated.metadata.where is None
+        assert evaluated.props == {}
+        assert evaluated.metadata.is_evaluated
+        assert len(evaluated.values) == 0 if body is None else 1
+
+    @pytest.mark.parametrize("expr, expected", [
+        ("", ""),
+        ("1", 1),
+        ("1+1", 2),
+        ("'one'", "one"),
+        ("'one' + 'two'", "onetwo"),
+        ("True", True),
+        ("1 > 2", False),
+    ])
+    def test_i_can_evaluate_the_other_metadata(self,expr, expected):
+        """
+        I only test WHERE, it's the same for the others
+        :param expr:
+        :param expected:
+        :return:
+        """
+
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", where=expr)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.metadata.body is None
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.post is None
+        assert evaluated.metadata.where == expr
+        assert evaluated.get_metadata_value(ConceptParts.WHERE) == expected
+        assert evaluated.props == {}
+        assert evaluated.metadata.is_evaluated
+        assert len(evaluated.values) == 0 if expr is None else 1
+
+    @pytest.mark.parametrize("expr, expected", [
+        (None, None),
+        ("", ""),
+        ("1", 1),
+        ("1+1", 2),
+        ("'one'", "one"),
+        ("'one' + 'two'", "onetwo"),
+        ("True", True),
+        ("1 > 2", False),
+    ])
+    def test_i_can_evaluate_a_concept_with_prop(self, expr, expected):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo").def_prop("a", expr)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.post is None
+        assert evaluated.metadata.where is None
+        assert evaluated.props == {"a": Property("a", expected)}
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_metadata_using_do_not_resolve(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo")
+        concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.body == "do not resolve"
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_property_using_do_not_resolve(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo").def_prop("a")
+        concept.compiled["a"] = DoNotResolve("do not resolve")
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.get_prop("a") == "do not resolve"
+        assert evaluated.metadata.is_evaluated
+
+    def test_original_value_is_overridden_when_using_do_no_resolve(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo", body="original value").set_prop("a", "original value")
+        concept.compiled["a"] = DoNotResolve("do not resolve")
+        concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.body == "do not resolve"
+        assert evaluated.get_prop("a") == "do not resolve"
+        assert evaluated.metadata.is_evaluated
+
+    def test_props_are_evaluated_before_body(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body="a+1").def_prop("a", "10").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == 11
+
+    def test_i_can_evaluate_when_another_concept_is_referenced(self):
+        sheerka = self.get_sheerka()
+        concept_a = Concept("a")
+        sheerka.add_in_cache(concept_a)
+
+        concept = Concept("foo", body="a").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated == simplec("foo", simplec("a", None))
+        assert id(evaluated.body) != id(concept_a)
+        assert evaluated.metadata.is_evaluated
+        assert evaluated.body.metadata.is_evaluated
+
+    def test_i_can_evaluate_when_the_referenced_concept_has_a_body(self):
+        sheerka = self.get_sheerka()
+        concept_a = Concept("a", body="1")
+        sheerka.add_in_cache(concept_a)
+
+        concept = Concept("foo", body="a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == simplec("a", 1)
+        assert not concept_a.metadata.is_evaluated
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="'a'"))
+        sheerka.add_in_cache(Concept(name="b", body="a"))
+        sheerka.add_in_cache(Concept(name="c", body="b"))
+        concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept_d)
+
+        assert evaluated.key == concept_d.key
+        expected = simplec("c", simplec("b", simplec("a", "a")))
+        assert evaluated.body == expected
+        assert sheerka.value(evaluated) == 'a'
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a"))
+        sheerka.add_in_cache(Concept(name="b", body="a"))
+        sheerka.add_in_cache(Concept(name="c", body="b"))
+        concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept_d)
+
+        assert evaluated.key == concept_d.key
+        expected = simplec("c", simplec("b", simplec("a", None)))
+        assert evaluated.body == expected
+        assert sheerka.value(evaluated) == Concept(name="a").init_key()
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_concept_when_properties_reference_others_concepts(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.add_in_cache(Concept(name="a").init_key())
+
+        concept = Concept("foo", body="a").def_prop("a", "a").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        # first prop a is evaluated to concept_a
+        # then body is evaluated to prop a -> concept_a
+        assert evaluated.key == concept.key
+        assert evaluated.body == concept_a
+
+    def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2(self):
+        """
+        Same test,
+        but the name of the property and the name of the concept are different
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.add_in_cache(Concept(name="a"))
+
+        concept = Concept("foo", body="concept_a").def_prop("concept_a", "a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == concept_a
+
+    def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="1"))
+        sheerka.add_in_cache(Concept(name="b", body="2"))
+
+        concept = Concept("foo", body="propA + propB").def_prop("propA", "a").def_prop("propB", "b")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == 3
+
+    def test_i_can_evaluate_concept_when_properties_is_a_concept(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.add_in_cache(Concept(name="a", body="'a'").init_key())
+
+        concept = Concept("foo").def_prop("a")
+        concept.compiled["a"] = concept_a
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.get_prop("a") == simplec("a", "a")
+
+    def test_i_can_evaluate_when_property_asts_is_a_list(self):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo", body="1")
+
+        concept = Concept("to_eval").def_prop("prop")
+        concept.compiled["prop"] = [foo, DoNotResolve("1")]
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        props = evaluated.get_prop("prop")
+        assert len(props) == 2
+        assert props[0] == simplec("foo", 1)
+        assert props[1] == "1"
+
+    def test_i_can_evaluate_when_compiled_is_set_up_with_return_value(self):
+        sheerka = self.get_sheerka()
+
+        python_node = PythonNode("1 +1 ")
+        parser_result = ParserResultConcept(parser="who", value=python_node)
+
+        concept = Concept("to_eval").def_prop("prop")
+        concept.compiled["prop"] = [ReturnValueConcept("who", True, parser_result)]
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.get_prop("prop") == 2
+
+        # also works when only one return value
+        concept = Concept("to_eval").def_prop("prop")
+        concept.compiled["prop"] = ReturnValueConcept("who", True, parser_result)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.get_prop("prop") == 2
+
+    def test_i_can_reference_sheerka(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body="sheerka.test()").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == sheerka.test()
+
+    def test_properties_values_takes_precedence_over_the_outside_world(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
+        sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
+
+        concept = Concept("foo", body="a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == simplec("a", "concept_a")  # this test was already done
+
+        # so check this one.
+        concept = Concept("foo", body="a").def_prop("a", "'property_a'")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == 'property_a'
+
+        # or this one.
+        concept = Concept("foo", body="a").def_prop("a", "b")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == simplec(name="b", body="concept_b")
+
+    def test_properties_values_takes_precedence(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
+        sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
+
+        concept = Concept("foo", body="a + b").def_prop("a", "'prop_a'")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == 'prop_aconcept_b'
+
+    def test_i_can_reference_sub_property_of_a_property(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="concept_a").def_prop("subProp", "'sub_a'"))
+
+        concept = Concept("foo", body="a.props['subProp'].value").def_prop("a", "concept_a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated == simplec(concept.key, "sub_a")
+
+    def test_i_cannot_evaluate_concept_if_property_is_in_error(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept(name="concept_a").def_prop("subProp", "undef_concept")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
+
+    def test_key_is_initialized_by_evaluation(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.init_key().key
+

tests/core/test_SheerkaSetsManager.py

@@ -0,0 +1,124 @@
+import os
+import shutil
+from os import path
+
+import pytest
+from core.builtin_concepts import ConceptAlreadyInSet, BuiltinConcepts
+from core.concept import Concept
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
+
+
+
+    def test_i_can_add_concept_to_set(self):
+        sheerka = self.get_sheerka(False, False)
+
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+
+        all_foos = Concept("all_foos")
+        sheerka.set_id_if_needed(all_foos, False)
+
+        context = self.get_context(sheerka)
+        res = sheerka.add_concept_to_set(context, foo, all_foos)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 1
+        assert foo.id in all_entries
+
+    def test_i_can_add_several_concepts_to_set(self):
+        sheerka = self.get_sheerka(False, False)
+
+        foo1 = Concept("foo1")
+        sheerka.set_id_if_needed(foo1, False)
+
+        foo2 = Concept("foo1")
+        sheerka.set_id_if_needed(foo2, False)
+
+        all_foos = Concept("all_foos")
+        sheerka.set_id_if_needed(all_foos, False)
+
+        context = self.get_context(sheerka)
+        sheerka.add_concept_to_set(context, foo1, all_foos)
+        res = sheerka.add_concept_to_set(context, foo2, all_foos)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 2
+        assert foo1.id in all_entries
+        assert foo2.id in all_entries
+
+    def test_i_cannot_add_the_same_concept_twice_in_a_set(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+
+        all_foos = Concept("all_foos")
+        sheerka.set_id_if_needed(all_foos, False)
+
+        context = self.get_context(sheerka)
+        sheerka.add_concept_to_set(context, foo, all_foos)
+        res = sheerka.add_concept_to_set(context, foo, all_foos)
+
+        assert not res.status
+        assert res.body == ConceptAlreadyInSet(foo, all_foos)
+
+        all_entries = sheerka.sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 1
+        assert foo.id in all_entries
+
+    def test_i_get_elements_from_a_set(self):
+        sheerka = self.get_sheerka()
+
+        one = Concept("one")
+        two = Concept("two")
+        three = Concept("three")
+        number = Concept("number")
+
+        for c in [one, two, three, number]:
+            sheerka.set_id_if_needed(c, False)
+            sheerka.add_in_cache(c)
+
+        context = self.get_context(sheerka)
+        for c in [one, two, three]:
+            sheerka.add_concept_to_set(context, c, number)
+
+        elements = sheerka.get_set_elements(number)
+
+        assert set(elements) == {one, two, three}
+
+    def test_i_cannot_get_elements_if_not_a_set(self):
+        sheerka = self.get_sheerka()
+        one = Concept("one")
+        sheerka.set_id_if_needed(one, False)
+        sheerka.add_in_cache(one)
+
+        error = sheerka.get_set_elements(one)
+
+        assert sheerka.isinstance(error, BuiltinConcepts.NOT_A_SET)
+        assert error.body == one
+
+    def test_isa_and_isa_group(self):
+        sheerka = self.get_sheerka()
+
+        group = Concept("group").init_key()
+        group.metadata.id = "1001"
+        assert not sheerka.isagroup(group)
+
+        foo = Concept("foo").init_key()
+        foo.metadata.id = "1002"
+        assert not sheerka.isa(foo, group)
+
+        context = self.get_context(sheerka)
+        sheerka.add_concept_to_set(context, foo, group)
+        assert sheerka.isagroup(group)
+        assert sheerka.isa(foo, group)

tests/core/test_ast.py

@@ -0,0 +1,175 @@
+import ast
+
+import pytest
+
+from core.ast.nodes import NodeParent, GenericNodeConcept
+import core.ast.nodes
+from core.ast.visitors import ConceptNodeVisitor, UnreferencedNamesVisitor
+from core.builtin_concepts import BuiltinConcepts
+from core.sheerka.Sheerka import Sheerka
+
+
+def get_sheerka():
+    sheerka = Sheerka(skip_builtins_in_db=True)
+    sheerka.initialize("mem://")
+
+    return sheerka
+
+
+class TestNameVisitor(ConceptNodeVisitor):
+    """
+    Test class for a basic Visitor test
+    """
+
+    def __init__(self):
+        self.names = []
+
+    def visit_Name(self, node):
+        self.names.append(node)
+
+
+def test_i_can_transform_simple_ast_using_generic_node():
+    source = """
+def my_function(a,b):
+    for i in range(b):
+        a = a+b
+    return a
+"""
+    tree = ast.parse(source)
+    tree_as_concept = core.ast.nodes.python_to_concept(tree)
+    sheerka = get_sheerka()
+
+    assert tree_as_concept.node_type == "Module"
+    assert sheerka.isinstance(tree_as_concept.get_prop("body"), BuiltinConcepts.LIST)
+
+    def_func = tree_as_concept.get_prop("body")[0]
+    assert sheerka.isinstance(def_func, BuiltinConcepts.GENERIC_NODE)
+    assert def_func.node_type == "FunctionDef"
+    assert def_func.parent == NodeParent(tree_as_concept, "body")
+    assert def_func.get_prop("name") == "my_function"
+
+    def_func_args = def_func.get_prop("args")
+    assert sheerka.isinstance(def_func_args, BuiltinConcepts.GENERIC_NODE)
+    assert def_func_args.node_type == "arguments"
+
+    def_func_args_real_args = def_func_args.get_prop("args")
+    assert sheerka.isinstance(def_func_args_real_args, BuiltinConcepts.LIST)
+    assert len(def_func_args_real_args) == 2
+
+    assert sheerka.isinstance(def_func_args_real_args[0], BuiltinConcepts.GENERIC_NODE)
+    assert def_func_args_real_args[0].node_type == "arg"
+    assert def_func_args_real_args[0].parent == NodeParent(def_func_args, "args")
+    assert def_func_args_real_args[0].get_prop("arg") == "a"
+    assert sheerka.isinstance(def_func_args_real_args[1], BuiltinConcepts.GENERIC_NODE)
+    assert def_func_args_real_args[1].node_type == "arg"
+    assert def_func_args_real_args[1].parent == NodeParent(def_func_args, "args")
+    assert def_func_args_real_args[1].get_prop("arg") == "b"
+
+    def_fun_body = def_func.get_prop("body")
+    assert sheerka.isinstance(def_fun_body, BuiltinConcepts.LIST)
+    assert len(def_fun_body) == 2
+
+    def_fun_body_for = def_fun_body[0]
+    assert sheerka.isinstance(def_fun_body_for, BuiltinConcepts.GENERIC_NODE)
+    assert def_fun_body_for.node_type == "For"
+    assert def_fun_body_for.parent == NodeParent(def_func, "body")
+
+    def_fun_body_return = def_fun_body[1]
+    assert sheerka.isinstance(def_fun_body_return, BuiltinConcepts.GENERIC_NODE)
+    assert def_fun_body_return.node_type == "Return"
+    assert def_fun_body_return.parent == NodeParent(def_func, "body")
+
+
+def test_i_can_visit_concept_node():
+    source = """
+def my_function(a,b):
+    for i in range(b):
+        a = a+b
+    return a
+    """
+
+    node = ast.parse(source)
+    concept_node = core.ast.nodes.python_to_concept(node)
+
+    visitor = TestNameVisitor()
+    visitor.visit(concept_node)
+
+    sheerka = get_sheerka()
+    assert sheerka.value(visitor.names[0]) == "i"
+    assert sheerka.value(visitor.names[1]) == "range"
+    assert sheerka.value(visitor.names[2]) == "b"
+    assert sheerka.value(visitor.names[3]) == "a"
+    assert sheerka.value(visitor.names[4]) == "a"
+    assert sheerka.value(visitor.names[5]) == "b"
+    assert sheerka.value(visitor.names[6]) == "a"
+
+
+def test_i_can_get_unreferenced_variables():
+    source = """
+def my_function(a,b):
+    for i in range(b):
+        a = a+b
+    return a
+
+my_function(x,y)
+"""
+
+    sheerka = get_sheerka()
+
+    node = ast.parse(source)
+    concept_node = core.ast.nodes.python_to_concept(node)
+
+    visitor = UnreferencedNamesVisitor(sheerka)
+    visitor.visit(concept_node)
+    values = visitor.names
+
+    assert len(visitor.names) == 2
+    assert "x" in values
+    assert "y" in values
+
+
+@pytest.mark.parametrize("source, expected", [
+    ("a,b", ["a", "b"]),
+    ("isinstance(a, int)", ["a", "int"])
+
+])
+def test_i_can_get_unreferenced_variables_from_simple_expressions(source, expected):
+    sheerka = get_sheerka()
+
+    node = ast.parse(source)
+    concept_node = core.ast.nodes.python_to_concept(node)
+
+    visitor = UnreferencedNamesVisitor(sheerka)
+    visitor.visit(concept_node)
+
+    assert sorted(list(visitor.names)) == expected
+
+
+def test_i_can_compare_NodeParent_with_tuple():
+    node_parent = NodeParent(GenericNodeConcept("For", None), "target")
+    assert node_parent == ("For", "target")
+
+
+def test_i_can_transform_back():
+    source = """
+def my_function(a,b):
+    for i in range(b):
+        a = a + b
+    return a
+
+
+my_function(x, y)
+    """
+
+    node = ast.parse(source)
+    concept_node = core.ast.nodes.python_to_concept(node)
+
+    transformed_back = core.ast.nodes.concept_to_python(concept_node)
+    assert dump_ast(transformed_back) == dump_ast(node)
+
+
+def dump_ast(node):
+    dump = ast.dump(node)
+    for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
+        dump = dump.replace(to_remove, "")
+    return dump

tests/core/test_builtin_helpers.py

@@ -0,0 +1,145 @@
+import ast
+import pytest
+import core.builtin_helpers
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_use_expect_one_when_empty(self):
+        sheerka = self.get_sheerka()
+
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), [])
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY)
+
+    def test_i_can_use_expect_one_when_too_many_success(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", True, "value1"),
+            ReturnValueConcept("who", True, "value2"),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS)
+        assert res.value.body == items
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_same_success(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", True, "value"),
+            ReturnValueConcept("who", True, "value"),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert res.status
+        assert res.value == items[0].value
+        assert res.parents == items
+
+    def test_i_can_use_expect_when_only_errors_1(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", False, sheerka.new(BuiltinConcepts.ERROR)),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert not res.status
+        assert res.value, items[0]
+        assert res.parents == items
+
+    def test_i_can_use_expect_when_only_errors_2(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", False, None),
+            ReturnValueConcept("who", False, None),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
+        assert res.value.body == items
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_one_success_1(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", True, None),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert res.status
+        assert res.body == items[0].body
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_one_success_2(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", False, None),
+            ReturnValueConcept("who", True, None),
+            ReturnValueConcept("who", False, None),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert res.status
+        assert res.body == items[1].body
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_not_a_list_true(self):
+        sheerka = self.get_sheerka()
+
+        item = ReturnValueConcept("who", True, None)
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), item)
+        assert res.status
+        assert res == item
+
+    def test_i_can_use_expect_one_when_not_a_list_false(self):
+        sheerka = self.get_sheerka()
+
+        item = ReturnValueConcept("who", False, None)
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), item)
+
+        assert not res.status
+        assert res == item
+
+    @pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [
+        ("a == 1", [], [], []),
+        ("a == 1", ["a"], [], ["a == 1"]),
+        ("a == 1", [], ["a"], []),
+        ("predicate(a)", [], [], []),
+        ("predicate(a)", ["a"], [], ["predicate(a)"]),
+        ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]),
+        ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]),
+        ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]),
+        ("predicate(a, b)", ["a"], ["b"], []),
+        ("a + b == 1", [], [], []),
+        ("a + b == 1", ["a"], [], ["a + b == 1"]),
+        ("a + b == 1", ["a"], ["b"], []),
+        ("a + b == 1", ["b"], [], ["a + b == 1"]),
+        ("a + b == 1", ["a", "b"], [], ["a + b == 1"]),
+        ("a == 1 and b == 2", [], [], []),
+        ("a == 1 and b == 2", ["a"], [], ["a == 1"]),
+        ("a == 1 and b == 2", ["b"], [], ["b == 2"]),
+        ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]),
+        ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]),
+        ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]),
+        ("not(a == 1)", [], [], []),
+        ("not(a == 1)", ["a"], [], ["not(a==1)"]),
+        ("a == 1 or b == 2", [], [], []),
+        ("a == 1 or b == 2", ["a"], [], ["a == 1"]),
+        ("a == 1 or b == 2", ["b"], [], ["b == 2"]),
+        ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]),
+        ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]),
+    ])
+    def test_i_can_extract_predicates(self, expression, vars_to_include, vars_to_exclude, expected_expr):
+        sheerka = self.get_sheerka()
+        expected = [ast.parse(expr, mode="eval") for expr in expected_expr]
+
+        actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude)
+        assert len(actual) == len(expected)
+        for i in range(len(actual)):
+            assert self.dump_ast(actual[i]) == self.dump_ast(expected[i])

tests/core/test_concept.py

@@ -0,0 +1,210 @@
+import pytest
+
+from core.concept import Concept, ConceptParts
+
+
+@pytest.mark.parametrize("name, properties, expected", [
+    ("foo", [], "foo"),
+    ("foo(bar)", [], "foo ( bar )"),
+    ("foo a", ["a"], "foo __var__0"),
+    ("a foo b", ["a", "b"], "__var__0 foo __var__1"),
+    ("a foo b", ["b", "a"], "__var__1 foo __var__0"),
+    ("foo", ["foo"], "foo"),
+    ("foo a", ["foo"], "__var__0 a"),
+    ("foo       a   b", ["a"], "foo __var__0 b"),
+    ("'foo'", [], "foo"),
+    ("my name is a", ["a"], "my name is __var__0"),
+    ("a b c d", ["b", "c"], "a __var__0 __var__1 d"),
+    ("a 'b c' d", ["b", "c"], "a b c d"),
+    ("a | b", ["a", "b"], "__var__0 | __var__1"),
+    ("a b a c", ["a", "b"], "__var__0 __var__1 __var__0 c"),
+    ("a b a c", ["b", "a"], "__var__1 __var__0 __var__1 c"),
+])
+def test_i_can_compute_the_key(name, properties, expected):
+    concept = Concept(name)
+    for prop in properties:
+        concept.metadata.props.append((prop, None))
+    concept.init_key()
+
+    assert concept.key == expected
+
+
+def test_key_does_not_use_variable_when_definition_is_set():
+    concept = Concept("plus").def_prop('plus')
+
+    concept.init_key()
+    assert concept.metadata.key == "plus"
+
+
+def test_i_can_serialize():
+    """
+    Test concept.to_dict()
+    :return:
+    """
+    concept = Concept(
+        name="concept_name",
+        is_builtin=True,
+        is_unique=True,
+        key="concept_key",
+        body="definition of the body",
+        where="definition of the where",
+        pre="definition of the pre",
+        post="definition of the post",
+        definition="bnf definition",
+        definition_type="def type",
+        desc="this this the desc",
+        id="123456"
+    ).def_prop("a", "10").def_prop("b", None)
+
+    to_dict = concept.to_dict()
+    assert to_dict == {
+        'body': 'definition of the body',
+        'definition': 'bnf definition',
+        'definition_type': 'def type',
+        'desc': 'this this the desc',
+        'id': '123456',
+        'is_builtin': True,
+        'is_unique': True,
+        'key': 'concept_key',
+        'name': 'concept_name',
+        'post': 'definition of the post',
+        'pre': 'definition of the pre',
+        'props': [('a', "10"), ('b', None)],
+        'where': 'definition of the where'
+    }
+
+
+def test_i_can_deserialize():
+    """
+    Test concept.from_dict()
+    :return:
+    """
+
+    from_dict = {
+        'body': 'definition of the body',
+        'definition': 'bnf definition',
+        'definition_type': 'def type',
+        'desc': 'this this the desc',
+        'id': '123456',
+        'is_builtin': True,
+        'is_unique': True,
+        'key': 'concept_key',
+        'name': 'concept_name',
+        'post': 'definition of the post',
+        'pre': 'definition of the pre',
+        'props': [('a', "10"), ('b', None)],
+        'where': 'definition of the where'
+    }
+
+    concept = Concept().from_dict(from_dict)
+
+    assert concept == Concept(
+        name="concept_name",
+        is_builtin=True,
+        is_unique=True,
+        key="concept_key",
+        body="definition of the body",
+        where="definition of the where",
+        pre="definition of the pre",
+        post="definition of the post",
+        definition="bnf definition",
+        definition_type="def type",
+        desc="this this the desc",
+        id="123456"
+    ).def_prop("a", "10").def_prop("b", None)
+
+
+def test_i_can_deserialize_props_coming_from_sdp():
+    from_dict = {
+        'props': [['a', "10"], ['b', None]],  # JSON transform set into list
+    }
+
+    concept = Concept().from_dict(from_dict)
+    assert concept == Concept().def_prop("a", "10").def_prop("b", None)
+
+
+def test_i_can_compare_concepts():
+    concept_a = Concept(
+        name="concept_name",
+        is_builtin=True,
+        is_unique=True,
+        key="concept_key",
+        body="definition of the body",
+        where="definition of the where",
+        pre="definition of the pre",
+        post="definition of the post",
+        definition="bnf definition",
+        definition_type="def type",
+        desc="this this the desc",
+        id="123456"
+    ).def_prop("a", "10").def_prop("b", None)
+
+    concept_b = Concept(
+        name="concept_name",
+        is_builtin=True,
+        is_unique=True,
+        key="concept_key",
+        body="definition of the body",
+        where="definition of the where",
+        pre="definition of the pre",
+        post="definition of the post",
+        definition="bnf definition",
+        definition_type="def type",
+        desc="this this the desc",
+        id="123456"
+    ).def_prop("a", "10").def_prop("b", None)
+
+    assert concept_a == concept_b
+
+
+def test_i_can_compare_concept_with_circular_reference():
+    foo = Concept("foo")
+    foo.metadata.body = foo
+
+    assert foo == foo
+
+
+def test_i_can_compare_concept_with_sophisticated_circular_reference():
+    foo = Concept("foo")
+    bar = Concept("foo", body=foo)
+    baz = Concept("foo", body=bar)
+    foo.metadata.body = baz
+
+    assert foo != bar
+
+
+def test_i_can_compare_concept_with_sophisticated_circular_reference_in_other_metadata():
+    foo = Concept("foo")
+    bar = Concept("foo", pre=foo)
+    baz = Concept("foo", pre=bar)
+    foo.metadata.pre = baz
+
+    assert foo != bar
+
+
+def test_i_can_update_from():
+    template = Concept(
+        name="concept_name",
+        is_builtin=True,
+        is_unique=True,
+        key="concept_key",
+        body="definition of the body",
+        where="definition of the where",
+        pre="definition of the pre",
+        post="definition of the post",
+        definition="bnf definition",
+        definition_type="def type",
+        desc="this this the desc",
+        id="123456"
+    ).def_prop("a", "10").def_prop("b", None)
+
+    template.values[ConceptParts.BODY] = "value in body"
+    template.values[ConceptParts.WHERE] = "value in where"
+    template.values[ConceptParts.PRE] = "value in pre"
+    template.values[ConceptParts.POST] = "value in post"
+    template.set_prop("a", 10)
+    template.set_prop("b", 20)
+
+    concept = Concept().update_from(template)
+
+    assert concept == template

tests/core/test_sheerka.py

@@ -0,0 +1,244 @@
+import pytest
+import os
+
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE
+from core.sheerka.Sheerka import Sheerka
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class ConceptWithGetValue(Concept):
+    def get_value(self):
+        return self.get_prop("my_prop")
+
+
+class TestSheerka(TestUsingFileBasedSheerka):
+
+    def test_root_folder_is_created_after_initialization(self):
+        return_value = Sheerka().initialize(self.root_folder)
+        assert return_value.status, "initialisation should be successful"
+        assert os.path.exists(self.root_folder), "init folder should be created"
+
+    def test_i_can_list_builtin_concepts(self):
+        sheerka = self.get_sheerka()
+        builtins = list(sheerka.get_builtins_classes_as_dict())
+
+        assert str(BuiltinConcepts.ERROR) in builtins
+        assert str(BuiltinConcepts.RETURN_VALUE) in builtins
+
+    def test_builtin_concepts_are_initialized(self):
+        sheerka = self.get_sheerka(skip_builtins_in_db=False)
+        assert len(sheerka.cache_by_key) == len(BuiltinConcepts)
+        for concept_name in BuiltinConcepts:
+            assert str(concept_name) in sheerka.cache_by_key
+            assert sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, str(concept_name)) is not None
+
+        for key, concept_class in sheerka.get_builtins_classes_as_dict().items():
+            assert isinstance(sheerka.cache_by_key[key], concept_class)
+
+    def test_builtin_concepts_can_be_updated(self):
+        sheerka = self.get_sheerka(False, False)
+        loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
+        loaded_sheerka.metadata.desc = "I have a description"
+        sheerka.sdp.modify("Test", sheerka.CONCEPTS_ENTRY, loaded_sheerka.key, loaded_sheerka)
+
+        sheerka = self.get_sheerka(False, False)
+        loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
+
+        assert loaded_sheerka.metadata.desc == "I have a description"
+
+    def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string(self):
+        """
+        Checks that a concept can be found its name
+        even when there are variables in the name (ex 'hello + a' or 'a + b' )
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        for key in sheerka.get_builtins_classes_as_dict():
+            assert sheerka.get(key) is not None
+            assert sheerka.get(str(key)) is not None
+
+    def test_i_cannot_get_when_key_is_none(self):
+        sheerka = self.get_sheerka()
+
+        res = sheerka.get(None)
+        assert sheerka.isinstance(res, BuiltinConcepts.ERROR)
+        assert res.body == "Concept key is undefined."
+
+    def test_unknown_concept_is_return_when_the_concept_key_is_not_found(self):
+        sheerka = self.get_sheerka()
+
+        loaded = sheerka.get("key_that_does_not_exist")
+
+        assert loaded is not None
+        assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert loaded.body == ("key", "key_that_does_not_exist")
+        assert loaded.metadata.is_evaluated
+
+    def test_unknown_concept_is_return_when_the_concept_id_is_not_found(self):
+        sheerka = self.get_sheerka()
+
+        loaded = sheerka.get_by_id("id_that_does_not_exist")
+
+        assert loaded is not None
+        assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert loaded.body == ("id", "id_that_does_not_exist")
+        assert loaded.metadata.is_evaluated
+
+    def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self):
+        sheerka = self.get_sheerka()
+        ret = sheerka.new(BuiltinConcepts.RETURN_VALUE, who="who", status="status", value="value", message="message")
+
+        assert isinstance(ret, ReturnValueConcept)
+        assert ret.key == str(BuiltinConcepts.RETURN_VALUE)
+        assert ret.who == "who"
+        assert ret.status == "status"
+        assert ret.value == "value"
+        assert ret.message == "message"
+
+    def test_i_can_instantiate_a_builtin_concept_when_no_specific_class(self):
+        sheerka = self.get_sheerka()
+        ret = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body="fake_concept")
+
+        assert isinstance(ret, Concept)
+        assert ret.key == str(BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert ret.body == "fake_concept"
+
+    def test_i_can_instantiate_a_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new = sheerka.new(concept.key, a=10, b="value")
+
+        assert sheerka.isinstance(new, concept)
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(new.metadata, prop) == getattr(concept.metadata, prop)
+
+        assert new.props["a"].value == 10
+        assert new.props["b"].value == "value"
+
+    def test_i_can_instantiate_with_the_name_and_the_id(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
+
+        concepts = sheerka.new("foo")
+        assert len(concepts) == 2
+
+        foo1 = sheerka.new(("foo", "1001"))
+        assert foo1.metadata.body == "foo1"
+
+        foo2 = sheerka.new(("foo", "1002"))
+        assert foo2.metadata.body == "foo2"
+
+    def test_instances_are_different_when_asking_for_new(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new1 = sheerka.new(concept.key, a=10, b="value")
+        new2 = sheerka.new(concept.key, a=10, b="value")
+
+        assert new1 == new2
+        assert id(new1) != id(new2)
+
+    def test_i_get_the_same_instance_when_is_unique_is_true(self):
+        sheerka = self.get_sheerka()
+        concept = Concept(name="unique", is_unique=True)
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new1 = sheerka.new(concept.key, a=10, b="value")
+        new2 = sheerka.new(concept.key, a=10, b="value")
+
+        assert new1 == new2
+        assert id(new1) == id(new2)
+
+    def test_i_cannot_instantiate_an_unknown_concept(self):
+        sheerka = self.get_sheerka()
+
+        new = sheerka.new("fake_concept")
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert new.body == ('key', 'fake_concept')
+
+    def test_i_cannot_instantiate_with_invalid_id(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
+
+        new = sheerka.new(("foo", "invalid_id"))
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert new.body == [('key', 'foo'), ('id', 'invalid_id')]
+
+    def test_i_cannot_instantiate_with_invalid_key(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
+
+        new = sheerka.new(("invalid_key", "1001"))
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert new.body == [('key', 'invalid_key'), ('id', '1001')]
+
+    def test_concept_id_is_irrelevant_when_only_one_concept(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+
+        new = sheerka.new(("foo", "invalid_id"))
+
+        assert sheerka.isinstance(new, "foo")
+        assert new.metadata.body == "foo1"
+
+    def test_i_cannot_instantiate_when_properties_are_not_recognized(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new = sheerka.new(concept.key, a=10, c="value")
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_PROPERTY)
+        assert new.property_name == "c"
+        assert sheerka.isinstance(new.concept, concept)
+
+    @pytest.mark.parametrize("concept, reduce_simple_list, expected", [
+        (None, False, None),
+        (3.14, False, 3.14),
+        ("foo", False, "foo"),
+        (True, False, True),
+        (Concept("name", body="foo"), False, "foo"),
+        (Concept("name"), False, Concept("name")),
+        (ConceptWithGetValue("name").set_prop("my_prop", "my_value"), False, "my_value"),
+        (ReturnValueConcept(value="return_value"), False, "return_value"),
+        (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
+        (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
+        (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]),
+        (Concept("name", body=["foo"]), True, "foo"),
+        (Concept("name", body=Concept("foo")), False, Concept("foo")),
+        (Concept("name", body=Concept("foo", body="value")), False, "value"),
+        (Concept("name", body=Concept("foo", body=Concept("bar", body="value"))), False, "value"),
+        (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"),
+    ])
+    def test_i_can_get_value(self, concept, reduce_simple_list, expected):
+        sheerka = self.get_sheerka()
+
+        # I use auto_init() instead of evaluate_concept() to be quicker
+        c = concept
+        while isinstance(c, Concept):
+            c.auto_init()
+            c = c.body
+
+        assert sheerka.value(concept, reduce_simple_list) == expected
+
+    def test_list_of_concept_is_sorted_by_id(self):
+        sheerka = self.get_sheerka(False, False)
+        concepts = sheerka.concepts()
+
+        assert concepts[0].id < concepts[-1].id
+
+    def test_builtin_error_concept_are_errors(self):
+        # only test a random one, it will be the same for the others
+        sheerka = self.get_sheerka()
+        assert not sheerka.is_success(sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS))

tests/core/test_sheerka_call_evaluators.py

@@ -0,0 +1,375 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator
+
+from tests.BaseTest import BaseTest
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class Out:
+    debug_out = []
+
+    def out(self, method, name, context, return_value):
+        name = name[len(BaseEvaluator.PREFIX):]
+        if isinstance(return_value, list):
+            target = [str(r.body.key) for r in return_value]
+        else:
+            target = str(return_value.body.key)
+        step = str(context.step)
+        text = f"{step} [{context.iteration}] "
+        text += f"{name} - {method} - target={target}"
+        self.debug_out.append(text)
+
+    def out_all(self, method, name, context, return_values):
+        name = name[len(BaseEvaluator.PREFIX):]
+        target = [str(r.body.key) for r in return_values]
+        step = str(context.step)
+        text = f"{step} [{context.iteration}] "
+        text += f"{name} - {method} - target={target}"
+        self.debug_out.append(text)
+
+
+class OneReturnValueEvaluatorForTestingPurpose(OneReturnValueEvaluator, Out):
+    def __init__(self, name, steps, priority):
+        super().__init__(name, steps, priority)
+
+    def matches(self, context, return_value):
+        self.out("matches", self.name, context, return_value)
+        return True
+
+    def eval(self, context, return_value):
+        self.out("eval", self.name, context, return_value)
+
+
+class AllReturnValueEvaluatorForTestingPurpose(AllReturnValuesEvaluator, Out):
+    def __init__(self, name, steps, priority):
+        super().__init__(name, steps, priority)
+
+    def matches(self, context, return_values):
+        self.out("matches", self.name, context, return_values)
+        return True
+
+    def eval(self, context, return_values):
+        self.out("eval", self.name, context, return_values)
+
+
+class EvaluatorOneWithPriority(OneReturnValueEvaluatorForTestingPurpose):
+    def __init__(self, name, priority):
+        super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
+
+
+class EvaluatorOneWithPriority10(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("priority10", 10)
+
+
+class EvaluatorOneWithPriority15(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("priority15", 15)
+
+
+class EvaluatorOneWithPriority20(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("priority20", 20)
+
+
+class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose):
+    def __init__(self, name, priority):
+        super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
+
+
+class EvaluatorAllWithPriority10(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_priority10", 10)
+
+
+class EvaluatorAllWithPriority15(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_priority15", 15)
+
+
+class EvaluatorAllWithPriority20(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_priority20", 20)
+
+
+class EvaluatorOneModifyFoo(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("modifyFoo", 10)
+
+    def matches(self, context, return_value):
+        super().matches(context, return_value)
+        return context.sheerka.isinstance(return_value.body, "foo")
+
+    def eval(self, context, return_value):
+        super().eval(context, return_value)
+        return BaseTest.tretval(context.sheerka, Concept("bar"))
+
+
+class EvaluatorOneModifyBar(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("modifyBar", 10)
+
+    def matches(self, context, return_value):
+        super().matches(context, return_value)
+        return context.sheerka.isinstance(return_value.body, "bar")
+
+    def eval(self, context, return_value):
+        super().eval(context, return_value)
+        return BaseTest.tretval(context.sheerka, Concept("baz"))
+
+
+class EvaluatorOnePreEvaluation(OneReturnValueEvaluatorForTestingPurpose):
+    def __init__(self):
+        super().__init__("preEval", [BuiltinConcepts.BEFORE_EVALUATION], 10)
+
+
+class EvaluatorOneMultiSteps(OneReturnValueEvaluatorForTestingPurpose):
+    def __init__(self):
+        super().__init__("multiStep", [BuiltinConcepts.EVALUATION, BuiltinConcepts.BEFORE_EVALUATION], 10)
+
+
+class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_reduce_foobar", 10)
+
+    def matches(self, context, return_values):
+        super().matches(context, return_values)
+        keys = [c.body.key for c in return_values]
+        return "foo" in keys and "bar" in keys
+
+    def eval(self, context, return_values):
+        super().eval(context, return_values)
+        ret = BaseTest.tretval(context.sheerka, context.sheerka.new(BuiltinConcepts.SUCCESS))
+        ret.parents = return_values
+        return ret
+
+
+class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority):
+
+    def __init__(self):
+        super().__init__("suppress", 100)
+
+    def matches(self, context, return_values):
+        super().matches(context, return_values)
+        return True
+
+    def eval(self, context, return_values):
+        super().eval(context, return_values)
+        foo = None
+        for ret in return_values:
+            if ret.body.name == "foo":
+                foo = ret
+
+        if foo:
+            return context.sheerka.ret(self.name, False, Concept("does not matter"), parents=[foo])
+        else:
+            return None
+
+
+class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
+
+    def test_that_return_values_is_unchanged_when_no_evaluator(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = []
+
+        entries = self.tretval(sheerka, Concept("foo"))
+        return_values = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert return_values == [entries]
+
+    def test_i_can_use_a_list_as_input(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = []
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        return_values = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert return_values == entries
+
+    def test_step_concept_is_removed_after_processing_if_not_reduced(self):
+        """
+        No evaluator
+        """
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = []
+
+        entry = self.tretval(sheerka, Concept("foo"))
+        return_values = sheerka.execute(self.get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
+
+        assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
+
+    def test_step_concept_is_removed_after_processing_if_not_reduced_2(self):
+        """
+        This time the entry is modified by an evaluator,
+        nevertheless, step concept is removed
+        """
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+        entry = self.tretval(sheerka, Concept("foo"))
+        return_values = sheerka.execute(self.get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
+
+        assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
+
+    def test_that_higher_priority_evaluators_are_evaluated_first(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [
+            EvaluatorAllWithPriority10,
+            EvaluatorOneWithPriority20,
+            EvaluatorAllWithPriority15,
+            EvaluatorOneWithPriority10,
+            EvaluatorOneWithPriority15,
+            EvaluatorAllWithPriority20, ]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__EVALUATION [0] priority20 - matches - target=foo',
+            '__EVALUATION [0] priority20 - eval - target=foo',
+            '__EVALUATION [0] priority20 - matches - target=__EVALUATION',
+            '__EVALUATION [0] priority20 - eval - target=__EVALUATION',
+            "__EVALUATION [0] all_priority20 - matches - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority20 - eval - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority15 - matches - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority15 - eval - target=['foo', '__EVALUATION']",
+            '__EVALUATION [0] priority15 - matches - target=foo',
+            '__EVALUATION [0] priority15 - eval - target=foo',
+            '__EVALUATION [0] priority15 - matches - target=__EVALUATION',
+            '__EVALUATION [0] priority15 - eval - target=__EVALUATION',
+            "__EVALUATION [0] all_priority10 - matches - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority10 - eval - target=['foo', '__EVALUATION']",
+            '__EVALUATION [0] priority10 - matches - target=foo',
+            '__EVALUATION [0] priority10 - eval - target=foo',
+            '__EVALUATION [0] priority10 - matches - target=__EVALUATION',
+            '__EVALUATION [0] priority10 - eval - target=__EVALUATION'
+        ]
+
+    def test_that_predicate_is_checked_before_evaluation_for_one_return(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+        entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__EVALUATION [0] modifyFoo - matches - target=foo',
+            '__EVALUATION [0] modifyFoo - eval - target=foo',
+            '__EVALUATION [0] modifyFoo - matches - target=baz',
+            '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyFoo - matches - target=bar',
+            '__EVALUATION [1] modifyFoo - matches - target=baz',
+            '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
+        ]
+
+    def test_that_predicate_is_checked_before_evaluation_for_all_return(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorAllReduceFooBar]
+
+        entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', 'bar', '__EVALUATION']",
+            "__EVALUATION [0] all_reduce_foobar - eval - target=['foo', 'bar', '__EVALUATION']",
+            "__EVALUATION [1] all_reduce_foobar - matches - target=['__SUCCESS']"
+        ]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', '__EVALUATION']"
+        ]
+
+    def test_evaluation_continue_until_no_more_modification(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar]
+
+        entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__EVALUATION [0] modifyFoo - matches - target=foo',
+            '__EVALUATION [0] modifyFoo - eval - target=foo',
+            '__EVALUATION [0] modifyFoo - matches - target=baz',
+            '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [0] modifyBar - matches - target=foo',
+            '__EVALUATION [0] modifyBar - matches - target=baz',
+            '__EVALUATION [0] modifyBar - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyFoo - matches - target=bar',
+            '__EVALUATION [1] modifyFoo - matches - target=baz',
+            '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyBar - matches - target=bar',
+            '__EVALUATION [1] modifyBar - eval - target=bar',
+            '__EVALUATION [1] modifyBar - matches - target=baz',
+            '__EVALUATION [1] modifyBar - matches - target=__EVALUATION',
+            '__EVALUATION [2] modifyFoo - matches - target=baz',
+            '__EVALUATION [2] modifyFoo - matches - target=baz',
+            '__EVALUATION [2] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [2] modifyBar - matches - target=baz',
+            '__EVALUATION [2] modifyBar - matches - target=baz',
+            '__EVALUATION [2] modifyBar - matches - target=__EVALUATION'
+        ]
+
+    def test_evaluation_steps_are_respected(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION])
+
+        assert Out.debug_out == [
+            '__BEFORE_EVALUATION [0] preEval - matches - target=foo',
+            '__BEFORE_EVALUATION [0] preEval - eval - target=foo',
+            '__BEFORE_EVALUATION [0] preEval - matches - target=__BEFORE_EVALUATION',
+            '__BEFORE_EVALUATION [0] preEval - eval - target=__BEFORE_EVALUATION']
+
+    def test_evaluation_multi_steps_are_respected(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneMultiSteps]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries,
+                        [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__BEFORE_EVALUATION [0] multiStep - matches - target=foo',
+            '__BEFORE_EVALUATION [0] multiStep - eval - target=foo',
+            '__BEFORE_EVALUATION [0] multiStep - matches - target=__BEFORE_EVALUATION',
+            '__BEFORE_EVALUATION [0] multiStep - eval - target=__BEFORE_EVALUATION',
+            '__EVALUATION [0] multiStep - matches - target=foo',
+            '__EVALUATION [0] multiStep - eval - target=foo',
+            '__EVALUATION [0] multiStep - matches - target=__EVALUATION',
+            '__EVALUATION [0] multiStep - eval - target=__EVALUATION'
+        ]
+
+    def test_evaluators_can_be_pre_processed(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+
+        # disable evaluator
+        context = self.get_context(sheerka)
+        context.add_preprocess(EvaluatorOneModifyFoo().name, enabled=False)  # disabled  for this exec context
+        Out.debug_out = []
+        sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == []
+
+        # other contextes are not impacted
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            '__EVALUATION [0] modifyFoo - matches - target=foo',
+            '__EVALUATION [0] modifyFoo - eval - target=foo',
+            '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyFoo - matches - target=bar',
+            '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
+        ]

tests/core/test_sheerka_call_parsers.py

@@ -0,0 +1,363 @@
+from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept
+from parsers.BaseParser import BaseParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_ret_val(text, who="who"):
+    return ReturnValueConcept(who, True, UserInputConcept(text, "user_name"))
+
+
+class BaseTestParser(BaseParser):
+    debug_out = []
+
+    def __init__(self, name, priority, status=None, parser_result=None):
+        super().__init__(name, priority)
+        self.status = status
+        self.parser_result = parser_result
+
+    @staticmethod
+    def _get_name(name):
+        return name[8:] if name.startswith("parsers.") else name
+
+    @staticmethod
+    def _get_source(text_):
+        return text_ if isinstance(text_, str) else text_.body
+
+    def _out(self, name, priority, status, source):
+        debug = f"name={name}"
+        debug += f", priority={priority}"
+        debug += f", status={status}"
+        debug += f", source={source}"
+        self.debug_out.append(debug)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        parser_result = ParserResultConcept(parser=self, value=value)
+        return ReturnValueConcept(self, self.status, self.parser_result or parser_result)
+
+
+class Enabled90FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled90False", 90, False)
+
+
+class Enabled80FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled80False", 80, False)
+
+
+class Enabled80MultipleFalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled80MultipleFalse", 80, False)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
+        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        return [
+            ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)),
+            ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)),
+        ]
+
+
+class Enabled80MultipleTrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled80MultipleTrue", 80)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
+        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        return [
+            ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)),
+            ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)),
+        ]
+
+
+class Enabled70FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled70False", 70, False, "Not a ParserResult")
+
+
+class Enabled50TrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled50True", 50, True)
+
+    def parse(self, context, text):
+        source = self._get_source(text)
+        status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world"
+        self._out(self._get_name(self.name), self.priority, status, source)
+
+        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        return_value = ParserResultConcept(parser=self, value=value)
+        return ReturnValueConcept(self, status, return_value)
+
+
+class Enabled50bisTrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled50BisTrue", 50, True)
+
+
+class Enabled50FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled50False", 50, False)
+
+
+class Enabled10TrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled10True", 10, True)
+
+
+class DisabledParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Disabled", 90, True)
+        self.enabled = False
+
+
+class NoneParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("None", 90, True, None)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        return None
+
+
+class ListOfNoneParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("ListOfNone", 90, True, None)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        return [None, None]
+
+
+class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
+
+    def test_disabled_parsers_are_not_executed(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled": Enabled10TrueParser,
+            "Disabled": DisabledParser
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == ['name=Enabled10True, priority=10, status=True, source=hello world']
+
+    def test_parser_are_executed_by_priority(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled90False, priority=90, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
+        ]
+
+    def test_parsing_stop_at_the_first_success(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50bisTrue": Enabled50bisTrueParser,
+            "Enabled10True": Enabled10TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
+        ]
+
+    def test_parsing_stop_at_the_first_success_2(self):
+        """
+        Same test than before, but Enabled50True takes more time to find a match
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+            "Enabled10True": Enabled10TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled90False, priority=90, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
+        ]
+
+    def test_all_parsers_of_a_given_priority_are_executed(self):
+        """
+        Make sure that all parsers with priority 50 are executed
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+            "Enabled50bisTrue": Enabled50bisTrueParser,
+            "Enabled50False": Enabled50FalseParser,
+            "Enabled10True": Enabled10TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled90False, priority=90, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
+            'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
+            'name=Enabled50False, priority=50, status=False, source=hello world',
+            'name=Enabled50False, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50False, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50False, priority=50, status=False, source=Enabled80False:Enabled90False:hello world',
+        ]
+
+    def test_a_parser_has_access_to_the_output_of_its_predecessors(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+        res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
+        assert res_as_tuple == [
+            ('Enabled90False', False, 'Enabled90False:hello world'),
+            ('Enabled80False', False, 'Enabled80False:hello world'),
+            ('Enabled80False', False, 'Enabled80False:Enabled90False:hello world'),
+            ('Enabled50True', False, 'Enabled50True:hello world'),
+            ('Enabled50True', False, 'Enabled50True:Enabled90False:hello world'),
+            ('Enabled50True', False, 'Enabled50True:Enabled80False:hello world'),
+            ('Enabled50True', True, 'Enabled50True:Enabled80False:Enabled90False:hello world'),
+        ]
+
+    def test_none_return_values_are_discarded(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "NoneParser": NoneParser,
+            "ListOfNone": ListOfNoneParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+
+        BaseTestParser.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+        assert res == []
+        assert BaseTestParser.debug_out == [
+            'name=None, priority=90, status=True, source=hello world',
+            'name=ListOfNone, priority=90, status=True, source=hello world'
+        ]
+
+    def test_following_priorities_can_only_see_parser_result_return_values(self):
+        """
+        Normally, lower priority parsers can see the result of the higher priority parsers
+        This is true only if the higher priority parser return a ParserResultConcept
+        :return:
+        """
+
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled70False": Enabled70FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled70False, priority=70, status=False, source=hello world',
+            'name=Enabled70False, priority=70, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+        ]
+
+    def test_i_can_manage_parser_with_multiple_results(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80MultipleFalse": Enabled80MultipleFalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80MultipleFalse, priority=80, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_1',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_2',
+        ]
+
+        res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
+        assert res_as_tuple == [
+            ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_1'),
+            ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_2'),
+            ('Enabled50True', False, 'Enabled50True:hello world'),
+            ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_1'),
+            ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'),
+        ]
+
+    def test_i_can_manage_parser_with_multiple_results_and_a_sucess(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80MultipleTrue": Enabled80MultipleTrueParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80MultipleTrue, priority=80, status=None, source=hello world',
+        ]
+
+        res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
+        assert res_as_tuple == [
+            ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'),
+            ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'),
+        ]

tests/core/test_sheerka_transform.py

@@ -0,0 +1,318 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY
+from sdp.sheerkaDataProvider import Event
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaTransform(TestUsingMemoryBasedSheerka):
+    def test_i_can_transform_an_unknown_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo", body="body")
+        concept_with_sub = Concept("concept_with_sub", body=foo)
+
+        concept = Concept(
+            name="concept_name",
+            is_builtin=True,
+            is_unique=True,
+            key="concept_key",
+            body=concept_with_sub,
+            where=[foo, 1, "1", True, 1.0],
+            pre=foo,
+            post=None,  # will not appear
+            definition="it is a definition",
+            definition_type="def type",
+            desc="this this the desc"
+        ).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub")
+
+        # add values and props
+        concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init()
+        concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0]
+        concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init()
+        concept.values[ConceptParts.POST] = "a value for POST"
+        concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub)
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(concept)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            'meta.name': 'concept_name',
+            'meta.key': 'concept_key',
+            'meta.is_builtin': True,
+            'meta.is_unique': True,
+            'meta.definition': 'it is a definition',
+            'meta.definition_type': 'def type',
+            'meta.desc': 'this this the desc',
+            'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                            OBJ_ID_KEY: 1,
+                            'meta.body': 'body',
+                            'meta.name': 'foo'}, 1, '1', True, 1.0],
+            'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1},
+            'meta.body': {
+                '__type__': SheerkaTransformType.Concept,
+                '__id__': 2,
+                'meta.name': 'concept_with_sub',
+                'meta.body': {
+                    '__type__': SheerkaTransformType.Reference,
+                    '__id__': 1}},
+            'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']],
+            'pre': {'__type__': SheerkaTransformType.Concept,
+                    '__id__': 4,
+                    'meta.body': 'body',
+                    'meta.name': 'foo',
+                    'body': 'body'},
+            'post': "a value for POST",
+            'body': {'__type__': SheerkaTransformType.Concept,
+                     '__id__': 3,
+                     'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference},
+                     'meta.name': 'concept_with_sub',
+                     'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}},
+            'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0],
+            'props': [('a', 10),
+                      ('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}),
+                      ('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})],
+        }
+
+    def test_i_can_transform_unknown_concept_with_almost_same_value(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo")
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(concept)
+
+        assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'}
+
+    def test_i_can_transform_known_concept_when_the_values_are_the_same(self):
+        """
+        Values are the same means that we are serializing a concept which has kept all its default values
+        There is not diff between the concept to serialize and the one which was registered with create_new_concept()
+        We serialize only the id of the concept
+        :return:
+        """
+        sheerka = self.get_sheerka()
+
+        concept = Concept(
+            name="concept_name",
+            is_builtin=True,
+            is_unique=False,
+            key="concept_key",
+            body="body definition",
+            where="where definition",
+            pre="pre definition",
+            post="post definition",
+            definition="it is a definition",
+            definition_type="def type",
+            desc="this this the desc"
+        ).def_prop("a").def_prop("b")
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new_concept = sheerka.new(concept.key)
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(new_concept)
+
+        assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"}
+
+    def test_i_can_transform_known_concept_when_the_values_are_different(self):
+        """
+        Values are the different means the concept was modified.
+        It's different from the one which was registered with create_new_concept()
+        We serialize only the differences
+        :return:
+        """
+        sheerka = self.get_sheerka()
+
+        concept = Concept(
+            name="concept_name",
+            is_builtin=True,
+            is_unique=False,
+            key="concept_key",
+            body="body definition",
+            where="where definition",
+            pre="pre definition",
+            post="post definition",
+            definition="it is a definition",
+            definition_type="def type",
+            desc="this this the desc"
+        ).def_prop("a").def_prop("b")
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre")
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(new_concept)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            "id": "1001",
+            'pre': 'another pre',
+            "body": "another",
+            'props': [('a', 10)]
+        }
+
+    def test_i_can_transform_concept_with_circular_reference(self):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo")
+        bar = Concept("bar", body=foo)
+        foo.metadata.body = bar
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(foo)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            'meta.name': 'foo',
+            'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                          OBJ_ID_KEY: 1,
+                          'meta.name': 'bar',
+                          'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                                        OBJ_ID_KEY: 0},
+                          },
+        }
+
+    def test_i_can_transform_concept_with_circular_reference_2(self):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo")
+        bar = Concept("foo", body=foo)
+        foo.metadata.body = bar
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(foo)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            'meta.name': 'foo',
+            'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                          OBJ_ID_KEY: 1,
+                          'meta.name': 'foo',
+                          'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                                        OBJ_ID_KEY: 0},
+                          },
+        }
+
+    def test_i_can_transform_the_unknown_concept(self):
+        sheerka = self.get_sheerka(False)
+
+        unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(unknown)
+
+        assert len(to_dict) == 3
+        assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+        assert to_dict[OBJ_ID_KEY] == 0
+        assert "id" in to_dict
+
+    def test_i_can_transform_simple_execution_context(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(context)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
+            OBJ_ID_KEY: 0,
+            '_parent': None,
+            '_id': 0,
+            '_tab': '',
+            '_bag': {},
+            '_start': 0,
+            '_stop': 0,
+            'who': 'requester',
+            'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'},
+            'desc': 'this is the desc',
+            'children': [],
+            'preprocess': None,
+            'inputs': {},
+            'values': {},
+            'obj': None,
+            'concepts': {}
+        }
+
+    def test_i_can_transform_list(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict([context])
+
+        assert len(to_dict) == 1
+        assert isinstance(to_dict, list)
+        assert to_dict[0]["who"] == "requester"
+        assert to_dict[0]["desc"] == "this is the desc"
+
+    def test_i_can_transform_set(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict({context})
+
+        assert len(to_dict) == 1
+        assert isinstance(to_dict, list)
+        assert to_dict[0]["who"] == "requester"
+        assert to_dict[0]["desc"] == "this is the desc"
+
+    def test_i_can_transform_dict(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+        known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key()
+        sheerka.create_new_concept(self.get_context(sheerka), known_concept)
+        unknown_concept = Concept("bar")
+        known = sheerka.new("foo")
+
+        bag = {
+            "context": context,
+            "known_concept": known_concept,
+            "unknown_concept": unknown_concept,
+            "True": True,
+            "Number": 1.1,
+            "String": "a string value",
+            "None": None,
+            unknown_concept: "hello",
+            known: "world"
+        }
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(bag)
+
+        assert isinstance(to_dict, dict)
+        assert to_dict['Number'] == 1.1
+        assert to_dict['String'] == 'a string value'
+        assert to_dict['True']
+        assert to_dict['None'] is None
+        assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+        assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+        assert to_dict["known_concept"]["id"] == '1001'
+        assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+        assert to_dict["(None)bar"] == "hello"
+        assert to_dict["(1001)foo"] == "world"
+
+    def test_i_can_transform_when_circular_references(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+        context.push("another requester", "another desc")
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(context)
+
+        assert isinstance(to_dict, dict)
+        assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+        assert len(to_dict["children"]) == 1
+        assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+        assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
+        assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0
+        assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
+        assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1

tests/core/test_tokenizer.py

@@ -0,0 +1,141 @@
+import pytest
+from core.tokenizer import Tokenizer, Token, TokenKind, LexerError, Keywords
+
+
+def test_i_can_tokenize():
+    source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&<>c:name:"
+    source += "$£€!_identifier°~_^\\`#"
+    tokens = list(Tokenizer(source))
+    assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
+    assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
+    assert tokens[2] == Token(TokenKind.MINUS, "-", 2, 1, 3)
+    assert tokens[3] == Token(TokenKind.SLASH, "/", 3, 1, 4)
+    assert tokens[4] == Token(TokenKind.LBRACE, "{", 4, 1, 5)
+    assert tokens[5] == Token(TokenKind.RBRACE, "}", 5, 1, 6)
+    assert tokens[6] == Token(TokenKind.LBRACKET, "[", 6, 1, 7)
+    assert tokens[7] == Token(TokenKind.RBRACKET, "]", 7, 1, 8)
+    assert tokens[8] == Token(TokenKind.LPAR, "(", 8, 1, 9)
+    assert tokens[9] == Token(TokenKind.RPAR, ")", 9, 1, 10)
+    assert tokens[10] == Token(TokenKind.WHITESPACE, "    ", 10, 1, 11)
+    assert tokens[11] == Token(TokenKind.COMMA, ",", 14, 1, 15)
+    assert tokens[12] == Token(TokenKind.SEMICOLON, ";", 15, 1, 16)
+    assert tokens[13] == Token(TokenKind.COLON, ":", 16, 1, 17)
+    assert tokens[14] == Token(TokenKind.DOT, ".", 17, 1, 18)
+    assert tokens[15] == Token(TokenKind.QMARK, "?", 18, 1, 19)
+    assert tokens[16] == Token(TokenKind.NEWLINE, "\n", 19, 1, 20)
+    assert tokens[17] == Token(TokenKind.NEWLINE, "\n\r", 20, 2, 1)
+    assert tokens[18] == Token(TokenKind.NEWLINE, "\r", 22, 3, 1)
+    assert tokens[19] == Token(TokenKind.NEWLINE, "\r\n", 23, 4, 1)
+    assert tokens[20] == Token(TokenKind.IDENTIFIER, "identifier_0", 25, 5, 1)
+    assert tokens[21] == Token(TokenKind.WHITESPACE, "\t  \t", 37, 5, 13)
+    assert tokens[22] == Token(TokenKind.NUMBER, "10.15", 41, 5, 17)
+    assert tokens[23] == Token(TokenKind.WHITESPACE, " ", 46, 5, 22)
+    assert tokens[24] == Token(TokenKind.NUMBER, "10", 47, 5, 23)
+    assert tokens[25] == Token(TokenKind.WHITESPACE, " ", 49, 5, 25)
+    assert tokens[26] == Token(TokenKind.STRING, "'string\n'", 50, 5, 26)
+    assert tokens[27] == Token(TokenKind.WHITESPACE, " ", 59, 6, 1)
+    assert tokens[28] == Token(TokenKind.STRING, '"another string"', 60, 6, 2)
+    assert tokens[29] == Token(TokenKind.EQUALS, '=', 76, 6, 18)
+    assert tokens[30] == Token(TokenKind.VBAR, '|', 77, 6, 19)
+    assert tokens[31] == Token(TokenKind.AMPER, '&', 78, 6, 20)
+    assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 21)
+    assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 22)
+    assert tokens[34] == Token(TokenKind.CONCEPT, 'name', 81, 6, 23)
+    assert tokens[35] == Token(TokenKind.DOLLAR, '$', 88, 6, 30)
+    assert tokens[36] == Token(TokenKind.STERLING, '£', 89, 6, 31)
+    assert tokens[37] == Token(TokenKind.EURO, '€', 90, 6, 32)
+    assert tokens[38] == Token(TokenKind.EMARK, '!', 91, 6, 33)
+    assert tokens[39] == Token(TokenKind.IDENTIFIER, '_identifier', 92, 6, 34)
+    assert tokens[40] == Token(TokenKind.DEGREE, '°', 103, 6, 45)
+    assert tokens[41] == Token(TokenKind.TILDE, '~', 104, 6, 46)
+    assert tokens[42] == Token(TokenKind.UNDERSCORE, '_', 105, 6, 47)
+    assert tokens[43] == Token(TokenKind.CARAT, '^', 106, 6, 48)
+    assert tokens[44] == Token(TokenKind.BACK_SLASH, '\\', 107, 6, 49)
+    assert tokens[45] == Token(TokenKind.BACK_QUOTE, '`', 108, 6, 50)
+    assert tokens[46] == Token(TokenKind.HASH, '#', 109, 6, 51)
+
+    assert tokens[47] == Token(TokenKind.EOF, '', 110, 6, 52)
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("_ident", True),
+    ("ident", True),
+    ("ident123", True),
+    ("ident_123", True),
+    ("ident-like-this", True),
+    ("àèùéû", True),
+    ("011254", False),
+    ("0abcd", False),
+    ("-abcd", False)
+])
+def test_i_can_tokenize_identifiers(text, expected):
+    tokens = list(Tokenizer(text))
+    comparison = tokens[0].type == TokenKind.IDENTIFIER
+    assert comparison == expected
+
+
+@pytest.mark.parametrize("text, message, error_text, index, line, column", [
+    ("'string", "Missing Trailing quote", "'string", 7, 1, 8),
+    ('"string', "Missing Trailing quote", '"string', 7, 1, 8),
+    ('"a" + "string', "Missing Trailing quote", '"string', 13, 1, 14),
+    ('"a"\n\n"string', "Missing Trailing quote", '"string', 12, 3, 8),
+    ("c::", "Concept name not found", "", 2, 1, 3),
+    ("c:foo\nbar:", "New line is forbidden in concept name", "foo", 5, 1, 6),
+    ("c:foo", "Missing ending colon", "foo", 5, 1, 6)
+])
+def test_i_can_detect_tokenizer_errors(text, message, error_text, index, line, column):
+    with pytest.raises(LexerError) as e:
+        list(Tokenizer(text))
+    assert e.value.message == message
+    assert e.value.text == error_text
+    assert e.value.index == index
+    assert e.value.line == line
+    assert e.value.column == column
+
+
+@pytest.mark.parametrize("text, expected_text, expected_newlines", [
+    ("'foo'", "'foo'", 0),
+    ('"foo"', '"foo"', 0),
+    ("'foo\rbar'", "'foo\rbar'", 1),
+    ("'foo\nbar'", "'foo\nbar'", 1),
+    ("'foo\n\rbar'", "'foo\n\rbar'", 1),
+    ("'foo\r\nbar'", "'foo\r\nbar'", 1),
+    ("'foo\r\rbar'", "'foo\r\rbar'", 2),
+    ("'foo\n\nbar'", "'foo\n\nbar'", 2),
+    ("'foo\r\n\n\rbar'", "'foo\r\n\n\rbar'", 2),
+    ("'\rfoo\rbar\r'", "'\rfoo\rbar\r'", 3),
+    ("'\nfoo\nbar\n'", "'\nfoo\nbar\n'", 3),
+    ("'\n\rfoo\r\n'", "'\n\rfoo\r\n'", 2),
+    (r"'foo\'bar'", r"'foo\'bar'", 0),
+    (r'"foo\"bar"', r'"foo\"bar"', 0),
+    ('"foo"bar"', '"foo"', 0),
+    ("'foo'bar'", "'foo'", 0),
+])
+def test_i_can_parse_strings(text, expected_text, expected_newlines):
+    lexer = Tokenizer(text)
+    text_found, nb_of_newlines = lexer.eat_string(0, 1, 1)
+
+    assert nb_of_newlines == expected_newlines
+    assert text_found == expected_text
+
+
+@pytest.mark.parametrize("text", [
+    "1", "3.1415", "0.5", "01", "-5", "-5.10"
+])
+def test_i_can_parse_numbers(text):
+    tokens = list(Tokenizer(text))
+    assert tokens[0].type == TokenKind.NUMBER
+    assert tokens[0].value == text
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("def", Keywords.DEF),
+    ("concept", Keywords.CONCEPT),
+    ("as", Keywords.AS),
+    ("pre", Keywords.PRE),
+    ("post", Keywords.POST)
+])
+def test_i_can_recognize_keywords(text, expected):
+    tokens = list(Tokenizer(text))
+    assert tokens[0].type == TokenKind.KEYWORD
+    assert tokens[0].value == expected

tests/core/test_utils.py

@@ -0,0 +1,145 @@
+import core.utils
+import pytest
+
+from core.tokenizer import Token, TokenKind
+
+
+@pytest.mark.parametrize("lst, as_string", [
+    (None, "",),
+    ([], ""),
+    (["hello", "world"], "hello world"),
+    # (["hello world", "my friend"], '"hello world" "my friend"')
+])
+def test_i_can_create_string_from_a_list(lst, as_string):
+    assert core.utils.sysarg_to_string(lst) == as_string
+
+
+def test_i_can_get_classes():
+    classes = list(core.utils.get_classes("core.builtin_concepts"))
+    error_concept = core.utils.get_class("core.builtin_concepts.ErrorConcept")
+    return_value_concept = core.utils.get_class("core.builtin_concepts.ReturnValueConcept")
+
+    assert len(classes) > 2
+    assert error_concept in classes
+    assert return_value_concept in classes
+
+
+def test_i_can_get_base_classes():
+    classes = list(core.utils.get_classes_from_package("parsers"))
+
+    # example of classes that should be in the result
+    base_parser = core.utils.get_class("parsers.BaseParser.BaseParser")
+    default_parser = core.utils.get_class("parsers.DefaultParser.DefaultParser")
+    exact_concept_parser = core.utils.get_class("parsers.ExactConceptParser.ExactConceptParser")
+    python_parser = core.utils.get_class("parsers.PythonParser.PythonParser")
+    node = core.utils.get_class("parsers.BaseParser.Node")
+    def_concept_node = core.utils.get_class("parsers.DefaultParser.DefConceptNode")
+    python_node = core.utils.get_class("parsers.PythonParser.PythonNode")
+
+    assert base_parser in classes
+    assert default_parser in classes
+    assert exact_concept_parser in classes
+    assert python_parser in classes
+    assert node in classes
+    assert def_concept_node in classes
+    assert python_node in classes
+
+
+def test_i_can_get_sub_classes():
+    sub_classes = core.utils.get_sub_classes("parsers", "parsers.BaseParser.BaseParser")
+
+    # example of classes that should be (or not) in the result
+    base_parser = core.utils.get_class("parsers.BaseParser.BaseParser")
+    default_parser = core.utils.get_class("parsers.DefaultParser.DefaultParser")
+    exact_concept_parser = core.utils.get_class("parsers.ExactConceptParser.ExactConceptParser")
+    python_parser = core.utils.get_class("parsers.PythonParser.PythonParser")
+    concept_lexer_parser = core.utils.get_class("parsers.ConceptLexerParser.ConceptLexerParser")
+
+    assert base_parser not in sub_classes
+    assert default_parser in sub_classes
+    assert exact_concept_parser in sub_classes
+    assert python_parser in sub_classes
+    assert concept_lexer_parser in sub_classes
+
+
+@pytest.mark.parametrize("a,b, expected", [
+    ([], [], []),
+    ([], ['a'], ['a']),
+    ([[]], ['a'], [['a']]),
+    (['a'], [], ['a']),
+    ([['a']], [], [['a']]),
+
+    ([['a']], ['b'], [['a', 'b']]),
+    ([['a'], ['b']], ['c'], [['a', 'c'], ['b', 'c']]),
+    ([['a1', 'a2'], ['b1', 'b2', 'b3']], ['c'], [['a1', 'a2', 'c'], ['b1', 'b2', 'b3', 'c']]),
+    ([[]], ['a', 'b'], [['a'], ['b']]),
+    ([['a'], ['b']], ['c', 'd', 'e'], [['a', 'c'], ['b', 'c'], ['a', 'd'], ['b', 'd'], ['a', 'e'], ['b', 'e']]),
+])
+def test_i_can_product(a, b, expected):
+    res = core.utils.product(a, b)
+    assert res == expected
+
+
+@pytest.mark.parametrize("input_as_list, expected_as_list", [
+    ([" "], []),
+    ([" ", "one"], ["one"]),
+    (["one", " "], ["one"]),
+    ([" ", "one", " "], ["one"]),
+
+    (["\n", "one"], ["one"]),
+    (["one", "\n"], ["one"]),
+    (["\n", "one", "\n"], ["one"]),
+
+    ([" ", "\n", "one"], ["one"]),
+    (["one", " ", "\n"], ["one"]),
+    ([" ", "\n", "one", " ", "\n"], ["one"]),
+
+    (["\n", " ", "one"], ["one"]),
+    (["one", "\n", " "], ["one"]),
+    (["\n", " ", "one", "\n", " "], ["one"]),
+
+    ([" ", "\n", " ", "one"], ["one"]),
+    (["one", " ", "\n", " "], ["one"]),
+    ([" ", "\n", " ", "one", " ", "\n", " "], ["one"]),
+
+    (["\n", " ", "\n", "one"], ["one"]),
+    (["one", "\n", " ", "\n"], ["one"]),
+    (["\n", " ", "\n", "one", "\n", " ", "\n"], ["one"]),
+
+])
+def test_i_can_strip(input_as_list, expected_as_list):
+    actual = core.utils.strip_tokens(get_tokens(input_as_list))
+    expected = get_tokens(expected_as_list)
+    assert actual == expected
+
+
+def test_by_default_eof_is_not_stripped():
+    actual = core.utils.strip_tokens(get_tokens(["one", "two", " ", "\n", "<EOF>"]))
+    expected = get_tokens(["one", "two", " ", "\n", "<EOF>"])
+    assert actual == expected
+
+
+def test_i_can_strip_eof():
+    actual = core.utils.strip_tokens(get_tokens(["one", "two", " ", "\n", "<EOF>"]), True)
+    expected = get_tokens(["one", "two"])
+    assert actual == expected
+
+
+def test_i_can_escape():
+    actual = core.utils.escape_char("hello 'world' my friend", "'")
+    assert actual == "hello \\'world\\' my friend"
+
+
+def get_tokens(lst):
+    res = []
+    for e in lst:
+        if e == " ":
+            res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0))
+        elif e == "\n":
+            res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0))
+        elif e == "<EOF>":
+            res.append(Token(TokenKind.EOF, "\n", 0, 0, 0))
+        else:
+            res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0))
+
+    return res