Added first version of DebugManager. Implemented draft of the rule engine

tests/BaseTest.py

@@ -2,9 +2,11 @@ import ast
 
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
+from core.rule import Rule
 from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
+from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.BnfNodeParser import StrMatch
-from parsers.BnfParser import BnfParser
 from sdp.sheerkaDataProvider import Event
 
 
@@ -40,6 +42,10 @@ class BaseTest:
             dump = dump.replace(to_remove, "")
         return dump
 
+    @staticmethod
+    def dump_tokens(tokens):
+        return [t.repr_value for t in tokens]
+
     def init_concepts(self, *concepts, **kwargs):
         sheerka = self.get_sheerka(**kwargs)
 
@@ -52,20 +58,21 @@ class BaseTest:
             if isinstance(c, str):
                 c = Concept(c)
 
-            if c.metadata.definition and c.metadata.definition_type != DEFINITION_TYPE_DEF:
-                desc = f"Resolving BNF {c.metadata.definition}"
+            if c.get_metadata().definition and c.get_metadata().definition_type != DEFINITION_TYPE_DEF:
+                desc = f"Resolving BNF {c.get_metadata().definition}"
                 with context.push(BuiltinConcepts.INIT_BNF,
                                   c,
                                   obj=c,
                                   desc=desc) as sub_context:
 
-                    bnf_parser = BnfParser()
-                    res = bnf_parser.parse(sub_context, c.metadata.definition)
+                    bnf_parser = BnfDefinitionParser()
+                    res = bnf_parser.parse(sub_context, c.get_metadata().definition)
                     if res.status:
-                        c.bnf = res.value.value
-                        c.metadata.definition_type = DEFINITION_TYPE_BNF
+                        c.set_bnf(res.value.value)
+                        c.get_metadata().definition_type = DEFINITION_TYPE_BNF
                     else:
-                        raise Exception(f"Error in bnf definition '{c.metadata.definition}'", sheerka.get_error(res))
+                        raise Exception(f"Error in bnf definition '{c.get_metadata().definition}'",
+                                        sheerka.get_error(res))
 
             if create_new:
                 sheerka.create_new_concept(context, c)
@@ -78,6 +85,40 @@ class BaseTest:
 
         return sheerka, context, *result
 
+    def init_format_rules(self, *rules, create_new=True, compile_rule=True, concepts=None, **kwargs):
+        if concepts:
+            sheerka, context, *concepts = self.init_concepts(*concepts, **kwargs)
+        else:
+            sheerka, context, *concepts = self.init_concepts(**kwargs)
+
+        if create_new:
+            sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache.clear()
+            sheerka.cache_manager.delete(sheerka.CONCEPTS_KEYS_ENTRY, SheerkaRuleManager.RULE_IDS)
+            with sheerka.sdp.get_transaction(context.event.get_digest()) as transaction:
+                transaction.clear(SheerkaRuleManager.FORMAT_RULE_ENTRY)
+
+        initialized = []
+        for rule_blue_print in rules:
+            if isinstance(rule_blue_print, tuple):
+                rule = Rule("print", None, rule_blue_print[0], rule_blue_print[1])
+                if not compile_rule:
+                    rule.metadata.is_compiled = True
+            else:
+                rule = rule_blue_print
+
+            is_enabled = rule.metadata.is_enabled
+            sheerka.services[SheerkaRuleManager.NAME].init_rule(context, rule)
+            if create_new:
+                res = sheerka.create_new_rule(context, rule)
+                initialized.append(res.body.body)
+            else:
+                initialized.append(rule)
+
+            if is_enabled is not None:
+                rule.metadata.is_enabled = is_enabled
+
+        return sheerka, context, *initialized
+
     @staticmethod
     def get_concept_instance(sheerka, concept, **kwargs):
         """
@@ -88,10 +129,10 @@ class BaseTest:
         :return:
         """
         instance = sheerka.new(concept.key if isinstance(concept, Concept) else concept)
-        for i, var in enumerate(instance.metadata.variables):
+        for i, var in enumerate(instance.get_metadata().variables):
             if var[0] in kwargs:
                 assert isinstance(kwargs[var[0]], str), "variables definitions must be string"
-                instance.metadata.variables[i] = (var[0], kwargs[var[0]])
+                instance.get_metadata().variables[i] = (var[0], kwargs[var[0]])
 
         return instance
 
@@ -110,10 +151,10 @@ class BaseTest:
         return sheerka.ret(who, True, obj)
 
     @staticmethod
-    def pretval(concept, source=None, parser="parsers.name", who="some_name", status=True):
+    def pretval(concept, source=None, parser="parsers.name", who=None, status=True):
         """ParserResult ret_val (p stands for ParserResult)"""
         return ReturnValueConcept(
-            who,
+            who or parser,
             status,
             ParserResultConcept(parser=parser,
                                 source=source or concept.name,
@@ -135,8 +176,8 @@ class BaseTest:
             for v in variables:
                 concept.def_var(v)
         if bnf:
-            concept.bnf = bnf
-            concept.metadata.definition_type = DEFINITION_TYPE_BNF
+            concept.set_bnf(bnf)
+            concept.get_metadata().definition_type = DEFINITION_TYPE_BNF
         concept.init_key()
         sheerka.set_id_if_needed(concept, False)
         sheerka.add_in_cache(concept)
@@ -150,8 +191,8 @@ class BaseTest:
             name = concept
             concept = Concept(concept)
 
-        concept.bnf = expression or StrMatch(name)
-        concept.metadata.definition_type = DEFINITION_TYPE_BNF
+        concept.set_bnf(expression or StrMatch(name))
+        concept.get_metadata().definition_type = DEFINITION_TYPE_BNF
         return concept
 
     @staticmethod
@@ -164,9 +205,9 @@ class BaseTest:
         if kwargs:
             for k, v in kwargs.items():
                 if k in ("body", "pre", "post", "where"):
-                    setattr(concept.metadata, k, v)
+                    setattr(concept.get_metadata(), k, v)
                 else:
-                    concept.metadata.variables[k] = v
+                    concept.get_metadata().variables[k] = v
         return concept
 
     def init_scenario(self, init_expressions):

tests/TestUsingFileBasedSheerka.py

@@ -3,6 +3,7 @@ import shutil
 from os import path
 
 import pytest
+from core.concept import ALL_ATTRIBUTES
 from core.sheerka.Sheerka import Sheerka
 
 from tests.BaseTest import BaseTest
@@ -27,14 +28,16 @@ class TestUsingFileBasedSheerka(BaseTest):
         os.chdir(current_pwd)
 
     def get_sheerka(self, **kwargs):
+        reset_attrs = kwargs.get("reset_attrs", True)
+        if reset_attrs:
+            ALL_ATTRIBUTES.clear()
 
+        use_dict = kwargs.get("use_dict", False)
         # use dictionary based io instead of file
         # If you do so, information between two different instances of sheerka
         #   won't be shared
-        use_dict = kwargs.get("use_dict", False)
-
         root = "mem://" if use_dict else self.root_folder
         sheerka = Sheerka()
-        sheerka.initialize(root, save_execution_context=False)
+        sheerka.initialize(root, save_execution_context=False, enable_process_return_values=False)
 
         return sheerka

tests/TestUsingMemoryBasedSheerka.py

@@ -1,3 +1,4 @@
+from core.concept import ALL_ATTRIBUTES
 from core.sheerka.Sheerka import Sheerka
 
 from tests.BaseTest import BaseTest
@@ -9,23 +10,28 @@ class TestUsingMemoryBasedSheerka(BaseTest):
 
     @staticmethod
     def _inner_get_sheerka(cache_only):
+        ALL_ATTRIBUTES.clear()
         sheerka = Sheerka(cache_only=cache_only)
-        sheerka.initialize("mem://", save_execution_context=False)
+        sheerka.initialize("mem://", save_execution_context=False, enable_process_return_values=False)
         return sheerka
 
     def get_sheerka(self, **kwargs):
         cache_only = kwargs.get("cache_only", True)
         use_singleton = kwargs.get("singleton", True)
+        reset_attrs = kwargs.get("reset_attrs", True)
 
         sheerka = kwargs.get("sheerka", None)
         if sheerka:
             return sheerka
 
+        if reset_attrs:
+            ALL_ATTRIBUTES.clear()
+
         if use_singleton:
             singleton_instance = TestUsingMemoryBasedSheerka.singleton_instance
             if singleton_instance:
                 singleton_instance.reset(cache_only)
-                singleton_instance.cache_manager.init_from(TestUsingMemoryBasedSheerka.dump)
+                singleton_instance.cache_manager.init_from_dump(TestUsingMemoryBasedSheerka.dump)
                 return singleton_instance
             else:
                 new_instance = self._inner_get_sheerka(cache_only)

tests/cache/test_FastCache.py

@@ -0,0 +1,67 @@
+from cache.FastCache import FastCache
+
+
+def test_i_can_put_an_retrieve_values():
+    cache = FastCache()
+    cache.put("key", "value")
+
+    assert cache.get("key") == "value"
+    assert cache.cache == {"key": "value"}
+    assert cache.lru == ["key"]
+
+
+def test_i_can_put_and_retrieve_multiple_items():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    assert cache.cache == {"key1": "value1", "key2": "value2", "key3": "value3"}
+    assert cache.lru == ["key1", "key2", "key3"]
+
+
+def test_i_the_least_used_is_remove_first():
+    cache = FastCache(3)
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    cache.put("key4", "value4")
+    assert cache.cache == {"key4": "value4", "key2": "value2", "key3": "value3"}
+    assert cache.lru == ["key2", "key3", "key4"]
+
+    cache.put("key5", "value5")
+    assert cache.cache == {"key4": "value4", "key5": "value5", "key3": "value3"}
+    assert cache.lru == ["key3", "key4", "key5"]
+
+
+def test_i_can_put_the_same_key_several_times():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key1", "value3")
+
+    assert cache.cache == {"key1": "value3", "key2": "value2"}
+    assert cache.lru == ["key2", "key1"]
+
+
+def test_none_is_returned_when_not_found():
+    cache = FastCache()
+    assert cache.get("foo") is None
+
+
+def test_i_can_evict_by_key():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("to_keep1", "to_keep_value1")
+    cache.put("key2", "value2")
+    cache.put("to_keep2", "to_keep_value2")
+    cache.put("key3", "value3")
+    cache.put("to_keep3", "to_keep_value3")
+
+    cache.evict_by_key(lambda k: k.startswith("key"))
+    assert cache.cache == {"to_keep1": "to_keep_value1",
+                           "to_keep2": "to_keep_value2",
+                           "to_keep3": "to_keep_value3"}
+
+    assert cache.lru == ["to_keep1", "to_keep2", "to_keep3"]

tests/cache/test_cache.py

@@ -565,3 +565,35 @@ class TestCache(TestUsingMemoryBasedSheerka):
 
         cache.get(str(MAX_INITIALIZED_KEY + 1))
         assert len(cache._initialized_keys) == 1
+
+    def test_i_can_populate(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache()
+
+        cache.populate(lambda: items, lambda item: item[0])
+
+        assert len(cache) == 3
+        assert cache.get("1") == ("1", "1")
+        assert cache.get("2") == ("2", "2")
+        assert cache.get("3") == ("3", "3")
+
+    def test_max_size_is_respected_when_populate(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3"), ("4", "4"), ("5", "5")]
+        cache = Cache(max_size=3)
+
+        cache.populate(lambda: items, lambda item: item[0])
+
+        assert len(cache) == 3
+        assert cache.get("3") == ("3", "3")
+        assert cache.get("4") == ("4", "4")
+        assert cache.get("5") == ("5", "5")
+
+    def test_i_can_get_all(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache()
+
+        cache.populate(lambda: items, lambda item: item[0])
+
+        res = cache.get_all()
+        assert len(res) == 3
+        assert list(res) == [('1', '1'), ('2', '2'), ('3', '3')]

tests/core/test_ExecutionContext.py

@@ -1,140 +1,196 @@
-import pytest
-
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from sdp.sheerkaDataProvider import Event
 
-
-def test_id_is_incremented_by_event_digest():
-    a = ExecutionContext("foo", Event("event_1"), None, BuiltinConcepts.NOP, None)
-    b = ExecutionContext("foo", Event("event_1"), None, BuiltinConcepts.NOP, None)
-    c = ExecutionContext("foo", Event("event_2"), None, BuiltinConcepts.NOP, None)
-    d = b.push(BuiltinConcepts.NOP, None)
-    e = c.push(BuiltinConcepts.NOP, None)
-
-    assert a.id == 0
-    assert b.id == 1
-    assert c.id == 0
-    assert d.id == 2
-    assert e.id == 1
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
-def test_i_can_use_with_statement():
-    with ExecutionContext("who_", Event("event"), "fake_sheerka", BuiltinConcepts.NOP, None) as e:
-        pass
-    assert e.elapsed > 0
+class TestExecutionContext(TestUsingMemoryBasedSheerka):
 
+    def test_id_is_incremented_by_event_digest(self):
+        sheerka = self.get_sheerka()
 
-def test_i_can_push():
-    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None,
-                         desc="some description",
-                         obj=Concept("foo"),
-                         step=BuiltinConcepts.EVALUATION,
-                         iteration=15,
-                         concepts={"bar": Concept("bar")})
-    a.preprocess = set()
-    a.preprocess.add("preprocess")
-    a.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
-    a.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
+        b = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
+        c = ExecutionContext("foo", Event("event_2"), sheerka, BuiltinConcepts.NOP, None)
+        d = b.push(BuiltinConcepts.NOP, None)
+        e = c.push(BuiltinConcepts.NOP, None)
 
-    b = a.push(BuiltinConcepts.EVALUATION, "sub action context", desc="sub description")
+        assert a.id == 0
+        assert b.id == 1
+        assert c.id == 0
+        assert d.id == 2
+        assert e.id == 1
 
-    assert b._parent == a
-    assert b.who == a.who
-    assert b.event == a.event
-    assert b.sheerka == a.sheerka
-    assert b.action == BuiltinConcepts.EVALUATION
-    assert b.action_context == "sub action context"
-    assert b.desc == "sub description"
-    assert b.obj == a.obj
-    assert b.step == a.step
-    assert b.iteration == a.iteration
-    assert b.concepts == a.concepts
-    assert b.id == a.id + 1
-    assert b._tab == a._tab + "    "
-    assert b.preprocess == a.preprocess
-    assert b.protected_hints == {BuiltinConcepts.EVAL_BODY_REQUESTED}
-    assert b.global_hints == a.global_hints
+    def test_i_can_use_with_statement(self):
+        sheerka = self.get_sheerka()
 
+        with ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None) as e:
+            pass
+        assert e.elapsed > 0
 
-def test_children_i_created_when_i_push():
-    e = ExecutionContext("who_", Event("event"), "fake_sheerka", BuiltinConcepts.NOP, None)
-    e.push(BuiltinConcepts.NOP, None, who="a", desc="I do something")
-    e.push(BuiltinConcepts.NOP, None, who="b", desc="oups! I did a again")
-    e.push(BuiltinConcepts.NOP, None, who="c", desc="I do something else")
+    def test_i_can_push(self):
+        sheerka = self.get_sheerka()
 
-    assert len(e._children) == 3
-    assert e._children[0].who, e._children[0].who == ("a", "I do something")
-    assert e._children[1].who, e._children[1].who == ("b", "oups! I did a again")
-    assert e._children[2].who, e._children[2].who == ("c", "I do something else")
+        a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None,
+                             desc="some description",
+                             obj=Concept("foo"),
+                             concepts={"bar": Concept("bar")})
+        a.preprocess = set()
+        a.preprocess.add("preprocess")
+        a.preprocess_parsers = ["list of parsers"]
+        a.preprocess_evaluators = ["list of evaluators"]
+        a.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        a.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
+        b = a.push(BuiltinConcepts.EVALUATION, "sub action context", desc="sub description")
 
-def test_i_can_add_variable_when_i_push():
-    e = ExecutionContext("who_", Event("event"), "fake_sheerka", BuiltinConcepts.NOP, None)
-    sub_e = e.push(BuiltinConcepts.NOP, None, who="a", my_new_var="new var value")
+        assert b._parent == a
+        assert b.who == a.who
+        assert b.event == a.event
+        assert b.sheerka == a.sheerka
+        assert b.action == BuiltinConcepts.EVALUATION
+        assert b.action_context == "sub action context"
+        assert b.desc == "sub description"
+        assert b.obj == a.obj
+        assert b.concepts == a.concepts
+        assert b.id == a.id + 1
+        assert b.preprocess == a.preprocess
+        assert b.preprocess_parsers == a.preprocess_parsers
+        assert b.preprocess_evaluators == a.preprocess_evaluators
+        assert b.protected_hints == {BuiltinConcepts.EVAL_BODY_REQUESTED}
+        assert b.global_hints == a.global_hints
 
-    assert sub_e.my_new_var == "new var value"
-    with pytest.raises(AttributeError):
-        assert e.my_new_var == ""  # my_new_var does not exist in parent
+    def test_children_i_created_when_i_push(self):
+        sheerka = self.get_sheerka()
 
+        e = ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None)
+        e.push(BuiltinConcepts.NOP, None, who="a", desc="I do something")
+        e.push(BuiltinConcepts.NOP, None, who="b", desc="oups! I did a again")
+        e.push(BuiltinConcepts.NOP, None, who="c", desc="I do something else")
 
-def test_local_hints_are_local_and_global_hints_are_global():
-    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None)
-    a.protected_hints.add("local hint 1")
-    a.global_hints.add("global hint 1")
+        assert len(e._children) == 3
+        assert e._children[0].who, e._children[0].who == ("a", "I do something")
+        assert e._children[1].who, e._children[1].who == ("b", "oups! I did a again")
+        assert e._children[2].who, e._children[2].who == ("c", "I do something else")
 
-    b = a.push(BuiltinConcepts.NOP, None)
-    b.protected_hints.add("local hint 2")
-    b.global_hints.add("global hint 2")
+    # def test_i_can_add_variable_when_i_push(self):
+    #     sheerka = self.get_sheerka()
+    #
+    #     e = ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None)
+    #     sub_e = e.push(BuiltinConcepts.NOP, None, who="a", my_new_var="new var value")
+    #
+    #     assert sub_e.my_new_var == "new var value"
+    #     with pytest.raises(AttributeError):
+    #         assert e.my_new_var == ""  # my_new_var does not exist in parent
 
-    assert a.protected_hints == {"local hint 1"}
-    assert a.global_hints == {"global hint 1", "global hint 2"}
+    def test_local_hints_are_local_and_global_hints_are_global(self):
+        sheerka = self.get_sheerka()
 
-    assert b.protected_hints == {"local hint 1", "local hint 2"}
-    assert b.global_hints == {"global hint 1", "global hint 2"}
+        a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
+        a.protected_hints.add("local hint 1")
+        a.global_hints.add("global hint 1")
 
+        b = a.push(BuiltinConcepts.NOP, None)
+        b.protected_hints.add("local hint 2")
+        b.global_hints.add("global hint 2")
 
-def test_global_hits_are_global_even_when_empty():
-    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None)
+        assert a.protected_hints == {"local hint 1"}
+        assert a.global_hints == {"global hint 1", "global hint 2"}
 
-    b = a.push(BuiltinConcepts.NOP, None)
-    b.global_hints.add("global hint 2")
+        assert b.protected_hints == {"local hint 1", "local hint 2"}
+        assert b.global_hints == {"global hint 1", "global hint 2"}
 
-    assert a.global_hints == {"global hint 2"}
-    assert b.global_hints == {"global hint 2"}
+    def test_global_hits_are_global_even_when_empty(self):
+        sheerka = self.get_sheerka()
 
+        a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
 
-def test_i_can_search():
-    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.TESTING, "a")
-    ab = a.push(BuiltinConcepts.TESTING, "ab", obj="obj_ab")
-    ac = a.push(BuiltinConcepts.TESTING, "ac", obj="obj_ac")
-    abb = ab.push(BuiltinConcepts.TESTING, "abb", obj="skip")
-    abbb = abb.push(BuiltinConcepts.TESTING, "abbb", obj="obj_abbb")
+        b = a.push(BuiltinConcepts.NOP, None)
+        b.global_hints.add("global hint 2")
 
-    assert list(abbb.search()) == [abb, ab, a]
-    assert list(abbb.search(start_with_self=True)) == [abbb, abb, ab, a]
-    assert list(abbb.search(lambda ec: ec.obj != "skip")) == [ab, a]
-    assert list(abbb.search(lambda ec: ec.obj != "skip", lambda ec: ec.action_context)) == ["ab", "a"]
-    assert list(abbb.search(stop=lambda ec: ec.obj == "skip")) == [abb]
-    assert list(abbb.search(
-        stop=lambda ec: ec.obj == "skip",
-        start_with_self=True,
-        get_obj=lambda ec: ec.obj)) == ["obj_abbb", "skip"]
+        assert a.global_hints == {"global hint 2"}
+        assert b.global_hints == {"global hint 2"}
 
+    def test_i_can_search(self):
+        sheerka = self.get_sheerka()
 
-def test_variables_are_passed_to_children_but_not_to_parents():
-    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None)
-    assert not hasattr(a, "var")
+        a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.TESTING, "a")
+        ab = a.push(BuiltinConcepts.TESTING, "ab", obj="obj_ab")
+        ac = a.push(BuiltinConcepts.TESTING, "ac", obj="obj_ac")
+        abb = ab.push(BuiltinConcepts.TESTING, "abb", obj="skip")
+        abbb = abb.push(BuiltinConcepts.TESTING, "abbb", obj="obj_abbb")
 
-    b = a.push(BuiltinConcepts.NOP, None, var="foo")
-    assert b.var == "foo"
-    assert not hasattr(a, "var")
+        assert list(abbb.search()) == [abb, ab, a]
+        assert list(abbb.search(start_with_self=True)) == [abbb, abb, ab, a]
+        assert list(abbb.search(lambda ec: ec.obj != "skip")) == [ab, a]
+        assert list(abbb.search(lambda ec: ec.obj != "skip", lambda ec: ec.action_context)) == ["ab", "a"]
+        assert list(abbb.search(stop=lambda ec: ec.obj == "skip")) == [abb]
+        assert list(abbb.search(
+            stop=lambda ec: ec.obj == "skip",
+            start_with_self=True,
+            get_obj=lambda ec: ec.obj)) == ["obj_abbb", "skip"]
 
-    c = b.push(BuiltinConcepts.NOP, None)
-    assert c.var == "foo"
+    def test_i_can_deactivate_push(self):
+        sheerka = self.get_sheerka()
 
-    c.var = "bar"
-    assert c.var == "bar"
-    assert b.var != "bar"
-    assert not hasattr(a, "var")
+        context = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
+        context.deactivate_push()
+        sub_context1 = context.push(BuiltinConcepts.NOP, None)
+        sub_context2 = context.push(BuiltinConcepts.NOP, None)
+        sub_context3 = sub_context1.push(BuiltinConcepts.NOP, None)
+
+        assert id(sub_context1) == id(sub_context2)
+        assert id(sub_context1) == id(sub_context3)
+
+    def test_stm_are_copied_when_deactivate_push(self):
+        sheerka = self.get_sheerka()
+
+        context = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
+        context.add_to_short_term_memory("key1", "value1")
+        context.add_to_short_term_memory("key2", "value2")
+
+        context.deactivate_push()
+
+        stm_cache = sheerka.services[SheerkaMemory.NAME].short_term_objects.cache
+        assert stm_cache == {
+            context.id: {'key1': 'value1', 'key2': 'value2'},
+            context.id + 1: {'key1': 'value1', 'key2': 'value2'}
+        }
+
+    def test_has_parent(self):
+        sheerka = self.get_sheerka()
+        root = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
+
+        sub1 = root.push(BuiltinConcepts.TESTING, None)
+        sub2 = root.push(BuiltinConcepts.TESTING, None)
+
+        sub11 = sub1.push(BuiltinConcepts.TESTING, None)
+        sub111 = sub11.push(BuiltinConcepts.TESTING, None)
+
+        assert not root.has_parent(sub1.id)
+        assert sub1.has_parent(root.id)
+        assert sub11.has_parent(root.id)
+        assert sub111.has_parent(root.id)
+        assert sub2.has_parent(root.id)
+        assert not sub1.has_parent(sub2.id)
+        assert not sub2.has_parent(sub1.id)
+    # def test_variables_are_passed_to_children_but_not_to_parents(self):
+    #     sheerka = self.get_sheerka()
+    #
+    #     a = ExecutionContext("foo", Event("event_1"), sheerka, BuiltinConcepts.NOP, None)
+    #     assert not hasattr(a, "var")
+    #
+    #     b = a.push(BuiltinConcepts.NOP, None, var="foo")
+    #     assert b.var == "foo"
+    #     assert not hasattr(a, "var")
+    #
+    #     c = b.push(BuiltinConcepts.NOP, None)
+    #     assert c.var == "foo"
+    #
+    #     c.var = "bar"
+    #     assert c.var == "bar"
+    #     assert b.var != "bar"
+    #     assert not hasattr(a, "var")

tests/core/test_SheerkaAdmin.py

@@ -0,0 +1,65 @@
+from core.builtin_concepts import BuiltinConcepts
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaAdmin(TestUsingMemoryBasedSheerka):
+    def test_i_can_get_last_ret(self):
+        pass
+
+    def test_i_can_get_last_error_ret(self):
+        sheerka, context = self.init_concepts()
+
+        # nothing in history
+        last_error_ret = sheerka.last_error_ret(context)
+        assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE)
+        assert not last_error_ret.status
+        assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND)
+
+        # only success in history
+        r_success = sheerka.ret("Test", True, "success")
+        sheerka.last_return_values.append([r_success])
+        last_error_ret = sheerka.last_error_ret(context)
+        assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE)
+        assert not last_error_ret.status
+        assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.NOT_FOUND)
+
+        # at least on error
+        r_failure = sheerka.ret("Test", False, "failure")
+        sheerka.last_return_values.append([r_failure])
+        assert sheerka.last_error_ret(context) == r_failure
+
+        # add another success and make sure we get the same error
+        sheerka.last_return_values.append([r_success])
+        assert sheerka.last_error_ret(context) == r_failure
+
+        # and I only get the last failure
+        r_failure2 = sheerka.ret("Test", False, "another failure")
+        sheerka.last_return_values.append([r_failure2])
+        assert sheerka.last_error_ret(context) == r_failure2
+
+        # but I still can get the previous error
+        assert sheerka.last_error_ret(context, -2) == r_failure
+
+    def test_i_can_get_last_error_ret_when_only_one_ret_has_failed(self):
+        sheerka, context = self.init_concepts()
+
+        r_success = sheerka.ret("Test", True, "success")
+        r_failure = sheerka.ret("Test", False, "False1")
+
+        sheerka.last_return_values.append([r_success, r_failure])
+        assert sheerka.last_error_ret(context) == r_failure
+
+    def test_i_cannot_get_last_error_ret_when_too_many_errors(self):
+        sheerka, context = self.init_concepts()
+
+        r1 = sheerka.ret("Test", True, "success")
+        r2 = sheerka.ret("Test", False, "False1")
+        r3 = sheerka.ret("Test", False, "False2")
+
+        sheerka.last_return_values.append([r1, r2, r3])
+        last_error_ret = sheerka.last_error_ret(context)
+        assert sheerka.isinstance(last_error_ret, BuiltinConcepts.RETURN_VALUE)
+        assert not last_error_ret.status
+        assert sheerka.isinstance(last_error_ret.body, BuiltinConcepts.TOO_MANY_ERRORS)
+        assert last_error_ret.body.body == [r2, r3]

tests/core/test_SheerkaComparisonManager.py

@@ -1,5 +1,8 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.global_symbols import CONCEPT_PRECEDENCE_MODIFIED, CONCEPT_COMPARISON_CONTEXT, RULE_PRECEDENCE_MODIFIED, \
+    RULE_COMPARISON_CONTEXT
 from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager, ComparisonObj
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -31,15 +34,36 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
         in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
-        assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#")]
+        assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#")]
 
         weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
-        assert weighted == {"1001": 1, "1002": 2}
+        assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2}
 
         # I can commit
         sheerka.cache_manager.commit(context)
         in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
-        assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#")]
+        assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#")]
+
+    def test_i_can_add_is_greater_than_for_rules(self):
+        sheerka, context, r1, r2 = self.init_format_rules(("True", "true"), ("False", "false"),
+                                                          cache_only=False,
+                                                          compile_rule=False)
+        service = sheerka.services[SheerkaComparisonManager.NAME]
+
+        res = service.set_is_greater_than(context, "prop_name", r2, r1)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", r2.str_id, r1.str_id, ">", "#")]
+
+        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        assert weighted == {"r:|1:": 1, "r:|2:": 2}
+
+        # I can commit
+        sheerka.cache_manager.commit(context)
+        in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", r2.str_id, r1.str_id, ">", "#")]
 
     def test_i_can_add_a_is_less_than(self):
         sheerka, context, one, two = self.init_concepts("one", "two", cache_only=False)
@@ -50,15 +74,36 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
         in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
-        assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", one.id, two.id, "<", "#")]
+        assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", one.str_id, two.str_id, "<", "#")]
 
         weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
-        assert weighted == {"1001": 1, "1002": 2}
+        assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2}
 
         # I can commit
         sheerka.cache_manager.commit(context)
         in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
-        assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", one.id, two.id, "<", "#")]
+        assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", one.str_id, two.str_id, "<", "#")]
+
+    def test_i_can_add_is_less_than_for_rules(self):
+        sheerka, context, r1, r2 = self.init_format_rules(("True", "true"), ("False", "false"),
+                                                          cache_only=False,
+                                                          compile_rule=False)
+        service = sheerka.services[SheerkaComparisonManager.NAME]
+
+        res = service.set_is_less_than(context, "prop_name", r1, r2)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        assert in_cache == [ComparisonObj(context.event.get_digest(), "prop_name", r1.str_id, r2.str_id, "<", "#")]
+
+        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        assert weighted == {"r:|1:": 1, "r:|2:": 2}
+
+        # I can commit
+        sheerka.cache_manager.commit(context)
+        in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
+        assert in_db == [ComparisonObj(context.event.get_digest(), "prop_name", r1.str_id, r2.str_id, "<", "#")]
 
     def test_i_can_add_multiples_constraints(self):
         sheerka, context, one, two, three, four = self.init_concepts("one", "two", "three", "four", cache_only=False)
@@ -69,16 +114,16 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
 
         in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [
-            ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"),
-            ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#")
+            ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"),
+            ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#")
         ]
 
         # I can commit
         sheerka.cache_manager.commit(context)
         in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_db == [
-            ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"),
-            ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#")
+            ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"),
+            ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#")
         ]
 
         sheerka.cache_manager.clear(SheerkaComparisonManager.COMPARISON_ENTRY)  # reset the cache
@@ -86,21 +131,45 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         service.set_is_greater_than(context, "prop_name", four, three)
         in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [
-            ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"),
-            ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#"),
-            ComparisonObj(context.event.get_digest(), "prop_name", four.id, three.id, ">", "#"),
+            ComparisonObj(context.event.get_digest(), "prop_name", two.str_id, one.str_id, ">", "#"),
+            ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#"),
+            ComparisonObj(context.event.get_digest(), "prop_name", four.str_id, three.str_id, ">", "#"),
         ]
 
         weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
-        assert weighted == {"1001": 1, "1002": 2, "1003": 3, "1004": 4}
+        assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3, "c:four|1004:": 4}
+
+    def test_i_can_add_multiple_constraints_2(self):
+        sheerka, context, one, two, three = self.init_concepts("one", "two", "three")
+        service = sheerka.services[SheerkaComparisonManager.NAME]
+
+        service.set_is_greater_than(context, "prop_name", two, one)
+        service.set_is_greater_than(context, "prop_name", three, two)
+
+        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3}
+
+        service.set_is_greater_than(context, "prop_name", three, one)  # should not change order
+        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3}
+
+    def test_i_lesser_than_and_opposite_greater_than(self):
+        sheerka, context, one, two = self.init_concepts("one", "two")
+        service = sheerka.services[SheerkaComparisonManager.NAME]
+
+        service.set_is_greater_than(context, "prop_name", two, one)
+        service.set_is_less_than(context, "prop_name", one, two)
+
+        weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
+        assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2}
 
     @pytest.mark.parametrize("entries, expected", [
-        (["two > one"], {'1001': 1, '1002': 2}),
-        (["one < two"], {'1001': 1, '1002': 2}),
-        (["three > two", "one < two"], {'1001': 1, '1002': 2, '1003': 3}),
-        (["three > one", "one < two"], {'1001': 1, '1002': 2, '1003': 2}),
-        (["three >>"], {'1003': 2}),
-        (["one <<"], {'1001': 0}),
+        (["two > one"], {'c:one|1001:': 1, 'c:two|1002:': 2}),
+        (["one < two"], {'c:one|1001:': 1, 'c:two|1002:': 2}),
+        (["three > two", "one < two"], {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3}),
+        (["three > one", "one < two"], {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 2}),
+        (["three >>"], {'c:three|1003:': 2}),
+        (["one <<"], {'c:one|1001:': 0}),
     ])
     def test_i_can_get_concept_weight(self, entries, expected):
         sheerka, context, *concepts = self.init_concepts("one", "two", "three")
@@ -125,7 +194,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         service.set_is_lesser(context, "prop_name", one)
         sheerka.cache_manager.clear(service.RESOLVED_COMPARISON_ENTRY)
 
-        assert service.get_concepts_weights("prop_name") == {"1001": 0}
+        assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0}
 
     def test_i_can_get_partition(self):
         sheerka, context, one, two, three = self.init_concepts("one", "two", "three")
@@ -137,9 +206,9 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         res = service.get_partition("prop_name")
 
         assert res == {
-            1: ["1001"],
-            2: ["1002"],
-            3: ["1003"],
+            1: ["c:one|1001:"],
+            2: ["c:two|1002:"],
+            3: ["c:three|1003:"],
         }
 
     def test_i_can_detect_chicken_and_egg(self):
@@ -176,7 +245,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         service.set_is_less_than(context, "prop_name", one, two)
 
         weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
-        assert weighted == {"1001": 1, "1002": 2}
+        assert weighted == {"c:one|1001:": 1, "c:two|1002:": 2}
 
     def test_methods_are_correctly_bound(self):
         sheerka, context, one, two = self.init_concepts("one", "two")
@@ -188,17 +257,17 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
         service.set_is_lesser(context, "prop_name", one)
-        assert service.get_concepts_weights("prop_name") == {"1001": 0}  # DEFAULT_COMPARISON_VALUE - 1
+        assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0}  # DEFAULT_COMPARISON_VALUE - 1
 
         sheerka.set_is_greater_than(context, "prop_name", three, two)
-        assert service.get_concepts_weights("prop_name") == {"1001": 0, "1002": 1, "1003": 2}
+        assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 0, "c:two|1002:": 1, "c:three|1003:": 2}
 
         # I can commit
         sheerka.cache_manager.commit(context)
         in_db = sheerka.sdp.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_db == [
-            ComparisonObj(context.event.get_digest(), "prop_name", one.id, None, "<<", "#"),
-            ComparisonObj(context.event.get_digest(), "prop_name", three.id, two.id, ">", "#")
+            ComparisonObj(context.event.get_digest(), "prop_name", one.str_id, None, "<<", "#"),
+            ComparisonObj(context.event.get_digest(), "prop_name", three.str_id, two.str_id, ">", "#")
         ]
 
     def test_i_can_define_an_order_for_lesser_concepts(self):
@@ -211,7 +280,9 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
 
         sheerka.set_is_less_than(context, "prop_name", less_l, lesser)
         sheerka.set_is_greater_than(context, "prop_name", less_l, even_more_l)
-        assert service.get_concepts_weights("prop_name") == {"1001": 0, "1002": -1, "1003": -2}
+        assert service.get_concepts_weights("prop_name") == {"c:lesser|1001:": 0,
+                                                             "c:less_l|1002:": -1,
+                                                             "c:even_less_l|1003:": -2}
 
     def test_i_cannot_define_less_than_a_lesser_if_not_a_lesser_itself(self):
         """
@@ -237,10 +308,10 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
         service.set_is_greatest(context, "prop_name", three)
-        assert service.get_concepts_weights("prop_name") == {"1003": 2}  # DEFAULT_COMPARISON_VALUE + 1
+        assert service.get_concepts_weights("prop_name") == {"c:three|1003:": 2}  # DEFAULT_COMPARISON_VALUE + 1
 
         sheerka.set_is_greater_than(context, "prop_name", two, one)
-        assert service.get_concepts_weights("prop_name") == {"1001": 1, "1002": 2, "1003": 3}
+        assert service.get_concepts_weights("prop_name") == {"c:one|1001:": 1, "c:two|1002:": 2, "c:three|1003:": 3}
 
     def test_i_cannot_define_greater_than_a_greatest_if_not_a_greater_itself(self):
         sheerka, context, greatest, foo = self.init_concepts("greatest", "foo")
@@ -280,7 +351,9 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
 
         sheerka.set_is_less_than(context, "prop_name", greatest, more_g)
         sheerka.set_is_greater_than(context, "prop_name", even_more_g, more_g)
-        assert service.get_concepts_weights("prop_name") == {"1001": 2, "1002": 3, "1003": 4}
+        assert service.get_concepts_weights("prop_name") == {"c:greatest|1001:": 2,
+                                                             "c:more_g|1002:": 3,
+                                                             "c:even_more_g|1003:": 4}
 
     def test_i_can_mix_all_comparisons(self):
         sheerka, context, one, two, three, four, five, six, three_and_half = self.init_concepts(
@@ -298,25 +371,54 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         sheerka.set_is_less_than(context, "prop_name", three, four)
 
         assert service.get_concepts_weights("prop_name") == {
-            "1001": -1,
-            "1002": 0,
-            "1003": 1,
-            "1004": 2,
-            "1005": 3,
-            "1006": 4
+            "c:one|1001:": -1,
+            "c:two|1002:": 0,
+            "c:three|1003:": 1,
+            "c:four|1004:": 2,
+            "c:five|1005:": 3,
+            "c:six|1006:": 4
         }
 
         sheerka.set_is_less_than(context, "prop_name", three_and_half, four)
         sheerka.set_is_greater_than(context, "prop_name", three_and_half, three)
 
         assert service.get_concepts_weights("prop_name") == {
-            "1001": -1,
-            "1002": 0,
-            "1003": 1,
-            "1007": 2,
-            "1004": 3,
-            "1005": 4,
-            "1006": 5
+            "c:one|1001:": -1,
+            "c:two|1002:": 0,
+            "c:three|1003:": 1,
+            "c:two_and_half|1007:": 2,
+            "c:four|1004:": 3,
+            "c:five|1005:": 4,
+            "c:six|1006:": 5
+        }
+
+    def test_i_can_mix_all_comparisons_2(self):
+        sheerka, context, zero, one, two, three, four, five, thousand = self.init_concepts(
+            "zero", "one", "two", "three", "four", "five", "thousand")
+        service = sheerka.services[SheerkaComparisonManager.NAME]
+
+        service.set_is_greater_than(context, "prop_name", four, one)
+        service.set_is_less_than(context, "prop_name", one, five)
+        service.set_is_less_than(context, "prop_name", one, three)
+        service.set_is_less_than(context, "prop_name", two, three)
+        service.set_is_greater_than(context, "prop_name", two, one)
+        service.set_is_greater_than(context, "prop_name", five, four)
+        service.set_is_less_than(context, "prop_name", three, four)
+        service.set_is_greatest(context, "prop_name", thousand)
+        service.set_is_lesser(context, "prop_name", zero)
+        service.set_is_less_than(context, "prop_name", two, five)
+        service.set_is_less_than(context, "prop_name", two, four)
+        service.set_is_greater_than(context, "prop_name", five, one)
+        service.set_is_greater_than(context, "prop_name", five, three)
+
+        assert service.get_concepts_weights("prop_name") == {
+            "c:zero|1001:": 0,
+            "c:one|1002:": 1,
+            "c:two|1003:": 2,
+            "c:three|1004:": 3,
+            "c:four|1005:": 4,
+            "c:five|1006:": 5,
+            "c:thousand|1007:": 6
         }
 
     @pytest.mark.parametrize("definition", [
@@ -335,3 +437,47 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+
+    def test_an_event_is_fired_when_modifying_concepts_precedence(self):
+        sheerka, context, one, two, foo = self.init_concepts("one", "two", "foo")
+        event_received = False
+
+        def receive_event(c):
+            nonlocal event_received
+            event_received = True
+
+        sheerka.subscribe(CONCEPT_PRECEDENCE_MODIFIED, receive_event)
+
+        sheerka.set_is_greater_than(context, foo, one, two)
+        assert not event_received
+
+        sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, foo, one, two)
+        assert not event_received
+
+        sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, one, two, CONCEPT_COMPARISON_CONTEXT)
+        assert event_received
+
+    def test_an_event_is_fired_when_modifying_rule_precedence(self):
+        sheerka, context, r1, r2 = self.init_format_rules(
+            ("True", "True"),
+            ("False", "False"),
+            compile_rule=False,
+        )
+        foo = Concept("foo")
+        event_received = False
+        sheerka.cache_manager.clear(SheerkaComparisonManager.COMPARISON_ENTRY)
+
+        def receive_event(c):
+            nonlocal event_received
+            event_received = True
+
+        sheerka.subscribe(RULE_PRECEDENCE_MODIFIED, receive_event)
+
+        sheerka.set_is_greater_than(context, foo, r1, r2)
+        assert not event_received
+
+        sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, foo, r1, r2)
+        assert not event_received
+
+        sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, r1, r2, RULE_COMPARISON_CONTEXT)
+        assert event_received

tests/core/test_SheerkaConceptAlgebra.py

@@ -19,7 +19,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka):
         res = sheerka.cadd(context, sheerka.new("man"), sheerka.new("driver"))
 
         assert isinstance(res, Concept)
-        assert res.metadata.props == {BuiltinConcepts.ISA: {male, human},
+        assert res.get_metadata().props == {BuiltinConcepts.ISA: {male, human},
                                       BuiltinConcepts.HASA: {car, licence}, }
 
     def test_can_add_concepts_when_property_already_exist(self):
@@ -32,7 +32,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka):
         res = sheerka.cadd(context, sheerka.new("man"), sheerka.new("king"))
 
         assert isinstance(res, Concept)
-        assert res.metadata.props == {BuiltinConcepts.ISA: {male, human}}
+        assert res.get_metadata().props == {BuiltinConcepts.ISA: {male, human}}
 
     def test_i_can_subtract_concepts(self):
         sheerka, context, foo, bar, isa1, isa2, hasa1, hasa2 = self.init_concepts(
@@ -53,14 +53,14 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka):
 
         res = sheerka.csub(context, new_foo)
         assert isinstance(res, Concept)
-        assert res.metadata.props == res.metadata.props
+        assert res.get_metadata().props == res.get_metadata().props
 
         res = sheerka.csub(context, new_foo, new_bar)
         assert isinstance(res, Concept)
-        assert res.metadata.props == {BuiltinConcepts.ISA: {isa2},
+        assert res.get_metadata().props == {BuiltinConcepts.ISA: {isa2},
                                       BuiltinConcepts.HASA: {hasa2}, }
 
-    def test_i_can_recognize_myself_when_cache_only_is_not_set(self):
+    def test_i_can_recognize_myself_when_using_sdp_repository(self):
         sheerka, context, foo, isa1, hasa1, = self.init_concepts("foo", "isa1", "has1",
                                                                  cache_only=False,
                                                                  create_new=True)
@@ -73,7 +73,7 @@ class TestSheerkaConceptsAlgebra(TestUsingMemoryBasedSheerka):
 
         assert sheerka.recognize(new_foo, all_scores=True) == [ConceptScore(1, new_foo, new_foo)]
 
-    def test_i_can_recognize_myself_when_cache_only_is_set(self):
+    def test_i_can_recognize_myself_when_not_using_sdp_repository(self):
         sheerka, context, foo, isa1, hasa1, = self.init_concepts("foo", "isa1", "has1")
 
         new_foo = sheerka.new("foo")

tests/core/test_SheerkaCreateNewConcept.py

@@ -1,6 +1,6 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF
+from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs
 from core.sheerka.Sheerka import Sheerka
 
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
@@ -22,10 +22,11 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
 
         concept_found = res.value.body
         for prop in PROPERTIES_TO_SERIALIZE:
-            assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop)
+            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
 
         assert concept_found.key == "__var__0 + __var__1"
         assert concept_found.id == "1001"
+        assert get_concept_attrs(concept) == ['a', 'b']
 
         # saved in cache
         assert sheerka.has_id(concept.id)
@@ -63,7 +64,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
 
         concept_found = res.value.body
         for prop in PROPERTIES_TO_SERIALIZE:
-            assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop)
+            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
 
         assert concept_found.key == "hello __var__0"
         assert concept_found.id == "1001"
@@ -120,7 +121,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
         sheerka = self.get_sheerka()
         concept1 = self.get_default_concept()
         concept2 = self.get_default_concept()
-        concept2.metadata.body = "a+b"
+        concept2.get_metadata().body = "a+b"
 
         res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
         res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
@@ -149,7 +150,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
 
         assert res.status
         assert sheerka.get_by_name(concept.name) == concept
-        assert sheerka.get_by_name(concept.metadata.definition) == concept
+        assert sheerka.get_by_name(concept.get_metadata().definition) == concept
 
         concept = Concept(name="foo", definition="foo", definition_type=DEFINITION_TYPE_DEF)
         res = sheerka.create_new_concept(context, concept)
@@ -273,6 +274,7 @@ class TestSheerkaCreateNewConceptFileBased(TestUsingFileBasedSheerka):
         assert len(sheerka.sdp.get(Sheerka.CONCEPTS_BY_KEY_ENTRY, "foo")) == 2
 
         sheerka = self.get_sheerka()  # new instance
+        context = self.get_context(sheerka)
         sheerka.create_new_concept(context, Concept("foo", body="3"))
         sheerka.cache_manager.commit(context)
 

tests/core/test_SheerkaDebugManager.py

@@ -0,0 +1,425 @@
+import pytest
+from core.builtin_concepts import BuiltinConcepts
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugVarSetting, DebugRuleSetting
+from sdp.sheerkaDataProvider import Event
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_activate_debug(self):
+        sheerka, context = self.init_concepts()
+
+        sheerka.set_debug(context, True)
+        assert sheerka.debug_activated()
+
+        sheerka.set_debug(context, False)
+        assert not sheerka.debug_activated()
+
+    def test_when_debug_mode_is_activated_context_are_in_debug_mode(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+
+        sheerka.set_debug(root_context, True)
+
+        context = root_context.push(BuiltinConcepts.NOP, None)  # sub_context.id = 1
+        sub_context = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context.parent = 1
+        sub_context2 = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context2.parent = 1
+        sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None)  # is a child
+
+        assert context.id == 1
+        assert context.debug_enabled
+        assert sub_context.debug_enabled
+        assert sub_context2.debug_enabled
+        assert sub_sub_context.debug_enabled
+        assert not root_context.debug_enabled
+
+    def test_i_can_activate_debug_for_new_context(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+
+        sheerka.set_debug(root_context, True)
+        sheerka.set_explicit(root_context, True)
+        sheerka.activate_debug_for(root_context, 1)
+
+        context = ExecutionContext("test", Event(), sheerka, BuiltinConcepts.TESTING, None)  # context.id = 1
+        assert context.debug_enabled
+
+    def test_i_can_activate_debug_for_new_context_2(self):
+        """
+        This time children is also requested
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+
+        sheerka.set_debug(root_context, True)
+        sheerka.set_explicit(root_context, True)
+        sheerka.activate_debug_for(root_context, 1, children=True)
+
+        context = ExecutionContext("test", Event(), sheerka, BuiltinConcepts.TESTING, None)  # context.id = 1
+        assert context.debug_enabled
+
+    def test_i_can_activate_debug_for_a_context_using_push(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        sheerka.set_debug(root_context, True)
+        sheerka.set_explicit(root_context, True)
+        sheerka.activate_debug_for(root_context, 1)
+
+        context = root_context.push(BuiltinConcepts.NOP, None)  # sub_context.id = 1
+        sub_context = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context.parent = 1
+
+        assert context.id == 1
+        assert 1 in service.context_cache
+        assert context.debug_enabled
+        assert not sub_context.debug_enabled
+        assert not root_context.debug_enabled
+
+    def test_global_debug_must_be_activated_to_activate_context_debug(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        sheerka.set_debug(root_context, False)
+        sheerka.activate_debug_for(root_context, 1)
+
+        context = root_context.push(BuiltinConcepts.NOP, None)  # sub_context.id = 1
+
+        assert context.id == 1
+        assert 1 in service.context_cache
+        assert not context.debug_enabled
+
+    def test_i_can_activate_debug_for_sub_children(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        sheerka.set_debug(root_context, True)
+        sheerka.set_explicit(root_context, True)
+        sheerka.activate_debug_for(root_context, 1, children=True)
+
+        context = root_context.push(BuiltinConcepts.NOP, None)  # sub_context.id = 1
+        sub_context = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context.parent = 1
+        sub_context2 = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context2.parent = 1
+        sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None)  # is a child
+
+        assert context.id == 1
+        assert 1 in service.context_cache
+        assert "1+" in service.context_cache
+        assert context.debug_enabled
+        assert sub_context.debug_enabled
+        assert sub_context2.debug_enabled
+        assert sub_sub_context.debug_enabled
+        assert not root_context.debug_enabled
+
+    def test_i_can_deactivate_debug_for_a_context(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        sheerka.set_debug(root_context, True)
+        sheerka.set_explicit(root_context, True)
+        sheerka.activate_debug_for(root_context, 1)
+        sheerka.deactivate_debug_for(root_context, 1)
+
+        context = root_context.push(BuiltinConcepts.NOP, None)
+
+        assert context.id == 1
+        assert 1 not in service.context_cache
+        assert not context.debug_enabled
+
+    def test_i_can_deactivate_context_but_not_children(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        sheerka.set_debug(root_context, True)
+        sheerka.set_explicit(root_context, True)
+        sheerka.activate_debug_for(root_context, 1, True)
+        sheerka.deactivate_debug_for(root_context, 1)
+
+        context = root_context.push(BuiltinConcepts.NOP, None)  # sub_context.id = 1
+        sub_context = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context.parent = 1
+        sub_context2 = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context2.parent = 1
+        sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None)  # is a child
+
+        assert context.id == 1
+        assert 1 not in service.context_cache
+        assert "1+" in service.context_cache
+        assert not context.debug_enabled
+        assert sub_context.debug_enabled
+        assert sub_context2.debug_enabled
+        assert sub_sub_context.debug_enabled
+
+    def test_i_can_deactivate_context_and_children(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        sheerka.set_debug(root_context, True)
+        sheerka.set_explicit(root_context, True)
+        sheerka.activate_debug_for(root_context, 1)
+        sheerka.deactivate_debug_for(root_context, 1, children=True)
+
+        context = root_context.push(BuiltinConcepts.NOP, None)  # sub_context.id = 1
+        sub_context = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context.parent = 1
+        sub_context2 = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context2.parent = 1
+        sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None)  # is a child
+
+        assert 1 not in service.context_cache
+        assert "1+" not in service.context_cache
+        assert not context.debug_enabled
+        assert not sub_context.debug_enabled
+        assert not sub_context2.debug_enabled
+        assert not sub_sub_context.debug_enabled
+
+    def test_i_can_activate_debug_for_a_variable(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        sheerka.set_debug(context)
+
+        sheerka.activate_debug_for(context, "Out")
+        assert "Out" in service.variable_cache
+        assert sheerka.debug_activated_for("Out")
+
+        sheerka.deactivate_debug_for(context, "Out")
+        assert "Out" not in service.variable_cache
+        assert not sheerka.debug_activated_for("Out")
+
+    def test_i_can_activate_debug_for_sub_children_using_the_simplified_form(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        sheerka.set_debug(root_context, True)
+        sheerka.set_explicit(root_context, True)
+        sheerka.activate_debug_for(root_context, "1+")
+
+        context = root_context.push(BuiltinConcepts.NOP, None)  # sub_context.id = 1
+        sub_context = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context.parent = 1
+        sub_context2 = context.push(BuiltinConcepts.NOP, None)  # sub_sub_context2.parent = 1
+        sub_sub_context = sub_context.push(BuiltinConcepts.NOP, None)  # is a child
+
+        assert context.id == 1
+        assert 1 in service.context_cache
+        assert "1+" in service.context_cache
+        assert context.debug_enabled
+        assert sub_context.debug_enabled
+        assert sub_context2.debug_enabled
+        assert sub_sub_context.debug_enabled
+        assert not root_context.debug_enabled
+
+    @pytest.mark.parametrize("settings, expected", [
+        ({"service": "my_service"}, True),
+        ({"service": "other_service"}, False),
+        ({"method": "my_method"}, True),
+        ({"method": "other_method"}, False),
+    ])
+    def test_i_can_compute_debug(self, settings, expected):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.debug_var(context, **settings)
+        assert service.compute_debug("my_service", "my_method", context) == expected
+
+    def test_i_can_compute_debug_for_context(self):
+        sheerka, root_context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(root_context, True)
+
+        context1 = root_context.push(BuiltinConcepts.TESTING, None)
+        context2 = root_context.push(BuiltinConcepts.TESTING, None)
+        service.debug_var(root_context, context_id=context1.id)
+
+        assert service.compute_debug("my_service", "my_method", context1)
+        assert not service.compute_debug("my_service", "my_method", context2)
+
+    def test_i_can_disable_debug_setting(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        service.debug_var(context, service="my_service", enabled=False)
+        assert not service.compute_debug("my_service", "my_method", context)
+
+    def test_i_can_compute_combination_of_debug_settings(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.debug_var(context, service="my_service", enabled=True)
+        service.debug_var(context, method="my_method", enabled=False)
+        service.debug_var(context, variable="xxx", enabled=False)  # is not used
+        service.debug_var(context, debug_id=1, enabled=False)  # is not used
+
+        assert not service.compute_debug("my_service", "my_method", context)  # False > True
+        assert service.compute_debug("my_service", "another_method", context)
+        assert not service.compute_debug("another_service", "my_method", context)
+
+    def test_variable_debug_is_deactivated_by_default(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        assert not service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0)
+
+    def test_i_can_activate_variable_debug_for_all_variables(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.debug_var(context, variable="*")
+
+        assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == True
+
+    def test_i_can_activate_variable_display_for_all_variables(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.debug_var(context, variable="*", enabled='list')
+
+        assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == 'list'
+
+    def test_i_can_disable_variable_debug(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.debug_var(context, variable="my_variable", enabled=False)
+        assert not service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0)
+
+    @pytest.mark.parametrize("enabled_1, enabled_2, expected", [
+        (False, False, False),
+        (False, True, False),
+        (False, 'list', False),
+        (True, False, False),
+        (True, True, True),
+        (True, 'list', 'list'),
+        ('list', False, False),
+        ('list', True, 'list'),
+        ('list', 'list', 'list'),
+    ])
+    def test_i_can_compute_combination_of_debug_var_settings(self, enabled_1, enabled_2, expected):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.debug_var(context, variable="my_variable", enabled=enabled_1)
+        service.debug_var(context, variable="*", enabled=enabled_2)
+
+        assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == expected
+
+    @pytest.mark.parametrize("settings, expected", [
+        ({"service": "my_service"}, True),
+        ({"service": "other_service"}, False),
+        ({"method": "my_method"}, True),
+        ({"method": "other_method"}, False),
+        ({"context_id": 0}, True),
+        ({"context_id": 1}, False),
+    ])
+    def test_service_and_method_and_context_id_are_tested_before_disabling_a_variable(self, settings, expected):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        settings["variable"] = "my_variable"
+
+        service.debug_var(context, **settings, enabled=True)
+        assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == expected
+
+    @pytest.mark.parametrize("context_children, expected", [
+        (True, True),
+        (False, False),
+    ])
+    def test_i_can_compute_debug_var_for_context_children(self, context_children, expected):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        sub_context = context.push(BuiltinConcepts.TESTING, None)
+        sub_sub_context = sub_context.push(BuiltinConcepts.TESTING, None)
+
+        service.debug_var(context, context_id=sub_context.id, context_children=context_children)
+        assert service.compute_debug("my_service", "my_method", sub_sub_context) == expected
+
+    def test_i_can_update_debug_setting(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+
+        service.debug_var(context, service="my_service", enabled=True)
+        service.debug_var(context, service="my_service", enabled=False)
+
+        assert len(service.debug_vars_settings) == 1
+        assert not service.debug_vars_settings[0].enabled
+
+        service.debug_var(context, service="my_service", enabled=True)
+        assert len(service.debug_vars_settings) == 1
+        assert service.debug_vars_settings[0].enabled
+
+    def test_i_can_reset_debug_settings(self):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.debug_var(context, service="my_service")
+        assert service.compute_debug("my_service", "my_method", context)
+
+        service.reset_debug(context)
+        assert not service.compute_debug("my_service", "my_method", context)
+
+    @pytest.mark.parametrize("settings, expected", [
+        ({"rule": "1"}, True),
+        ({"rule": "2"}, False),
+        ({"context_id": 0}, True),
+        ({"context_id": 1}, False),
+        ({"debug_id": 0}, True),
+        ({"debug_id": 1}, False),
+    ])
+    def test_i_can_compute_rules_debug_settings(self, settings, expected):
+        sheerka, context = self.init_concepts()
+        service = sheerka.services[SheerkaDebugManager.NAME]
+        service.set_debug(context, True)
+
+        service.debug_rule(context, **settings)
+        assert service.compute_debug_rule("1", 0, 0) == expected
+
+    def test_state_is_saved_and_restored(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids.clear()
+        root_context = self.get_context(sheerka)
+
+        sheerka.set_debug(root_context, True)
+        sheerka.set_explicit(root_context, False)
+        sheerka.activate_debug_for(root_context, 1, children=True)
+        sheerka.activate_debug_for(root_context, "SomeVar")
+        sheerka.debug_rule(root_context, "1", 10, 15)
+        sheerka.debug_var(root_context, service="service_name")
+
+        another_service = SheerkaDebugManager(sheerka)
+        another_service.initialize_deferred(root_context, True)
+
+        assert another_service.activated
+        assert not another_service.explicit
+        assert another_service.context_cache == {1, "1+"}
+        assert another_service.variable_cache == {"SomeVar"}
+        assert another_service.debug_rules_settings == [
+            DebugRuleSetting("1", 10, 15, True)
+        ]
+        assert another_service.debug_vars_settings == [
+            DebugVarSetting('service_name', None, None, None, False, None, False, True)]

tests/core/test_SheerkaEvaluateConcept.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
-from core.concept import Concept, DoNotResolve, ConceptParts, Property, InfiniteRecursionResolved, CB, NotInit
+from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, CB, NotInit, \
+    concept_part_value
 from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from parsers.PythonParser import PythonNode
@@ -11,7 +12,7 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("body, expected", [
-        (None, BuiltinConcepts.NOT_INITIALIZED),
+        (None, NotInit),
         ("", ""),
         ("1", 1),
         ("1+1", 2),
@@ -27,13 +28,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         assert evaluated.key == concept.key
         assert evaluated.body == expected
-        assert evaluated.metadata.body == body
-        assert evaluated.metadata.pre is None
-        assert evaluated.metadata.post is None
-        assert evaluated.metadata.where is None
+        assert evaluated.get_metadata().body == body
+        assert evaluated.get_metadata().pre is None
+        assert evaluated.get_metadata().post is None
+        assert evaluated.get_metadata().where is None
         assert evaluated.variables() == {}
-        assert evaluated.metadata.is_evaluated
-        assert len(evaluated.values) == 0 if body is None else 1
+        assert evaluated.get_metadata().is_evaluated
+        assert len(evaluated.values()) == 0 if body is None else 1
 
         assert "foo" in sheerka.services[SheerkaMemory.NAME].registration
 
@@ -59,13 +60,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept)
 
         assert evaluated.key == concept.key
-        assert evaluated.metadata.body is None
-        assert evaluated.metadata.post == expr
-        assert evaluated.metadata.pre is None
-        assert evaluated.metadata.where is None
+        assert evaluated.get_metadata().body is None
+        assert evaluated.get_metadata().post == expr
+        assert evaluated.get_metadata().pre is None
+        assert evaluated.get_metadata().where is None
         assert evaluated.get_value(ConceptParts.POST) == expected
         assert evaluated.variables() == {}
-        assert not evaluated.metadata.is_evaluated
+        assert not evaluated.get_metadata().is_evaluated
         assert len(evaluated.values) == 0 if expr is None else 1
 
     @pytest.mark.parametrize("expr, expected", [
@@ -84,12 +85,12 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept)
 
         assert evaluated.key == concept.key
-        assert evaluated.metadata.pre is None
-        assert evaluated.metadata.pre is None
-        assert evaluated.metadata.post is None
-        assert evaluated.metadata.where is None
-        assert evaluated.variables() == {"a": Property("a", expected)}
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().pre is None
+        assert evaluated.get_metadata().pre is None
+        assert evaluated.get_metadata().post is None
+        assert evaluated.get_metadata().where is None
+        assert evaluated.variables() == {"a": expected}
+        assert evaluated.get_metadata().is_evaluated
 
     def test_i_can_evaluate_when_the_body_is_the_name_of_the_concept(self):
         # to prove that I can distinguish from a string
@@ -102,33 +103,33 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
     def test_i_can_evaluate_metadata_using_do_not_resolve(self):
         sheerka, context, concept = self.init_concepts(Concept("foo"), eval_body=True)
-        concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
+        concept.get_compiled()[ConceptParts.BODY] = DoNotResolve("do not resolve")
 
         evaluated = sheerka.evaluate_concept(context, concept)
 
         assert evaluated.body == "do not resolve"
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().is_evaluated
 
     def test_i_can_evaluate_variable_using_do_not_resolve(self):
         sheerka, context, concept = self.init_concepts(Concept("foo").def_var("a"), eval_body=True)
-        concept.compiled["a"] = DoNotResolve("do not resolve")
+        concept.get_compiled()["a"] = DoNotResolve("do not resolve")
 
         evaluated = sheerka.evaluate_concept(context, concept)
 
         assert evaluated.get_value("a") == "do not resolve"
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().is_evaluated
 
     def test_original_value_is_overridden_when_using_do_no_resolve(self):
         concept = Concept("foo", body="original value").def_var("a", "original value")
         sheerka, context, concept = self.init_concepts(concept, eval_body=True)
-        concept.compiled["a"] = DoNotResolve("do not resolve")
-        concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
+        concept.get_compiled()["a"] = DoNotResolve("do not resolve")
+        concept.get_compiled()[ConceptParts.BODY] = DoNotResolve("do not resolve")
 
         evaluated = sheerka.evaluate_concept(context, concept)
 
         assert evaluated.body == "do not resolve"
         assert evaluated.get_value("a") == "do not resolve"
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().is_evaluated
 
     def test_variables_are_evaluated_before_body(self):
         sheerka, context, concept = self.init_concepts(Concept("foo", body="a+1").def_var("a", "10"), eval_body=True)
@@ -145,9 +146,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         evaluated = sheerka.evaluate_concept(context, concept)
 
-        assert evaluated == CB("foo", CB("a", BuiltinConcepts.NOT_INITIALIZED))
-        assert evaluated.metadata.is_evaluated
-        assert evaluated.body.metadata.is_evaluated
+        assert evaluated == CB("foo", CB("a", NotInit))
+        assert evaluated.get_metadata().is_evaluated
+        assert evaluated.body.get_metadata().is_evaluated
 
     def test_i_can_evaluate_when_the_referenced_concept_has_a_body(self):
         sheerka, context, concept_a, concept = self.init_concepts(
@@ -159,8 +160,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         assert evaluated.key == concept.key
         assert evaluated.body == CB("a", 1)
-        assert not concept_a.metadata.is_evaluated
-        assert evaluated.metadata.is_evaluated
+        assert not concept_a.get_metadata().is_evaluated
+        assert evaluated.get_metadata().is_evaluated
 
     def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(self):
         sheerka, context, concept_a, concept_b, concept_c, concept_d = self.init_concepts(
@@ -176,7 +177,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         expected = CB("c", CB("b", CB("a", "a")))
         assert evaluated.body == expected
         assert sheerka.objvalue(evaluated) == 'a'
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().is_evaluated
 
     def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(self):
         sheerka, context, concept_a, concept_b, concept_c, concept_d = self.init_concepts(
@@ -189,10 +190,10 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept_d)
 
         assert evaluated.key == concept_d.key
-        expected = CB("c", CB("b", CB("a", BuiltinConcepts.NOT_INITIALIZED)))
+        expected = CB("c", CB("b", CB("a", NotInit)))
         assert evaluated.body == expected
-        assert sheerka.objvalue(evaluated) == CB("a", BuiltinConcepts.NOT_INITIALIZED)
-        assert evaluated.metadata.is_evaluated
+        assert sheerka.objvalue(evaluated) == CB("a", NotInit)
+        assert evaluated.get_metadata().is_evaluated
 
     def test_i_can_evaluate_concept_when_variables_reference_others_concepts_1(self):
         """
@@ -273,7 +274,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         sheerka, context, concept_a = self.init_concepts(Concept(name="a", body="'a'"), eval_body=True)
 
         concept = Concept("foo").def_var("a")
-        concept.compiled["a"] = concept_a
+        concept.get_compiled()["a"] = concept_a
         evaluated = sheerka.evaluate_concept(context, concept)
 
         assert evaluated.key == concept.key
@@ -306,7 +307,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         foo = Concept("foo", body="1")
 
         concept = Concept("to_eval").def_var("prop")
-        concept.compiled["prop"] = [foo, DoNotResolve("1")]
+        concept.get_compiled()["prop"] = [foo, DoNotResolve("1")]
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka, True), concept)
 
         variables = evaluated.get_value("prop")
@@ -321,13 +322,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         parser_result = ParserResultConcept(parser="who", value=python_node)
 
         concept = Concept("to_eval").def_var("prop")
-        concept.compiled["prop"] = [ReturnValueConcept("who", True, parser_result)]
+        concept.get_compiled()["prop"] = [ReturnValueConcept("who", True, parser_result)]
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka, True), concept)
         assert evaluated.get_value("prop") == 2
 
         # also works when only one return value
         concept = Concept("to_eval").def_var("prop")
-        concept.compiled["prop"] = ReturnValueConcept("who", True, parser_result)
+        concept.get_compiled()["prop"] = ReturnValueConcept("who", True, parser_result)
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka, True), concept)
         assert evaluated.get_value("prop") == 2
 
@@ -342,7 +343,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, add_instance)
 
         assert evaluated.key == add_instance.key
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().is_evaluated
         assert sheerka.objvalue(evaluated) == 3
 
     def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_and_different_names(self):
@@ -356,7 +357,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, add_instance)
 
         assert evaluated.key == add_instance.key
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().is_evaluated
         assert sheerka.objvalue(evaluated) == 3
 
     def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_multiple_levels(self):
@@ -371,7 +372,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, inc_instance)
 
         assert evaluated.key == inc_instance.key
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().is_evaluated
         assert sheerka.objvalue(evaluated) == 2
 
     def test_i_can_reference_sheerka(self):
@@ -445,12 +446,12 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.key == concept.init_key().key
 
     @pytest.mark.parametrize("where_clause, expected, expected_prop, expected_body", [
-        ("True", True, None, BuiltinConcepts.NOT_INITIALIZED),
-        ("False", False, ConceptParts.WHERE, BuiltinConcepts.NOT_INITIALIZED),
+        ("True", True, None, NotInit),
+        ("False", False, ConceptParts.WHERE, NotInit),
         ("self < 10", False, ConceptParts.WHERE, 10),
         ("self < 11", True, None, 10),
-        ("a < 20", False, "a", BuiltinConcepts.NOT_INITIALIZED),
-        ("a > 19", True, None, BuiltinConcepts.NOT_INITIALIZED),
+        ("a < 20", False, "a", NotInit),
+        ("a > 19", True, None, NotInit),
         ("a + self > 20", True, None, 10),
         ("a + self < 20", False, ConceptParts.WHERE, 10),
     ])
@@ -602,7 +603,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         )
 
         evaluated = sheerka.evaluate_concept(context, plus)
-        assert evaluated.key == BuiltinConcepts.CONDITION_FAILED
+        assert evaluated.key == str(BuiltinConcepts.CONDITION_FAILED)
         assert evaluated.body == "a is an int"
 
     def test_i_can_enable_disable_where_clause_evaluation(self):
@@ -728,18 +729,18 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         one = sheerka.new("one")
         number2 = sheerka.new("number")
-        number2.compiled["one"] = one
-        number2.compiled[ConceptParts.BODY] = one
+        number2.get_compiled()["one"] = one
+        number2.get_compiled()[ConceptParts.BODY] = one
         forties = sheerka.new("forties")
-        forties.compiled["forty"] = sheerka.new("forty")
-        forties.compiled["number"] = number2
+        forties.get_compiled()["forty"] = sheerka.new("forty")
+        forties.get_compiled()["number"] = number2
 
         number1 = sheerka.new("number")
-        number1.compiled["forties"] = forties
-        number1.compiled[ConceptParts.BODY] = forties
+        number1.get_compiled()["forties"] = forties
+        number1.get_compiled()[ConceptParts.BODY] = forties
 
         forty_one_thousand = sheerka.new("thousand")
-        forty_one_thousand.compiled["number"] = number1
+        forty_one_thousand.get_compiled()["number"] = number1
 
         evaluated = sheerka.evaluate_concept(context, forty_one_thousand)
 
@@ -758,15 +759,15 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert "a" in sheerka.locals
 
     @pytest.mark.parametrize("metadata", [
-        "where",
-        "pre",
-        "post",
-        "ret"
+        ConceptParts.WHERE,
+        ConceptParts.PRE,
+        ConceptParts.POST,
+        ConceptParts.RET
     ])
     def test_i_cannot_evaluate_python_statement_in_where_pre_post_ret(self, metadata, capsys):
         sheerka, context, foo = self.init_concepts("foo")
-        setattr(foo.metadata, metadata, "a=10; print('10')")
-        foo.metadata.need_validation = True
+        setattr(foo.get_metadata(), concept_part_value(metadata), "a=10; print('10')")
+        foo.get_metadata().need_validation = True
 
         evaluated = sheerka.evaluate_concept(context, foo, eval_body=True)
         captured = capsys.readouterr()
@@ -774,7 +775,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
         error = evaluated.body
         assert sheerka.isinstance(error, BuiltinConcepts.PYTHON_SECURITY_ERROR)
-        assert error.prop.value == metadata
+        assert error.prop == metadata
         assert error.body == "a=10; print('10')"
         assert captured.out == ""
 
@@ -813,13 +814,14 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("concept, eval_body, expected", [
         (Concept("foo"), False, []),
-        (Concept("foo", pre="pre", post="post", ret="ret", where="where"), False, ["pre", "post"]),
-        (Concept("foo", pre="pr", post="p", ret="r", where="w"), True, ["pre", "ret", "post", "variables", "body"]),
-        (Concept("foo", pre="a").def_var("a"), False, ["variables", "pre"]),
-        (Concept("foo", pre="self"), False, ["body", "pre"]),
-        (Concept("foo", pre="self + a").def_var("a"), False, ["variables", "body", "pre"]),
-        (Concept("foo", pre="self + a", ret="ret").def_var("a"), False, ["variables", "body", "pre"]),
-        (Concept("foo", pre="self + a", ret="ret").def_var("a"), True, ["variables", "body", "pre", "ret"]),
+        (Concept("foo", pre="pre", post="post", ret="ret", where="where"), False, ["#pre#", "#post#"]),
+        (Concept("foo", pre="pr", post="p", ret="r", where="w"), True,
+         ["#pre#", "#ret#", "#post#", "variables", "#body#"]),
+        (Concept("foo", pre="a").def_var("a"), False, ["variables", "#pre#"]),
+        (Concept("foo", pre="self"), False, ["#body#", "#pre#"]),
+        (Concept("foo", pre="self + a").def_var("a"), False, ["variables", "#body#", "#pre#"]),
+        (Concept("foo", pre="self + a", ret="ret").def_var("a"), False, ["variables", "#body#", "#pre#"]),
+        (Concept("foo", pre="self + a", ret="ret").def_var("a"), True, ["variables", "#body#", "#pre#", "#ret#"]),
         (Concept("foo", body="body"), False, [])
     ])
     def test_i_can_compute_metadata_to_eval(self, concept, eval_body, expected):
@@ -839,27 +841,27 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         context = self.get_context(sheerka, eval_where=True)
         concept = Concept("foo", where="where")
         service.initialize_concept_asts(context, concept)
-        assert service.compute_metadata_to_eval(context, concept) == ["where"]
+        assert service.compute_metadata_to_eval(context, concept) == ["#where#"]
 
         concept = Concept("foo", where="where a").def_var("a")
         service.initialize_concept_asts(context, concept)
-        assert service.compute_metadata_to_eval(context, concept) == ["variables", "where"]
+        assert service.compute_metadata_to_eval(context, concept) == ["variables", "#where#"]
 
         concept = Concept("foo", where="where self")
         service.initialize_concept_asts(context, concept)
-        assert service.compute_metadata_to_eval(context, concept) == ["body", "where"]
+        assert service.compute_metadata_to_eval(context, concept) == ["#body#", "#where#"]
 
         context = self.get_context(sheerka, eval_body=True)
         concept = Concept("foo")
-        assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"]
+        assert service.compute_metadata_to_eval(context, concept) == ["variables", "#body#"]
 
         context = self.get_context(sheerka, eval_body=True)
         concept = Concept("foo").def_var("a")
-        assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"]
+        assert service.compute_metadata_to_eval(context, concept) == ["variables", "#body#"]
 
         context = self.get_context(sheerka, eval_body=True)
         concept = Concept("foo", body="body").def_var("a")
-        assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"]
+        assert service.compute_metadata_to_eval(context, concept) == ["variables", "#body#"]
 
     @pytest.mark.parametrize("concept, expected", [
         (Concept("foo"), True),
@@ -870,7 +872,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluated = sheerka.evaluate_concept(context, concept, eval_body=True)
 
         assert evaluated.key == concept.key
-        assert concept.metadata.is_evaluated == expected
+        assert concept.get_metadata().is_evaluated == expected
 
     def test_i_only_compute_the_requested_metadata(self):
         sheerka, context, concept = self.init_concepts(
@@ -879,8 +881,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)  # to prove that we do not care
         context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)  # to prove that we do not care
 
-        evaluated = sheerka.evaluate_concept(context, concept, metadata=['pre'])
-        assert evaluated.values == {"a": Property("a", NotInit), ConceptParts.PRE: Property(ConceptParts.PRE, 'pre')}
+        evaluated = sheerka.evaluate_concept(context, concept, metadata=[ConceptParts.PRE])
+        assert evaluated.values() == {"a": NotInit, ConceptParts.PRE: 'pre'}
 
     def test_i_can_manage_ret(self):
         sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="foo"))
@@ -897,14 +899,20 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         res = sheerka.evaluate_concept(context, bar, eval_body=False)
         assert res.id == bar.id
 
+    def test_i_can_eval_concept_with_rules(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo a", body="a.name").def_var("a", "r:|1:"))
+
+        res = sheerka.evaluate_concept(context, foo, eval_body=True)
+        assert res.body == "Print return values"
+
     # I cannot implement value cache for now
     # def test_values_when_no_variables_are_computed_only_once(self):
     #     sheerka, context, foo = self.init_concepts(Concept("foo", body="10"))
     #
     #     evaluated = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True)
     #     assert evaluated.body == 10
-    #     assert len(evaluated.compiled) > 0
+    #     assert len(evaluated.get_compiled()) > 0
     #
     #     evaluated_2 = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True)
     #     assert evaluated_2.body == 10
-    #     assert len(evaluated_2.compiled) == 0
+    #     assert len(evaluated_2.get_compiled()) == 0

tests/core/test_SheerkaEvaluateRules.py

@@ -0,0 +1,74 @@
+import operator
+
+from core.concept import Concept
+from core.rule import Rule
+from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules, LOW_PRIORITY_RULES, DISABLED_RULES
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
+    def test_i_can_evaluate_python_rules(self):
+        sheerka, context, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_format_rules(
+            Rule(predicate="a == 1", action="", priority=1),  # r1
+            Rule(predicate="a == 2", action="", priority=1),  # r2
+            Rule(predicate="a == 3", action="", priority=0),  # r3
+            Rule(predicate="a == 4", action="", priority=0),  # r4
+            Rule(predicate="a == 5", action="", priority=0),  # r5
+            Rule(predicate="a == 1", action="", priority=1),  # r6
+            Rule(predicate="a == 7", action="", priority=1, is_enabled=False),  # r7
+            Rule(predicate="a == 8", action="", priority=1),  # r8
+            Rule(predicate="a == 9", action="", priority=2),  # r9
+        )
+        service = sheerka.services[SheerkaEvaluateRules.NAME]
+        rules = sorted([r1, r2, r3, r4, r5, r6, r7, r8, r9], key=operator.attrgetter('priority'), reverse=True)
+
+        res = service.evaluate_rules(context, rules, {"a": 1}, set())
+
+        assert res == {
+            True: [r1, r6],
+            False: [r9, r2, r8],
+            LOW_PRIORITY_RULES: [r3, r4, r5],
+            DISABLED_RULES: [r7]
+        }
+
+    def test_i_can_evaluate_concept_rules(self):
+        sheerka, context, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_format_rules(
+            Rule(predicate="a equals 1", action="", priority=1),  # r1
+            Rule(predicate="a equals 2", action="", priority=1),  # r2
+            Rule(predicate="a equals 3", action="", priority=0),  # r3
+            Rule(predicate="a equals 4", action="", priority=0),  # r4
+            Rule(predicate="a equals 5", action="", priority=0),  # r5
+            Rule(predicate="a equals 1", action="", priority=1),  # r6
+            Rule(predicate="a equals 7", action="", priority=1, is_enabled=False),  # r7
+            Rule(predicate="a equals 8", action="", priority=1),  # r8
+            Rule(predicate="a equals 9", action="", priority=2),  # r9
+            concepts=[Concept("x equals y", body="x == y").def_var("x").def_var("y")],
+        )
+
+        service = sheerka.services[SheerkaEvaluateRules.NAME]
+        rules = sorted([r1, r2, r3, r4, r5, r6, r7, r8, r9], key=operator.attrgetter('priority'), reverse=True)
+
+        res = service.evaluate_rules(context, rules, {"a": 1}, set())
+
+        assert res == {
+            True: [r1, r6],
+            False: [r9, r2, r8],
+            LOW_PRIORITY_RULES: [r3, r4, r5],
+            DISABLED_RULES: [r7]
+        }
+
+    def test_i_can_disable_rules_at_runtime(self):
+        sheerka, context, r1, r2, = self.init_format_rules(
+            Rule(predicate="a == 1", action="", priority=2),  # r1
+            Rule(predicate="a == 1", action="", priority=1),  # r2
+        )
+        service = sheerka.services[SheerkaEvaluateRules.NAME]
+        rules = sorted([r1, r2], key=operator.attrgetter('priority'), reverse=True)
+
+        res = service.evaluate_rules(context, rules, {"a": 1}, {r1.id})
+
+        assert res == {
+            True: [r2],
+            DISABLED_RULES: [r1]
+        }

tests/core/test_SheerkaEventManager.py

@@ -0,0 +1,66 @@
+from core.sheerka.services.SheerkaEventManager import SheerkaEventManager
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def example_of_function(context):
+    print(f"example_of_class_method. event={context.event.get_digest()}")
+
+
+def example_of_function_with_data(context, data):
+    print(f"example_of_class_method. event={context.event.get_digest()}, {data=}")
+
+
+class TestSheerkaEventManager(TestUsingMemoryBasedSheerka):
+
+    def example_of_class_method(self, context):
+        print(f"example_of_class_method. event={context.event.get_digest()}")
+
+    @staticmethod
+    def example_of_static_method(context):
+        print(f"example_of_static_method. event={context.event.get_digest()}")
+
+    def example_of_class_method_with_data(self, context, data):
+        print(f"example_of_class_method. event={context.event.get_digest()}, {data=}")
+
+    @staticmethod
+    def example_of_static_method_with_data(context, data):
+        print(f"example_of_static_method. event={context.event.get_digest()}, {data=}")
+
+    def test_i_can_subscribe_and_publish(self, capsys):
+        sheerka, context = self.init_concepts()
+        topic = "my topic"
+
+        sheerka.subscribe(topic, self.example_of_class_method)
+        sheerka.subscribe(topic, self.example_of_static_method)
+        sheerka.subscribe(topic, example_of_function)
+
+        sheerka.publish(context, topic)
+
+        captured = capsys.readouterr()
+        assert captured.out == """example_of_class_method. event=xxx
+example_of_static_method. event=xxx
+example_of_class_method. event=xxx
+"""
+
+        service = sheerka.services[SheerkaEventManager.NAME]
+        service.reset_topic(topic)
+
+    def test_i_can_subscribe_and_publish_with_data(self, capsys):
+        sheerka, context = self.init_concepts()
+        topic = "my topic"
+
+        sheerka.subscribe(topic, self.example_of_class_method_with_data)
+        sheerka.subscribe(topic, self.example_of_static_method_with_data)
+        sheerka.subscribe(topic, example_of_function_with_data)
+
+        sheerka.publish(context, topic, "42")
+
+        captured = capsys.readouterr()
+        assert captured.out == """example_of_class_method. event=xxx, data='42'
+example_of_static_method. event=xxx, data='42'
+example_of_class_method. event=xxx, data='42'
+"""
+
+        service = sheerka.services[SheerkaEventManager.NAME]
+        service.reset_topic(topic)

tests/core/test_SheerkaHistoryManager.py

@@ -32,7 +32,8 @@ class TestSheerkaHistoryManager(TestUsingMemoryBasedSheerka):
             hist("xxx", False),
             hist("one", True),
             hist("def concept one as 1", True),
-            hist("Initializing Sheerka.", None)]
+            hist("Initializing Sheerka.", None)
+        ]
 
         h = list(sheerka.history(2))
         assert h == [

tests/core/test_SheerkaMemory.py

@@ -15,7 +15,7 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         foo = Concept("foo")
         sheerka.add_to_short_term_memory(None, "a", foo)
 
-        assert service.short_term_objects.copy() == {":a": foo}
+        assert service.short_term_objects.copy() == {'global': {'a': foo}}
         assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo)
 
     def test_i_can_add_context_short_term_memory(self):
@@ -26,23 +26,36 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
         sheerka.add_to_short_term_memory(context, "a", foo)
 
         context_id = ExecutionContext.ids[context.event.get_digest()]
-        assert service.short_term_objects.copy() == {f"{context_id}:a": foo}
+        assert service.short_term_objects.copy() == {context_id: {'a': foo}}
         assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
         assert sheerka.get_from_short_term_memory(None, "a") is None
 
+    def test_i_can_add_many(self):
+        sheerka, context = self.init_concepts()
+        bag = {"a": "foo", "b": "bar", }
+        context_id = ExecutionContext.ids[context.event.get_digest()]
+        service = sheerka.services[SheerkaMemory.NAME]
+
+        sheerka.add_many_to_short_term_memory(context, bag)
+        assert service.short_term_objects.copy() == {context_id: bag}
+
     def test_i_can_get_obj_from_parents(self):
         sheerka, context = self.init_concepts()
-        service = sheerka.services[SheerkaMemory.NAME]
         foo = Concept("foo")
+
         sheerka.add_to_short_term_memory(None, "a", foo)
+        sheerka.add_to_short_term_memory(context, "b", foo)
 
         with context.push(BuiltinConcepts.TESTING, None) as sub_context:
-            assert service.short_term_objects.copy() == {":a": foo}
             assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
             assert id(sheerka.get_from_short_term_memory(context, "a")) == id(foo)
             assert id(sheerka.get_from_short_term_memory(None, "a")) == id(foo)
 
-    def test_entry_are_removed_on_context_exit(self):
+            assert id(sheerka.get_from_short_term_memory(sub_context, "b")) == id(foo)
+            assert id(sheerka.get_from_short_term_memory(context, "b")) == id(foo)
+            assert sheerka.get_from_short_term_memory(None, "b") is None
+
+    def test_short_term_memory_entries_are_removed_on_context_exit(self):
         sheerka, context = self.init_concepts()
 
         with context.push(BuiltinConcepts.TESTING, None) as sub_context:
@@ -52,6 +65,17 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
 
         assert sheerka.get_from_short_term_memory(sub_context, "a") is None
 
+    def test_short_term_memory_entries_are_removed_on_context_exit_2(self):
+        # this time we test the bulk insert
+        sheerka, context = self.init_concepts()
+
+        with context.push(BuiltinConcepts.TESTING, None) as sub_context:
+            foo = Concept("foo")
+            sheerka.add_many_to_short_term_memory(sub_context, {"a": foo})
+            assert id(sheerka.get_from_short_term_memory(sub_context, "a")) == id(foo)
+
+        assert sheerka.get_from_short_term_memory(sub_context, "a") is None
+
     def test_i_can_add_and_retrieve_from_memory(self):
         sheerka, context = self.init_concepts()
         service = sheerka.services[SheerkaMemory.NAME]

tests/core/test_SheerkaModifyConcept.py

@@ -1,5 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts
+from core.concept import Concept, ConceptParts, get_concept_attrs
 
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -10,8 +10,10 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka):
     def test_i_can_modify_a_concept(self):
         sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True, cache_only=False)
 
+        assert get_concept_attrs(foo) == []
+
         foo_instance = sheerka.new("foo")
-        foo_instance.metadata.body = "metadata value"  # modify metadata
+        foo_instance.get_metadata().body = "metadata value"  # modify metadata
         foo_instance.def_var("var_name", "default value")  # modify definition of variables
         foo_instance.add_prop(BuiltinConcepts.ISA, bar)  # modify property
         foo_instance.set_value(ConceptParts.BODY, "body value")  # modify value
@@ -20,30 +22,31 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka):
 
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
-        assert res.body.body.metadata.body == "metadata value"
-        assert res.body.body.metadata.variables == [("var_name", "default value")]
+        assert res.body.body.get_metadata().body == "metadata value"
+        assert res.body.body.get_metadata().variables == [("var_name", "default value")]
         assert res.body.body.get_prop(BuiltinConcepts.ISA) == {bar}
         assert res.body.body.body == "body value"
         assert res.body.body.get_value("var_name") == "var value"
+        assert get_concept_attrs(foo) == ["var_name"]
 
         # test that object
         foo_from_sheerka = sheerka.get_by_key("foo")
-        assert foo_from_sheerka.metadata.body == "metadata value"
-        assert foo_from_sheerka.metadata.variables == [("var_name", "default value")]
+        assert foo_from_sheerka.get_metadata().body == "metadata value"
+        assert foo_from_sheerka.get_metadata().variables == [("var_name", "default value")]
         assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == {bar}
         assert foo_from_sheerka.body == "body value"
         assert foo_from_sheerka.get_value("var_name") == "var value"
 
         # other caches are also updated
-        assert sheerka.get_by_id(foo.id).metadata.body == "metadata value"
-        assert sheerka.get_by_name(foo.name).metadata.body == "metadata value"
-        assert sheerka.get_by_hash(foo_instance.get_definition_hash()).metadata.body == "metadata value"
+        assert sheerka.get_by_id(foo.id).get_metadata().body == "metadata value"
+        assert sheerka.get_by_name(foo.name).get_metadata().body == "metadata value"
+        assert sheerka.get_by_hash(foo_instance.get_definition_hash()).get_metadata().body == "metadata value"
 
         # sdp can be updated
         sheerka.cache_manager.commit(context)
         from_sdp = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, foo.id)
-        assert from_sdp.metadata.body == "metadata value"
-        assert from_sdp.metadata.variables == [("var_name", "default value")]
+        assert from_sdp.get_metadata().body == "metadata value"
+        assert from_sdp.get_metadata().variables == [("var_name", "default value")]
         assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}
         assert from_sdp.body == "body value"
         assert from_sdp.get_value("var_name") == "var value"
@@ -81,16 +84,16 @@ class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka):
             Concept("foo", body="2"), create_new=True)
 
         foo2_instance = sheerka.new("foo")[1]
-        foo2_instance.metadata.body = "value"
+        foo2_instance.get_metadata().body = "value"
 
         res = sheerka.modify_concept(context, foo2_instance)
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
-        assert res.body.body.metadata.body == "value"
+        assert res.body.body.get_metadata().body == "value"
 
         foo_from_sheerka = sheerka.new("foo")
-        assert foo_from_sheerka[0].metadata.body == "1"
-        assert foo_from_sheerka[1].metadata.body == "value"
+        assert foo_from_sheerka[0].get_metadata().body == "1"
+        assert foo_from_sheerka[1].get_metadata().body == "value"
 
     def test_i_can_get_and_set_attribute(self):
         sheerka, context = self.init_concepts()

tests/core/test_SheerkaRuleManager.py

@@ -0,0 +1,355 @@
+import ast
+
+import pytest
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.global_symbols import RULE_COMPARISON_CONTEXT
+from core.rule import Rule
+from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleParser, \
+    FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \
+    FormatRuleSyntaxError, FormatAstList, UnexpectedEof, FormatAstColor, RulePredicate
+from core.tokenizer import Token, TokenKind
+from parsers.BaseNodeParser import SourceCodeWithConceptNode, SourceCodeNode
+from parsers.PythonParser import PythonNode
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+seq = FormatAstSequence
+raw = FormatAstRawText
+var = FormatAstVariable
+func = FormatAstFunction
+lst = FormatAstList
+
+PYTHON_EVALUATOR_NAME = "Python"
+CONCEPT_EVALUATOR_NAME = "Concept"
+
+
+class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("action_type, cache_entry", [
+        ("print", SheerkaRuleManager.FORMAT_RULE_ENTRY),
+        ("exec", SheerkaRuleManager.EXEC_RULE_ENTRY),
+    ])
+    def test_i_can_create_a_new_rule(self, action_type, cache_entry):
+        sheerka, context = self.init_concepts(cache_only=False)
+        previous_rules_number = sheerka.cache_manager.caches[sheerka.CONCEPTS_KEYS_ENTRY].cache.copy()[
+            SheerkaRuleManager.RULE_IDS]
+
+        rule = Rule(action_type, "name", "True", "Hello world")
+
+        res = sheerka.create_new_rule(context, rule)
+        expected_id = str(previous_rules_number + 1)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_RULE)
+
+        created_rule = res.body.body
+        assert created_rule.metadata.id == expected_id
+        assert created_rule.metadata.name == "name"
+        assert created_rule.metadata.predicate == "True"
+        assert created_rule.metadata.action_type == action_type
+        assert created_rule.metadata.action == "Hello world"
+
+        # saved in cache
+        assert len(sheerka.cache_manager.caches[cache_entry].cache) > 0
+        from_cache = sheerka.cache_manager.get(cache_entry, expected_id)
+        assert from_cache.metadata.id == expected_id
+        assert from_cache.metadata.name == "name"
+        assert from_cache.metadata.predicate == "True"
+        assert from_cache.metadata.action_type == action_type
+        assert from_cache.metadata.action == "Hello world"
+
+        sheerka.cache_manager.commit(context)
+
+        # saved in sdp
+        from_sdp = sheerka.sdp.get(cache_entry, expected_id)
+        assert from_sdp.metadata.id == expected_id
+        assert from_sdp.metadata.name == "name"
+        assert from_sdp.metadata.predicate == "True"
+        assert from_sdp.metadata.action_type == action_type
+        assert from_sdp.metadata.action == "Hello world"
+
+    def test_i_can_create_multiple_rules(self):
+        sheerka, context = self.init_concepts(cache_only=False)
+        previous_rules_number = len(sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache)
+
+        sheerka.create_new_rule(context, Rule("print", "name1", "True", "Hello world"))
+        sheerka.create_new_rule(context, Rule("print", "name2", "value() is __EXPLANATION", "list(value())"))
+
+        assert len(
+            sheerka.cache_manager.caches[SheerkaRuleManager.FORMAT_RULE_ENTRY].cache) == 2 + previous_rules_number
+
+    @pytest.mark.parametrize("text, expected", [
+        ("", FormatAstRawText("")),
+        (" ", FormatAstRawText(" ")),
+        ("  raw text  ", FormatAstRawText("  raw text  ")),
+        ("{variable}", FormatAstVariable("variable")),
+        ("{ variable }", FormatAstVariable("variable")),
+        (" xy {v} z", seq([raw(" xy "), var("v"), raw(" z")])),
+        (r"\{variable}", FormatAstRawText("{variable}")),
+        (r"\\{variable}", seq([raw("\\"), var("variable")])),
+        (r"\\\{variable}", FormatAstRawText(r"\{variable}")),
+        (r"{var1}{var2}", seq([var("var1"), var("var2")])),
+        ("func()", FormatAstFunction("func", [], {})),
+        ("func(a, 'string value', c)", FormatAstFunction("func", ["a", "'string value'", "c"], {})),
+        ("func(a=10, b='string value')", FormatAstFunction("func", [], {"a": "10", "b": "'string value'"})),
+        ("func('string value'='another string value')", func("func", [], {"'string value'": "'another string value'"})),
+        ("red(' xy {v}')", FormatAstColor("red", seq([raw(" xy "), var("v")]))),
+        ('blue(" xy {v}")', FormatAstColor("blue", seq([raw(" xy "), var("v")]))),
+        ('green(  xy )', FormatAstColor("green", var("xy"))),
+        ('green()', FormatAstColor("green", raw(""))),
+        ('green("")', FormatAstColor("green", raw(""))),
+        ("list(var_name, 2, 'children')", FormatAstList("var_name", recurse_on="children", recursion_depth=2)),
+        ("list(var_name, recursion_depth=2, recurse_on='children')", FormatAstList("var_name",
+                                                                                 recurse_on="children",
+                                                                                 recursion_depth=2)),
+        ("list(var_name, recursion_depth=2, 'children')", FormatAstList("var_name", recursion_depth=2)),
+        ("list(var_name, 'children', recursion_depth=2)", FormatAstList("var_name", recursion_depth=2)),
+        ("list(var_name)", FormatAstList("var_name")),
+        ("{obj.prop1.prop2[0].prop3['value']}", FormatAstVariable("obj.prop1.prop2[0].prop3['value']")),
+        ("[{id}]", seq([raw("["), var("id"), raw("]")])),
+        ("{variable:format}", FormatAstVariable("variable", "format")),
+        ("{variable:3}", FormatAstVariable("variable", "3")),
+        (r"\not_a_function(a={var})", seq([raw("not_a_function(a="), var("var"), raw(")")])),
+    ])
+    def test_i_can_parse_format_rule(self, text, expected):
+        assert FormatRuleParser(text).parse() == expected
+
+    @pytest.mark.parametrize("text, expected_error", [
+        ("{", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
+        ("{var_name", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))),
+        ("{}", FormatRuleSyntaxError("variable name not found", None)),
+        ("func(", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
+        ("func(a,b,c", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))),
+        ("func(a,,c", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
+        ("func(a,,c)", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))),
+        ("red(a,b)", FormatRuleSyntaxError("only one parameter supported", Token(TokenKind.IDENTIFIER, "b", 6, 1, 7))),
+        ("red(a=b)", FormatRuleSyntaxError("keyword arguments are not supported", None)),
+        ("red(xy {v})", FormatRuleSyntaxError("Invalid identifier", None)),
+        ("list()", FormatRuleSyntaxError("variable name not found", None)),
+        ("list(recursion_depth=2)", FormatRuleSyntaxError("variable name not found", None)),
+        ("list(a,b,c,d)", FormatRuleSyntaxError("too many positional arguments",
+                                                Token(TokenKind.IDENTIFIER, "d", 11, 1, 12))),
+        ("list(a, recursion_depth=hello)", FormatRuleSyntaxError("'hello' is not numeric", None)),
+        ("list(a, recursion_depth='hello')", FormatRuleSyntaxError("'recursion_depth' must be an integer", None)),
+    ])
+    def test_i_cannot_parse_invalid_format(self, text, expected_error):
+        parser = FormatRuleParser(text)
+        parser.parse()
+
+        assert parser.error_sink == expected_error
+
+    @pytest.mark.parametrize("text", [
+        "a == 5",
+        "foo == 5",
+        "func() == 5",
+    ])
+    def test_i_can_compile_predicate_when_pure_python(self, text):
+        sheerka, context, *concepts = self.init_concepts("foo")
+        service = sheerka.services[SheerkaRuleManager.NAME]
+        ast_ = ast.parse(text, "<source>", 'eval')
+        expected_python_node = PythonNode(text, ast_)
+
+        res = service.compile_when(context, "test", text)
+
+        assert len(res) == 1
+        assert isinstance(res[0], RulePredicate)
+        assert res[0].evaluator == PYTHON_EVALUATOR_NAME
+        assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.objvalue(res[0].predicate) == expected_python_node
+        assert res[0].concept is None
+
+    @pytest.mark.parametrize("text, expected_type", [
+        ("isinstance(a, int)", SourceCodeWithConceptNode),
+        ("func()", SourceCodeNode),
+    ])
+    def test_i_can_compile_predicates_that_resolve_to_python(self, text, expected_type):
+        sheerka, context, *concepts = self.init_concepts()
+        service = sheerka.services[SheerkaRuleManager.NAME]
+        ast_ = ast.parse(text, "<source>", 'eval')
+        expected_python_node = PythonNode(text, ast_)
+
+        res = service.compile_when(context, "test", text)
+
+        assert len(res) == 1
+        assert isinstance(res[0], RulePredicate)
+        assert res[0].evaluator == PYTHON_EVALUATOR_NAME
+        assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
+        assert isinstance(sheerka.objvalue(res[0].predicate), expected_type)
+        assert sheerka.objvalue(res[0].predicate).python_node == expected_python_node
+        assert res[0].concept is None
+
+    def test_i_can_compile_predicate_when_python_and_concept(self):
+        sheerka, context, *concepts = self.init_concepts(Concept("foo bar"), create_new=True)
+        service = sheerka.services[SheerkaRuleManager.NAME]
+        text = "foo bar == 5"
+        ast_ = ast.parse("__C__foo0bar__1001__C__ == 5", "<source>", 'eval')
+        resolved_expected = PythonNode(text, ast_)
+
+        res = service.compile_when(context, "test", text)
+
+        assert len(res) == 1
+        assert isinstance(res[0], RulePredicate)
+        assert res[0].evaluator == PYTHON_EVALUATOR_NAME
+        assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.objvalue(res[0].predicate) == resolved_expected
+        assert res[0].concept is None
+
+    @pytest.mark.parametrize("text, expected_variables", [
+        ("cat is an animal", ["cat", "animal"]),
+        ("a is an animal", ["a", "animal"]),
+        ("cat is an b", ["cat", "b"]),
+    ])
+    def test_i_can_compile_predicate_when_exact_concept(self, text, expected_variables):
+        sheerka, context, *concepts = self.init_concepts(
+            Concept("x is an y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+            Concept("cat"),
+            Concept("animal"),
+            create_new=True
+        )
+        service = sheerka.services[SheerkaRuleManager.NAME]
+
+        expected = concepts[0]
+        expected.get_metadata().variables = [('x', expected_variables[0]), ('y', expected_variables[1])]
+
+        res = service.compile_when(context, "test", text)
+
+        assert len(res) == 1
+        assert isinstance(res[0], RulePredicate)
+        assert res[0].evaluator == CONCEPT_EVALUATOR_NAME
+        assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.objvalue(res[0].predicate) == expected
+        assert res[0].concept == expected
+
+    @pytest.mark.parametrize("text, expected_variables", [
+        ("a cat is an animal", ["cat", "animal"]),
+        ("a cat is an b", ["a", "animal"]),
+    ])
+    def test_i_can_compile_predicate_when_sya_node_parser(self, text, expected_variables):
+        sheerka, context, *concepts = self.init_concepts(
+            Concept("x is an y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+            Concept("a cat"),
+            Concept("animal"),
+            create_new=True
+        )
+        service = sheerka.services[SheerkaRuleManager.NAME]
+        expected = concepts[0]
+
+        res = service.compile_when(context, "test", text)
+
+        assert len(res) == 1
+        assert isinstance(res[0], RulePredicate)
+        assert res[0].evaluator == CONCEPT_EVALUATOR_NAME
+        assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.objvalue(res[0].predicate)[0].concept == expected
+        assert res[0].concept == expected
+
+    def test_i_can_compile_predicate_when_bnf_node_parser(self):
+        sheerka, context, *concepts = self.init_concepts(
+            Concept("animal"),
+            Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
+            create_new=True
+        )
+        service = sheerka.services[SheerkaRuleManager.NAME]
+        expected = concepts[1]
+
+        res = service.compile_when(context, "test", "cat is an animal")
+
+        assert len(res) == 1
+        assert isinstance(res[0], RulePredicate)
+        assert res[0].evaluator == CONCEPT_EVALUATOR_NAME
+        assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.objvalue(res[0].predicate)[0].concept == expected
+        assert res[0].concept == expected
+
+    def test_i_can_compile_predicate_when_multiple_choices(self):
+        sheerka, context, *concepts = self.init_concepts(
+            Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+            Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
+            create_new=True
+        )
+        service = sheerka.services[SheerkaRuleManager.NAME]
+
+        res = service.compile_when(context, "test", "a is a b")
+
+        assert len(res) == 2
+        assert isinstance(res[0], RulePredicate)
+        assert res[0].evaluator == CONCEPT_EVALUATOR_NAME
+        assert sheerka.isinstance(res[0].predicate, BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.objvalue(res[0].predicate)[0].concept == concepts[0]
+        assert res[0].concept == concepts[0]
+
+        assert isinstance(res[1], RulePredicate)
+        assert res[1].evaluator == CONCEPT_EVALUATOR_NAME
+        assert sheerka.isinstance(res[1].predicate, BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.objvalue(res[1].predicate)[0].concept == concepts[1]
+        assert res[1].concept == concepts[1]
+
+    # @pytest.mark.skip
+    # @pytest.mark.parametrize("text, expected", [
+    #     ("cat is an animal", set()),
+    #     ("a is an animal", {"a"}),
+    #     ("a is an b", {"a", "b"}),
+    #     ("cat is an b", {"b"}),
+    #     ("a cat is an b", {"b"}),
+    #
+    #     ("cat is a animal", set()),
+    #     ("a is a animal", {"a"}),
+    #     ("a is a b", {"a", "b"}),
+    #     ("cat is a b", {"b"}),
+    #     ("a cat is an b", {"b"}),
+    #
+    #     ("a == 5", {"a"}),
+    #     ("isinstance(a, int)", {"a"}),
+    #     ("a cat == b", {"b"})
+    # ])
+    # def test_i_can_get_rules_variables(self, text, expected):
+    #     sheerka, context, *concepts = self.init_concepts(
+    #         Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+    #         Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
+    #         Concept("x is an y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+    #         Concept("cat"),
+    #         Concept("animal"),
+    #         Concept("a cat"),
+    #         create_new=True
+    #     )
+    #     service = sheerka.services[SheerkaRuleManager.NAME]
+    #
+    #     compiled = service.compile_when(context, "test", "a is a b")
+    #
+    #     assert service.get_unknown_variables(compiled) == expected
+
+
+class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
+    def test_rules_are_initialized_at_startup(self):
+        sheerka, context, *rules = self.init_format_rules(
+            Rule("print", "name1", "True", "Hello world"),
+            Rule("print", "name2", "value() is __EXPLANATION", "list(value())")
+        )
+        sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE,
+                                    rules[0],
+                                    rules[1],
+                                    RULE_COMPARISON_CONTEXT)
+
+        sheerka.cache_manager.commit(context)
+        assert len(sheerka.cache_manager.copy(SheerkaRuleManager.FORMAT_RULE_ENTRY)) == len(rules)
+
+        sheerka = self.get_sheerka()  # new instance
+        assert len(sheerka.cache_manager.copy(SheerkaRuleManager.FORMAT_RULE_ENTRY)) == len(rules)
+
+        # manually update the rules (I need their new priorities)
+        service = sheerka.services[SheerkaRuleManager.NAME]
+        rules = [service.format_rule_cache.get(rule_id) for rule_id in service.format_rule_cache]
+
+        # check if the rules are correctly initialized
+        rules_as_map = {rule.id: rule for rule in rules}
+        for rule_id in service.format_rule_cache:
+            actual = service.format_rule_cache.get(rule_id)
+            expected = rules_as_map[rule_id]
+            assert actual.metadata.is_compiled == expected.metadata.is_compiled
+            assert actual.metadata.is_enabled == expected.metadata.is_enabled
+            assert actual.compiled_action == expected.compiled_action
+            assert actual.compiled_predicate == expected.compiled_predicate
+            assert actual.priority is not None
+            assert actual.priority == expected.priority

tests/core/test_SheerkaSetsManager.py

@@ -268,12 +268,12 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
             create_new=True
         )
 
-        assert twenties.bnf.elements[1].recurse_id is None
+        assert twenties.get_bnf().elements[1].recurse_id is None
 
         # update number
         sheerka.set_isa(context, sheerka.new("one"), number)
 
-        assert twenties.bnf.elements[1].recurse_id == "1003#1002(number)"
+        assert twenties.get_bnf().elements[1].recurse_id == "1003#1002(number)"
 
     def test_concepts_in_group_cache_is_updated(self):
         sheerka, context, one, two, number = self.init_concepts("one", "two", "number")
@@ -306,7 +306,8 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
         assert sheerka.add_concept_to_set(context, foo, group).status
         sheerka.cache_manager.commit(context)
 
-        sheerka = self.get_sheerka()  # another session
+        sheerka = self.get_sheerka(reset_attrs=False)  # another session
+        context = self.get_context(sheerka)
         assert sheerka.add_concept_to_set(context, bar, group).status
 
         # I can get the elements
@@ -319,7 +320,8 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
         }
 
         # I can also add a group another elements
-        sheerka = self.get_sheerka()
+        sheerka = self.get_sheerka(reset_attrs=False)
+        context = self.get_context(sheerka)
         foo3 = Concept("foo3")
         foo4 = Concept("foo4")
         for c in [foo3, foo4]:
@@ -348,19 +350,19 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
         # nothing was previously in ISA
         foo = sheerka.new(foo.key)
-        assert BuiltinConcepts.ISA not in foo.metadata.props
+        assert BuiltinConcepts.ISA not in foo.get_metadata().props
         res = sheerka.set_isa(context, foo, group)
         assert res.status
         sheerka.cache_manager.commit(context)
 
-        sheerka = self.get_sheerka()
+        sheerka = self.get_sheerka(reset_attrs=False)
         assert foo.get_prop(BuiltinConcepts.ISA) == {group}
         assert sheerka.isa(foo, group)
         assert sheerka.isinset(foo, group)
         assert sheerka.isaset(context, group)
 
         # I can do the same for bar
-        sheerka = self.get_sheerka()
+        sheerka = self.get_sheerka(reset_attrs=False)
         res = sheerka.set_isa(context, bar, group)
         assert res.status
         assert bar.get_prop(BuiltinConcepts.ISA) == {group}
@@ -371,7 +373,7 @@ class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
         sheerka.cache_manager.commit(context)
 
         # they are both in the same group
-        sheerka = self.get_sheerka()
+        sheerka = self.get_sheerka(reset_attrs=False)
         all_entries = sheerka.sdp.get(SheerkaSetsManager.CONCEPTS_GROUPS_ENTRY)
         assert all_entries == {
             group.id: {foo.id, bar.id}

tests/core/test_SheerkaVariableManager.py

@@ -1,6 +1,7 @@
 from core.concept import Concept, ConceptParts
 from core.sheerka.services.SheerkaVariableManager import SheerkaVariableManager
 
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -9,8 +10,8 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
         sheerka = self.get_sheerka(cache_only=False)
         context = self.get_context(sheerka)
 
-        sheerka.record(context, "TestSheerkaVariable", "my_variable", 1)
-        res = sheerka.load("TestSheerkaVariable", "my_variable")
+        sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1)
+        res = sheerka.load_var("TestSheerkaVariable", "my_variable")
         assert res == 1
 
         # I can persist in db
@@ -30,8 +31,8 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
 
         concept = Concept("foo").set_value("a", "alpha").set_value(ConceptParts.BODY, 3.14)
 
-        sheerka.record(context, "TestSheerkaVariable", "my_variable", concept)
-        res = sheerka.load("TestSheerkaVariable", "my_variable")
+        sheerka.record_var(context, "TestSheerkaVariable", "my_variable", concept)
+        res = sheerka.load_var("TestSheerkaVariable", "my_variable")
 
         assert res == concept
         assert res.body == concept.body
@@ -43,19 +44,19 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
 
         concept = Concept("foo")
 
-        sheerka.record(context, "TestSheerkaVariable", "my_variable", concept)
-        assert sheerka.load("TestSheerkaVariable", "my_variable") is not None
+        sheerka.record_var(context, "TestSheerkaVariable", "my_variable", concept)
+        assert sheerka.load_var("TestSheerkaVariable", "my_variable") is not None
 
-        sheerka.delete(context, "TestSheerkaVariable", "my_variable")
-        assert sheerka.load("TestSheerkaVariable", "my_variable") is None
+        sheerka.delete_var(context, "TestSheerkaVariable", "my_variable")
+        assert sheerka.load_var("TestSheerkaVariable", "my_variable") is None
 
     def test_i_can_set_and_get_a_value(self):
         sheerka = self.get_sheerka(cache_only=False)
         context = self.get_context(sheerka)
         context.event.user_id = "Test_user"
 
-        sheerka.set(context, "my_variable", "my value")
-        res = sheerka.get(context, "my_variable")
+        sheerka.set_var(context, "my_variable", "my value")
+        res = sheerka.get_var(context, "my_variable")
         assert res == "my value"
 
         # I can persist in db
@@ -69,13 +70,33 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
         assert loaded.who == "Test_user"
         assert loaded.parents is None
 
+
+class TestSheerkaVariableUsingFileBasedSdp(TestUsingFileBasedSheerka):
+
+    def test_i_can_bound_variables_to_sheerka(self):
+        sheerka, context = self.init_concepts()
+
+        old_value = sheerka.enable_process_return_values
+        new_value = not old_value
+        sheerka.record_var(context, "TestSheerkaVariable", "sheerka.enable_process_return_values", new_value)
+        sheerka.cache_manager.commit(context)
+
+        assert sheerka.enable_process_return_values == new_value
+
+        # the modification is persisted upon new Sheerka creation
+        sheerka = self.get_sheerka()
+        assert sheerka.enable_process_return_values == new_value
+
+        # reset old value
+        sheerka.enable_process_return_values = old_value
+
     # def test_i_can_get_the_parent_when_modified(self):
     #     sheerka = self.get_sheerka()
     #     context = self.get_context(sheerka)
     #
-    #     sheerka.record(context, "TestSheerkaVariable", "my_variable", 1)
-    #     sheerka.record(context, "TestSheerkaVariable", "my_variable", 2)
-    #     res = sheerka.load("TestSheerkaVariable", "my_variable")
+    #     sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1)
+    #     sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 2)
+    #     res = sheerka.load_var("TestSheerkaVariable", "my_variable")
     #     assert res == 2
     #
     #     loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
@@ -96,8 +117,8 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
     #     sheerka = self.get_sheerka(singleton=True)
     #     context = self.get_context(sheerka)
     #
-    #     sheerka.record(context, "TestSheerkaVariable", "my_variable", 1)
-    #     sheerka.record(context, "TestSheerkaVariable", "my_variable", 1)
+    #     sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1)
+    #     sheerka.record_var(context, "TestSheerkaVariable", "my_variable", 1)
     #
     #     loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
     #     assert loaded.event_id == context.event.get_digest()

tests/core/test_ast.py

@@ -1,174 +0,0 @@
-import ast
-
-import core.ast.nodes
-import pytest
-from core.ast.nodes import NodeParent, GenericNodeConcept
-from core.ast.visitors import ConceptNodeVisitor, UnreferencedNamesVisitor
-from core.builtin_concepts import BuiltinConcepts
-from core.sheerka.Sheerka import Sheerka
-
-
-def get_sheerka():
-    sheerka = Sheerka(cache_only=True)
-    sheerka.initialize("mem://")
-
-    return sheerka
-
-
-class TestNameVisitor(ConceptNodeVisitor):
-    """
-    Test class for a basic Visitor test
-    """
-
-    def __init__(self):
-        self.names = []
-
-    def visit_Name(self, node):
-        self.names.append(node)
-
-
-def test_i_can_transform_simple_ast_using_generic_node():
-    source = """
-def my_function(a,b):
-    for i in range(b):
-        a = a+b
-    return a
-"""
-    tree = ast.parse(source)
-    tree_as_concept = core.ast.nodes.python_to_concept(tree)
-    sheerka = get_sheerka()
-
-    assert tree_as_concept.node_type == "Module"
-    assert sheerka.isinstance(tree_as_concept.get_value("body"), BuiltinConcepts.LIST)
-
-    def_func = tree_as_concept.get_value("body").body[0]
-    assert sheerka.isinstance(def_func, BuiltinConcepts.GENERIC_NODE)
-    assert def_func.node_type == "FunctionDef"
-    assert def_func.parent == NodeParent(tree_as_concept, "body")
-    assert def_func.get_value("name") == "my_function"
-
-    def_func_args = def_func.get_value("args")
-    assert sheerka.isinstance(def_func_args, BuiltinConcepts.GENERIC_NODE)
-    assert def_func_args.node_type == "arguments"
-
-    def_func_args_real_args = def_func_args.get_value("args")
-    assert sheerka.isinstance(def_func_args_real_args, BuiltinConcepts.LIST)
-    assert len(def_func_args_real_args.body) == 2
-
-    assert sheerka.isinstance(def_func_args_real_args.body[0], BuiltinConcepts.GENERIC_NODE)
-    assert def_func_args_real_args.body[0].node_type == "arg"
-    assert def_func_args_real_args.body[0].parent == NodeParent(def_func_args, "args")
-    assert def_func_args_real_args.body[0].get_value("arg") == "a"
-    assert sheerka.isinstance(def_func_args_real_args.body[1], BuiltinConcepts.GENERIC_NODE)
-    assert def_func_args_real_args.body[1].node_type == "arg"
-    assert def_func_args_real_args.body[1].parent == NodeParent(def_func_args, "args")
-    assert def_func_args_real_args.body[1].get_value("arg") == "b"
-
-    def_fun_body = def_func.get_value("body")
-    assert sheerka.isinstance(def_fun_body, BuiltinConcepts.LIST)
-    assert len(def_fun_body.body) == 2
-
-    def_fun_body_for = def_fun_body.body[0]
-    assert sheerka.isinstance(def_fun_body_for, BuiltinConcepts.GENERIC_NODE)
-    assert def_fun_body_for.node_type == "For"
-    assert def_fun_body_for.parent == NodeParent(def_func, "body")
-
-    def_fun_body_return = def_fun_body.body[1]
-    assert sheerka.isinstance(def_fun_body_return, BuiltinConcepts.GENERIC_NODE)
-    assert def_fun_body_return.node_type == "Return"
-    assert def_fun_body_return.parent == NodeParent(def_func, "body")
-
-
-def test_i_can_visit_concept_node():
-    source = """
-def my_function(a,b):
-    for i in range(b):
-        a = a+b
-    return a
-    """
-
-    node = ast.parse(source)
-    concept_node = core.ast.nodes.python_to_concept(node)
-
-    visitor = TestNameVisitor()
-    visitor.visit(concept_node)
-
-    sheerka = get_sheerka()
-    assert sheerka.objvalue(visitor.names[0]) == "i"
-    assert sheerka.objvalue(visitor.names[1]) == "range"
-    assert sheerka.objvalue(visitor.names[2]) == "b"
-    assert sheerka.objvalue(visitor.names[3]) == "a"
-    assert sheerka.objvalue(visitor.names[4]) == "a"
-    assert sheerka.objvalue(visitor.names[5]) == "b"
-    assert sheerka.objvalue(visitor.names[6]) == "a"
-
-
-def test_i_can_get_unreferenced_variables():
-    source = """
-def my_function(a,b):
-    for i in range(b):
-        a = a+b
-    return a
-
-my_function(x,y)
-"""
-
-    sheerka = get_sheerka()
-
-    node = ast.parse(source)
-    concept_node = core.ast.nodes.python_to_concept(node)
-
-    visitor = UnreferencedNamesVisitor(sheerka)
-    visitor.visit(concept_node)
-    values = visitor.names
-
-    assert len(visitor.names) == 2
-    assert "x" in values
-    assert "y" in values
-
-
-@pytest.mark.parametrize("source, expected", [
-    ("a,b", ["a", "b"]),
-    ("isinstance(a, int)", ["a", "int"])
-
-])
-def test_i_can_get_unreferenced_variables_from_simple_expressions(source, expected):
-    sheerka = get_sheerka()
-
-    node = ast.parse(source)
-    concept_node = core.ast.nodes.python_to_concept(node)
-
-    visitor = UnreferencedNamesVisitor(sheerka)
-    visitor.visit(concept_node)
-
-    assert sorted(list(visitor.names)) == expected
-
-
-def test_i_can_compare_NodeParent_with_tuple():
-    node_parent = NodeParent(GenericNodeConcept("For", None), "target")
-    assert node_parent == ("For", "target")
-
-
-def test_i_can_transform_back():
-    source = """
-def my_function(a,b):
-    for i in range(b):
-        a = a + b
-    return a
-
-
-my_function(x, y)
-    """
-
-    node = ast.parse(source)
-    concept_node = core.ast.nodes.python_to_concept(node)
-
-    transformed_back = core.ast.nodes.concept_to_python(concept_node)
-    assert dump_ast(transformed_back) == dump_ast(node)
-
-
-def dump_ast(node):
-    dump = ast.dump(node)
-    for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
-        dump = dump.replace(to_remove, "")
-    return dump

tests/core/test_ast_helper.py

@@ -0,0 +1,44 @@
+import ast
+
+import pytest
+from core.ast_helpers import UnreferencedNamesVisitor, UnreferencedVariablesVisitor
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestAstHelper(TestUsingMemoryBasedSheerka):
+    @pytest.mark.parametrize("source, expected", [
+        ("a,b", {"a", "b"}),
+        ("isinstance(a, int)", {"isinstance", "a", "int"}),
+        ("date.today()", {"date"}),
+        ("test()", {"test"}),
+        ("sheerka.test()", {"sheerka"}),
+        ("for i in range(10): pass", set())
+
+    ])
+    def test_i_can_get_unreferenced_names_from_simple_expressions(self, source, expected):
+        context = self.get_context()
+
+        ast_ = ast.parse(source)
+        visitor = UnreferencedNamesVisitor(context)
+        visitor.visit(ast_)
+
+        assert visitor.names == expected
+
+    @pytest.mark.parametrize("source, expected", [
+        ("a,b", {"a", "b"}),
+        ("isinstance(a, int)", {"a", "int"}),
+        ("date.today()", set()),
+        ("test()", set()),
+        ("sheerka.test()", set()),
+        ("for i in range(10): pass", set())
+
+    ])
+    def test_i_can_get_unreferenced_variables_from_simple_expressions(self, source, expected):
+        context = self.get_context()
+
+        ast_ = ast.parse(source)
+        visitor = UnreferencedVariablesVisitor(context)
+        visitor.visit(ast_)
+
+        assert visitor.names == expected

tests/core/test_builtin_helpers.py

@@ -105,44 +105,44 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert res == item
 
-    @pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [
-        ("a == 1", [], [], []),
-        ("a == 1", ["a"], [], ["a == 1"]),
-        ("a == 1", [], ["a"], []),
-        ("predicate(a)", [], [], []),
-        ("predicate(a)", ["a"], [], ["predicate(a)"]),
-        ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]),
-        ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]),
-        ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]),
-        ("predicate(a, b)", ["a"], ["b"], []),
-        ("a + b == 1", [], [], []),
-        ("a + b == 1", ["a"], [], ["a + b == 1"]),
-        ("a + b == 1", ["a"], ["b"], []),
-        ("a + b == 1", ["b"], [], ["a + b == 1"]),
-        ("a + b == 1", ["a", "b"], [], ["a + b == 1"]),
-        ("a == 1 and b == 2", [], [], []),
-        ("a == 1 and b == 2", ["a"], [], ["a == 1"]),
-        ("a == 1 and b == 2", ["b"], [], ["b == 2"]),
-        ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]),
-        ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]),
-        ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]),
-        ("not(a == 1)", [], [], []),
-        ("not(a == 1)", ["a"], [], ["not(a==1)"]),
-        ("a == 1 or b == 2", [], [], []),
-        ("a == 1 or b == 2", ["a"], [], ["a == 1"]),
-        ("a == 1 or b == 2", ["b"], [], ["b == 2"]),
-        ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]),
-        ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]),
-        ("a < 1 and a > b", ["a"], [], ["a < 1 and a > b"]),
-    ])
-    def test_i_can_extract_predicates(self, expression, vars_to_include, vars_to_exclude, expected_expr):
-        sheerka = self.get_sheerka()
-        expected = [ast.parse(expr, mode="eval") for expr in expected_expr]
-
-        actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude)
-        assert len(actual) == len(expected)
-        for i in range(len(actual)):
-            assert self.dump_ast(actual[i]) == self.dump_ast(expected[i])
+    # @pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [
+    #     ("a == 1", [], [], []),
+    #     ("a == 1", ["a"], [], ["a == 1"]),
+    #     ("a == 1", [], ["a"], []),
+    #     ("predicate(a)", [], [], []),
+    #     ("predicate(a)", ["a"], [], ["predicate(a)"]),
+    #     ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]),
+    #     ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]),
+    #     ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]),
+    #     ("predicate(a, b)", ["a"], ["b"], []),
+    #     ("a + b == 1", [], [], []),
+    #     ("a + b == 1", ["a"], [], ["a + b == 1"]),
+    #     ("a + b == 1", ["a"], ["b"], []),
+    #     ("a + b == 1", ["b"], [], ["a + b == 1"]),
+    #     ("a + b == 1", ["a", "b"], [], ["a + b == 1"]),
+    #     ("a == 1 and b == 2", [], [], []),
+    #     ("a == 1 and b == 2", ["a"], [], ["a == 1"]),
+    #     ("a == 1 and b == 2", ["b"], [], ["b == 2"]),
+    #     ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]),
+    #     ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]),
+    #     ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]),
+    #     ("not(a == 1)", [], [], []),
+    #     ("not(a == 1)", ["a"], [], ["not(a==1)"]),
+    #     ("a == 1 or b == 2", [], [], []),
+    #     ("a == 1 or b == 2", ["a"], [], ["a == 1"]),
+    #     ("a == 1 or b == 2", ["b"], [], ["b == 2"]),
+    #     ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]),
+    #     ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]),
+    #     ("a < 1 and a > b", ["a"], [], ["a < 1 and a > b"]),
+    # ])
+    # def test_i_can_extract_predicates(self, expression, vars_to_include, vars_to_exclude, expected_expr):
+    #     sheerka = self.get_sheerka()
+    #     expected = [ast.parse(expr, mode="eval") for expr in expected_expr]
+    #
+    #     actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude)
+    #     assert len(actual) == len(expected)
+    #     for i in range(len(actual)):
+    #         assert self.dump_ast(actual[i]) == self.dump_ast(expected[i])
 
     @pytest.mark.parametrize("concepts, expected", [
         ([], []),

tests/core/test_concept.py

@@ -1,6 +1,7 @@
 import pytest
+from core.builtin_concepts import BuiltinConcepts
 
-from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF
+from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF, ALL_ATTRIBUTES, get_concept_attrs
 
 
 @pytest.mark.parametrize("name, variables, expected", [
@@ -24,7 +25,7 @@ from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF
 def test_i_can_compute_the_key(name, variables, expected):
     concept = Concept(name)
     for var_name in variables:
-        concept.metadata.variables.append((var_name, None))
+        concept.get_metadata().variables.append((var_name, None))
     concept.init_key()
 
     assert concept.key == expected
@@ -33,17 +34,17 @@ def test_i_can_compute_the_key(name, variables, expected):
 def test_i_can_compute_the_key_when_from_definition():
     # if definition is not defined, use the name
     concept = Concept()
-    concept.metadata.name = "hello a"
-    concept.metadata.variables = [("a", None)]
+    concept.get_metadata().name = "hello a"
+    concept.get_metadata().variables = [("a", None)]
     concept.init_key()
     assert concept.key == "hello __var__0"
 
     # if definition is defined, use it
     concept = Concept()
-    concept.metadata.name = "greetings"
-    concept.metadata.definition = "hello a"
-    concept.metadata.definition_type = DEFINITION_TYPE_DEF
-    concept.metadata.variables = [("a", None)]
+    concept.get_metadata().name = "greetings"
+    concept.get_metadata().definition = "hello a"
+    concept.get_metadata().definition_type = DEFINITION_TYPE_DEF
+    concept.get_metadata().variables = [("a", None)]
     concept.init_key()
     assert concept.key == "hello __var__0"
 
@@ -52,7 +53,7 @@ def test_key_does_not_use_variable_when_definition_is_set():
     concept = Concept("plus").def_var('plus')
 
     concept.init_key()
-    assert concept.metadata.key == "plus"
+    assert concept.get_metadata().key == "plus"
 
 
 def test_i_can_serialize():
@@ -181,6 +182,8 @@ def test_i_can_compare_concepts():
 
 
 def test_i_can_detect_concept_differences():
+    ALL_ATTRIBUTES.clear()
+
     assert Concept(name="concept_name") != Concept()
     assert Concept(is_builtin=True) != Concept()
     assert Concept(is_unique=True) != Concept()
@@ -197,14 +200,18 @@ def test_i_can_detect_concept_differences():
     assert Concept().add_prop("a", "b") != Concept()
     assert Concept().set_value("a", "b") != Concept()
 
+
+def test_compiled_is_not_used_when_comparing_concepts():
+    ALL_ATTRIBUTES.clear()
+
     concept = Concept()
-    concept.compiled["foo"] = "value"
+    concept.get_compiled()["foo"] = "value"
     assert concept == Concept()  # compiled is not used in the comparison
 
 
 def test_i_can_compare_concept_with_circular_reference():
     foo = Concept("foo")
-    foo.metadata.body = foo
+    foo.get_metadata().body = foo
 
     assert foo == foo
 
@@ -213,7 +220,7 @@ def test_i_can_compare_concept_with_sophisticated_circular_reference():
     foo = Concept("foo")
     bar = Concept("foo", body=foo)
     baz = Concept("foo", body=bar)
-    foo.metadata.body = baz
+    foo.get_metadata().body = baz
 
     assert foo != bar
 
@@ -222,12 +229,14 @@ def test_i_can_compare_concept_with_sophisticated_circular_reference_in_other_me
     foo = Concept("foo")
     bar = Concept("foo", pre=foo)
     baz = Concept("foo", pre=bar)
-    foo.metadata.pre = baz
+    foo.get_metadata().pre = baz
 
     assert foo != bar
 
 
 def test_i_can_update_from():
+    ALL_ATTRIBUTES.clear()
+
     template = Concept(
         name="concept_name",
         is_builtin=True,
@@ -242,6 +251,9 @@ def test_i_can_update_from():
         desc="this this the desc",
         id="123456"
     ).def_var("a", "10").def_var("b", None)
+    template.add_prop(BuiltinConcepts.ISA, Concept("foo").def_var("x", "value_x"))
+    template.add_prop(BuiltinConcepts.HASA, Concept("bar").def_var("x", "value_x"))
+    template.set_prop(BuiltinConcepts.AUTO_EVAL, True)
 
     # make sure origin is preserved
     setattr(template, "##origin##", "digest")
@@ -257,3 +269,28 @@ def test_i_can_update_from():
 
     assert concept == template
     assert getattr(concept, "##origin##") == "digest"
+
+
+def test_i_can_manage_concepts_attributes():
+    ALL_ATTRIBUTES.clear()
+    concept = Concept("foo")
+    assert get_concept_attrs(concept) == []
+    assert concept.values() == {}
+
+    concept.set_value(ConceptParts.BODY, "I have a body!")
+    assert concept.values() == {"#body#": "I have a body!"}
+
+
+def test_attributes_are_generated_once_for_all():
+    ALL_ATTRIBUTES.clear()
+    concept = Concept("foo")
+    concept.get_metadata().id = "id"
+
+    concept.set_value("key1", "value1")
+    concept.set_value("key2", "value2")
+    assert get_concept_attrs(concept) == ["key1", "key2"]
+    assert concept.values() == {"key1": "value1", "key2": "value2"}
+
+    concept.set_value("key3", "value3")  # too late for it !
+    assert get_concept_attrs(concept) == ["key1", "key2"]
+    assert concept.values() == {"key1": "value1", "key2": "value2"}

tests/core/test_sheerka.py

@@ -1,8 +1,8 @@
 import os
 
 import pytest
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept, AllBuiltinConcepts
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit
 from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS
 from core.tokenizer import Token, TokenKind
 
@@ -31,7 +31,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert "parsers.DefConceptParser.DefConceptParser" in sheerka.parsers
         assert "parsers.BnfNodeParser.BnfNodeParser" in sheerka.parsers
         assert "parsers.SyaNodeParser.SyaNodeParser" in sheerka.parsers
-        assert "parsers.AtomNodeParser.AtomNodeParser" in sheerka.parsers
+        assert "parsers.SequenceNodeParser.SequenceNodeParser" in sheerka.parsers
 
         # make sure BaseNodeParser is properly initialized
         assert BASE_NODE_PARSER_CLASS not in sheerka.parsers
@@ -70,7 +70,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert loaded is not None
         assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
         assert loaded.body == ("key", "key_that_does_not_exist")
-        assert loaded.metadata.is_evaluated
+        assert loaded.get_metadata().is_evaluated
 
     def test_i_cannot_get_when_id_is_not_found(self):
         sheerka = self.get_sheerka()
@@ -80,7 +80,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert loaded is not None
         assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
         assert loaded.body == ("id", "id_that_does_not_exist")
-        assert loaded.metadata.is_evaluated
+        assert loaded.get_metadata().is_evaluated
 
     def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self):
         sheerka = self.get_sheerka()
@@ -112,7 +112,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
 
         assert sheerka.isinstance(new, concept)
         for prop in PROPERTIES_TO_SERIALIZE:
-            assert getattr(new.metadata, prop) == getattr(concept.metadata, prop)
+            assert getattr(new.get_metadata(), prop) == getattr(concept.get_metadata(), prop)
 
         assert new.get_value("a") == 10
         assert new.get_value("b") == "value"
@@ -127,17 +127,17 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         concepts = sheerka.new("foo")
         assert len(concepts) == 2
         assert concepts[0].id == "1001"
-        assert concepts[0].metadata.body == "foo1"
+        assert concepts[0].get_metadata().body == "foo1"
         assert concepts[1].id == "1002"
-        assert concepts[1].metadata.body == "foo2"
+        assert concepts[1].get_metadata().body == "foo2"
 
         # only one instance if the id is given
         foo1 = sheerka.new(("foo", "1001"))
-        assert foo1.metadata.body == "foo1"
+        assert foo1.get_metadata().body == "foo1"
 
         # only one instance if the id is given
         foo2 = sheerka.new(("foo", "1002"))
-        assert foo2.metadata.body == "foo2"
+        assert foo2.get_metadata().body == "foo2"
 
     def test_instances_are_different_when_asking_for_new(self):
         sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
@@ -152,18 +152,18 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
 
         new_foo = sheerka.new("foo")
-        new_foo.metadata.body = "metadata value"  # modify metadata
+        new_foo.get_metadata().body = "metadata value"  # modify metadata
         new_foo.def_var("var_name", "default value")  # modify definition of variables
         new_foo.add_prop(BuiltinConcepts.ISA, bar)  # modify property
-        new_foo.compiled["var_name"] = "'var value'"
+        new_foo.get_compiled()["var_name"] = "'var value'"
         new_foo.set_value(ConceptParts.BODY, "body value")  # modify value
         new_foo.set_value("var_name", "var value")  # modify value
 
-        assert new_foo.metadata.body != foo.metadata.body
-        assert new_foo.metadata.variables != foo.metadata.variables
-        assert new_foo.metadata.props != foo.metadata.props
+        assert new_foo.get_metadata().body != foo.get_metadata().body
+        assert new_foo.get_metadata().variables != foo.get_metadata().variables
+        assert new_foo.get_metadata().props != foo.get_metadata().props
         assert new_foo.values != foo.values
-        assert new_foo.compiled != foo.compiled
+        assert new_foo.get_compiled() != foo.get_compiled()
 
     def test_i_get_the_same_instance_when_is_unique_is_true(self):
         sheerka, context, concept = self.init_concepts(Concept(name="unique", is_unique=True), create_new=True)
@@ -181,16 +181,16 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
             eval_body=True)
 
         sheerka.evaluate_concept(context, sheerka.get_by_id(template.id))
-        assert template.metadata.is_evaluated
+        assert template.get_metadata().is_evaluated
         assert template.body == "foo body"
 
         new = sheerka.new(template.key)
-        assert not new.metadata.is_evaluated
-        assert new.body == BuiltinConcepts.NOT_INITIALIZED
+        assert not new.get_metadata().is_evaluated
+        assert new.body == NotInit
 
         new = sheerka.new((None, template.id))
-        assert not new.metadata.is_evaluated
-        assert new.body == BuiltinConcepts.NOT_INITIALIZED
+        assert not new.get_metadata().is_evaluated
+        assert new.body == NotInit
 
     def test_i_cannot_instantiate_an_unknown_concept(self):
         sheerka = self.get_sheerka()
@@ -230,7 +230,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         new = sheerka.new(("foo", "invalid_id"))
 
         assert sheerka.isinstance(new, "foo")
-        assert new.metadata.body == "foo1"
+        assert new.get_metadata().body == "foo1"
 
     def test_i_cannot_instantiate_when_properties_are_not_recognized(self):
         sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
@@ -242,16 +242,16 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(new.concept, concept)
 
     @pytest.mark.parametrize("concept, reduce_simple_list, expected", [
-        # (None, False, None),
-        # (3.14, False, 3.14),
-        # ("foo", False, "foo"),
-        # (True, False, True),
-        # (Concept("name", body="foo"), False, "foo"),
-        # (Concept("name"), False, Concept("name")),
-        # (ConceptWithGetObjValue("name").set_value("my_prop", "my_value"), False, "my_value"),
-        # (ReturnValueConcept(value="return_value"), False, "return_value"),
-        # (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
-        # (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
+        (None, False, None),
+        (3.14, False, 3.14),
+        ("foo", False, "foo"),
+        (True, False, True),
+        (Concept("name", body="foo"), False, "foo"),
+        (Concept("name"), False, Concept("name")),
+        (ConceptWithGetObjValue("name").set_value("my_prop", "my_value"), False, "my_value"),
+        (ReturnValueConcept(value="return_value"), False, "return_value"),
+        (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
+        (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
         (ReturnValueConcept(value=Concept("foo", body=False).auto_init(), status=True), False, False),
         (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]),
         (Concept("name", body=["foo"]), True, "foo"),
@@ -342,7 +342,7 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
 
     def test_builtin_concepts_are_initialized(self):
         sheerka = self.get_sheerka()
-        for concept_name in BuiltinConcepts:
+        for concept_name in AllBuiltinConcepts:
             assert sheerka.has_key(str(concept_name))
             assert sheerka.sdp.get(sheerka.CONCEPTS_BY_KEY_ENTRY, str(concept_name)) is not None
 
@@ -360,15 +360,15 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
     def test_builtin_concepts_can_be_updated(self):
         sheerka = self.get_sheerka()
         before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
-        before_parsing.metadata.desc = "I have a description"
-        before_parsing.metadata.full_serialization = True
+        before_parsing.get_metadata().desc = "I have a description"
+        before_parsing.get_metadata().full_serialization = True
         with sheerka.sdp.get_transaction("Test") as transac:
             transac.add(sheerka.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True)
 
         sheerka = self.get_sheerka()  # another fresh new instance
         before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
 
-        assert before_parsing.metadata.desc == "I have a description"
+        assert before_parsing.get_metadata().desc == "I have a description"
 
     def test_i_first_look_in_local_cache(self):
         sheerka, context, concept = self.init_concepts("foo", create_new=True)

tests/core/test_sheerkaResultManager.py

@@ -1,6 +1,8 @@
 import types
 
+import pytest
 from core.builtin_concepts import BuiltinConcepts
+from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.services.SheerkaResultManager import SheerkaResultConcept
 from sdp.sheerkaDataProvider import Event
 
@@ -8,21 +10,28 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
 class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
+    io_cache = None
 
     @classmethod
     def setup_class(cls):
         sheerka = cls().get_sheerka()
         sheerka.save_execution_context = True
+        sheerka.evaluate_user_input("def concept one as 1")
+        cls.io_cache = sheerka.sdp.io.cache.copy()
 
     @classmethod
     def teardown_class(cls):
         sheerka = cls().get_sheerka()
         sheerka.save_execution_context = False
 
-    def test_i_can_get_the_result_by_digest(self):
+    def init_test(self):
         sheerka, context = self.init_concepts()
+        sheerka.sdp.io.cache = self.io_cache.copy()
+        return sheerka, context
+
+    def test_i_can_get_the_result_by_digest(self):
+        sheerka, context = self.init_test()
 
-        sheerka.evaluate_user_input("def concept one as 1")
         digest = sheerka.get_last_execution().event.get_digest()
 
         res = sheerka.get_results_by_digest(context, digest)
@@ -32,7 +41,7 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.digest == digest
         assert isinstance(res.body, types.GeneratorType)
 
-        assert sheerka.load(SheerkaResultConcept.NAME, "digest") == digest
+        assert sheerka.load_var(SheerkaResultConcept.NAME, "digest") == digest
 
         previous_results = list(res.body)
 
@@ -58,11 +67,9 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert sheerka.get_results(context) is None
 
     def test_i_can_get_the_result_by_command_name(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test()
 
-        sheerka.evaluate_user_input("def concept one as 1")
         digest = sheerka.get_last_execution().event.get_digest()
-
         sheerka.evaluate_user_input("one")  # another command
 
         res = sheerka.get_results_by_command(context, "def concept")
@@ -72,9 +79,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert isinstance(res.body, types.GeneratorType)
 
     def test_i_can_get_the_result_by_command_when_not_in_the_same_page_size(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test()
 
-        sheerka.evaluate_user_input("def concept one as 1")
         sheerka.evaluate_user_input("one")
         sheerka.evaluate_user_input("one")
         sheerka.evaluate_user_input("one")
@@ -92,9 +98,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.body == {'command': 'def concept'}
 
     def test_i_cannot_get_result_from_command_if_the_command_does_not_exists_multiple_pages(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test()
 
-        sheerka.evaluate_user_input("def concept one as 1")
         sheerka.evaluate_user_input("one")
         sheerka.evaluate_user_input("one")
         sheerka.evaluate_user_input("one")
@@ -105,9 +110,8 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert res.body == {'command': 'fake command'}
 
     def test_i_can_get_last_results(self):
-        sheerka, context = self.init_concepts()
+        sheerka, context = self.init_test()
 
-        sheerka.evaluate_user_input("def concept one as 1")
         sheerka.evaluate_user_input("one")
 
         res = sheerka.get_last_results(context)
@@ -144,4 +148,106 @@ class TestSheerkaResultManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
         assert res.body == {'query': 'last'}
 
+    @pytest.mark.parametrize('kwargs', [
+        {"desc": "Evaluating 'def concept one as 1'"},
+        {"id": 0},
+        {"desc": "Evaluating 'def concept one as 1'", "id": 0}
+    ])
+    def test_i_can_get_last_results_using_kwarg(self, kwargs):
+        sheerka, context = self.init_test()
+        ExecutionContext.ids.clear()
 
+        res = sheerka.get_last_results(context, **kwargs)
+        items = list(res.body)
+
+        assert len(items) == 1
+        for k, v in kwargs.items():
+            assert getattr(items[0], k) == v
+
+    @pytest.mark.parametrize('predicate, expected', [
+        ('desc.startswith("Evaluating \'def concept one as 1\'")', {"desc": "Evaluating 'def concept one as 1'"}),
+        ("id == 0", {"id": 0}),
+        ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
+    ])
+    def test_i_can_get_last_results_using_filter(self, predicate, expected):
+        sheerka, context = self.init_test()
+        ExecutionContext.ids.clear()
+
+        res = sheerka.get_last_results(context, filter=predicate)
+        items = list(res.body)
+
+        assert len(items) == 1
+        for k, v in expected.items():
+            assert getattr(items[0], k) == v
+
+    @pytest.mark.parametrize('predicate, expected', [
+        ('desc.startswith("Evaluating \'def concept one as 1\'")', {"desc": "Evaluating 'def concept one as 1'"}),
+        ("id == 0", {"id": 0}),
+        ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
+    ])
+    def test_i_can_get_last_results_using_the_first_argument_to_filter(self, predicate, expected):
+        sheerka, context = self.init_test()
+        ExecutionContext.ids.clear()
+
+        res = sheerka.get_last_results(context, predicate)
+        items = list(res.body)
+
+        assert len(items) == 1
+        for k, v in expected.items():
+            assert getattr(items[0], k) == v
+
+    @pytest.mark.parametrize('kwargs', [
+        {"desc": "Evaluating 'def concept one as 1'"},
+        {"id": 0},
+        {"desc": "Evaluating 'def concept one as 1'", "id": 0}
+    ])
+    def test_i_can_get_results_using_kwarg(self, kwargs):
+        sheerka, context = self.init_test()
+        ExecutionContext.ids.clear()
+        sheerka.get_last_results(context)
+
+        res = sheerka.get_results(context, **kwargs)
+        items = list(res.body)
+
+        assert len(items) == 1
+        for k, v in kwargs.items():
+            assert getattr(items[0], k) == v
+
+    @pytest.mark.parametrize('predicate, expected', [
+        ('desc.startswith("Evaluating \'def concept one as 1\'")', {"desc": "Evaluating 'def concept one as 1'"}),
+        ("id == 0", {"id": 0}),
+        ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
+    ])
+    def test_i_can_get_results_using_filter(self, predicate, expected):
+        sheerka, context = self.init_test()
+        ExecutionContext.ids.clear()
+        sheerka.get_last_results(context)
+
+        res = sheerka.get_results(context, filter=predicate)
+        items = list(res.body)
+
+        assert len(items) == 1
+        for k, v in expected.items():
+            assert getattr(items[0], k) == v
+
+    @pytest.mark.parametrize('predicate, expected', [
+        ('desc.startswith("Evaluating \'def concept one as 1\'")', {"desc": "Evaluating 'def concept one as 1'"}),
+        ("id == 0", {"id": 0}),
+        ("'def concept one as 1' in desc and id == 0", {"desc": "Evaluating 'def concept one as 1'", "id": 0})
+    ])
+    def test_i_can_get_results_using_the_first_argument_to_filter(self, predicate, expected):
+        sheerka, context = self.init_test()
+        ExecutionContext.ids.clear()
+        sheerka.get_last_results(context)
+
+        res = sheerka.get_results(context, predicate)
+        items = list(res.body)
+
+        assert len(items) == 1
+        for k, v in expected.items():
+            assert getattr(items[0], k) == v
+
+    def test_i_can_manage_invalid_predicates(self):
+        predicate = {"filter": "a b c"}
+        with pytest.raises(SyntaxError):
+            SheerkaResultConcept.get_predicate(**predicate)

tests/core/test_sheerka_call_evaluators.py

@@ -1,5 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
+from core.sheerka.services.SheerkaExecute import SheerkaExecute
 from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator
 
 from tests.BaseTest import BaseTest
@@ -15,16 +16,18 @@ class Out:
             target = [str(r.body.key) for r in return_value]
         else:
             target = str(return_value.body.key)
-        step = str(context.step)
-        text = f"{step} [{context.iteration}] "
+        step = str(context.action_context["step"])
+        iteration = context.action_context["iteration"]
+        text = f"{step} [{iteration}] "
         text += f"{name} - {method} - target={target}"
         self.debug_out.append(text)
 
     def out_all(self, method, name, context, return_values):
         name = name[len(BaseEvaluator.PREFIX):]
         target = [str(r.body.key) for r in return_values]
-        step = str(context.step)
-        text = f"{step} [{context.iteration}] "
+        step = str(context.action_context["step"])
+        iteration = context.action_context["iteration"]
+        text = f"{step} [{iteration}] "
         text += f"{name} - {method} - target={target}"
         self.debug_out.append(text)
 
@@ -295,6 +298,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
             EvaluatorOneWithPriority10,
             EvaluatorOneWithPriority15,
             EvaluatorAllWithPriority20, ]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo"))]
         Out.debug_out = []
@@ -318,6 +323,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
     def test_that_predicate_is_checked_before_evaluation_for_one_return(self):
         sheerka = self.get_sheerka()
         sheerka.evaluators = [EvaluatorOneModifyFoo]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
         Out.debug_out = []
@@ -334,6 +341,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
     def test_that_predicate_is_checked_before_evaluation_for_all_return(self):
         sheerka = self.get_sheerka()
         sheerka.evaluators = [EvaluatorAllReduceFooBar]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar"))]
         Out.debug_out = []
@@ -354,6 +363,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
     def test_evaluation_continue_until_no_more_modification(self):
         sheerka = self.get_sheerka()
         sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
         Out.debug_out = []
@@ -379,6 +390,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
     def test_evaluation_steps_are_respected(self):
         sheerka = self.get_sheerka()
         sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo"))]
         Out.debug_out = []
@@ -392,6 +405,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
     def test_evaluation_multi_steps_are_respected(self):
         sheerka = self.get_sheerka()
         sheerka.evaluators = [EvaluatorOneMultiSteps]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo"))]
         Out.debug_out = []
@@ -405,9 +420,88 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
             '__EVALUATION [0] multiStep - eval - target=foo',
         ]
 
+    def test_evaluators_can_be_selected_in_the_context(self):
+        sheerka, context = self.init_concepts()
+        sheerka.evaluators = [EvaluatorOneWithPriority20,
+                              EvaluatorAllWithPriority15,
+                              EvaluatorAllWithPriority10, ]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
+        context.preprocess_evaluators = [EvaluatorAllWithPriority10().short_name]  # it will be the only one to be evaluated
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+
+        Out.debug_out = []
+        sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            "__EVALUATION [0] all_priority10 - matches - target=['foo']",
+            "__EVALUATION [0] all_priority10 - eval - target=['foo']"
+        ]
+
+        # grouped evaluator is in cache
+        service = sheerka.services[SheerkaExecute.NAME]
+        assert "__EVALUATION|all_priority10" in service.grouped_evaluators_cache
+
+    def test_evaluators_priorities_can_be_tweaked_by_the_context(self):
+        sheerka, context = self.init_concepts()
+        sheerka.evaluators = [EvaluatorOneWithPriority20,
+                              EvaluatorOneWithPriority15,
+                              EvaluatorOneWithPriority10, ]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
+
+        # invert the priorities
+        context.add_preprocess(EvaluatorOneWithPriority20().name, priority=1)
+        context.add_preprocess(EvaluatorOneWithPriority15().name, priority=2)
+        context.add_preprocess(EvaluatorOneWithPriority10().name, priority=3)
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+
+        Out.debug_out = []
+        sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            "__EVALUATION [0] priority10 - matches - target=foo",
+            "__EVALUATION [0] priority10 - eval - target=foo",
+            "__EVALUATION [0] priority15 - matches - target=foo",
+            "__EVALUATION [0] priority15 - eval - target=foo",
+            '__EVALUATION [0] priority20 - matches - target=foo',
+            '__EVALUATION [0] priority20 - eval - target=foo',
+        ]
+
+        # grouped evaluator is in cache
+        service = sheerka.services[SheerkaExecute.NAME]
+        assert "__EVALUATION|evaluators.priority20|evaluators.priority15|evaluators.priority10" in service.grouped_evaluators_cache
+
+    def test_evaluators_enabled_can_be_tweaked_by_the_context(self):
+        sheerka, context = self.init_concepts()
+        sheerka.evaluators = [EvaluatorOneWithPriority20,
+                              EvaluatorOneWithPriority15,
+                              EvaluatorOneWithPriority10, ]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
+
+        # invert the priorities
+        context.add_preprocess(EvaluatorOneWithPriority20().name, enabled=False)
+        context.add_preprocess(EvaluatorOneWithPriority15().name, enabled=False)
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+
+        Out.debug_out = []
+        sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            "__EVALUATION [0] priority10 - matches - target=foo",
+            "__EVALUATION [0] priority10 - eval - target=foo",
+        ]
+
+        # grouped evaluator is in cache
+        service = sheerka.services[SheerkaExecute.NAME]
+        assert "__EVALUATION|evaluators.priority10" in service.grouped_evaluators_cache
+
     def test_evaluators_can_be_pre_processed(self):
         sheerka = self.get_sheerka()
         sheerka.evaluators = [EvaluatorOneModifyFoo]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo"))]
 
@@ -418,7 +512,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
         sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION])
         assert Out.debug_out == []
 
-        # other contextes are not impacted
+        # other evaluations are not impacted
         Out.debug_out = []
         sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
         assert Out.debug_out == [
@@ -430,6 +524,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
     def test_i_can_initialize_evaluator_if_initialize_evaluator_is_defined(self):
         sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz")
         sheerka.evaluators = [EvaluatorOneInitializationOnce]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [
             self.tretval(sheerka, foo),
@@ -451,6 +547,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
     def test_i_can_initialize_evaluators_multiple_times(self):
         sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz")
         sheerka.evaluators = [EvaluatorOneInitializationMultiple]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [
             self.tretval(sheerka, foo),
@@ -479,6 +577,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
         """
         sheerka = self.get_sheerka()
         sheerka.evaluators = [EvaluatorOneDoNotModifyExecutionFlow]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo"))]
         Out.debug_out = []
@@ -499,6 +599,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
         """
         sheerka = self.get_sheerka()
         sheerka.evaluators = [EvaluatorOneModifyFoo]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo"))]
         Out.debug_out = []
@@ -516,6 +618,8 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
     def test_i_can_remove_return_values_from_the_execution_workflow(self):
         sheerka = self.get_sheerka()
         sheerka.evaluators = [EvaluatorAllSuppressEntries]
+        service = sheerka.services[SheerkaExecute.NAME]
+        service.reset_evaluators()
 
         entries = [self.tretval(sheerka, Concept("foo")),
                    self.tretval(sheerka, Concept("bar")),

tests/core/test_sheerka_printer.py

@@ -353,25 +353,21 @@ second             : 'value d'
         sheerka.print(lst)
         captured = capsys.readouterr()
         assert captured.out == """(1001)foo a b
-a    : 'value a'
-var.a: *name 'var' is not defined*
-b    : 'value b'
-var.b: *name 'var' is not defined*
-id   : '1001'
-name : 'foo a b'
-key  : 'foo __var__0 __var__1'
-body : __NOT_INITIALIZED
-self : (1001)foo a b
+a   : 'value a'
+b   : 'value b'
+id  : '1001'
+name: 'foo a b'
+key : 'foo __var__0 __var__1'
+body: **NotInit**
+self: (1001)foo a b
 (1001)foo a b
-a    : 'value c'
-var.a: *name 'var' is not defined*
-b    : 'value d'
-var.b: *name 'var' is not defined*
-id   : '1001'
-name : 'foo a b'
-key  : 'foo __var__0 __var__1'
-body : __NOT_INITIALIZED
-self : (1001)foo a b
+a   : 'value c'
+b   : 'value d'
+id  : '1001'
+name: 'foo a b'
+key : 'foo __var__0 __var__1'
+body: **NotInit**
+self: (1001)foo a b
 """
 
     def test_i_can_format_d_when_dictionary(self, capsys):
@@ -391,34 +387,32 @@ self : (1001)foo a b
         sheerka.print(lst)
         captured = capsys.readouterr()
         assert captured.out == """(1001)foo a b
-a    : 'value a'
-var.a: *name 'var' is not defined*
-b    : {'a'      : 'value a'
-        'beta'   : {'b1': 10
-                    'b2': Obj(a='10', b=15)
-                    'b3': ['items',
-                           'in',
-                           'a',
-                           'list']}
-        'gamma'  : {'list' : ["'quoted string'",
-                              ""double" 'single'",
-                              'c3']
-                    'empty': []}
-        'epsilon': ['a',
-                    'b',
-                    'c']
-        'g'      : {'tuple': ('tuple-a',
-                              'tuple-b',
-                              'tuple-b')
-                    'empty': ()}
-        'h'      : {'set'  : {'set-a'}
-                    'empty': {}}}
-var.b: *name 'var' is not defined*
-id   : '1001'
-name : 'foo a b'
-key  : 'foo __var__0 __var__1'
-body : __NOT_INITIALIZED
-self : (1001)foo a b
+a   : 'value a'
+b   : {'a'      : 'value a'
+       'beta'   : {'b1': 10
+                   'b2': Obj(a='10', b=15)
+                   'b3': ['items',
+                          'in',
+                          'a',
+                          'list']}
+       'gamma'  : {'list' : ["'quoted string'",
+                             ""double" 'single'",
+                             'c3']
+                   'empty': []}
+       'epsilon': ['a',
+                   'b',
+                   'c']
+       'g'      : {'tuple': ('tuple-a',
+                             'tuple-b',
+                             'tuple-b')
+                   'empty': ()}
+       'h'      : {'set'  : {'set-a'}
+                   'empty': {}}}
+id  : '1001'
+name: 'foo a b'
+key : 'foo __var__0 __var__1'
+body: **NotInit**
+self: (1001)foo a b
 """
 
     def test_i_can_manage_when_property_does_not_exist(self, capsys):

tests/core/test_tokenizer.py

@@ -4,7 +4,7 @@ from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
 
 def test_i_can_tokenize():
     source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&<>c:name:"
-    source += "$£€!_identifier°~_^\\`==#__var__10r/regex\nregex/"
+    source += "$£€!_identifier°~_^\\`==#__var__10r/regex\nregex/r:xxx|1:"
     tokens = list(Tokenizer(source))
     assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
     assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
@@ -56,12 +56,15 @@ def test_i_can_tokenize():
     assert tokens[47] == Token(TokenKind.HASH, '#', 111, 6, 54)
     assert tokens[48] == Token(TokenKind.VAR_DEF, '__var__10', 112, 6, 55)
     assert tokens[49] == Token(TokenKind.REGEX, '/regex\nregex/', 121, 6, 64)
+    assert tokens[50] == Token(TokenKind.RULE, ("xxx", "1"), 135, 7, 7)
 
-    assert tokens[50] == Token(TokenKind.EOF, '', 135, 7, 7)
+    assert tokens[51] == Token(TokenKind.EOF, '', 143, 7, 15)
 
 
 @pytest.mark.parametrize("text, expected", [
     ("_ident", True),
+    ("__ident", True),
+    ("___ident", True),
     ("ident", True),
     ("ident123", True),
     ("ident_123", True),
@@ -168,3 +171,29 @@ def test_i_can_parse_concept_token(text, expected):
 
     assert tokens[0].type == TokenKind.CONCEPT
     assert tokens[0].value == expected
+
+@pytest.mark.parametrize("text, expected", [
+    ("r:key:", ("key", None)),
+    ("r:key|id:", ("key", "id")),
+    ("r:key|:", ("key", None)),
+    ("r:|id:", (None, "id")),
+    ("r:125:", ("125", None)),
+])
+def test_i_can_parse_concept_token(text, expected):
+    tokens = list(Tokenizer(text))
+
+    assert tokens[0].type == TokenKind.RULE
+    assert tokens[0].value == expected
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("r|regex|", "|regex|"),
+    ("r/regex/", "/regex/"),
+    ("r'regex'", "'regex'"),
+    ('r"regex"', '"regex"'),
+])
+def test_i_can_parse_regex_token(text, expected):
+    tokens = list(Tokenizer(text))
+
+    assert tokens[0].type == TokenKind.REGEX
+    assert tokens[0].value == expected

tests/core/test_utils.py

@@ -2,8 +2,8 @@ from dataclasses import dataclass
 
 import core.utils
 import pytest
-from core.concept import ConceptParts, Concept
-from core.tokenizer import Token, TokenKind
+from core.concept import Concept
+from core.tokenizer import Token, TokenKind, Tokenizer, Keywords
 
 
 @dataclass
@@ -15,6 +15,12 @@ class Obj:
         return hash((self.prop1, self.prop1))
 
 
+@dataclass
+class Obj2:
+    prop1: object
+    prop2: object
+
+
 def get_tokens(lst):
     res = []
     for e in lst:
@@ -134,11 +140,25 @@ def test_i_can_product(a, b, expected):
 
 ])
 def test_i_can_strip(input_as_list, expected_as_list):
-    actual = core.utils.strip_tokens(get_tokens(input_as_list))
+    actual = core.utils.strip_tokens(get_tokens(input_as_list))  # KSI 20201007 Why not use Tokenizer ?!! For perf ?
     expected = get_tokens(expected_as_list)
     assert actual == expected
 
 
+@pytest.mark.parametrize("text, value, expected", [
+    ("xxx=yyy", "=", 1),
+    ("xxx", "=", -1),
+    ("xxx = yyy", "=", 2),
+    ("xxx = yyy", " = ", -1),
+])
+def test_i_can_index(text, value, expected):
+    assert core.utils.index_tokens(Tokenizer(text), value) == expected
+
+
+def test_i_can_manage_non_in_index_tokens():
+    assert core.utils.index_tokens(None, "=") == -1
+
+
 def test_by_default_eof_is_not_stripped():
     actual = core.utils.strip_tokens(get_tokens(["one", "two", " ", "\n", "<EOF>"]))
     expected = get_tokens(["one", "two", " ", "\n", "<EOF>"])
@@ -176,6 +196,16 @@ def test_i_can_unstr_concept(text, expected_key, expected_id):
     assert i == expected_id
 
 
+@pytest.mark.parametrize("text, expected_key, expected_id", [
+    ("r:key:", "key", None),
+    ("r:key|id:", "key", "id"),
+])
+def test_i_can_unstr_concept_rules(text, expected_key, expected_id):
+    k, i = core.utils.unstr_concept(text, prefix="r:")
+    assert k == expected_key
+    assert i == expected_id
+
+
 def test_i_can_str_concept():
     assert core.utils.str_concept(("key", "id")) == "c:key|id:"
     assert core.utils.str_concept((None, "id")) == "c:|id:"
@@ -186,10 +216,12 @@ def test_i_can_str_concept():
     concept = Concept("foo").init_key()
     assert core.utils.str_concept(concept) == "c:foo:"
 
-    concept.metadata.id = "1001"
+    concept.get_metadata().id = "1001"
     assert core.utils.str_concept(concept) == "c:foo|1001:"
     assert core.utils.str_concept(concept, drop_name=True) == "c:|1001:"
 
+    assert core.utils.str_concept(("key", "id"), prefix='r:') == "r:key|id:"
+
 
 @pytest.mark.parametrize("text, expected", [
     (None, None),
@@ -198,7 +230,7 @@ def test_i_can_str_concept():
     ("xxx", None),
     ("xxx.", None),
     ("xxx.yyy", None),
-    ("core.concept.ConceptParts.BODY", ConceptParts.BODY),
+    ("core.tokenizer.Keywords.CONCEPT", Keywords.CONCEPT),
 ])
 def test_i_can_decode_enum(text, expected):
     actual = core.utils.decode_enum(text)
@@ -214,15 +246,24 @@ def test_encode_concept_key_id():
     concept = Concept("foo").init_key()
     assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_00None00__C__"
 
-    concept.metadata.id = "1001"
+    concept.get_metadata().id = "1001"
     assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_1001__C__"
 
+    assert core.utils.encode_concept(("key", "id"), "R") == "__R__KEY_key__ID_id__R__"
+    assert core.utils.encode_concept((None, "id"), "R") == "__R__KEY_00None00__ID_id__R__"
+    assert core.utils.encode_concept(("key", None), "R") == "__R__KEY_key__ID_00None00__R__"
+    assert core.utils.encode_concept(("k + y", "id"), "R") == "__R__KEY_k000y__ID_id__R__"
+
 
 def test_decode_concept_key_id():
     assert core.utils.decode_concept("__C__KEY_key__ID_id__C__") == ("key", "id")
     assert core.utils.decode_concept("__C__KEY_00None00__ID_id__C__") == (None, "id")
     assert core.utils.decode_concept("__C__KEY_key__ID_00None00__C__") == ("key", None)
 
+    assert core.utils.decode_concept("__R__KEY_key__ID_id__R__", "R") == ("key", "id")
+    assert core.utils.decode_concept("__R__KEY_00None00__ID_id__R__", "R") == (None, "id")
+    assert core.utils.decode_concept("__R__KEY_key__ID_00None00__R__", "R") == ("key", None)
+
 
 @pytest.mark.parametrize("a,b,expected", [
     ([], [], []),
@@ -246,3 +287,54 @@ def test_i_can_make_unique():
     assert core.utils.make_unique(["a", "a", "b", "c", "c"]) == ["a", "b", "c"]
     assert core.utils.make_unique([Obj("a", "b"), Obj("a", "c"), Obj("a", "b")]) == [Obj("a", "b"), Obj("a", "c")]
     assert core.utils.make_unique([Obj("a", "b"), Obj("a", "c")], lambda o: o.prop1) == [Obj("a", "b")]
+
+
+@pytest.mark.parametrize("expression, bag, expected", [
+    ("", {}, None),
+    (None, {}, None),
+    ("a", {"a": 1}, 1),
+    ("a.x", {"a.x": 1}, 1),
+    ("a.prop1", {"a": Obj("prop1", "prop2")}, "prop1"),
+    ("a.prop1.prop2.prop1", {"a": Obj2(Obj2("prop11", Obj2("prop121", "prop122")), "2")}, "prop121"),
+    ("a.prop1.prop2.prop1", {"a": Obj2(Obj2("prop11", None), "2")}, None),
+    ("a[1]", {"a": ['lst-first', 'lst-second']}, 'lst-second'),
+    ("a['bar']", {"a": {'foo': 'dict-first', 'bar': 'dict-second'}}, 'dict-second'),
+    ("a.prop1[0]", {"a": Obj2(['lst-first', 'lst-second'], None)}, 'lst-first'),
+    ("a.prop2['bar']", {"a": Obj2(None, {'foo': 'dict-first', 'bar': 'dict-second'})}, 'dict-second'),
+    ("a.bar", {"a": {'foo': 'dict-first', 'bar': 'dict-second'}}, 'dict-second'),
+    ("a.prop2.bar", {"a": Obj2(None, {'foo': 'dict-first', 'bar': 'dict-second'})}, 'dict-second'),
+])
+def test_i_can_evaluate_expression(expression, bag, expected):
+    assert core.utils.evaluate_expression(expression, bag) == expected
+
+
+@pytest.mark.parametrize("expression, bag, expected_error, prop_name", [
+    ("a", {}, NameError, "a"),
+    ("a.prop3", {"a": Obj("prop1", "prop2")}, NameError, "prop3"),
+])
+def test_i_cannot_evaluate_expression(expression, bag, expected_error, prop_name):
+    with pytest.raises(expected_error) as e:
+        core.utils.evaluate_expression(expression, bag)
+
+    assert e.value.args == (prop_name,)
+
+
+@pytest.mark.parametrize("text, expected_text", [
+    ("hello world", "hello world"),
+    ("'hello' 'world'", "'hello' 'world'"),
+    ("def concept a from", "def concept a from"),
+    ("()[]{}1=1.5+-/*><&é", "()[]{}1=1.5+-/*><&é"),
+    ("execute(c:concept_name:)", "execute(c:concept_name:)")
+
+])
+def test_i_can_get_text_from_tokens(text, expected_text):
+    tokens = list(Tokenizer(text))
+    assert core.utils.get_text_from_tokens(tokens) == expected_text
+
+
+@pytest.mark.parametrize("text, custom, expected_text", [
+    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)")
+])
+def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
+    tokens = list(Tokenizer(text))
+    assert core.utils.get_text_from_tokens(tokens, custom) == expected_text

tests/evaluators/test_DefConceptEvaluator.py

@@ -4,17 +4,17 @@ import pytest
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
 from core.tokenizer import Tokenizer
-from evaluators.AddConceptEvaluator import AddConceptEvaluator
+from evaluators.DefConceptEvaluator import DefConceptEvaluator
 from parsers.BaseParser import BaseParser
 from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
-from parsers.BnfParser import BnfParser
+from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.DefConceptParser import DefConceptNode, NameNode
 from parsers.PythonParser import PythonNode, PythonParser
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
-class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
+class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
 
     @staticmethod
     def get_concept_part(part):
@@ -47,7 +47,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
             status=True,
             value=ParserResultConcept(
                 source=source,
-                parser=BnfParser(),
+                parser=BnfDefinitionParser(),
                 value=parsing_expression
             )
         )
@@ -86,7 +86,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_match(self, ret_val, expected):
         context = self.get_context()
-        assert AddConceptEvaluator().matches(context, ret_val) == expected
+        assert DefConceptEvaluator().matches(context, ret_val) == expected
 
     def test_that_the_source_is_correctly_set_for_bnf_concept(self):
         context = self.get_context()
@@ -98,21 +98,21 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
             body="print('hello' + a)",
             ret="a")
 
-        evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
+        evaluated = DefConceptEvaluator().eval(context, def_concept_return_value)
 
         assert evaluated.status
         assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
 
         created_concept = evaluated.body.body
-        assert created_concept.metadata.name == "hello a"
-        assert created_concept.metadata.key == "hello __var__0"
-        assert created_concept.metadata.where == "isinstance(a, str )"
-        assert created_concept.metadata.pre == "a is not None"
-        assert created_concept.metadata.post is None  # test that NotInitialized is mapped into None
-        assert created_concept.metadata.body == "print('hello' + a)"
-        assert created_concept.metadata.ret == "a"
-        assert created_concept.metadata.definition == "hello a"
-        assert created_concept.metadata.definition_type == "bnf"
+        assert created_concept.get_metadata().name == "hello a"
+        assert created_concept.get_metadata().key == "hello __var__0"
+        assert created_concept.get_metadata().where == "isinstance(a, str )"
+        assert created_concept.get_metadata().pre == "a is not None"
+        assert created_concept.get_metadata().post is None  # test that NotInitialized is mapped into None
+        assert created_concept.get_metadata().body == "print('hello' + a)"
+        assert created_concept.get_metadata().ret == "a"
+        assert created_concept.get_metadata().definition == "hello a"
+        assert created_concept.get_metadata().definition_type == "bnf"
 
     def test_i_can_add_concept_with_the_correct_variables_when_referencing_other_concepts(self):
         context = self.get_context()
@@ -121,13 +121,13 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
             where=self.pretval(Concept("u is a v").def_var("u").def_var("v"), source="x is a number"),
             body=self.pretval(Concept("add a b").def_var("a").def_var("b"), source="add x y"), )
 
-        evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
+        evaluated = DefConceptEvaluator().eval(context, def_concept_return_value)
 
         assert evaluated.status
         assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
 
         created_concept = evaluated.body.body
-        assert created_concept.metadata.variables == [("x", None), ("y", None)]
+        assert created_concept.get_metadata().variables == [("x", None), ("y", None)]
 
     def test_that_the_source_is_correctly_set_for_concept_with_simple_definition(self):
         context = self.get_context()
@@ -138,20 +138,20 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
             pre="a is not None",
             body="print('hello' + a)")
 
-        evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
+        evaluated = DefConceptEvaluator().eval(context, def_concept_return_value)
 
         assert evaluated.status
         assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
 
         created_concept = evaluated.body.body
-        assert created_concept.metadata.name == "greetings"
-        assert created_concept.metadata.key == "hello __var__0"
-        assert created_concept.metadata.where == "isinstance(a, str )"
-        assert created_concept.metadata.pre == "a is not None"
-        assert created_concept.metadata.post is None
-        assert created_concept.metadata.body == "print('hello' + a)"
-        assert created_concept.metadata.definition == "hello a"
-        assert created_concept.metadata.definition_type == "def"
+        assert created_concept.get_metadata().name == "greetings"
+        assert created_concept.get_metadata().key == "hello __var__0"
+        assert created_concept.get_metadata().where == "isinstance(a, str )"
+        assert created_concept.get_metadata().pre == "a is not None"
+        assert created_concept.get_metadata().post is None
+        assert created_concept.get_metadata().body == "print('hello' + a)"
+        assert created_concept.get_metadata().definition == "hello a"
+        assert created_concept.get_metadata().definition_type == "def"
 
     def test_that_the_new_concept_is_correctly_saved_in_db(self):
         context = self.get_context()
@@ -166,23 +166,23 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         from_db = context.sheerka.get_by_key("hello " + VARIABLE_PREFIX + "0")
         assert context.sheerka.isinstance(from_db, BuiltinConcepts.UNKNOWN_CONCEPT)
 
-        AddConceptEvaluator().eval(context, def_concept_return_value)
+        DefConceptEvaluator().eval(context, def_concept_return_value)
         context.sheerka.concepts_cache = {}  # reset cache
         from_db = context.sheerka.get_by_key("hello " + VARIABLE_PREFIX + "0")
 
-        assert from_db.metadata.key == f"hello {VARIABLE_PREFIX}0"
-        assert from_db.metadata.name == "hello a"
-        assert from_db.metadata.where == "isinstance(a, str )"
-        assert from_db.metadata.pre == "a is not None"
-        assert from_db.metadata.post is None
-        assert from_db.metadata.body == "print('hello' + a)"
-        assert from_db.metadata.definition == "hello a"
-        assert from_db.metadata.definition_type == "bnf"
-        assert len(from_db.metadata.variables) == 1
-        assert from_db.metadata.variables[0] == ("a", None)
-        assert "a" in from_db.values
+        assert from_db.get_metadata().key == f"hello {VARIABLE_PREFIX}0"
+        assert from_db.get_metadata().name == "hello a"
+        assert from_db.get_metadata().where == "isinstance(a, str )"
+        assert from_db.get_metadata().pre == "a is not None"
+        assert from_db.get_metadata().post is None
+        assert from_db.get_metadata().body == "print('hello' + a)"
+        assert from_db.get_metadata().definition == "hello a"
+        assert from_db.get_metadata().definition_type == "bnf"
+        assert len(from_db.get_metadata().variables) == 1
+        assert from_db.get_metadata().variables[0] == ("a", None)
+        assert "a" in from_db.values()
 
-        assert from_db.compiled == {}  # ast is not saved in db
+        assert from_db.get_compiled() == {}  # ast is not saved in db
 
     @pytest.mark.parametrize("expression, name, expected", [
         ("isinstance(a, str)", "a b", {"a"}),
@@ -195,39 +195,39 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         ret_val = self.get_concept_part(expression)
         context = self.get_context()
 
-        assert AddConceptEvaluator.get_variables(context.sheerka, ret_val, name.split()) == expected
+        assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, name.split()) == expected
 
     def test_i_can_get_variables_when_keywords(self):
         sheerka, context = self.init_concepts()
 
         def_concept = self.get_def_concept("condition pre").value.value
-        name_to_use = AddConceptEvaluator.get_name_to_use(def_concept)
+        name_to_use = DefConceptEvaluator.get_name_to_use(def_concept)
         concept_part = self.get_concept_part("pre")
 
-        assert AddConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == {"pre"}
+        assert DefConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == {"pre"}
 
     def test_i_cannot_get_variables_from_python_node_when_name_has_only_one_token(self):
         ret_val = self.get_concept_part("isinstance(a, str)")
         context = self.get_context()
 
-        assert AddConceptEvaluator.get_variables(context.sheerka, ret_val, ["a"]) == []
+        assert DefConceptEvaluator.get_variables(context.sheerka, ret_val, ["a"]) == []
 
     def test_i_can_get_variables_from_definition(self):
         parsing_expression = Sequence(ConceptExpression('mult'),
                                       ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
         ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression)
 
-        assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"add", "mult"}
+        assert DefConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"add", "mult"}
 
     def test_concept_that_references_itself_is_correctly_created(self):
         context = self.get_context()
         def_concept_as_return_value = self.get_def_concept("foo", body="foo")
 
-        ret_val = AddConceptEvaluator().eval(context, def_concept_as_return_value)
+        ret_val = DefConceptEvaluator().eval(context, def_concept_as_return_value)
 
         assert ret_val.status
         new_concept = ret_val.body.body
         assert new_concept.name == 'foo'
-        assert new_concept.metadata.body == 'foo'
-        assert new_concept.values == {}
-        assert new_concept.metadata.variables == []
+        assert new_concept.get_metadata().body == 'foo'
+        assert new_concept.values() == {}
+        assert new_concept.get_metadata().variables == []

tests/evaluators/test_FormatRuleEvaluator.py

@@ -0,0 +1,51 @@
+import pytest
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.rule import Rule, RuleMetadata
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import Tokenizer
+from evaluators.FormatRuleEvaluator import FormatRuleEvaluator
+from parsers.FormatRuleParser import FormatRuleNode, FormatRuleParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestFormatRuleEvaluator(TestUsingMemoryBasedSheerka):
+
+    @staticmethod
+    def get_ret_val_from_rule(rule_def):
+        tokens = {k: list(Tokenizer(k.value + " " + v, yield_eof=False)) for k, v in rule_def.items()}
+        for v in tokens.values():
+            del v[1]
+
+        node = FormatRuleNode(tokens)
+        return ReturnValueConcept("parsers.FormatRule", True, ParserResultConcept(value=node))
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("name", True, ParserResultConcept(value=FormatRuleNode({}))), True),
+        (ReturnValueConcept("name", True, ParserResultConcept(value="other object")), False),
+        (ReturnValueConcept("name", False, ParserResultConcept(value=FormatRuleNode({}))), False),
+        (ReturnValueConcept("name", False, FormatRuleNode({})), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert FormatRuleEvaluator().matches(context, ret_val) == expected
+
+    def test_i_can_eval(self):
+        sheerka, context = self.init_concepts()
+        text = "when isinstance(value, __EXPLANATION) print list(value)"
+        ret_val = FormatRuleParser().parse(context, ParserInput(text))
+
+        res = FormatRuleEvaluator().eval(context, ret_val)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_RULE)
+        assert isinstance(res.body.body, Rule)
+        assert res.body.body.metadata == RuleMetadata("print",
+                                                      None,
+                                                      "isinstance(value, __EXPLANATION)",
+                                                      "list(value)",
+                                                      id=res.body.body.metadata.id,  # no need to compare the id
+                                                      is_compiled=True,
+                                                      is_enabled=True)
+        assert res.body.body.compiled_predicate is not None
+        assert res.body.body.compiled_action is not None

tests/evaluators/test_LexerNodeEvaluator.py

@@ -69,7 +69,7 @@ class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka):
         assert wrapper.parser == evaluator
         assert wrapper.source == "foo"
         assert return_value == foo
-        assert return_value.compiled[ConceptParts.BODY] == DoNotResolve("foo")
+        assert return_value.get_compiled()[ConceptParts.BODY] == DoNotResolve("foo")
         assert result.parents == [ret_val]
 
     def test_concept_python_node_is_returned_when_source_code(self):
@@ -88,5 +88,5 @@ class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka):
         assert wrapper.source == "foo + 1"
 
         assert return_value == PythonNode('foo + 1', ast.parse("__C__foo__C__ + 1", mode="eval"))
-        assert return_value.concepts == {"__C__foo__C__": foo}
+        assert return_value.objects == {"__C__foo__C__": foo}
         assert result.parents == [ret_val]

tests/evaluators/test_OneSuccessEvaluator.py

@@ -1,13 +1,16 @@
 import pytest
-
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts, ParserResultConcept
 from core.concept import Concept
+from core.rule import Rule
 from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
+
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
-def r(value, status=True):
-    return ReturnValueConcept(value, status, value)
+def r(name, status=True, value=None):
+    if isinstance(value, Rule):
+        value = ParserResultConcept(name, value=value, try_parsed=value)
+    return ReturnValueConcept(name, status, value or name)
 
 
 reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
@@ -25,6 +28,7 @@ class TestOneSuccessEvaluator(TestUsingMemoryBasedSheerka):
         ([r("evaluators.success"), r("evaluators.another success"), reduce_requested], False),
         ([r("evaluators.one success"), r("other success"), r("failed", False), reduce_requested], True),
         ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False),
+        ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False),
     ])
     def test_i_can_match(self, return_values, expected):
         context = self.get_context()

tests/evaluators/test_PythonEvaluator.py

@@ -2,19 +2,20 @@ import ast
 
 import pytest
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import Concept, CB, NotInit
+from core.concept import Concept, CB
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
-from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError
+from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError, NamesWithAttributesVisitor
 from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode
+from parsers.FunctionParser import FunctionParser
 from parsers.PythonParser import PythonNode, PythonParser
 from parsers.PythonWithConceptsParser import PythonWithConceptsParser
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
-def get_concept_name(concept):
-    return concept.name
+def get_obj_name(obj):
+    return obj.name
 
 
 def get_source_code_node(source_code, concepts=None):
@@ -36,7 +37,7 @@ def get_source_code_node(source_code, concepts=None):
         tokens = list(Tokenizer(source_code, yield_eof=False))
         return SourceCodeNode(0, len(tokens), tokens, python_node=python_node)
     else:
-        python_node.concepts = concepts
+        python_node.objects = concepts
         scwcn = SourceCodeWithConceptNode(None, None)
         scwcn.python_node = python_node
         return scwcn
@@ -62,10 +63,13 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert PythonEvaluator().matches(context, ret_val) == expected
 
     @pytest.mark.parametrize("text, expected", [
-        ("1 + 1", 2),
-        ("test()", "I have access to Sheerka !"),
+        # ("1 + 1", 2),
+        # ("test()", "I have access to Sheerka !"),
         ("sheerka.test()", "I have access to Sheerka !"),
         ("a=10\na", 10),
+        ("Concept('foo')", Concept('foo')),
+        ("BuiltinConcepts.NOP", BuiltinConcepts.NOP),
+        ("in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", False),
     ])
     def test_i_can_eval(self, text, expected):
         context = self.get_context()
@@ -76,6 +80,29 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.value == expected
 
+    def test_i_can_eval_isinstance(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        parsed = PythonParser().parse(context, ParserInput("isinstance('foo', str)"))
+        evaluated = PythonEvaluator().eval(context, parsed)
+        assert evaluated.status
+        assert evaluated.value
+
+        parsed = PythonParser().parse(context, ParserInput("isinstance(foo, 'foo')"))
+        evaluated = PythonEvaluator().eval(context, parsed)
+        assert evaluated.status
+        assert evaluated.value
+
+    def test_i_can_eval_context_obj_properties(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a", "hello world!").auto_init())
+        context = self.get_context()
+        parsed = PythonParser().parse(context, ParserInput("a"))
+
+        context.obj = foo
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == "hello world!"
+
     @pytest.mark.parametrize("source_code_node, expected", [
         (get_source_code_node("1 + 1"), 2),
         (get_source_code_node("one + one", {"one": Concept("one", body="1")}), 2)
@@ -89,14 +116,14 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.value == expected
 
-    def test_i_can_eval_using_context(self):
+    def test_i_can_eval_a_method_that_requires_the_context(self):
         context = self.get_context()
-        parsed = PythonParser().parse(context, ParserInput("test_using_context('value for param1', 10)"))
+        parsed = PythonParser().parse(context, ParserInput("test_using_context('value for param')"))
 
         evaluated = PythonEvaluator().eval(context, parsed)
 
         assert evaluated.status
-        assert evaluated.value.startswith("I have access to Sheerka ! param1='value for param1', param2=10, event=")
+        assert evaluated.value == "I have access to Sheerka ! param='value for param', event='xxx'."
 
     def test_i_can_eval_using_context_when_self_is_not_sheerka(self):
         sheerka, context = self.init_concepts()
@@ -107,21 +134,6 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert sheerka.has_key("foo")
 
-    @pytest.mark.parametrize("concept", [
-        Concept("foo"),
-        Concept("foo", body="2"),
-        Concept("foo").def_var("prop", "'a'"),
-        Concept("foo", body="bar")
-    ])
-    def test_simple_concepts_are_not_for_me(self, concept):
-        context = self.get_context()
-        context.sheerka.add_in_cache(Concept("foo"))
-
-        parsed = PythonParser().parse(context, ParserInput("foo"))
-        evaluated = PythonEvaluator().eval(context, parsed)
-
-        assert not evaluated.status
-        assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.NOT_FOR_ME)
 
     def test_i_can_eval_ast_expression_that_references_concepts(self):
         """
@@ -167,10 +179,10 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
     def test_i_can_eval_concept_token(self):
         context = self.get_context()
         context.sheerka.add_in_cache(Concept("foo", body="2"))
+        context.add_to_short_term_memory("get_obj_name", get_obj_name)
 
-        parsed = PythonParser().parse(context, ParserInput("get_concept_name(c:foo:)"))
+        parsed = PythonParser().parse(context, ParserInput("get_obj_name(c:foo:)"))
         python_evaluator = PythonEvaluator()
-        python_evaluator.globals["get_concept_name"] = get_concept_name
         evaluated = python_evaluator.eval(context, parsed)
 
         assert evaluated.status
@@ -210,7 +222,8 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         evaluated = python_evaluator.eval(context, parsed)
 
         assert evaluated.status
-        assert sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE) == {'1001': 1, '1002': 2}
+        assert sheerka.get_concepts_weights(BuiltinConcepts.PRECEDENCE) == {'c:__var__0 plus __var__1|1001:': 1,
+                                                                            'c:__var__0 mult __var__1|1002:': 2}
 
     def test_i_can_define_variables(self):
         sheerka, context = self.init_concepts()
@@ -261,7 +274,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         evaluated = PythonEvaluator().eval(context, parsed)
 
         assert not evaluated.status
-        assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.TOO_MANY_ERRORS)
+        assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.ERROR)
 
         error0 = evaluated.body.body[0]
         assert isinstance(error0, PythonEvalError)
@@ -275,21 +288,6 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert error1.error.args[0] == 'can only concatenate str (not "int") to str'
         assert error1.concepts == {'foo': 'string'}
 
-    def test_i_do_not_include_not_initialized_variables_when_evaluating(self):
-        sheerka, context, foo = self.init_concepts(
-            Concept("foo a", pre="a == 'True'").def_var("a", "'True'").def_var("b"))
-
-        foo.set_value("b", "'Initialized!'")
-        context.obj = foo
-
-        assert foo.get_value("a") == NotInit
-        assert foo.get_value("b") == "'Initialized!'"
-
-        my_globals = {}
-        PythonEvaluator().update_globals_with_context(my_globals, context)
-
-        assert my_globals == {"self": foo, "b": "'Initialized!'"}
-
     def test_i_can_use_sheerka_locals(self):
         sheerka, context = self.init_concepts()
 
@@ -307,11 +305,47 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_eval_concept_with_ret(self):
         sheerka, context, one, the = self.init_concepts("one", Concept("the a", ret="a").def_var("a"))
-        ret_val = get_ret_val_from_source_code(context, "test_using_context(one, the one)", {
-            "the one": self.get_concept_instance(sheerka, the, a="one"),
-            "one": self.get_concept_instance(sheerka, "one")
+        ret_val = get_ret_val_from_source_code(context, "test_using_context(the one)", {
+            "the one": self.get_concept_instance(sheerka, the, a="one")
         })
 
         evaluated = PythonEvaluator().eval(context, ret_val)
         assert evaluated.status
-        assert evaluated.value.startswith("I have access to Sheerka ! param1=(1001)one, param2=(1001)one, event=")
+        assert evaluated.value == "I have access to Sheerka ! param=(1001)one, event='xxx'."
+
+    def test_i_can_eval_rules_from_python_parser(self):
+        sheerka, context = self.init_concepts()
+        parsed_ret_val = PythonParser().parse(context, ParserInput("r:|1:.id"))
+
+        assert parsed_ret_val.status
+
+        evaluated = PythonEvaluator().eval(context, parsed_ret_val)
+        assert evaluated.status
+        assert evaluated.value == "1"
+
+    def test_i_can_eval_rules_from_function_parser(self):
+        context = self.get_context()
+        context.add_to_short_term_memory("get_obj_name", get_obj_name)
+
+        parsed = FunctionParser().parse(context, ParserInput("get_obj_name(r:|1:)"))
+        python_evaluator = PythonEvaluator()
+        evaluated = python_evaluator.eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == "Print return values"
+
+    @pytest.mark.parametrize("text, expected", [
+        ("foo.bar.baz", [["foo", "bar", "baz"]]),
+        ("foo.bar.baz; one.two.three", [["foo", "bar", "baz"]]),
+        ("foo.bar.baz; one.two.three; foo.bar", [["foo", "bar", "baz"], ["foo", "bar"]]),
+        ("one.two.three", []),
+        ("foo", [["foo"]]),
+        ("foo.bar[1].baz", [["foo", "bar", "baz"]]),
+        ("foo[1].bar.baz", [["foo", "bar", "baz"]]),
+    ])
+    def test_names_with_attributes_visitor(self, text, expected):
+        ast_ = ast.parse(text, "<src>", mode="exec")
+        visitor = NamesWithAttributesVisitor()
+
+        sequence = visitor.get_sequences(ast_, "foo")
+        assert sequence == expected

tests/evaluators/test_ReturnBodyEvaluator.py

@@ -0,0 +1,50 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from evaluators.ReturnBodyEvaluator import ReturnBodyEvaluator
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestReturnBodyEvaluator(TestUsingMemoryBasedSheerka):
+    def test_i_can_match(self):
+        sheerka, root_context = self.init_concepts()
+        root_context.add_to_global_hints(BuiltinConcepts.RETURN_BODY_REQUESTED)
+
+        context = root_context.push(BuiltinConcepts.PROCESSING, None)
+        sub_context = context.push(BuiltinConcepts.PROCESSING, None)
+        sub_sub_context = sub_context.push(BuiltinConcepts.PROCESSING, None)
+
+        evaluator = ReturnBodyEvaluator()
+
+        assert evaluator.matches(root_context, [])  # no parent, real root
+        assert evaluator.matches(context, [])  # no BuiltinConcepts.PROCESSING, is root for unit test
+        assert evaluator.matches(sub_context, [])  # only one BuiltinConcepts.PROCESSING, is root in prod
+        assert not evaluator.matches(sub_sub_context, [])  # not a root
+
+    def test_i_can_eval_concept(self):
+        sheerka, root_context = self.init_concepts()
+        root_context.add_to_global_hints(BuiltinConcepts.RETURN_BODY_REQUESTED)
+        context = root_context.push(BuiltinConcepts.PROCESSING, None)
+        evaluator = ReturnBodyEvaluator()
+
+        return_values = [
+            sheerka.ret("Who", True, Concept("foo", body="status -> will be reduce").auto_init()),
+            sheerka.ret("Who", False, Concept("foo", body="status -> won't be reduced").auto_init()),
+            sheerka.ret("Who", False, Concept("foo", body="body not init -> won't be reduced")),
+        ]
+
+        res = evaluator.eval(context, return_values)
+
+        assert res == [sheerka.ret(evaluator.name, True, "status -> will be reduce")]
+        assert res[0].parents == [return_values[0]]
+
+    def test_i_do_not_reduce_containers_concepts(self):
+        sheerka, root_context = self.init_concepts()
+        root_context.add_to_global_hints(BuiltinConcepts.RETURN_BODY_REQUESTED)
+        context = root_context.push(BuiltinConcepts.PROCESSING, None)
+        evaluator = ReturnBodyEvaluator()
+
+        container = sheerka.new(BuiltinConcepts.EXPLANATION, body=[])  # body is initialized
+        return_values = [sheerka.ret("who", True, container)]
+
+        assert evaluator.eval(context, return_values) is None

tests/evaluators/test_RuleEvaluator.py

@@ -0,0 +1,70 @@
+import pytest
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.rule import Rule, ACTION_TYPE_DEFERRED
+from evaluators.RuleEvaluator import RuleEvaluator
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestRuleEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=[Rule()])), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept")), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=[])), False),
+        (ReturnValueConcept("some_name", True, Rule), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=Rule())), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=[Rule()])), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert RuleEvaluator().matches(context, ret_val) == expected
+
+    @pytest.mark.parametrize("body, expected", [
+        ([Rule("exec", "rule1")], Rule("exec", "rule1")),
+        ([Rule("exec", "rule1"), Rule("exec", "rule2")], [Rule("exec", "rule1"), Rule("exec", "rule2")]),
+
+    ])
+    def test_i_can_evaluate(self, body, expected):
+        context = self.get_context()
+        ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=body), True)
+        evaluator = RuleEvaluator()
+
+        result = evaluator.eval(context, ret_val)
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert result.value == expected
+        assert result.parents == [ret_val]
+
+    def test_i_can_evaluate_deferred_rules(self):
+        context = self.get_context()
+
+        rule = Rule().set_id("i")
+        rule.metadata.action_type = ACTION_TYPE_DEFERRED
+        ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=[rule]), True)
+        context.add_to_short_term_memory("i", 1)
+        evaluator = RuleEvaluator()
+
+        result = evaluator.eval(context, ret_val)
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert result.value == context.sheerka.get_rule_by_id("1")
+        assert result.parents == [ret_val]
+
+    def test_i_cannot_evaluate_deferred_rule_if_the_rule_is_unknown(self):
+        sheerka, context = self.init_concepts()
+
+        rule = Rule().set_id("unknown variable")
+        rule.metadata.action_type = ACTION_TYPE_DEFERRED
+        ret_val = ReturnValueConcept("some_name", True, ParserResultConcept(value=[rule]), True)
+        evaluator = RuleEvaluator()
+
+        result = evaluator.eval(context, ret_val)
+
+        assert result.who == evaluator.name
+        assert not result.status
+        assert sheerka.isinstance(result.value, BuiltinConcepts.UNKNOWN_RULE)
+        assert result.value.body == [('id', 'unknown variable')]
+        assert result.parents == [ret_val]

tests/non_reg/test_sheerka_non_reg.py

@@ -56,7 +56,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
 
         # sanity check
         evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), return_value)
-        assert evaluated == simplec("un", simplec("one", BuiltinConcepts.NOT_INITIALIZED))
+        assert evaluated == simplec("un", simplec("one", NotInit))
 
     def test_i_can_recognize_concept_with_no_body(self):
         sheerka = self.get_sheerka()
@@ -95,9 +95,9 @@ as:
 """
 
         expected = self.get_default_concept()
-        expected.metadata.id = "1001"
-        expected.metadata.desc = None
-        expected.metadata.variables = [("a", None), ("b", None)]
+        expected.get_metadata().id = "1001"
+        expected.get_metadata().desc = None
+        expected.get_metadata().variables = [("a", None), ("b", None)]
         expected.init_key()
 
         sheerka = self.get_sheerka(cache_only=False)
@@ -110,7 +110,7 @@ as:
         concept_saved = res[0].value.body
 
         for prop in PROPERTIES_TO_SERIALIZE:
-            assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
+            assert getattr(concept_saved.get_metadata(), prop) == getattr(expected.get_metadata(), prop)
 
         # cache is up to date
         assert sheerka.has_key(concept_saved.key)
@@ -136,7 +136,7 @@ as:
 
         res = sheerka.evaluate_user_input("def concept a xx b as a plus b")
         expected = Concept(name="a xx b", body="a plus b").def_var("a").def_var("b").init_key()
-        expected.metadata.id = "1001"
+        expected.get_metadata().id = "1001"
 
         assert len(res) == 1
         assert res[0].status
@@ -145,7 +145,7 @@ as:
         concept_saved = res[0].value.body
 
         for prop in PROPERTIES_TO_SERIALIZE:
-            assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
+            assert getattr(concept_saved.get_metadata(), prop) == getattr(expected.get_metadata(), prop)
 
         assert sheerka.has_key(concept_saved.key)
 
@@ -212,9 +212,9 @@ as:
         # sanity check
         evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value)
         assert evaluated.body == "hello foo"
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().is_evaluated
         assert evaluated.get_value("a") == simplec("foo", "foo")
-        assert evaluated.get_value("a").metadata.is_evaluated
+        assert evaluated.get_value("a").get_metadata().is_evaluated
 
     def test_i_can_recognize_duplicate_concepts_with_same_value(self):
         # when multiple result, choose the one that is the more specific (that has the less variables)
@@ -227,7 +227,7 @@ as:
         assert len(res) == 1
         assert res[0].status
         assert sheerka.isinstance(res[0].body, "hello foo")
-        assert res[0].value.body == BuiltinConcepts.NOT_INITIALIZED
+        assert res[0].value.body == NotInit
         assert res[0].who == sheerka.get_evaluator_name(OneSuccessEvaluator.NAME)
 
     def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different(self):
@@ -297,9 +297,9 @@ as:
         # sanity check
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka=sheerka, eval_body=True), return_value)
         assert evaluated.body == "one three"
-        assert evaluated.metadata.is_evaluated
+        assert evaluated.get_metadata().is_evaluated
         assert evaluated.get_value("a") == sheerka.new(concept_a.key, body="one").init_key()
-        assert evaluated.get_value("a").metadata.is_evaluated
+        assert evaluated.get_value("a").get_metadata().is_evaluated
 
     @pytest.mark.parametrize("user_input", [
         "def concept greetings from def hello a where a",
@@ -315,7 +315,7 @@ as:
         concept_found = res[0].value
         assert sheerka.isinstance(concept_found, greetings)
         assert concept_found.get_value("a") == "foo"
-        assert concept_found.metadata.need_validation
+        assert concept_found.get_metadata().need_validation
 
         res = sheerka.evaluate_user_input("greetings")
         assert len(res) == 1
@@ -323,7 +323,7 @@ as:
         concept_found = res[0].value
         assert sheerka.isinstance(concept_found, greetings)
         assert concept_found.get_value("a") == NotInit
-        assert not concept_found.metadata.need_validation
+        assert not concept_found.get_metadata().need_validation
 
     @pytest.mark.parametrize("desc, definitions", [
         ("Simple form", [
@@ -718,7 +718,7 @@ as:
         assert len(res) == 1
         assert res[0].status
         assert sheerka.isinstance(res[0].body, "foobar")
-        assert res[0].body.body == BuiltinConcepts.NOT_INITIALIZED
+        assert res[0].body.body == NotInit
 
         res = sheerka.evaluate_user_input("eval foo bar")
         assert len(res) == 1
@@ -871,14 +871,17 @@ as:
 
         res = sheerka.evaluate_user_input("get_concepts_weights('some_prop')")
         assert res[0].status
-        assert res[0].body == {'1001': 1, '1002': 2, '1003': 3}
+        assert res[0].body == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3}
 
         # test i use a concept to define relation
         sheerka.evaluate_user_input("def concept a > b as set_is_greater_than('some_prop', a, b)")
         res = sheerka.evaluate_user_input("eval four > three")
         assert res[0].status
 
-        assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2, '1003': 3, '1004': 4}
+        assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1,
+                                                             'c:two|1002:': 2,
+                                                             'c:three|1003:': 3,
+                                                             'c:four|1004:': 4}
 
     def test_i_can_evaluate_expression_when_using_token_concept(self):
         sheerka, context, one, two, three, is_less_than = self.init_concepts(
@@ -899,14 +902,14 @@ as:
         res = sheerka.evaluate_user_input(expression)
         assert res[0].status
         assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
-        assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2}
+        assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2}
 
         # it now also works using the concepts names
         expression = "eval two < three"
         res = sheerka.evaluate_user_input(expression)
         assert res[0].status
         assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
-        assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2, '1003': 3}
+        assert sheerka.get_concepts_weights("some_prop") == {'c:one|1001:': 1, 'c:two|1002:': 2, 'c:three|1003:': 3}
 
     def test_i_can_detect_multiple_errors_when_evaluating_a_concept(self):
         sheerka, context, foo, plus_one = self.init_concepts(
@@ -918,7 +921,7 @@ as:
 
         assert not res[0].status
         assert context.sheerka.isinstance(res[0].body, BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        assert context.sheerka.isinstance(res[0].body.body, BuiltinConcepts.TOO_MANY_ERRORS)
+        assert context.sheerka.isinstance(res[0].body.body, BuiltinConcepts.ERROR)
 
         error0 = res[0].body.body.body[0]
         assert isinstance(error0, PythonEvalError)
@@ -951,7 +954,7 @@ as:
 
         # simulate that sheerka was stopped and restarted
         sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
-        sheerka.cache_manager.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "twenties").compiled = {}
+        sheerka.cache_manager.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "twenties").set_compiled({})
 
         res = sheerka.evaluate_user_input("eval twenty one")
         assert res[0].status
@@ -985,8 +988,8 @@ as:
         plus = res[0].body
         assert isinstance(plus, Concept)
         assert plus.name == "plus"
-        assert plus.compiled["a"] == sheerka.new("one")
-        assert plus.compiled["b"] == CC(the, a=sheerka.new("one"))
+        assert plus.get_compiled()["a"] == sheerka.new("one")
+        assert plus.get_compiled()["b"] == CC(the, a=sheerka.new("one"))
 
         res = sheerka.evaluate_user_input("eval one plus the one")
         assert res[0].status
@@ -1063,7 +1066,7 @@ as:
             "def concept q from q ? as question(q)",
             "set_auto_eval(q)",
             "def concept is_a from x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)",
-            "set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:)",
+            "set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:, 'Sya')",
         ]
 
         sheerka = self.init_scenario(init)
@@ -1185,20 +1188,20 @@ class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
 
         saved_concept = sheerka.sdp.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "plus")
         assert saved_concept.key == "plus"
-        assert saved_concept.metadata.definition == "a ('plus' plus)?"
-        assert "a" in saved_concept.values
-        assert "plus" in saved_concept.values
+        assert saved_concept.get_metadata().definition == "a ('plus' plus)?"
+        assert "a" in saved_concept.values()
+        assert "plus" in saved_concept.values()
 
         expected_bnf = Sequence(
             ConceptExpression(concept_a, rule_name="a"),
             Optional(Sequence(StrMatch("plus"), ConceptExpression("plus"))))
 
         new_concept = res[0].value.body
-        assert new_concept.metadata.name == "plus"
-        assert new_concept.metadata.definition == "a ('plus' plus)?"
-        assert new_concept.bnf == expected_bnf
-        assert "a" in new_concept.values
-        assert "plus" in new_concept.values
+        assert new_concept.get_metadata().name == "plus"
+        assert new_concept.get_metadata().definition == "a ('plus' plus)?"
+        assert new_concept.get_bnf() == expected_bnf
+        assert "a" in new_concept.values()
+        assert "plus" in new_concept.values()
 
     def test_i_can_recognize_bnf_definitions_from_separate_instances(self):
         sheerka = self.get_sheerka()

tests/out/test_SheerkaOut.py

@@ -0,0 +1,381 @@
+import pytest
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from core.rule import Rule
+from core.sheerka.services.SheerkaOut import SheerkaOut
+from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, FormatAstVariable, FormatAstSequence, \
+    FormatAstColor, FormatAstVariableNotFound, FormatAstList
+from core.utils import flatten_all_children
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+seq = FormatAstSequence
+raw = FormatAstRawText
+var = FormatAstVariable
+
+
+class TestSheerkaOut(TestUsingMemoryBasedSheerka):
+
+    def init_service_with_rules(self, *rules, **kwargs):
+        sheerka, context, *rules = self.init_format_rules(*rules, **kwargs)
+        service = sheerka.services[SheerkaOut.NAME]
+
+        return sheerka, context, service, *rules
+
+    @pytest.mark.parametrize("rule, expected", [
+        (("__ret", "hello world"), FormatAstRawText("hello world")),
+        (("__ret", "{__ret_value}"), FormatAstVariable("__ret_value", None, 1)),
+        (("__ret", "{status}"), FormatAstVariable("status", None, True)),
+        (("__ret", "{foo}"), FormatAstVariableNotFound("foo")),
+        (("__ret", "hello world {__ret_value} !"), seq([FormatAstRawText("hello world "),
+                                                        FormatAstVariable("__ret_value", None, 1),
+                                                        FormatAstRawText(" !")])),
+        (("__ret", "red(__ret_value)"), FormatAstColor("red", FormatAstVariable("__ret_value", None, 1))),
+        (("__ret", "blue('hello world {__ret_value} !')"), FormatAstColor("blue",
+                                                                          seq([FormatAstRawText("hello world "),
+                                                                               FormatAstVariable("__ret_value", None,
+                                                                                                 1),
+                                                                               FormatAstRawText(" !")]))),
+        (("__ret", "list(foo)"), FormatAstVariableNotFound("foo")),
+        (("__ret", "{__ret_value:3}"), FormatAstVariable("__ret_value", "3", "  1"))
+    ])
+    def test_i_can_develop_when_simple_rules(self, rule, expected):
+        sheerka, context, service, rule = self.init_service_with_rules(rule)
+        ret = sheerka.ret("Test", True, 1)
+
+        res = service.create_out_tree(context, ret)
+
+        assert res == expected
+
+    def test_i_can_develop_rules_of_rules_using_obj_type(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("__ret", "status: {status}, value: {__ret_value}"),  # ret_value is a 'foo'
+            ("isinstance(__obj, 'foo')", "{id}-{name}:{body}")
+        )
+        sheerka.create_new_concept(context, Concept("foo"))
+        foo = sheerka.new("foo", body="hello world")
+        ret = sheerka.ret("Test", True, foo)
+
+        res = service.create_out_tree(context, ret)
+
+        assert res == seq([FormatAstRawText("status: "),
+                           FormatAstVariable("status", None, True),
+                           FormatAstRawText(", value: "),
+                           FormatAstVariable("__ret_value", None, seq([FormatAstVariable("id", None, "1001"),
+                                                                       FormatAstRawText("-"),
+                                                                       FormatAstVariable("name", None, "foo"),
+                                                                       FormatAstRawText(":"),
+                                                                       FormatAstVariable("body", None,
+                                                                                         "hello world")]))])
+
+    @pytest.mark.parametrize('second_rule', [
+        ("__ret.body", "{id}-{name}:{body}"),
+        ("body", "{id}-{name}:{body}"),
+    ])
+    def test_i_can_develop_rules_of_rules_using_obj_names(self, second_rule):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("__ret", "status: {status}, value: {__ret.body}"),
+            second_rule
+        )
+        sheerka.create_new_concept(context, Concept("foo"))
+        foo = sheerka.new("foo", body="hello world")
+        ret = sheerka.ret("Test", True, foo)
+
+        res = service.create_out_tree(context, ret)
+
+        assert res == seq([FormatAstRawText("status: "),
+                           FormatAstVariable("status", None, True),
+                           FormatAstRawText(", value: "),
+                           FormatAstVariable("__ret.body", None, seq([FormatAstVariable("id", None, "1001"),
+                                                                      FormatAstRawText("-"),
+                                                                      FormatAstVariable("name", None, "foo"),
+                                                                      FormatAstRawText(":"),
+                                                                      FormatAstVariable("body", None,
+                                                                                        "hello world")]))])
+
+    def test_i_can_develop_list(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(("isinstance(__obj, list)", "list(__obj)"))
+        lst = list(flatten_all_children(context, lambda item: item.achildren))[:3]
+
+        res = service.create_out_tree(context, lst)
+        assert res == FormatAstList(variable="__obj", items=[FormatAstVariable("__item", None, lst[0], 0),
+                                                             FormatAstVariable("__item", None, lst[1], 1),
+                                                             FormatAstVariable("__item", None, lst[2], 2)])
+
+    def test_i_can_develop_list_using_the_default_items_prop(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(("isinstance(__obj, 'foo')", "list(__obj)"))
+        lst = list(flatten_all_children(context, lambda item: item.achildren))[:3]
+        foo = Concept("foo", body=lst, key="foo").auto_init()
+
+        res = service.create_out_tree(context, foo)
+        assert res == FormatAstList(variable="__obj", items=[FormatAstVariable("__item", None, lst[0], 0),
+                                                             FormatAstVariable("__item", None, lst[1], 1),
+                                                             FormatAstVariable("__item", None, lst[2], 2)])
+
+    def test_i_can_develop_list_using_the_custom_items_prop(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("isinstance(__obj, 'foo')", "list(__obj, items_prop='custom')"))
+        lst = list(flatten_all_children(context, lambda item: item.achildren))[:3]
+        foo = Concept("foo", key="foo").def_var("custom").auto_init()
+        foo.set_value("custom", lst)
+
+        res = service.create_out_tree(context, foo)
+        assert res == FormatAstList(variable="__obj", items_prop='custom',
+                                    items=[FormatAstVariable("__item", None, lst[0], 0),
+                                           FormatAstVariable("__item", None, lst[1], 1),
+                                           FormatAstVariable("__item", None, lst[2], 2)])
+
+    def test_not_a_list_are_resolved_into_variable_ast(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(("isinstance(__obj, 'foo')", "list(__obj)"))
+        foo = Concept("foo", key="foo")
+
+        res = service.create_out_tree(context, foo)
+        assert res == FormatAstVariable("__obj", None, foo)
+
+    def test_i_can_develop_list_of_return_values(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(("__rets", "list(__rets)"))
+        lst = [ReturnValueConcept("foo", True, "value for True"),
+               ReturnValueConcept("bar", True, Concept("bar", body="value for bar").auto_init()),
+               ReturnValueConcept("baz", False, sheerka.new(BuiltinConcepts.ERROR, body="baz in error")),
+               ]
+
+        res = service.create_out_tree(context, lst)
+        assert res == FormatAstList(variable="__rets", items=[FormatAstVariable("__item", None, lst[0], 0),
+                                                              FormatAstVariable("__item", None, lst[1], 1),
+                                                              FormatAstVariable("__item", None, lst[2], 2)])
+
+    def test_rules_are_correctly_reset_when_list(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("__rets", "list(__rets)"),
+            ("__ret and not __ret.status", "red(__ret)")
+        )
+        lst = [
+            ReturnValueConcept("who", False, "value1"),
+            ReturnValueConcept("who", False, "value2"),
+        ]
+
+        res = service.create_out_tree(context, lst)
+        assert res == FormatAstList(variable="__rets", items=[
+            FormatAstVariable("__item", None, FormatAstColor("red", FormatAstVariable("__ret", None, lst[0])), 0),
+            FormatAstVariable("__item", None, FormatAstColor("red", FormatAstVariable("__ret", None, lst[1])), 1),
+        ])
+
+    def test_i_can_develop_list_with_recurse(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("__rets", "list(__rets, recurse_on='parents', recursion_depth=2)"))
+        r1111 = ReturnValueConcept("Test", True, "r1111")
+        r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111])
+        r11 = ReturnValueConcept("Test", True, "r11", parents=[r111])
+        r1 = ReturnValueConcept("Test", True, "r1", parents=[r11])
+        r22 = ReturnValueConcept("Test", True, "r22")
+        r2 = ReturnValueConcept("Test", True, "r2", parents=[r22])
+
+        lst = [r1, r2]
+
+        res = service.create_out_tree(context, lst)
+        assert res == FormatAstList(variable="__rets", recurse_on='parents', recursion_depth=2, items=[
+            FormatAstVariable("__item", None, r1, 0),
+            FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=1, items=[
+                FormatAstVariable("__item", None, r11, 0),
+                FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=0, items=[
+                    FormatAstVariable("__item", None, r111, 0)])
+            ]),
+            FormatAstVariable("__item", None, r2, 1),
+            FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=1, items=[
+                FormatAstVariable("__item", None, r22, 0)]),
+        ])
+
+    def test_i_can_develop_list_with_recurse_using_container_format_instr(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("isinstance(__obj, 'foo')", "list(__obj)"))
+        r1111 = ReturnValueConcept("Test", True, "r1111")
+        r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111])
+        r11 = ReturnValueConcept("Test", True, "r11", parents=[r111])
+        r1 = ReturnValueConcept("Test", True, "r1", parents=[r11])
+        r22 = ReturnValueConcept("Test", True, "r22")
+        r2 = ReturnValueConcept("Test", True, "r2", parents=[r22])
+
+        foo = Concept("foo", key="foo", body=[r1, r2]).auto_init()
+        foo.set_format_instr(recurse_on='parents', recursion_depth=2)
+
+        res = service.create_out_tree(context, foo)
+        assert res == FormatAstList(variable="__obj", recurse_on='parents', recursion_depth=2, items=[
+            FormatAstVariable("__item", None, r1, 0),
+            FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=1, items=[
+                FormatAstVariable("__item", None, r11, 0),
+                FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=0, items=[
+                    FormatAstVariable("__item", None, r111, 0)])
+            ]),
+            FormatAstVariable("__item", None, r2, 1),
+            FormatAstList(variable="__parents", recurse_on='parents', recursion_depth=1, items=[
+                FormatAstVariable("__item", None, r22, 0)]),
+        ])
+
+    def test_i_can_develop_using_variable_properties(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("isinstance(__obj, Concept)", "{__obj.body}"),
+        )
+        lst = Concept("bar", body="value for bar").auto_init()
+
+        res = service.create_out_tree(context, lst)
+        assert res == FormatAstVariable("__obj.body", None, "value for bar")
+
+    @pytest.mark.parametrize("rule, expected", [
+        (("__ret", "red('hello world')"), FormatAstColor("red", FormatAstRawText("hello world"))),
+        (("__ret", "blue(__ret_value)"), FormatAstColor("blue", FormatAstVariable("__ret_value", None, 1))),
+    ])
+    def test_i_can_develop_color(self, rule, expected):
+        sheerka, context, service, rule = self.init_service_with_rules(rule)
+        ret = sheerka.ret("Test", True, 1)
+
+        res = service.create_out_tree(context, ret)
+
+        assert res == expected
+
+    @pytest.mark.parametrize("rule, expected", [
+        (("__ret", "{__ret}"), FormatAstVariable("__ret", None,
+                                                 ReturnValueConcept(who="Test", status=True, value=1))),
+        (("__ret", "magenta(__ret)"),
+         FormatAstColor("magenta",
+                        FormatAstVariable("__ret", None,
+                                          ReturnValueConcept(who="Test", status=True, value=1)))),
+    ])
+    def test_i_can_manage_infinite_recursion(self, rule, expected):
+        sheerka, context, service, rule = self.init_service_with_rules(rule)
+        ret = sheerka.ret("Test", True, 1)
+
+        res = service.create_out_tree(context, ret)
+
+        assert res == expected
+
+    def test_i_can_manage_infinite_recursion_when_multiple_rules(self):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            Rule("print", None, "__ret", "{__ret.value}", 1),
+            Rule("print", None, "__ret", "white(__ret)", 2),
+        )
+        ret = sheerka.ret("Test", True, "hello world!")
+
+        res = service.create_out_tree(context, ret)
+
+        assert res == FormatAstColor("white",
+                                     FormatAstVariable("__ret", None,
+                                                       FormatAstVariable("__ret.value", None, "hello world!")))
+
+    def test_i_can_print_out_the_result(self, capsys):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("__rets", "list(__rets)"),
+            ("__ret", "status: {status}, value: {__ret_value}"),
+            ("isinstance(__obj, 'foo')", "{id}-{name}:{body}")
+        )
+        sheerka.create_new_concept(context, Concept("foo"))
+        foo = sheerka.new("foo", body="hello world")
+        ret = sheerka.ret("Test", True, foo)
+
+        service.process_return_values(context, [ret])
+        captured = capsys.readouterr()
+        assert captured.out == "status: True, value: 1001-foo:hello world\n"
+
+    def test_i_can_print_out_a_list_of_results(self, capsys):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("__rets", "list(__rets)"),
+            ("isinstance(__ret_value, 'foo')", "foo: {body}"),
+            ("isinstance(__ret_value, 'bar')", "bar: {body}"),
+        )
+        sheerka.create_new_concept(context, Concept("foo"))
+        sheerka.create_new_concept(context, Concept("bar"))
+        sheerka.create_new_concept(context, Concept("baz"))
+
+        rets = [sheerka.ret("Test", True, sheerka.new("foo", body="foo value")),
+                sheerka.ret("Test", True, sheerka.new("bar", body="bar value")),
+                sheerka.ret("Test", True, sheerka.new("baz", body="baz value")), ]
+
+        service.process_return_values(context, rets)
+        captured = capsys.readouterr()
+        assert captured.out == """foo: (1001)foo
+bar: (1002)bar
+ReturnValue(who=Test, status=True, value=(1003)baz, message=None)
+"""
+
+    def test_i_can_print_out_a_list_with_recurse(self, capsys):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("__rets", "list(__rets, recurse_on='parents', recursion_depth=2)"),
+            ("__ret", "{__tab}{__ret}"),
+        )
+        r1111 = ReturnValueConcept("Test", True, "r1111")
+        r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111])
+        r11 = ReturnValueConcept("Test", True, "r11", parents=[r111])
+        r1 = ReturnValueConcept("Test", True, "r1", parents=[r11])
+        r22 = ReturnValueConcept("Test", True, "r22")
+        r2 = ReturnValueConcept("Test", True, "r2", parents=[r22])
+
+        rets = [r1, r2]
+
+        service.process_return_values(context, rets)
+        captured = capsys.readouterr()
+        assert captured.out == """ReturnValue(who=Test, status=True, value=r1, message=None)
+  ReturnValue(who=Test, status=True, value=r11, message=None)
+    ReturnValue(who=Test, status=True, value=r111, message=None)
+ReturnValue(who=Test, status=True, value=r2, message=None)
+  ReturnValue(who=Test, status=True, value=r22, message=None)
+"""
+
+    def test_i_can_print_out_a_list_with_recurse_using_format_instr(self, capsys):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("__rets", "list(__rets)"),
+            ("__ret", "{__tab}{__ret}"),
+        )
+        r1111 = ReturnValueConcept("Test", True, "r1111")
+        r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111])
+        r11 = ReturnValueConcept("Test", True, "r11", parents=[r111])
+        r1 = ReturnValueConcept("Test", True, "r1", parents=[r11])
+        r1.set_format_instr(recursion_depth=2, recurse_on="parents")
+        r22 = ReturnValueConcept("Test", True, "r22")
+        r2 = ReturnValueConcept("Test", True, "r2", parents=[r22])
+        r2.set_format_instr(recursion_depth=2, recurse_on="parents")
+
+        rets = [r1, r2]
+
+        service.process_return_values(context, rets)
+        captured = capsys.readouterr()
+        assert captured.out == """ReturnValue(who=Test, status=True, value=r1, message=None)
+  ReturnValue(who=Test, status=True, value=r11, message=None)
+    ReturnValue(who=Test, status=True, value=r111, message=None)
+ReturnValue(who=Test, status=True, value=r2, message=None)
+  ReturnValue(who=Test, status=True, value=r22, message=None)
+"""
+
+    def test_i_can_print_out_a_list_with_recurse_using_container_format_instr(self, capsys):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("isinstance(__obj, 'foo')", "list(__obj)"),
+            ("__ret", "{__tab}{__ret}"),
+        )
+        r1111 = ReturnValueConcept("Test", True, "r1111")
+        r111 = ReturnValueConcept("Test", True, "r111", parents=[r1111])
+        r11 = ReturnValueConcept("Test", True, "r11", parents=[r111])
+        r1 = ReturnValueConcept("Test", True, "r1", parents=[r11])
+        r22 = ReturnValueConcept("Test", True, "r22")
+        r2 = ReturnValueConcept("Test", True, "r2", parents=[r22])
+
+        foo = Concept("foo", key="foo", body=[r1, r2]).auto_init()
+        foo.set_format_instr(recurse_on='parents', recursion_depth=2)
+
+        service.process_return_values(context, foo)
+        captured = capsys.readouterr()
+        assert captured.out == """ReturnValue(who=Test, status=True, value=r1, message=None)
+  ReturnValue(who=Test, status=True, value=r11, message=None)
+    ReturnValue(who=Test, status=True, value=r111, message=None)
+ReturnValue(who=Test, status=True, value=r2, message=None)
+  ReturnValue(who=Test, status=True, value=r22, message=None)
+"""
+
+    def test_i_can_print_out_color(self, capsys):
+        sheerka, context, service, *rules = self.init_service_with_rules(
+            ("__ret", "status: green(status), blue('value: {__ret_value}')"),
+        )
+        sheerka.create_new_concept(context, Concept("foo"))
+        foo = sheerka.new("foo", body="hello world")
+        ret = sheerka.ret("Test", True, foo)
+
+        service.process_return_values(context, ret)
+        captured = capsys.readouterr()
+        assert captured.out == "status: \x1b[32mTrue\x1b[0m, \x1b[34mvalue: (1001)foo\x1b[0m\n"

tests/parsers/parsers_utils.py

@@ -1,7 +1,7 @@
 from core.concept import CC, Concept, ConceptParts, DoNotResolve, CIO
 from core.tokenizer import Tokenizer, TokenKind, Token
 from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN, \
-    SCN
+    SCN, RN
 from parsers.SyaNodeParser import SyaConceptParserHelper
 
 
@@ -108,6 +108,12 @@ def get_node(
         sub_expr.fix_pos(node)
         return sub_expr
 
+    if isinstance(sub_expr, RN):
+        start, length = _index(expression_as_tokens, sub_expr.source, skip)
+        sub_expr.start = start
+        sub_expr.end = start + length - 1
+        return sub_expr
+
     if isinstance(sub_expr, (CNC, CC, CN)):
         concept_node = get_node(
             concepts_map,
@@ -164,7 +170,7 @@ def get_node(
     concept_found = concepts_map.get(concept_key, None)
     if concept_found:
         concept_found = Concept().update_from(concept_found)  # make a copy when massively used in tests
-        if sya and len(concept_found.metadata.variables) > 0 and not is_bnf:
+        if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf:
             return SyaConceptParserHelper(concept_found, start, start + length - 1)
         elif init_empty_body:
             node = CNC(concept_found, start, start + length - 1, source=sub_expr, exclude_body=exclude_body)
@@ -183,7 +189,7 @@ def init_body(item, concept, value):
         del (item.compiled["body"])
         return
 
-    if not concept or concept.metadata.body or ConceptParts.BODY in item.compiled:
+    if not concept or concept.get_metadata().body or ConceptParts.BODY in item.compiled:
         return
 
     item.compiled[ConceptParts.BODY] = DoNotResolve(value)

tests/parsers/test_AtomsParser.py

@@ -2,7 +2,7 @@ import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_DEF
 from core.sheerka.services.SheerkaExecute import ParserInput
-from parsers.AtomNodeParser import AtomNodeParser
+from parsers.SequenceNodeParser import SequenceNodeParser
 from parsers.BaseNodeParser import cnode, utnode, CNC, SCN, CN
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -17,9 +17,9 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
             singleton=singleton)
 
         if use_sheerka:
-            parser = AtomNodeParser(sheerka=sheerka)
+            parser = SequenceNodeParser(sheerka=sheerka)
         else:
-            parser = AtomNodeParser()
+            parser = SequenceNodeParser()
             parser.init_from_concepts(context, updated_concepts)
 
         return sheerka, context, parser
@@ -351,7 +351,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         lexer_nodes = res.body.body
 
         assert res.status
-        assert lexer_nodes[0].concept.metadata.is_evaluated == expected_is_evaluated
+        assert lexer_nodes[0].concept.get_metadata().is_evaluated == expected_is_evaluated
 
     def test_the_parser_always_return_a_new_instance_of_the_concept(self):
         concepts_map = {
@@ -376,3 +376,9 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
 
+    def test_i_do_not_parse_rules_instead_of_concepts(self):
+        sheerka, context, parser = self.init_parser({})
+        res = parser.parse(context, ParserInput("r:|20:"))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)

tests/parsers/test_BaseCustomGrammarParser.py

@@ -38,22 +38,26 @@ class TestBaseCustomGrammarParser(TestUsingMemoryBasedSheerka):
 
         return sheerka, context, parser
 
-    @pytest.mark.parametrize("text, expected", [
-        ("when xxx yyy", {Keywords.WHEN: "when xxx yyy"}),
-        ("when uuu vvv print xxx yyy", {Keywords.WHEN: "when uuu vvv ", Keywords.PRINT: "print xxx yyy"}),
-        ("print xxx yyy when uuu vvv", {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy "}),
-        ("  when xxx", {Keywords.WHEN: "when xxx"}),
+    @pytest.mark.parametrize("text, strip_tokens, expected", [
+        ("when xxx yyy", False, {Keywords.WHEN: "when xxx yyy"}),
+        ("when uuu vvv print xxx yyy", False, {Keywords.WHEN: "when uuu vvv ", Keywords.PRINT: "print xxx yyy"}),
+        ("print xxx yyy when uuu vvv", False, {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy "}),
+        ("  when xxx", False, {Keywords.WHEN: "when xxx"}),
+        ("when xxx yyy", True, {Keywords.WHEN: "when xxx yyy"}),
+        ("when uuu vvv print xxx yyy", True, {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy"}),
+        ("print xxx yyy when uuu vvv", True, {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy"}),
+        ("  when xxx", True, {Keywords.WHEN: "when xxx"}),
     ])
-    def test_i_can_get_parts(self, text, expected):
+    def test_i_can_get_parts(self, text, strip_tokens, expected):
         sheerka, context, parser = self.init_parser(text)
 
-        res = parser.get_parts(["when", "print"])
+        res = parser.get_parts(["when", "print"], strip_tokens=strip_tokens)
         self.compare_results(res, expected)
 
     def test_i_can_get_parts_when_multilines(self):
         text = """when
 def func(x):
-    return x+1
+\treturn x+1
 func(a)
 """
         expected = {Keywords.WHEN: "when def func(x):\n\treturn x+1\nfunc(a)\n"}

tests/parsers/test_BaseNodeParser.py

@@ -51,7 +51,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         sheerka.add_in_cache(bar)
 
         concept = Concept("foo").init_key()
-        concept.bnf = bnf
+        concept.set_bnf(bnf)
         sheerka.set_id_if_needed(concept, False)
 
         res = BaseNodeParser.get_concepts_by_first_token(context, [concept])
@@ -72,7 +72,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         sheerka.add_in_cache(baz)
 
         foo = Concept("foo").init_key()
-        foo.bnf = OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux"))
+        foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux")))
         sheerka.set_id_if_needed(foo, False)
 
         res = BaseNodeParser.get_concepts_by_first_token(context, [bar, baz, foo])
@@ -99,7 +99,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         sheerka.add_in_cache(bar)
 
         foo = Concept("foo").init_key()
-        foo.bnf = OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux"))
+        foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux")))
         sheerka.set_id_if_needed(foo, False)
 
         res = BaseNodeParser.get_concepts_by_first_token(context, [bar, foo], use_sheerka=True)

tests/parsers/test_BaseParser.py

@@ -4,26 +4,6 @@ from core.tokenizer import Tokenizer, TokenKind, Token
 from parsers.BaseParser import BaseParser, BaseSplitIterParser
 
 
-@pytest.mark.parametrize("text, expected_text", [
-    ("hello world", "hello world"),
-    ("'hello' 'world'", "'hello' 'world'"),
-    ("def concept a from", "def concept a from"),
-    ("()[]{}1=1.5+-/*><&é", "()[]{}1=1.5+-/*><&é"),
-    ("execute(c:concept_name:)", "execute(c:concept_name:)")
-
-])
-def test_i_can_get_text_from_tokens(text, expected_text):
-    tokens = list(Tokenizer(text))
-    assert BaseParser.get_text_from_tokens(tokens) == expected_text
-
-
-@pytest.mark.parametrize("text, custom, expected_text", [
-    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)")
-])
-def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
-    tokens = list(Tokenizer(text))
-    assert BaseParser.get_text_from_tokens(tokens, custom) == expected_text
-
 
 @pytest.mark.parametrize("text, expected", [
     ("", ["<eof>"]),

tests/parsers/test_BnfNodeParser.py

@@ -1,11 +1,11 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF
+from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF, NotInit
 from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.BaseNodeParser import CNC, UTN, CN
 from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
     Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser
-from parsers.BnfParser import BnfParser
+from parsers.BnfDefinitionParser import BnfDefinitionParser
 
 import tests.parsers.parsers_utils
 from tests.BaseTest import BaseTest
@@ -137,11 +137,11 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
     @staticmethod
     def update_bnf(context, concept):
-        bnf_parser = BnfParser()
-        res = bnf_parser.parse(context, concept.metadata.definition)
+        bnf_parser = BnfDefinitionParser()
+        res = bnf_parser.parse(context, concept.get_metadata().definition)
         if res.status:
-            concept.bnf = res.value.value
-            concept.metadata.definition_type = DEFINITION_TYPE_BNF
+            concept.set_bnf(res.value.value)
+            concept.get_metadata().definition_type = DEFINITION_TYPE_BNF
         else:
             raise Exception(res)
         return concept
@@ -572,16 +572,16 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         # explicit validations of the compiled
         concept_foo = sequences[0][0].concept
-        assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED
-        assert concept_foo.compiled == {ConceptParts.BODY: DoNotResolve("one two")}
+        assert concept_foo.body == NotInit
+        assert concept_foo.get_compiled() == {ConceptParts.BODY: DoNotResolve("one two")}
 
         concept_bar = sequences[1][0].concept
-        assert concept_bar.body == BuiltinConcepts.NOT_INITIALIZED
-        assert concept_bar.compiled == {
+        assert concept_bar.body == NotInit
+        assert concept_bar.get_compiled() == {
             ConceptParts.BODY: concept_foo,
             "foo": concept_foo
         }
-        assert id(concept_bar.compiled[ConceptParts.BODY]) == id(concept_bar.compiled["foo"])
+        assert id(concept_bar.get_compiled()[ConceptParts.BODY]) == id(concept_bar.get_compiled()["foo"])
 
     def test_i_can_refer_to_other_concepts_with_body(self):
         my_map = {
@@ -598,12 +598,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         # explicit validations of the compiled
         concept_foo = sequences[0][0].concept
-        assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED
-        assert len(concept_foo.compiled) == 0  # because there is a body defined in the metadata
+        assert concept_foo.body == NotInit
+        assert len(concept_foo.get_compiled()) == 0  # because there is a body defined in the metadata
 
         concept_bar = sequences[1][0].concept
-        assert concept_bar.body == BuiltinConcepts.NOT_INITIALIZED
-        assert concept_bar.compiled == {
+        assert concept_bar.body == NotInit
+        assert concept_bar.get_compiled() == {
             ConceptParts.BODY: concept_foo,
             "foo": concept_foo
         }
@@ -625,19 +625,19 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         # explicit validations of the compiled
         concept_foo = sequences[0][0].concept
-        assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED
-        assert concept_foo.compiled == {ConceptParts.BODY: DoNotResolve("one two")}
+        assert concept_foo.body == NotInit
+        assert concept_foo.get_compiled() == {ConceptParts.BODY: DoNotResolve("one two")}
 
         concept_bar = sequences[1][0].concept
-        assert concept_bar.body == BuiltinConcepts.NOT_INITIALIZED
-        assert concept_bar.compiled == {
+        assert concept_bar.body == NotInit
+        assert concept_bar.get_compiled() == {
             ConceptParts.BODY: concept_foo,
             "foo": concept_foo
         }
 
         concept_baz = sequences[2][0].concept
-        assert concept_baz.body == BuiltinConcepts.NOT_INITIALIZED
-        assert concept_baz.compiled == {
+        assert concept_baz.body == NotInit
+        assert concept_baz.get_compiled() == {
             ConceptParts.BODY: concept_bar,
             "bar": concept_bar
         }
@@ -655,21 +655,21 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         expected = [CN("bar", source="twenty two")]
         sequences = self.validate_get_concepts_sequences(my_map, text, expected)
         concept_bar = sequences[0].concept
-        assert concept_bar.compiled == {
+        assert concept_bar.get_compiled() == {
             ConceptParts.BODY: DoNotResolve("twenty two"),
             "foo": my_map["foo"],
         }
-        assert concept_bar.compiled["foo"].compiled == {ConceptParts.BODY: DoNotResolve("twenty")}
+        assert concept_bar.get_compiled()["foo"].get_compiled() == {ConceptParts.BODY: DoNotResolve("twenty")}
 
         text = "thirty one"
         expected = [CN("bar", source="thirty one")]
         sequences = self.validate_get_concepts_sequences(my_map, text, expected)
         concept_bar = sequences[0].concept
-        assert concept_bar.compiled == {
+        assert concept_bar.get_compiled() == {
             ConceptParts.BODY: DoNotResolve("thirty one"),
             "foo": my_map["foo"],
         }
-        assert concept_bar.compiled["foo"].compiled == {ConceptParts.BODY: DoNotResolve("thirty")}
+        assert concept_bar.get_compiled()["foo"].get_compiled() == {ConceptParts.BODY: DoNotResolve("thirty")}
 
         text = "thirty three"
         expected = [[CN("foo", source="thirty"), CN("three")], []]
@@ -712,21 +712,21 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, sequences = self.exec_get_concepts_sequences(my_map, text, expected)
 
         concept_bar = sequences[0].concept
-        assert concept_bar.compiled == {
+        assert concept_bar.get_compiled() == {
             ConceptParts.BODY: DoNotResolve("twenty two"),
             "foo": sheerka.new("foo"),
         }
-        assert concept_bar.compiled["foo"].compiled == {}  # as foo as a body
+        assert concept_bar.get_compiled()["foo"].get_compiled() == {}  # as foo as a body
 
         text = "thirty one"
         expected = [CN("bar", source="thirty one")]
         sequences = self.validate_get_concepts_sequences(my_map, text, expected)
         concept_bar = sequences[0].concept
-        assert concept_bar.compiled == {
+        assert concept_bar.get_compiled() == {
             ConceptParts.BODY: DoNotResolve("thirty one"),
             "foo": sheerka.new("foo"),
         }
-        assert concept_bar.compiled["foo"].compiled == {}
+        assert concept_bar.get_compiled()["foo"].get_compiled() == {}
 
     def test_i_can_mix_zero_and_more_and_reference_to_other_concepts(self):
         my_map = {
@@ -738,13 +738,13 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         expected = [CN("bar", source="one two three")]
         sequences = self.validate_get_concepts_sequences(my_map, text, expected)
         concept_bar = sequences[0].concept
-        assert concept_bar.compiled == {
+        assert concept_bar.get_compiled() == {
             ConceptParts.BODY: DoNotResolve("one two three"),
             "foo": [my_map["foo"], my_map["foo"], my_map["foo"]]
         }
-        assert concept_bar.compiled["foo"][0].compiled == {ConceptParts.BODY: DoNotResolve("one")}
-        assert concept_bar.compiled["foo"][1].compiled == {ConceptParts.BODY: DoNotResolve("two")}
-        assert concept_bar.compiled["foo"][2].compiled == {ConceptParts.BODY: DoNotResolve("three")}
+        assert concept_bar.get_compiled()["foo"][0].get_compiled() == {ConceptParts.BODY: DoNotResolve("one")}
+        assert concept_bar.get_compiled()["foo"][1].get_compiled() == {ConceptParts.BODY: DoNotResolve("two")}
+        assert concept_bar.get_compiled()["foo"][2].get_compiled() == {ConceptParts.BODY: DoNotResolve("three")}
 
     def test_i_can_parse_concept_reference_that_is_not_in_grammar(self):
         my_map = {
@@ -763,7 +763,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         expected = [CN("foo", source="twenty one")]
         sequences = self.validate_get_concepts_sequences(my_map, text, expected)
         concept_foo = sequences[0].concept
-        assert concept_foo.compiled == {
+        assert concept_foo.get_compiled() == {
             ConceptParts.BODY: DoNotResolve("twenty one"),
             "unit": my_map["one"],
         }
@@ -786,8 +786,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         # explicit validations of the compiled
         concept_foo = sequences[0].concept
-        assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED
-        assert concept_foo.compiled == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]),
+        assert concept_foo.body == NotInit
+        assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]),
                                         ConceptParts.BODY: DoNotResolve(value='twenty two')}
 
         text = "twenty one"
@@ -796,8 +796,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         # explicit validations of the compiled
         concept_foo = sequences[0].concept
-        assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED
-        assert concept_foo.compiled == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]),
+        assert concept_foo.body == NotInit
+        assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]),
                                         ConceptParts.BODY: DoNotResolve(value='twenty one')}
 
     @pytest.mark.parametrize("bar_expr, expected", [
@@ -1241,7 +1241,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser(init_from_sheerka=True)
         sheerka.concepts_grammars.clear()  # simulate restart
         for c in cmap.values():
-            sheerka.get_by_id(c.id).bnf = None
+            sheerka.get_by_id(c.id).set_bnf(None)
 
         text = "thirty three"
         expected = CNC("thirties",

tests/parsers/test_BnfParser.py

@@ -8,7 +8,7 @@ from parsers.BaseParser import UnexpectedTokenErrorNode
 from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \
     OneOrMore, ConceptExpression
 from parsers.BnfNodeParser import BnfNodeParser
-from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
+from parsers.BnfDefinitionParser import BnfDefinitionParser, UnexpectedEndOfFileError
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -27,7 +27,7 @@ def update_concepts_ids(sheerka, parsing_expression):
     if isinstance(parsing_expression, ConceptExpression):
         if not parsing_expression.concept.id:
             concept = sheerka.get_by_key(parsing_expression.concept.key)
-            parsing_expression.concept.metadata.id = concept.id
+            parsing_expression.concept.get_metadata().id = concept.id
 
     for pe in parsing_expression.elements:
         update_concepts_ids(sheerka, pe)
@@ -40,7 +40,7 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
 
     def init_parser(self, *concepts):
         sheerka, context, *updated = self.init_concepts(*concepts, singleton=True)
-        parser = BnfParser()
+        parser = BnfDefinitionParser()
 
         return sheerka, context, parser, *updated
 
@@ -177,10 +177,10 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         sheerka, context, regex_parser, foo, bar = self.init_parser("foo", "bar")
 
         for concept in [foo, bar]:
-            concept.metadata.definition_type = DEFINITION_TYPE_BNF
+            concept.get_metadata().definition_type = DEFINITION_TYPE_BNF
 
-        foo.bnf = regex_parser.parse(context, "'twenty' | 'thirty'").value.value
-        bar.bnf = regex_parser.parse(context, "foo ('one' | 'two')").value.value
+        foo.set_bnf(regex_parser.parse(context, "'twenty' | 'thirty'").value.value)
+        bar.set_bnf(regex_parser.parse(context, "foo ('one' | 'two')").value.value)
 
         bnf_parser = BnfNodeParser()
         bnf_parser.init_from_concepts(context, [foo, bar])

tests/parsers/test_DefConceptParser.py

@@ -9,7 +9,7 @@ from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.BaseNodeParser import SCWC
 from parsers.BaseParser import NotInitializedNode, UnexpectedEofNode
 from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence
-from parsers.BnfParser import BnfParser
+from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode
 from parsers.DefConceptParser import UnexpectedTokenErrorNode, DefConceptNode
 from parsers.FunctionParser import FunctionParser
@@ -34,7 +34,7 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition
         def_concept.ret = get_concept_part(ret)
     if bnf_def:
         def_concept.definition = ReturnValueConcept(
-            "parsers.Bnf",
+            "parsers.BnfDefinition",
             True,
             bnf_def)
         def_concept.definition_type = DEFINITION_TYPE_BNF
@@ -346,7 +346,7 @@ def concept add one to a as:
 
         node = res.value.value
         definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
-        parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", None, definition, definition)
+        parser_result = ParserResultConcept(BnfDefinitionParser(), "a_concept | 'a_string'", None, definition, definition)
         expected = get_def_concept(name="name", body="__definition[0]", bnf_def=parser_result)
 
         assert res.status
@@ -515,7 +515,7 @@ from give me the date !
         assert res.status
         assert concept_defined.name.tokens == list(Tokenizer("def concept x", yield_eof=False))
 
-    def test_i_can_parse_when_ambiguity_in_where_pre_clause(self):
+    def test_i_can_parse_when_ambiguity_in_where_or_pre_clause(self):
         sheerka, context, parser, *concepts = self.init_parser(
             Concept("x is a y", pre="in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)"),
             Concept("x is a y")

tests/parsers/test_ExactConceptParser.py

@@ -7,7 +7,7 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
 def variable_def(concept, prop_name):
-    for name, value in concept.metadata.variables:
+    for name, value in concept.get_metadata().variables:
         if name == prop_name:
             return value
 
@@ -61,8 +61,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         assert concept_found == concept
-        assert not concept_found.metadata.need_validation
-        assert not concept_found.metadata.is_evaluated
+        assert not concept_found.get_metadata().need_validation
+        assert not concept_found.get_metadata().is_evaluated
 
     def test_i_can_parse_concepts_defined_several_times(self):
         sheerka = self.get_sheerka(singleton=True)
@@ -79,11 +79,11 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert results[0].status
         assert results[0].value.value.name == "hello a"
         assert variable_def(results[0].value.value, "a") == "world"
-        assert results[0].value.value.metadata.need_validation
+        assert results[0].value.value.get_metadata().need_validation
 
         assert results[1].status
         assert results[1].value.value.name == "hello world"
-        assert not results[1].value.value.metadata.need_validation
+        assert not results[1].value.value.get_metadata().need_validation
 
     def test_i_can_parse_a_concept_with_variables(self):
         sheerka = self.get_sheerka(singleton=True)
@@ -98,8 +98,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
 
         concept_found = results[0].value.value
         assert concept_found == CMV(concept, a="10", b="5")
-        assert concept_found.metadata.need_validation
-        assert not concept_found.metadata.is_evaluated
+        assert concept_found.get_metadata().need_validation
+        assert not concept_found.get_metadata().is_evaluated
 
     def test_i_can_parse_a_concept_with_duplicate_variables(self):
         sheerka = self.get_sheerka(singleton=True)
@@ -114,7 +114,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
 
         concept_found = results[0].value.value
         assert concept_found == CMV(concept, a="10", b="5")
-        assert concept_found.metadata.need_validation
+        assert concept_found.get_metadata().need_validation
 
     def test_i_can_parse_concept_when_defined_using_from_def(self):
         sheerka, context, plus = self.init_concepts(
@@ -128,8 +128,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         assert concept_found == CMV(plus, a="10", b="5")
-        assert concept_found.metadata.need_validation
-        assert not concept_found.metadata.is_evaluated
+        assert concept_found.get_metadata().need_validation
+        assert not concept_found.get_metadata().is_evaluated
 
     def test_i_can_parse_concept_token(self):
         sheerka, context, foo = self.init_concepts("foo")
@@ -141,8 +141,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         assert concept_found == foo
-        assert not concept_found.metadata.need_validation
-        assert concept_found.metadata.is_evaluated
+        assert not concept_found.get_metadata().need_validation
+        assert concept_found.get_metadata().is_evaluated
 
     def test_i_can_parse_concept_with_concept_tokens(self):
         sheerka, context, one, two, plus = self.init_concepts(
@@ -158,8 +158,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         assert concept_found == CMV(plus, a="c:one:", b="c:two:")
-        assert concept_found.metadata.need_validation
-        assert not concept_found.metadata.is_evaluated
+        assert concept_found.get_metadata().need_validation
+        assert not concept_found.get_metadata().is_evaluated
 
     def test_i_can_parse_when_expression_contains_keyword(self):
         sheerka, context, isa, def_concept = self.init_concepts(
@@ -174,8 +174,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         assert concept_found == CMV(isa, c="z")
-        assert concept_found.metadata.need_validation
-        assert not concept_found.metadata.is_evaluated
+        assert concept_found.get_metadata().need_validation
+        assert not concept_found.get_metadata().is_evaluated
 
         source = "def concept z"
         results = ExactConceptParser().parse(context, ParserInput(source))
@@ -184,8 +184,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert len(results) == 1
         assert results[0].status
         assert concept_found == CMV(def_concept, a="z")
-        assert concept_found.metadata.need_validation
-        assert not concept_found.metadata.is_evaluated
+        assert concept_found.get_metadata().need_validation
+        assert not concept_found.get_metadata().is_evaluated
 
     def test_i_can_manage_unknown_concept(self):
         context = self.get_context(self.get_sheerka(singleton=True))
@@ -218,4 +218,4 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
     #     assert len(results) == 1
     #     assert results[0].status
     #     assert results[0].value.value == concept
-    #     assert not results[0].value.value.metadata.need_validation
+    #     assert not results[0].value.value.get_metadata().need_validation

tests/parsers/test_ExpressionParser.py

@@ -33,14 +33,14 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("expression, expected", [
         ("one complicated expression", n("one complicated expression")),
-        # ("function_call(a,b,c)", n("function_call(a,b,c)")),
-        # ("one expression or another expression", OrNode(n("one expression"), n("another expression"))),
-        # ("one expression and another expression", AndNode(n("one expression"), n("another expression"))),
-        # ("one or two or three", OrNode(n("one"), n("two"), n("three"))),
-        # ("one and two and three", AndNode(n("one"), n("two"), n("three"))),
-        # ("one or two and three", OrNode(n("one"), AndNode(n("two"), n("three")))),
-        # ("one and two or three", OrNode(AndNode(n("one"), n("two")), n("three"))),
-        # ("one and (two or three)", AndNode(n("one"), OrNode(n("two"), n("three")))),
+        ("function_call(a,b,c)", n("function_call(a,b,c)")),
+        ("one expression or another expression", OrNode(n("one expression"), n("another expression"))),
+        ("one expression and another expression", AndNode(n("one expression"), n("another expression"))),
+        ("one or two or three", OrNode(n("one"), n("two"), n("three"))),
+        ("one and two and three", AndNode(n("one"), n("two"), n("three"))),
+        ("one or two and three", OrNode(n("one"), AndNode(n("two"), n("three")))),
+        ("one and two or three", OrNode(AndNode(n("one"), n("two")), n("three"))),
+        ("one and (two or three)", AndNode(n("one"), OrNode(n("two"), n("three")))),
     ])
     def test_i_can_parse_expression(self, expression, expected):
         sheerka, context, parser = self.init_parser()

tests/parsers/test_FormatRuleParser.py

@@ -1,12 +1,20 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
 from core.sheerka.services.SheerkaExecute import ParserInput
+from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, RulePredicate, FormatAstVariable
+from core.tokenizer import Keywords, Tokenizer
 from parsers.BaseCustomGrammarParser import KeywordNotFound
-from parsers.FormatRuleParser import FormatRuleParser, FormatAstRawText, FormatRuleNode
+from parsers.FormatRuleParser import FormatRuleParser, FormatRuleNode
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
-cmap = {}
+cmap = {
+    "is a": Concept("x is a y").def_var("x").def_var("y"),
+    "is a question": Concept("x is a y", pre="is_question()").def_var("x").def_var("y"),
+    "a is good": Concept("a is good").def_var("a"),
+    "b is good": Concept("b is good").def_var("b"),
+}
 
 
 class TestFormatRuleParser(TestUsingMemoryBasedSheerka):
@@ -45,16 +53,47 @@ class TestFormatRuleParser(TestUsingMemoryBasedSheerka):
         res = parser.parse(context, ParserInput(text))
         parser_result = res.body
         format_rule = res.body.body
-        rule = format_rule.rule
+        rules = format_rule.rule
         format_ast = format_rule.format_ast
 
         assert res.status
         assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert isinstance(format_rule, FormatRuleNode)
+        assert self.dump_tokens(format_rule.tokens[Keywords.WHEN][1:]) == self.dump_tokens(
+            Tokenizer("isinstance(last_value(), Concept)", False))
+        assert self.dump_tokens(format_rule.tokens[Keywords.PRINT][1:]) == self.dump_tokens(
+            Tokenizer("hello world!", False))
 
-        assert sheerka.isinstance(rule, BuiltinConcepts.RETURN_VALUE)
+        assert len(rules) == 1
+        assert isinstance(rules[0], RulePredicate)
         assert format_ast == FormatAstRawText("hello world!")
 
+    def test_when_is_parsed_in_the_context_of_a_question(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "when foo is a bar print hello world"
+        res = parser.parse(context, ParserInput(text))
+        format_rule = res.body.body
+        rules = format_rule.rule
+
+        assert res.status
+        assert len(rules) == 1
+        assert isinstance(rules[0], RulePredicate)
+        assert rules[0].predicate.body.body.get_metadata().pre == "is_question()"
+
+    def test_when_can_support_multiple_possibilities(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "when foo is good print hello world"
+        res = parser.parse(context, ParserInput(text))
+        format_rule = res.body.body
+        rules = format_rule.rule
+
+        assert res.status
+        assert len(rules) == 2
+        assert rules[0].predicate.body.body.get_metadata().name == "a is good"
+        assert rules[1].predicate.body.body.get_metadata().name == "b is good"
+
     @pytest.mark.parametrize("text, error", [
         ("hello world", [KeywordNotFound(None, keywords=['when', 'print'])]),
         ("when True", [KeywordNotFound([], keywords=['print'])]),
@@ -69,3 +108,30 @@ class TestFormatRuleParser(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME)
         assert not_for_me.reason == error
+
+    @pytest.mark.parametrize("text, expected_error", [
+        ("when a b print hello world!", BuiltinConcepts.TOO_MANY_ERRORS),
+    ])
+    def test_i_cannot_parse_invalid_predicates(self, text, expected_error):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(text))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, expected_error)
+
+    @pytest.mark.parametrize("expr, expected", [
+        ("hello world", FormatAstRawText("hello world")),
+        ("{id}", FormatAstVariable("id")),
+    ])
+    def test_i_can_parse_valid_print_expression(self, expr, expected):
+        sheerka, context, parser = self.init_parser()
+
+        text = "when True print " + expr
+        res = parser.parse(context, ParserInput(text))
+        format_rule = res.body.body
+        format_ast = format_rule.format_ast
+
+        assert res.status
+        assert format_ast == expected
+

tests/parsers/test_FunctionParser.py

@@ -1,8 +1,9 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
+from core.rule import Rule
 from core.sheerka.services.SheerkaExecute import ParserInput
-from parsers.BaseNodeParser import SCN, SCWC, CN, UTN, CNC
+from parsers.BaseNodeParser import SCN, SCWC, CN, UTN, CNC, RN
 from parsers.FunctionParser import FunctionParser, FN
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -76,6 +77,16 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
 
         assert res == expected
 
+    def test_i_can_parse_function_when_rule(self):
+        sheerka, context, parser = self.init_parser()
+
+        parser.reset_parser(context, ParserInput("func(r:|1:)"))
+        parser.parser_input.next_token()
+
+        res = parser.parse_function()
+
+        assert res == FN("func(", ")", ["r:|1:"])
+
     @pytest.mark.parametrize("text, expected", [
         ("func()", SCN("func()")),
         (" func()", SCN("func()")),
@@ -118,6 +129,63 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
             assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
             assert expressions == resolved_expected
 
+    def test_i_can_parse_when_the_parameter_is_not_a_concept(self):
+        """
+        It's not a concept, but it can be a valid short term memory object
+        :return:
+        """
+        sheerka, context, parser = self.init_parser()
+        text = "func(unknown_concept)"
+
+        res = parser.parse(context, ParserInput(text))
+
+        assert res.status
+
+    def test_i_cannot_parse_when_the_concept_is_not_found(self):
+        """
+        We do not check yet if it's a valid concept
+        If you find a cheap way to do so, simply remove this test
+        :return:
+        """
+        sheerka, context, parser = self.init_parser()
+        text = "func(c:|xxx:)"
+
+        res = parser.parse(context, ParserInput(text))
+
+        assert res.status
+
+    def test_i_can_parse_when_rules(self):
+        sheerka, context, parser = self.init_parser()
+        text = "func(r:|1:)"
+        expected = SCWC("func(", ")", RN("1"))
+        resolved_expected = compute_expected_array(cmap, text, [expected])[0]
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.body
+        expression = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert expression == resolved_expected
+        assert expression.python_node is not None
+        assert expression.return_value is not None
+
+    # def test_i_cannot_parse_when_rule_not_found(self):
+    #     sheerka, context, parser = self.init_parser()
+    #     text = "func(r:|fake:)"
+    #     expected = SCWC("func(", ")", RN("fake"))
+    #     resolved_expected = compute_expected_array(cmap, text, [expected])[0]
+    #
+    #     res = parser.parse(context, ParserInput(text))
+    #     parser_result = res.body
+    #     expression = res.body.body
+    #
+    #     assert not res.status
+    #     assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+    #     assert expression == resolved_expected
+    #     assert expression.python_node is not None
+    #     assert expression.return_value is not None
+
     @pytest.mark.parametrize("text, expected_error_type", [
         ("one", BuiltinConcepts.NOT_FOR_ME),
         ("$*!", BuiltinConcepts.NOT_FOR_ME),
@@ -138,7 +206,7 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
     @pytest.mark.parametrize("text, expected", [
         ("func(one two)", SCWC("func(", ")", "one", "two")),
     ])
-    def test_i_can_detect_non_function(self, text, expected):
+    def test_i_can_detect_none_function(self, text, expected):
         sheerka, context, parser = self.init_parser()
         resolved_expected = compute_expected_array(cmap, text, [expected])[0]
 
@@ -169,11 +237,11 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
         concept = res.body.body.nodes[0].concept
 
         assert res.status
-        assert isinstance(concept.compiled["a"], Concept)
+        assert isinstance(concept.get_compiled()["a"], Concept)
 
         # three is not recognized,
         # so it will be transformed into list of ReturnValueConcept that indicate how to recognized it
-        assert isinstance(concept.compiled["b"], list)
-        for item in concept.compiled["b"]:
+        assert isinstance(concept.get_compiled()["b"], list)
+        for item in concept.get_compiled()["b"]:
             assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)
 

tests/parsers/test_PythonParser.py

@@ -2,10 +2,10 @@ import ast
 
 import core.utils
 import pytest
-from core.builtin_concepts import ParserResultConcept, NotForMeConcept
+from core.builtin_concepts import ParserResultConcept, NotForMeConcept, BuiltinConcepts
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import LexerError
-from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
+from core.tokenizer import LexerError, TokenKind
+from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode, ConceptDetected
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -70,6 +70,52 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
         assert res.value.value == PythonNode(
             "__C__KEY_name__ID_id__C__ + 1",
             ast.parse(encoded + "+1", mode="eval"))
-        assert res.value.value.concepts == {
-            encoded: ("name", "id")
-        }
+        assert len(res.value.value.objects) == 1
+        assert res.value.value.objects[encoded].type == TokenKind.CONCEPT
+        assert res.value.value.objects[encoded].value == ("name", "id")
+
+    def test_i_can_parse_a_rule(self):
+        text = "r:name|id: + 1"
+
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), ParserInput(text))
+        encoded = core.utils.encode_concept(("name", "id"), "R")
+
+        assert res
+        assert res.value.source == "r:name|id: + 1"
+        assert res.value.value == PythonNode(
+            "__R__KEY_name__ID_id__R__ + 1",
+            ast.parse(encoded + "+1", mode="eval"))
+        assert len(res.value.value.objects) == 1
+        assert res.value.value.objects[encoded].type == TokenKind.RULE
+        assert res.value.value.objects[encoded].value == ("name", "id")
+
+    def test_i_can_parse_a_rule_2(self):
+        text = "r:name|id:.name"
+
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), ParserInput(text))
+        encoded = core.utils.encode_concept(("name", "id"), "R")
+
+        assert res
+        assert res.value.source == "r:name|id:.name"
+        assert res.value.value == PythonNode(
+            "__R__KEY_name__ID_id__R__.name",
+            ast.parse(encoded + ".name", mode="eval"))
+        assert len(res.value.value.objects) == 1
+        assert res.value.value.objects[encoded].type == TokenKind.RULE
+        assert res.value.value.objects[encoded].value == ("name", "id")
+
+    @pytest.mark.parametrize("text, expected_id", [
+        ("foo", "foo"),
+        ("c:foo:", "__C__KEY_foo__ID_00None00__C__")
+    ])
+    def test_i_cannot_parse_simple_concepts(self, text, expected_id):
+        sheerka, context, foo = self.init_concepts("foo")
+
+        parser = PythonParser()
+        res = parser.parse(context, ParserInput(text))
+
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
+        assert res.value.reason == [ConceptDetected(expected_id)]

tests/parsers/test_PythonWithConceptsParser.py

@@ -1,11 +1,13 @@
 import ast
 
 import pytest
+import core.utils
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept
+from core.rule import Rule
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Token, TokenKind, Tokenizer
-from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode
+from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode
 from parsers.PythonParser import PythonNode
 from parsers.PythonWithConceptsParser import PythonWithConceptsParser
 from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
@@ -23,6 +25,10 @@ def ret_val(*args):
             tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
             result.append(ConceptNode(item, index, index, tokens, item.name))
             index += 1
+        elif isinstance(item, Rule):
+            tokens = [Token(TokenKind.RULE, (None, item.id), 0, 0, 0)]
+            result.append(RuleNode(item, index, index, tokens, f"r:|{item.id}:"))
+            index += 1
         else:
             tokens = list(Tokenizer(item))
             result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
@@ -69,7 +75,7 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         assert isinstance(return_value, PythonNode)
         assert return_value.source == "foo + 1"
         assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1")
-        assert return_value.concepts["__C__foo__C__"] == foo
+        assert return_value.objects["__C__foo__C__"] == foo
 
     def test_i_can_parse_concepts_and_python_when_concept_is_known(self):
         context = self.get_context()
@@ -89,7 +95,7 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         assert isinstance(return_value, PythonNode)
         assert return_value.source == "foo + 1"
         assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__1001__C__ + 1")
-        assert return_value.concepts["__C__foo__1001__C__"] == foo
+        assert return_value.objects["__C__foo__1001__C__"] == foo
 
     def test_i_can_parse_when_concept_name_has_invalid_characters(self):
         context = self.get_context()
@@ -102,7 +108,7 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         return_value = result.value.value
 
         assert result.status
-        assert return_value.concepts["__C__foo0et000000__1001__C__"] == foo
+        assert return_value.objects["__C__foo0et000000__1001__C__"] == foo
 
     def test_i_can_parse_when_multiple_concepts(self):
         sheerka, context, foo, bar = self.init_concepts("foo", "bar")
@@ -120,8 +126,42 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         assert isinstance(return_value, PythonNode)
         assert return_value.source == "func(foo, bar)"
         assert return_value.get_dump(return_value.ast_) == to_str_ast("func(__C__foo__1001__C__, __C__bar__1002__C__)")
-        assert return_value.concepts["__C__foo__1001__C__"] == foo
-        assert return_value.concepts["__C__bar__1002__C__"] == bar
+        assert return_value.objects["__C__foo__1001__C__"] == foo
+        assert return_value.objects["__C__bar__1002__C__"] == bar
+
+    def test_i_can_parse_when_python_and_rule(self):
+        context = self.get_context()
+        rule = Rule().set_id("rule_id")
+        input_return_value = ret_val(rule, " + 1")
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        wrapper = result.value
+        return_value = result.value.value
+
+        assert result.status
+        assert result.who == parser.name
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.source == "r:|rule_id: + 1"
+        assert isinstance(return_value, PythonNode)
+        assert return_value.source == "r:|rule_id: + 1"
+        assert return_value.get_dump(return_value.ast_) == to_str_ast("__R____rule_id__R__ + 1")
+        assert return_value.objects["__R____rule_id__R__"] == rule
+
+    def test_python_ids_mappings_are_correct_when_rules_with_the_same_id(self):
+        context = self.get_context()
+        rule1 = Rule().set_id("rule_id")
+        rule2 = Rule().set_id("rule_id")
+
+        input_return_value = ret_val(rule1, "+", rule2)
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        return_value = result.value.value
+
+        assert result.status
+        assert return_value.objects["__R____rule_id__R__"] == rule1
+        assert return_value.objects["__R____rule_id_1__R__"] == rule2
 
     def test_python_ids_mappings_are_correct_when_concepts_with_the_same_name(self):
         context = self.get_context()
@@ -137,10 +177,10 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         return_value = result.value.value
 
         assert result.status
-        assert return_value.concepts["__C__foo__C__"] == foo1
-        assert return_value.concepts["__C__foo_1__C__"] == foo2
-        assert return_value.concepts["__C__foo__1001__C__"] == foo3
-        assert return_value.concepts["__C__foo__1002__C__"] == foo4
+        assert return_value.objects["__C__foo__C__"] == foo1
+        assert return_value.objects["__C__foo_1__C__"] == foo2
+        assert return_value.objects["__C__foo__1001__C__"] == foo3
+        assert return_value.objects["__C__foo__1002__C__"] == foo4
 
     def test_i_cannot_parse_if_syntax_error(self):
         context = self.get_context()

tests/parsers/test_RuleParser.py

@@ -0,0 +1,101 @@
+import pytest
+from core.builtin_concepts import BuiltinConcepts
+from core.sheerka.services.SheerkaExecute import ParserInput
+from parsers.RuleParser import RuleParser, RuleNotFound
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+my_rules = {("__rets", "list(__rets)")}
+
+
+class TestRuleParser(TestUsingMemoryBasedSheerka):
+    sheerka = None
+
+    @classmethod
+    def setup_class(cls):
+        t = cls()
+        cls.sheerka, context, _ = t.init_parser(my_rules)
+
+    def init_parser(self, rules=None):
+        if rules is not None:
+            sheerka, context, *concepts = self.init_format_rules(*rules)
+        else:
+            sheerka = TestRuleParser.sheerka
+            context = self.get_context(sheerka)
+
+        parser = RuleParser()
+        return sheerka, context, parser
+
+    def test_i_cannot_parse_is_empty(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(""))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
+
+    def test_i_cannot_parse_when_not_for_me(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput("hello world!"))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+
+    def test_i_cannot_parse_a_rule_by_name(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput("r:1:"))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_IMPLEMENTED)
+
+    def test_i_cannot_parse_an_unknown_rule(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput("r:|999999:"))
+        error = res.body
+        errors_causes = res.body.body
+
+        assert not res.status
+        assert sheerka.isinstance(error, BuiltinConcepts.ERROR)
+        assert errors_causes == [RuleNotFound("999999")]
+
+    def test_i_can_parse_rule(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput("r:|1:"))
+        parser_result = res.body
+        rules = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert len(rules) == 1
+        assert rules[0].id == "1"
+
+    def test_rule_recognition_is_deferred_when_id_is_a_string(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput("r:|xxx:"))
+        parser_result = res.body
+        rules = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert len(rules) == 1
+        assert rules[0].id == "xxx"
+        assert rules[0].metadata.action_type == "deferred"
+
+    @pytest.mark.parametrize("text", [
+        "r:|1:xxx",
+        "xxxr:|1:",
+        "r:|1: + 1"
+    ])
+    def test_i_can_only_parse_simple_rule(self, text):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(text))
+        not_for_me = res.body
+
+        assert not res.status
+        assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME)

tests/parsers/test_SyaNodeParser.py

@@ -1,7 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, CIO
-from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
+from core.concept import Concept, CIO, ALL_ATTRIBUTES
+from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
 from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \
@@ -54,14 +54,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         cmap["plus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
         cmap["mult"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
         cmap["minus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
-        TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context,
-                                                                                              BuiltinConcepts.PRECEDENCE,
-                                                                                              cmap["mult"],
-                                                                                              cmap["plus"])
-        TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context,
-                                                                                              BuiltinConcepts.PRECEDENCE,
-                                                                                              cmap["mult"],
-                                                                                              cmap["minus"])
+        TestSyaNodeParser.sheerka.set_is_greater_than(context,
+                                                      BuiltinConcepts.PRECEDENCE,
+                                                      cmap["mult"],
+                                                      cmap["plus"],
+                                                      CONCEPT_COMPARISON_CONTEXT)
+        TestSyaNodeParser.sheerka.set_is_greater_than(context,
+                                                      BuiltinConcepts.PRECEDENCE,
+                                                      cmap["mult"],
+                                                      cmap["minus"],
+                                                      CONCEPT_COMPARISON_CONTEXT)
 
         # TestSyaNodeParser.sheerka.force_sya_def(context, [
         #     (cmap["plus"].id, 5, SyaAssociativity.Right),
@@ -91,6 +93,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
             sheerka = TestSyaNodeParser.sheerka
             context = self.get_context(sheerka)
             concepts = cmap.values()
+            ALL_ATTRIBUTES.clear()
 
         if post_init_concepts:
             post_init_concepts(sheerka, context)
@@ -967,7 +970,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
             '  7:<ws> => PUSH_UNREC',
             '  8:: => ??',
             "  _: => RECOG [[UTN('twenty '), CN((1001)one)], [CN((1016)twenties)]]",
-            "  _: => POP ConceptNode(concept='(1016)twenties', source='twenty one', start=4, end=6, ConceptParts.BODY='DoNotResolve(value='twenty one')', unit='(1001)one')",
+            "  _: => POP ConceptNode(concept='(1016)twenties', source='twenty one', start=4, end=6, #body#='DoNotResolve(value='twenty one')', unit='(1001)one')",
             '  9:<ws> => EAT',
             ' 10:three => PUSH_UNREC',
             ' 11:<EOF> => ??',
@@ -978,7 +981,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_debug(self, expression, expected_debugs):
         sheerka, context, parser = self.init_parser()
-        context.add_to_private_hints(BuiltinConcepts.DEBUG)
+        context.debug_enabled = True
         res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_debugs)
@@ -1000,10 +1003,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         # check the compiled
         expected_concept = lexer_nodes[0].concept
-        assert expected_concept.compiled["a"] == cmap["one"]
-        assert expected_concept.compiled["b"] == cmap["mult"]
-        assert expected_concept.compiled["b"].compiled["a"] == cmap["two"]
-        assert expected_concept.compiled["b"].compiled["b"] == cmap["three"]
+        assert expected_concept.get_compiled()["a"] == cmap["one"]
+        assert expected_concept.get_compiled()["b"] == cmap["mult"]
+        assert expected_concept.get_compiled()["b"].get_compiled()["a"] == cmap["two"]
+        assert expected_concept.get_compiled()["b"].get_compiled()["b"] == cmap["three"]
 
     def test_i_can_parse_when_python_code(self):
         sheerka, context, parser = self.init_parser()
@@ -1019,9 +1022,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         # check the compiled
         expected_concept = lexer_nodes[0].concept
-        assert len(expected_concept.compiled["a"]) == 1
+        assert len(expected_concept.get_compiled()["a"]) == 1
 
-        return_value_a = expected_concept.compiled["a"][0]
+        return_value_a = expected_concept.get_compiled()["a"][0]
         assert sheerka.isinstance(return_value_a, BuiltinConcepts.RETURN_VALUE)
         assert return_value_a.status
         assert sheerka.isinstance(return_value_a.body, BuiltinConcepts.PARSER_RESULT)
@@ -1044,8 +1047,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         # check the compiled
         expected_concept = lexer_nodes[0].concept
-        assert sheerka.isinstance(expected_concept.compiled["a"], "twenties")
-        assert expected_concept.compiled["a"].compiled["unit"] == cmap["one"]
+        assert sheerka.isinstance(expected_concept.get_compiled()["a"], "twenties")
+        assert expected_concept.get_compiled()["a"].get_compiled()["unit"] == cmap["one"]
 
     def test_i_can_parse_sequences(self):
         sheerka, context, parser = self.init_parser()
@@ -1062,9 +1065,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
             ConceptNode(cmap["suffixed"], 10, 12, source="suffixed two")]
 
         # check the compiled
-        concept_plus_a = lexer_nodes[0].concept.compiled["a"]
-        concept_plus_b = lexer_nodes[0].concept.compiled["b"]
-        concept_suffixed_a = lexer_nodes[1].concept.compiled["a"]
+        concept_plus_a = lexer_nodes[0].concept.get_compiled()["a"]
+        concept_plus_b = lexer_nodes[0].concept.get_compiled()["b"]
+        concept_suffixed_a = lexer_nodes[1].concept.get_compiled()["a"]
 
         assert concept_plus_a == cmap["one"]
         assert len(concept_plus_b) == 1
@@ -1199,7 +1202,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         concept_found = lexer_nodes[0].concept
         for unrecognized in expected_unrecognized:
-            assert isinstance(concept_found.compiled[unrecognized], UnrecognizedTokensNode)
+            assert isinstance(concept_found.get_compiled()[unrecognized], UnrecognizedTokensNode)
 
     @pytest.mark.parametrize("text, expected", [
         ("x$!# suffixed one", [utnode(0, 4, "x$!# "), cnode("suffixed __var__0", 5, 7, "suffixed one")]),

tests/parsers/test_UnrecognizedNodeParser.py

@@ -1,8 +1,11 @@
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
 from core.concept import Concept, CC
 from core.tokenizer import Tokenizer, TokenKind
+from parsers.SequenceNodeParser import SequenceNodeParser
 from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, scnode, cnode, \
     utnode, SyaAssociativity, CN, CNC, UTN, SourceCodeWithConceptNode, SCWC, SourceCodeNode
+from parsers.BnfNodeParser import BnfNodeParser
+from parsers.SyaNodeParser import SyaNodeParser
 from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -14,7 +17,7 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args):
         if isinstance(n, CC):
             concept = n.concept or Concept.update_from(my_concepts_map[n.concept_key])
             for k, v in n.compiled.items():
-                concept.compiled[k] = _get_real_node(v)
+                concept.get_compiled()[k] = _get_real_node(v)
             return concept
 
         if isinstance(n, (utnode, UTN)):
@@ -26,7 +29,7 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args):
             tokens = full_expr_as_tokens[n.start: n.end + 1]
             if hasattr(n, "compiled"):
                 for k, v in n.compiled.items():
-                    concept.compiled[k] = _get_real_node(v)
+                    concept.get_compiled()[k] = _get_real_node(v)
             return ConceptNode(concept, n.start, n.end, tokens)
 
         if isinstance(n, SCWC):
@@ -118,45 +121,45 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         concept = res.body.concept
         assert concept == concepts_map["5params"]
 
-        assert len(concept.compiled["a"]) == 1
-        assert sheerka.isinstance(concept.compiled["a"][0], BuiltinConcepts.RETURN_VALUE)
-        assert concept.compiled["a"][0].status
-        assert concept.compiled["a"][0].who == "parsers.AtomNode"
-        assert concept.compiled["a"][0].body.body == [cnode("one", 1, 1, "one")]
+        assert len(concept.get_compiled()["a"]) == 1
+        assert sheerka.isinstance(concept.get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE)
+        assert concept.get_compiled()["a"][0].status
+        assert concept.get_compiled()["a"][0].who == "parsers." + SequenceNodeParser.NAME
+        assert concept.get_compiled()["a"][0].body.body == [cnode("one", 1, 1, "one")]
 
-        assert len(concept.compiled["b"]) == 1
-        assert sheerka.isinstance(concept.compiled["b"][0], BuiltinConcepts.RETURN_VALUE)
-        assert concept.compiled["b"][0].status
-        assert concept.compiled["b"][0].who == "parsers.AtomNode"
-        assert concept.compiled["b"][0].body.body == [cnode("two", 1, 1, "two"), cnode("three", 3, 3, "three")]
+        assert len(concept.get_compiled()["b"]) == 1
+        assert sheerka.isinstance(concept.get_compiled()["b"][0], BuiltinConcepts.RETURN_VALUE)
+        assert concept.get_compiled()["b"][0].status
+        assert concept.get_compiled()["b"][0].who == "parsers." + SequenceNodeParser.NAME
+        assert concept.get_compiled()["b"][0].body.body == [cnode("two", 1, 1, "two"), cnode("three", 3, 3, "three")]
 
-        assert len(concept.compiled["c"]) == 1
-        assert sheerka.isinstance(concept.compiled["c"][0], BuiltinConcepts.RETURN_VALUE)
-        assert concept.compiled["c"][0].status
-        assert concept.compiled["c"][0].who == "parsers.BnfNode"
+        assert len(concept.get_compiled()["c"]) == 1
+        assert sheerka.isinstance(concept.get_compiled()["c"][0], BuiltinConcepts.RETURN_VALUE)
+        assert concept.get_compiled()["c"][0].status
+        assert concept.get_compiled()["c"][0].who == "parsers." + BnfNodeParser.NAME
         expected_nodes = compute_expected_array(
             concepts_map,
             " twenty one ",
             [CNC("twenties", source="twenty one", unit="one")])
-        assert concept.compiled["c"][0].body.body == expected_nodes
+        assert concept.get_compiled()["c"][0].body.body == expected_nodes
 
-        assert len(concept.compiled["d"]) == 1
-        assert sheerka.isinstance(concept.compiled["d"][0], BuiltinConcepts.RETURN_VALUE)
-        assert concept.compiled["d"][0].status
-        assert concept.compiled["d"][0].who == "parsers.Python"
-        assert concept.compiled["d"][0].body.source == " 1 + 2 "
+        assert len(concept.get_compiled()["d"]) == 1
+        assert sheerka.isinstance(concept.get_compiled()["d"][0], BuiltinConcepts.RETURN_VALUE)
+        assert concept.get_compiled()["d"][0].status
+        assert concept.get_compiled()["d"][0].who == "parsers.Python"
+        assert concept.get_compiled()["d"][0].body.source == " 1 + 2 "
 
-        assert len(concept.compiled["e"]) == 1
-        assert sheerka.isinstance(concept.compiled["e"][0], BuiltinConcepts.RETURN_VALUE)
-        assert concept.compiled["e"][0].status
-        assert concept.compiled["e"][0].who == "parsers.SyaNode"
+        assert len(concept.get_compiled()["e"]) == 1
+        assert sheerka.isinstance(concept.get_compiled()["e"][0], BuiltinConcepts.RETURN_VALUE)
+        assert concept.get_compiled()["e"][0].status
+        assert concept.get_compiled()["e"][0].who == "parsers." + SyaNodeParser.NAME
         expected_nodes = compute_expected_array(
             concepts_map,
             " one plus two mult three ",
             [CNC("plus", a="one", b=CC("mult", a="two", b="three"))],
             exclude_body=True)
 
-        assert concept.compiled["e"][0].body.body == expected_nodes
+        assert concept.get_compiled()["e"][0].body.body == expected_nodes
 
         # # sanity check, I can evaluate the concept
         # evaluated = sheerka.evaluate_concept(self.get_context(sheerka, eval_body=True), concept)
@@ -177,25 +180,25 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
 
         assert res.status
         assert res.body.concept == concepts_map["plus"]
-        assert len(res.body.concept.compiled["a"]) == 1
-        assert res.body.concept.compiled["a"][0].status
-        assert res.body.concept.compiled["a"][0].who == "parsers.Python"
-        assert res.body.concept.compiled["a"][0].body.source == "1 "
+        assert len(res.body.concept.get_compiled()["a"]) == 1
+        assert res.body.concept.get_compiled()["a"][0].status
+        assert res.body.concept.get_compiled()["a"][0].who == "parsers.Python"
+        assert res.body.concept.get_compiled()["a"][0].body.source == "1 "
 
-        assert res.body.concept.compiled["b"] == concepts_map["mult"]
-        assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["a"][0], BuiltinConcepts.RETURN_VALUE)
-        assert res.body.concept.compiled["b"].compiled["a"][0].status
-        assert res.body.concept.compiled["b"].compiled["a"][0].who == "parsers.Python"
-        assert res.body.concept.compiled["b"].compiled["a"][0].body.source == " 2 "
+        assert res.body.concept.get_compiled()["b"] == concepts_map["mult"]
+        assert sheerka.isinstance(res.body.concept.get_compiled()["b"].get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE)
+        assert res.body.concept.get_compiled()["b"].get_compiled()["a"][0].status
+        assert res.body.concept.get_compiled()["b"].get_compiled()["a"][0].who == "parsers.Python"
+        assert res.body.concept.get_compiled()["b"].get_compiled()["a"][0].body.source == " 2 "
 
-        assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["b"][0], BuiltinConcepts.RETURN_VALUE)
-        assert res.body.concept.compiled["b"].compiled["b"][0].status
-        assert res.body.concept.compiled["b"].compiled["b"][0].who == "parsers.BnfNode"
+        assert sheerka.isinstance(res.body.concept.get_compiled()["b"].get_compiled()["b"][0], BuiltinConcepts.RETURN_VALUE)
+        assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].status
+        assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].who == "parsers.Bnf"
         expected_nodes = compute_expected_array(
             concepts_map,
             " twenty two",
             [CNC("twenties", source="twenty two", unit="two")])
-        assert res.body.concept.compiled["b"].compiled["b"][0].body.body == expected_nodes
+        assert res.body.concept.get_compiled()["b"].get_compiled()["b"][0].body.body == expected_nodes
 
     # def test_i_can_validate_and_evaluate_a_concept_node_with_python(self):
     #     sheerka, context, parser = self.init_parser()
@@ -211,12 +214,12 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
     #
     #     assert res.status
     #     assert res.body.concept == concepts_map["plus"]
-    #     assert res.body.concept.compiled["a"] == concepts_map["one"]
-    #     assert len(res.body.concept.compiled["b"]) == 1
-    #     assert sheerka.isinstance(res.body.concept.compiled["b"][0], BuiltinConcepts.RETURN_VALUE)
-    #     assert res.body.concept.compiled["b"][0].status
-    #     assert res.body.concept.compiled["b"][0].who == "parsers.Python"
-    #     assert res.body.concept.compiled["b"][0].body.source == "1 + 1"
+    #     assert res.body.concept.get_compiled()["a"] == concepts_map["one"]
+    #     assert len(res.body.concept.get_compiled()["b"]) == 1
+    #     assert sheerka.isinstance(res.body.concept.get_compiled()["b"][0], BuiltinConcepts.RETURN_VALUE)
+    #     assert res.body.concept.get_compiled()["b"][0].status
+    #     assert res.body.concept.get_compiled()["b"][0].who == "parsers.Python"
+    #     assert res.body.concept.get_compiled()["b"][0].body.source == "1 + 1"
     #
     #     # # evaluate
     #     # context = self.get_context(sheerka, eval_body=True)
@@ -231,10 +234,10 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
     #
     #     assert res.status
     #     assert res.body.concept == concepts_map["plus"]
-    #     assert res.body.concept.compiled["a"] == concepts_map["one"]
-    #     assert len(res.body.concept.compiled["b"]) == 1
-    #     assert res.body.concept.compiled["b"][0].status
-    #     assert res.body.concept.compiled["b"][0].who == "parsers.BnfNode"
+    #     assert res.body.concept.get_compiled()["a"] == concepts_map["one"]
+    #     assert len(res.body.concept.get_compiled()["b"]) == 1
+    #     assert res.body.concept.get_compiled()["b"][0].status
+    #     assert res.body.concept.get_compiled()["b"][0].who == "parsers.BnfNode"
     #
     #     # evaluate
     #     context = self.get_context(sheerka, eval_body=True)
@@ -336,7 +339,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert parser_result.source == expression
         assert len(actual_nodes) == 1
-        assert actual_nodes[0].nodes[0].concept.metadata.is_evaluated  # 'a plus b' is recognized as concept definition
+        assert actual_nodes[0].nodes[0].concept.get_metadata().is_evaluated  # 'a plus b' is recognized as concept definition
 
     def test_i_can_parse_unrecognized_source_code_with_concept_node_when_var_in_short_term_memory(self):
         sheerka, context, parser = self.init_parser()
@@ -355,7 +358,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert parser_result.source == expression
         assert len(actual_nodes) == 1
-        assert not actual_nodes[0].nodes[0].concept.metadata.is_evaluated  # 'a plus b' need to be evaluated
+        assert not actual_nodes[0].nodes[0].concept.get_metadata().is_evaluated  # 'a plus b' need to be evaluated
 
     def test_i_can_parse_unrecognized_sya_concept_that_references_source_code(self):
         sheerka, context, parser = self.init_parser()
@@ -381,8 +384,8 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
             expression, [CN("hello_sya", source="hello get_user_name(twenty one)")],
             exclude_body=True)
         assert actual_nodes == expected_array
-        assert isinstance(actual_nodes[0].concept.compiled["a"], list)
-        assert sheerka.isinstance(actual_nodes[0].concept.compiled["a"][0], BuiltinConcepts.RETURN_VALUE)
+        assert isinstance(actual_nodes[0].concept.get_compiled()["a"], list)
+        assert sheerka.isinstance(actual_nodes[0].concept.get_compiled()["a"][0], BuiltinConcepts.RETURN_VALUE)
 
     def test_i_can_parse_sequences(self):
         sheerka, context, parser = self.init_parser()

tests/repl/test_SheerkaPromptCompleter.py

@@ -19,10 +19,13 @@ class TestSheerkaPromptCompleter(TestUsingMemoryBasedSheerka):
         assert as_dict["get_partition"].display_text == "get_partition"
         assert as_dict["get_partition"].display_meta_text == "builtin"
 
-        assert "get_results" in as_dict
-        assert as_dict["get_results"].text == "get_results()"
-        assert as_dict["get_results"].display_text == "get_results"
-        assert as_dict["get_results"].display_meta_text == "builtin"
+        completions = SheerkaPromptCompleter(sheerka).get_completions(Document("cache"), CompleteEvent())
+        as_dict = {c.display_text: c for c in completions}
+
+        assert "caches_names" in as_dict
+        assert as_dict["caches_names"].text == "caches_names()"
+        assert as_dict["caches_names"].display_text == "caches_names"
+        assert as_dict["caches_names"].display_meta_text == "builtin"
 
     def test_i_can_complete_with_commands(self):
         sheerka = self.get_sheerka()

tests/sdp/test_sheerkaDataProvider.py

@@ -242,8 +242,10 @@ def test_i_can_load_an_entry(root):
         transaction.add("entry", "key2", "bar")
         transaction.add("entry", "key3", "baz")
 
-    load_entry = sdp.get("entry")
+    item = sdp.get("entry", "key1")
+    assert item == "foo"
 
+    load_entry = sdp.get("entry")
     assert load_entry == {
         "key1": "foo",
         "key2": "bar",
@@ -369,6 +371,11 @@ def test_i_can_add_an_object_and_save_it_as_a_reference(root):
     assert sdp.get("entry", "key2") == [ObjNoKey("a", "b"), ObjNoKey("c", "d")]
     assert sdp.get("entry", "key3") == {ObjNoKey("a", "b"), ObjNoKey("c", "d")}
 
+    # I can ask for the whole entry
+    assert sdp.get("entry") == {"key1": ObjNoKey("a", "b"),
+                                "key2": [ObjNoKey("a", "b"), ObjNoKey("c", "d")],
+                                "key3": {ObjNoKey("a", "b"), ObjNoKey("c", "d")}}
+
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {
         "entry": {'key1': '##REF##:8fac7e801d08361c3449c594b4261ab9c45ef47f1a08df68eb717db2b6919774',

tests/sheerkapickle/test_SheerkaPickler.py

@@ -1,12 +1,13 @@
 import logging
 
 import pytest
-from core.concept import Concept, ConceptParts
+from core.concept import Concept
+from core.tokenizer import Keywords
 from sheerkapickle import tags
 from sheerkapickle.SheerkaPickler import SheerkaPickler
 from sheerkapickle.SheerkaUnpickler import SheerkaUnpickler
 
-from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
 class Obj:
@@ -28,7 +29,7 @@ class Obj:
         return hash((self.a, self.b, self.c))
 
 
-class TestSheerkaPickler(TestUsingFileBasedSheerka):
+class TestSheerkaPickler(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("obj, expected", [
         (1, 1),
@@ -44,7 +45,7 @@ class TestSheerkaPickler(TestUsingFileBasedSheerka):
         ([1, [3.14, "a string"]], [1, [3.14, "a string"]]),
         ([1, (3.14, "a string")], [1, {tags.TUPLE: [3.14, "a string"]}]),
         ([], []),
-        (ConceptParts.BODY, {tags.ENUM: "core.concept.ConceptParts.BODY"}),
+        (Keywords.DEF, {tags.ENUM: 'core.tokenizer.Keywords.DEF'}),
     ])
     def test_i_can_flatten_and_restore_primitives(self, obj, expected):
         sheerka = self.get_sheerka()
@@ -105,9 +106,22 @@ class TestSheerkaPickler(TestUsingFileBasedSheerka):
         decoded = SheerkaUnpickler(sheerka).restore(flatten)
         assert decoded == obj
 
+    def test_i_cannot_correctly_flatten_compiled_and_generator(self):
+        sheerka = self.get_sheerka()
+
+        obj = Obj((i for i in range(3)), compile("a + b", "<src>", mode="eval"), None)
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+
+        assert isinstance(flatten["a"], str)
+        assert flatten["a"].startswith("<generator object")
+        assert isinstance(flatten["b"], str)
+        assert flatten["b"].startswith("<code object")
+        assert flatten["c"] is None
+
     @pytest.mark.parametrize("obj, expected", [
         ({None: "a"}, {'null': "a"}),
-        ({ConceptParts.BODY: "a"}, {'core.concept.ConceptParts.BODY': 'a'}),
+        ({Keywords.DEF: "a"}, {'core.tokenizer.Keywords.DEF': 'a'}),
         ({(1, 2): "a"}, {(1, 2): "a"}),
     ])
     def test_i_can_manage_specific_keys_in_dictionaries(self, obj, expected):
@@ -136,15 +150,15 @@ class TestSheerkaPickler(TestUsingFileBasedSheerka):
         sheerka = self.get_sheerka()
 
         foo = Obj("foo", "bar", "baz")
-        obj = [ConceptParts.BODY, foo, ConceptParts.WHERE, ConceptParts.BODY, foo]
+        obj = [Keywords.DEF, foo, Keywords.WHERE, Keywords.DEF, foo]
         flatten = SheerkaPickler(sheerka).flatten(obj)
 
-        assert flatten == [{'_sheerka/enum': 'core.concept.ConceptParts.BODY'},
+        assert flatten == [{'_sheerka/enum': 'core.tokenizer.Keywords.DEF'},
                            {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
                             'a': 'foo',
                             'b': 'bar',
                             'c': 'baz'},
-                           {'_sheerka/enum': 'core.concept.ConceptParts.WHERE'},
+                           {'_sheerka/enum': 'core.tokenizer.Keywords.WHERE'},
                            {'_sheerka/id': 0},
                            {'_sheerka/id': 1}]
 

tests/sheerkapickle/test_sheerka_handlers.py

@@ -1,6 +1,7 @@
 import sheerkapickle
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept, ConceptParts
+from core.rule import Rule
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.tokenizer import Tokenizer
 from evaluators.ConceptEvaluator import ConceptEvaluator
@@ -11,7 +12,7 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
 def set_full_serialization(concept):
-    concept.metadata.full_serialization = True
+    concept.get_metadata().full_serialization = True
     return concept
 
 
@@ -72,7 +73,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
 
         concept = Concept("foo").init_key()
         sheerka.create_new_concept(self.get_context(sheerka), concept)
-        concept.metadata.full_serialization = True
+        concept.get_metadata().full_serialization = True
         to_string = sheerkapickle.encode(sheerka, concept)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == concept
@@ -86,35 +87,35 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, concept)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == concept
-        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.pre", 10]]}'
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#pre#", 10]]}'
 
         concept = set_full_serialization(Concept("foo"))
         concept.set_value(ConceptParts.POST, 'a string')  # an string
         to_string = sheerkapickle.encode(sheerka, concept)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == concept
-        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.post", "a string"]]}'
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#post#", "a string"]]}'
 
         concept = set_full_serialization(Concept("foo"))
         concept.set_value(ConceptParts.WHERE, ['a string', 3.14])  # a list
         to_string = sheerkapickle.encode(sheerka, concept)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == concept
-        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.where", ["a string", 3.14]]]}'
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#where#", ["a string", 3.14]]]}'
 
         concept = set_full_serialization(Concept("foo"))
         concept.set_value(ConceptParts.WHERE, ('a string', 3.14))  # a tuple
         to_string = sheerkapickle.encode(sheerka, concept)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == concept
-        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.where", {"_sheerka/tuple": ["a string", 3.14]}]]}'
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#where#", {"_sheerka/tuple": ["a string", 3.14]}]]}'
 
         concept = set_full_serialization(Concept("foo"))
         concept.set_value(ConceptParts.BODY, set_full_serialization(Concept("foo", body="foo_body")))
         to_string = sheerkapickle.encode(sheerka, concept)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == concept
-        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.body", {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "foo_body"}]]}'
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#body#", {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "foo_body"}]]}'
 
     def test_i_can_encode_decode_unknown_concept_variables(self):
         sheerka = self.get_sheerka()
@@ -166,26 +167,26 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, concept)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == concept
-        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"], "values": [["cParts.body", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}'
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"], "values": [["#body#", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}'
 
-    def test_i_can_encode_decode_when_variable_is_a_concept(self):
-        sheerka = self.get_sheerka()
-
-        foo = Concept("foo")
-        sheerka.create_new_concept(self.get_context(sheerka), foo)
-
-        concept = Concept("my_name")
-        sheerka.create_new_concept(self.get_context(sheerka), concept)
-        concept.def_var(foo, "a value")
-        concept.set_value(foo, "another value")
-        concept.metadata.full_serialization = True
-
-        to_string = sheerkapickle.encode(sheerka, concept)
-        decoded = sheerkapickle.decode(sheerka, to_string)
-        assert decoded == concept
-        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "my_name", "meta.key": "my_name", ' + \
-               '"meta.variables": [[{"_sheerka/obj": "core.concept.Concept", "concept/id": ["foo", "1001"]}, "a value"]], ' + \
-               '"meta.id": "1002", "values": [[{"_sheerka/id": 1}, "another value"]]}'
+    # def test_i_can_encode_decode_when_variable_is_a_concept(self):
+    #     sheerka = self.get_sheerka()
+    #
+    #     foo = Concept("foo")
+    #     sheerka.create_new_concept(self.get_context(sheerka), foo)
+    #
+    #     concept = Concept("my_name")
+    #     sheerka.create_new_concept(self.get_context(sheerka), concept)
+    #     concept.def_var(foo, "a value")
+    #     concept.set_value(foo, "another value")
+    #     concept.get_metadata().full_serialization = True
+    #
+    #     to_string = sheerkapickle.encode(sheerka, concept)
+    #     decoded = sheerkapickle.decode(sheerka, to_string)
+    #     assert decoded == concept
+    #     assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "my_name", "meta.key": "my_name", ' + \
+    #            '"meta.variables": [[{"_sheerka/obj": "core.concept.Concept", "concept/id": ["foo", "1001"]}, "a value"]], ' + \
+    #            '"meta.id": "1002", "values": [[{"_sheerka/id": 1}, "another value"]]}'
 
     def test_i_can_manage_reference_of_the_same_object(self):
         sheerka = self.get_sheerka()
@@ -199,7 +200,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, concept)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == concept
-        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "bar", "values": [["cParts.pre", {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo"}], ["cParts.body", {"_sheerka/id": 1}]]}'
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "bar", "values": [["#pre#", {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo"}], ["#body#", {"_sheerka/id": 1}]]}'
 
     def test_i_can_encode_decode_user_input(self):
         sheerka = self.get_sheerka()
@@ -326,3 +327,15 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         decoded = sheerkapickle.decode(sheerka, to_string)
 
         assert decoded == foo
+
+    def test_i_can_encode_decode_rule(self):
+        sheerka = self.get_sheerka()
+
+        rule = Rule("print", "my rule", "True","Hello world")
+        rule.metadata.id = "1"
+
+        to_string = sheerkapickle.encode(sheerka, rule)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+
+        assert to_string == '{"_sheerka/obj": "core.rule.Rule", "rule/id": "1", "name": "my rule", "predicate": "True", "action_type": "print", "action": "Hello world"}'
+        assert decoded == rule