First version of explain. Creating a new parser was a wrong approach. Need to reimplement

tests/TestUsingMemoryBasedSheerka.py

@@ -16,7 +16,7 @@ class TestUsingMemoryBasedSheerka(BaseTest):
         skip_builtins_in_db = kwargs.get("skip_builtins_in_db", True)
         use_singleton = kwargs.get("singleton", False)
 
-        sheerka = kwargs.get("sheerka", False)
+        sheerka = kwargs.get("sheerka", None)
         if sheerka:
             return sheerka
 

tests/core/test_SheerkaHistoryManager.py

@@ -31,7 +31,7 @@ class TestSheerkaHistoryManager(TestUsingMemoryBasedSheerka):
             hist("xxx", False),
             hist("one", True),
             hist("def concept one as 1", True),
-            hist("Initializing Sheerka.", True)]
+            hist("Initializing Sheerka.", None)]
 
         h = list(sheerka.history(2))
         assert h == [

tests/core/test_SheerkaVariableManager.py

@@ -0,0 +1,71 @@
+from core.concept import Concept, ConceptParts
+from core.sheerka.Services.SheerkaVariableManager import SheerkaVariableManager
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
+    def test_i_can_record_and_load_a_constant(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+
+        sheerka.record(context, "TestSheerkaVariable", "my_variable", 1)
+        res = sheerka.load("TestSheerkaVariable", "my_variable")
+        assert res == 1
+
+        assert sheerka.sdp.exists(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
+        loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
+        assert loaded.event_id == context.event.get_digest()
+        assert loaded.key == "my_variable"
+        assert loaded.value == 1
+        assert loaded.who == "TestSheerkaVariable"
+        assert loaded.parents is None
+
+    def test_i_can_record_and_load_a_concept(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+
+        concept = Concept("foo").set_prop("a", "alpha").set_metadata_value(ConceptParts.BODY, 3.14)
+
+        sheerka.record(context, "TestSheerkaVariable", "my_variable", concept)
+        res = sheerka.load("TestSheerkaVariable", "my_variable")
+
+        assert res == concept
+        assert res.body == concept.body
+
+    def test_i_can_get_the_parent_when_modified(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+
+        sheerka.record(context, "TestSheerkaVariable", "my_variable", 1)
+        sheerka.record(context, "TestSheerkaVariable", "my_variable", 2)
+        res = sheerka.load("TestSheerkaVariable", "my_variable")
+        assert res == 2
+
+        loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
+        assert loaded.event_id == context.event.get_digest()
+        assert loaded.key == "my_variable"
+        assert loaded.value == 2
+        assert loaded.who == "TestSheerkaVariable"
+        assert loaded.parents == ['8c9ada7bf488d84229f6539f76042431638f16d600fe3b7ec7e7161043a40d59']
+
+        parent = sheerka.sdp.load_obj(loaded.parents[0])
+        assert parent.event_id == context.event.get_digest()
+        assert parent.key == "my_variable"
+        assert parent.value == 1
+        assert parent.who == "TestSheerkaVariable"
+        assert parent.parents is None
+
+    def test_variable_is_not_persisted_if_the_value_is_the_same(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+
+        sheerka.record(context, "TestSheerkaVariable", "my_variable", 1)
+        sheerka.record(context, "TestSheerkaVariable", "my_variable", 1)
+
+        loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "TestSheerkaVariable.my_variable")
+        assert loaded.event_id == context.event.get_digest()
+        assert loaded.key == "my_variable"
+        assert loaded.value == 1
+        assert loaded.who == "TestSheerkaVariable"
+        assert loaded.parents is None

tests/core/test_sheerka_printer.py

@@ -0,0 +1,278 @@
+from dataclasses import dataclass
+
+import pytest
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from parsers.ExpressionParser import TrueNode, LambdaNode
+from printer.SheerkaPrinter import FormatInstructions
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+@dataclass
+class Obj:
+    a: object
+    b: object
+
+
+@dataclass()
+class ObjLongProp:
+    first_property_name: object
+    second: object
+
+
+class TestSheerkaPrinter(TestUsingMemoryBasedSheerka):
+    @pytest.mark.parametrize("text, expected", [
+        ("Hello world!", "Hello world!\n"),
+        ("%black%%red%%green%%yellow%%reset%", "\x1b[30m\x1b[31m\x1b[32m\x1b[33m\x1b[0m\n"),
+        ("%blue%%magenta%%cyan%%white%%reset%", "\x1b[34m\x1b[35m\x1b[36m\x1b[37m\x1b[0m\n"),
+        (["Hello", "world!"], "Hello\nworld!\n"),
+        (("Hello", "world!"), "Hello\nworld!\n"),
+    ])
+    def test_i_can_print(self, capsys, text, expected):
+        sheerka = self.get_sheerka()
+        sheerka.print(text)
+
+        captured = capsys.readouterr()
+        assert captured.out == expected
+
+    def test_i_can_disable_color(self, capsys):
+        sheerka = self.get_sheerka()
+        sheerka.print("%red%Hello world !%reset%", FormatInstructions(no_color=True))
+
+        captured = capsys.readouterr()
+        assert captured.out == "Hello world !\n"
+
+    def test_i_can_print_concept(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo a b").def_prop("a").def_prop("b")
+        sheerka.print(foo)
+
+        captured = capsys.readouterr()
+        assert captured.out == str(foo) + "\n"
+
+    def test_i_can_use_custom_format(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo a b").def_prop("a").def_prop("b").init_key()
+        sheerka.printer_handler.register_custom_printer(
+            foo,
+            lambda printer, instr, item: printer.fp(instr, f"foo a={item.a}, b={item.b}"))
+        foo.set_prop("a", "value a").set_prop("b", "value b")
+        sheerka.print(foo)
+
+        captured = capsys.readouterr()
+        assert captured.out == "foo a=value a, b=value b\n"
+
+    def test_i_can_print_and_recurse(self, capsys):
+        sheerka = self.get_sheerka()
+        level3 = Concept("level3")
+        level2 = Concept("level2").set_metadata_value(ConceptParts.BODY, level3)
+        level1 = Concept("level1").set_metadata_value(ConceptParts.BODY, level2)
+
+        sheerka.print(level1)
+        captured = capsys.readouterr()
+        assert captured.out == f"{level1}\n"
+
+        sheerka.print(level1, FormatInstructions().set_recurse("body", 1))
+        captured = capsys.readouterr()
+        assert captured.out == f"{level1}\n  {level2}\n"
+
+        sheerka.print(level1, FormatInstructions().set_recurse("body", 2))
+        captured = capsys.readouterr()
+        assert captured.out == f"{level1}\n  {level2}\n    {level3}\n"
+
+        sheerka.print(level1, FormatInstructions().set_recurse("body", 10))
+        captured = capsys.readouterr()
+        assert captured.out == f"{level1}\n  {level2}\n    {level3}\n"
+
+    def test_i_can_print_and_recurse_list(self, capsys):
+        sheerka = self.get_sheerka()
+        level31 = Concept("level31")
+        level32 = Concept("level32")
+        level33 = Concept("level33")
+        level21 = Concept("level21").set_metadata_value(ConceptParts.BODY, [level31, level32])
+        level22 = Concept("level22").set_metadata_value(ConceptParts.BODY, [level33])
+        level1 = Concept("level1").set_metadata_value(ConceptParts.BODY, [level21, level22])
+
+        sheerka.print(level1)
+        captured = capsys.readouterr()
+        assert captured.out == f"{level1}\n"
+
+        sheerka.print(level1, FormatInstructions().set_recurse("body", 1))
+        captured = capsys.readouterr()
+        assert captured.out == f"{level1}\n  {level21}\n  {level22}\n"
+
+        sheerka.print(level1, FormatInstructions().set_recurse("body", 3))
+        captured = capsys.readouterr()
+        assert captured.out == f"{level1}\n  {level21}\n    {level31}\n    {level32}\n  {level22}\n    {level33}\n"
+
+    def test_explanation_concept_can_control_recursion(self, capsys):
+        sheerka = self.get_sheerka()
+        level31 = Concept("level31")
+        level32 = Concept("level32")
+        level33 = Concept("level33")
+        level21 = Concept("level21").set_metadata_value(ConceptParts.BODY, [level31, level32])
+        level22 = Concept("level22").set_metadata_value(ConceptParts.BODY, [level33])
+        level1 = Concept("level1").set_metadata_value(ConceptParts.BODY, [level21, level22])
+
+        instructions = FormatInstructions(no_color=True)
+        explanation = sheerka.new(
+            BuiltinConcepts.EXPLANATION,
+            digest="digest",
+            command="command",
+            title="title",
+            instructions=instructions,
+            body=[level1])
+
+        sheerka.print(explanation)
+        captured = capsys.readouterr()
+        assert captured.out == f"digest : command\n(None)level1\n"
+
+        instructions = FormatInstructions(no_color=True).set_recurse("body", 2)
+        explanation = sheerka.new(
+            BuiltinConcepts.EXPLANATION,
+            digest="digest",
+            command="command",
+            title="title",
+            instructions=instructions,
+            body=[level1])
+
+        sheerka.print(explanation)
+        captured = capsys.readouterr()
+        assert captured.out == """digest : command
+(None)level1
+  (None)level21
+    (None)level31
+    (None)level32
+  (None)level22
+    (None)level33
+"""
+
+    def test_i_can_format_concept(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo a b").def_prop("a").def_prop("b").init_key()
+        foo.set_prop("a", "value a").set_prop("b", "value b")
+        foo.set_metadata_value(ConceptParts.BODY, "body")
+        sheerka.set_id_if_needed(foo, False)
+
+        sheerka.printer_handler.register_format_l(foo, "{id}-{name}-{key}-{body}-{a}-{b}")
+
+        sheerka.print(foo)
+        captured = capsys.readouterr()
+        assert captured.out == "1001-foo a b-foo __var__0 __var__1-body-value a-value b\n"
+
+    def test_i_can_format_object(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = Obj("value a", "value b")
+
+        sheerka.printer_handler.register_format_l(foo, "{a}-{b}")
+
+        sheerka.print(foo)
+        captured = capsys.readouterr()
+        assert captured.out == "value a-value b\n"
+
+    def test_i_can_register_a_custom_format_by_its_name(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = Obj("value a", "value b")
+
+        sheerka.printer_handler.register_format_l("tests.core.test_sheerka_printer.Obj", "{a}-{b}")
+
+        sheerka.print(foo)
+        captured = capsys.readouterr()
+        assert captured.out == "value a-value b\n"
+
+    def test_i_can_define_format_in_print_instruction(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = Obj("value a", "value b")
+
+        instructions = FormatInstructions().set_format_l("tests.core.test_sheerka_printer.Obj", "{a}-{b}")
+
+        sheerka.print(foo, instructions)
+        captured = capsys.readouterr()
+        assert captured.out == "value a-value b\n"
+
+    def test_format_print_instruction_override_register_format(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = Obj("value a", "value b")
+
+        sheerka.printer_handler.register_format_l("tests.core.test_sheerka_printer.Obj", "{a}-{b}")
+        instructions = FormatInstructions().set_format_l("tests.core.test_sheerka_printer.Obj", "a={a} <> b={b}")
+
+        sheerka.print(foo, instructions)
+        captured = capsys.readouterr()
+        assert captured.out == "a=value a <> b=value b\n"
+
+    def test_i_can_format_d(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = [Obj("value a", "value b"), Obj("value c", "value d")]
+
+        sheerka.printer_handler.register_format_d(TrueNode(), ["a", "b"])
+        sheerka.print(foo)
+        captured = capsys.readouterr()
+        assert captured.out == """Obj(a='value a', b='value b')
+a: value a
+b: value b
+Obj(a='value c', b='value d')
+a: value c
+b: value d
+"""
+
+    def test_i_can_format_d_and_align_properties(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = [ObjLongProp("value a", "value b"), ObjLongProp("value c", "value d")]
+
+        sheerka.printer_handler.register_format_d(TrueNode(), ["first_property_name", "second"])
+        sheerka.print(foo)
+        captured = capsys.readouterr()
+        assert captured.out == """ObjLongProp(first_property_name='value a', second='value b')
+first_property_name: value a
+second             : value b
+ObjLongProp(first_property_name='value c', second='value d')
+first_property_name: value c
+second             : value d
+"""
+
+    def test_i_can_manage_when_property_does_not_exist(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = Obj("value a", "value b")
+
+        sheerka.printer_handler.register_format_d(TrueNode(), ["foo", "bar"])
+        sheerka.print(foo)
+        captured = capsys.readouterr()
+        assert captured.out == """Obj(a='value a', b='value b')
+foo: *Undefined*
+bar: *Undefined*
+"""
+
+    def test_i_can_select_the_object_to_format_d(self, capsys):
+        sheerka = self.get_sheerka()
+        foo = [Obj("value a", "value b"), ObjLongProp("value c", "value d")]
+
+        sheerka.printer_handler.register_format_d(LambdaNode(lambda o: isinstance(o, Obj)), ["a", "b"])
+        sheerka.print(foo)
+        captured = capsys.readouterr()
+        assert captured.out == """Obj(a='value a', b='value b')
+a: value a
+b: value b
+ObjLongProp(first_property_name='value c', second='value d')
+"""
+
+    @pytest.mark.parametrize("template, expected", [
+        ("+-{b}", "value a-value b\n"),
+        ("{b}-+", "value b-value a\n"),
+        ("\\+{b}", "+value b\n"),
+        ("{b}\\+", "value b+\n"),
+        ("+", "+\n"),
+        ("\\+", "\\+\n"),
+        ("+\\", "+\\\n"),
+    ])
+    def test_i_can_concat_print_instruction_and_register_format(self, capsys, template, expected):
+        sheerka = self.get_sheerka()
+        foo = Obj("value a", "value b")
+
+        sheerka.printer_handler.register_format_l("tests.core.test_sheerka_printer.Obj", "{a}")
+        instructions = FormatInstructions().set_format_l("tests.core.test_sheerka_printer.Obj", template)
+
+        sheerka.print(foo, instructions)
+        captured = capsys.readouterr()
+        assert captured.out == expected

tests/core/test_tokenizer.py

@@ -4,7 +4,7 @@ from core.tokenizer import Tokenizer, Token, TokenKind, LexerError, Keywords
 
 def test_i_can_tokenize():
     source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&<>c:name:"
-    source += "$£€!_identifier°~_^\\`#"
+    source += "$£€!_identifier°~_^\\`==#"
     tokens = list(Tokenizer(source))
     assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
     assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
@@ -52,9 +52,10 @@ def test_i_can_tokenize():
     assert tokens[43] == Token(TokenKind.CARAT, '^', 106, 6, 48)
     assert tokens[44] == Token(TokenKind.BACK_SLASH, '\\', 107, 6, 49)
     assert tokens[45] == Token(TokenKind.BACK_QUOTE, '`', 108, 6, 50)
-    assert tokens[46] == Token(TokenKind.HASH, '#', 109, 6, 51)
+    assert tokens[46] == Token(TokenKind.EQUALSEQUALS, '==', 109, 6, 51)
+    assert tokens[47] == Token(TokenKind.HASH, '#', 111, 6, 53)
 
-    assert tokens[47] == Token(TokenKind.EOF, '', 110, 6, 52)
+    assert tokens[48] == Token(TokenKind.EOF, '', 112, 6, 54)
 
 
 @pytest.mark.parametrize("text, expected", [
@@ -74,6 +75,19 @@ def test_i_can_tokenize_identifiers(text, expected):
     assert comparison == expected
 
 
+@pytest.mark.parametrize("text", [
+    "123abc",
+    "123",
+    "abc",
+    "abc123"
+])
+def test_i_can_parse_word(text):
+    tokens = list(Tokenizer(text, parse_word=True))
+    assert tokens[0].type == TokenKind.WORD
+    assert tokens[0].value == text
+    assert tokens[1].index == len(text)
+
+
 @pytest.mark.parametrize("text, message, error_text, index, line, column", [
     ("'string", "Missing Trailing quote", "'string", 7, 1, 8),
     ('"string', "Missing Trailing quote", '"string', 7, 1, 8),

tests/evaluators/test_ExplainEvaluator.py

@@ -0,0 +1,317 @@
+import os
+
+import pytest
+from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.ExecutionContext import ExecutionContext
+from evaluators.ExplainEvaluator import ExplainEvaluator
+from parsers.ExplainParser import ExplanationNode, RecurseDefNode, FormatLNode, UnionNode, FilterNode, FormatDNode
+from parsers.ExpressionParser import PropertyEqualsNode, PropertyEqualsSequenceNode, TrueNode, IsaNode
+from printer.FormatInstructions import FormatDetailDesc, FormatDetailType
+from pytest import fixture
+from sdp.sheerkaDataProvider import Event
+from sdp.sheerkaSerializer import Serializer, SerializerContext
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+@fixture(scope="module")
+def serializer():
+    """
+    Return a :class:`sdp.sheerkaSerializer.Serializer` instance for the module
+    """
+    return Serializer()
+
+
+class EC:
+    """
+    Helper to create execution context (AKA execution result)
+    """
+
+    def __init__(self, children=None, **props):
+        self.props = props
+        self.children = children
+
+
+def get_return_value(expr):
+    if isinstance(expr, ExplanationNode):
+        value = expr
+    else:
+        value = ExplanationNode("xxx_test_explain_evaluator_xxx", "", expr=expr)
+
+    return ReturnValueConcept(
+        "TestEvaluator",
+        True,
+        ParserResultConcept(parser="parser", value=value))
+
+
+def create_executions_results(context, list_of_ecs):
+    def update(execution_context, ec):
+        for prop_name, pro_value in ec.props.items():
+            setattr(execution_context, prop_name, pro_value)
+
+        if ec.children:
+            for child_ec in ec.children:
+                child_execution_context = execution_context.push("TestEvaluator")
+                update(child_execution_context, child_ec)
+
+    res = []
+    for ec in list_of_ecs:
+        execution_context = ExecutionContext("TestEvaluator", context.event, context.sheerka)
+        update(execution_context, ec)
+        res.append(execution_context)
+
+    return res
+
+
+def get_execution_result_from_file(sheerka, digest, serializer):
+    target_path = os.path.join("../_fixture/", digest) + "_result"
+    with open(target_path, "rb") as f:
+        context = SerializerContext(sheerka=sheerka)
+        return serializer.deserialize(f, context)
+
+
+def get_execution_result_from_list(executions_result):
+    return executions_result
+
+
+class TestExplainEvaluator(TestUsingMemoryBasedSheerka):
+
+    @staticmethod
+    def init_evaluator_with_file(self, serializer):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+        evaluator = ExplainEvaluator()
+        evaluator.get_execution_result = lambda s, d: get_execution_result_from_file(s, d, serializer)
+
+        return sheerka, context, evaluator
+
+    def init_evaluator_with_list(self, list_of_ecs):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+        evaluator = ExplainEvaluator()
+
+        executions_result = create_executions_results(context, list_of_ecs)
+        evaluator.get_execution_result = lambda s, d: get_execution_result_from_list(executions_result)
+
+        return sheerka, context, evaluator, executions_result
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=ExplanationNode("", ""))), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False),
+        (ReturnValueConcept("some_name", False, "not relevant"), False),
+        (ReturnValueConcept("some_name", True, Concept()), False)
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert ExplainEvaluator().matches(context, ret_val) == expected
+
+    def test_i_can_eval_in_list(self, serializer):
+        sheerka, context, evaluator, execution_results = self.init_evaluator_with_list(
+            [
+                EC(desc="correct desc"),
+                EC(desc="wrong desc"),
+            ]
+        )
+        ret_val = get_return_value(UnionNode(
+            [
+                FilterNode(TrueNode()),
+                FilterNode(PropertyEqualsNode("desc", "correct desc")),
+            ]))
+
+        res = evaluator.eval(context, ret_val)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION)
+
+        filtered = res.body.body
+        assert filtered == [execution_results[0]]
+
+    def test_i_can_eval_in_children(self):
+        sheerka, context, evaluator, execution_results = self.init_evaluator_with_list(
+            [
+                EC(desc="wrong desc", children=[EC(desc="wrong sub"), EC(desc="good sub")]),
+                EC(desc="wrong desc", children=[EC(desc="good sub")]),
+            ]
+        )
+        ret_val = get_return_value(UnionNode(
+            [
+                FilterNode(TrueNode()),
+                FilterNode(PropertyEqualsNode("desc", "good sub")),
+            ]))
+
+        res = evaluator.eval(context, ret_val)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION)
+
+        filtered = res.body.body
+        assert filtered == [
+            execution_results[0].children[1],
+            execution_results[1].children[0],
+        ]
+
+    def test_i_can_evaluate_multiple_filter_node(self):
+        sheerka, context, evaluator, execution_results = self.init_evaluator_with_list(
+            [
+                EC(desc="parent1", _id=1, children=[EC(desc="wrong sub"), EC(desc="good sub")]),
+                EC(desc="parent2", children=[EC(desc="wrong sub"), EC(desc="good sub")]),
+                EC(desc="good sub")
+            ])
+        ret_val = get_return_value(UnionNode(
+            [
+                FilterNode(TrueNode()),
+                FilterNode(PropertyEqualsNode("id", "1")),
+                FilterNode(PropertyEqualsNode("desc", "good sub")),
+            ]))
+
+        res = evaluator.eval(context, ret_val)
+        assert res.status
+        assert len(res.body) == 2
+
+        assert sheerka.isinstance(res.body[0], BuiltinConcepts.EXPLANATION)
+        assert sheerka.isinstance(res.body[1], BuiltinConcepts.EXPLANATION)
+
+        assert res.body[0].body == [execution_results[0]]
+        assert res.body[1].body == [
+            execution_results[0].children[1],
+            execution_results[1].children[1],
+            execution_results[2]
+        ]
+
+    def test_i_can_eval_parent_and_child(self):
+        sheerka, context, evaluator, execution_results = self.init_evaluator_with_list(
+            [
+                EC(desc="parent1", children=[EC(desc="wrong sub"), EC(desc="good sub")]),
+                EC(desc="parent2", children=[EC(desc="wrong sub"), EC(desc="good sub")]),
+                EC(desc="good sub")
+            ]
+        )
+        ret_val = get_return_value(UnionNode(
+            [
+                FilterNode(TrueNode()),
+                FilterNode(PropertyEqualsSequenceNode(["desc", "desc"], ["parent1", "good sub"])),
+            ]))
+
+        res = evaluator.eval(context, ret_val)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION)
+
+        filtered = res.body.body
+        assert filtered == [
+            execution_results[0].children[1],
+        ]
+
+    def test_i_correctly_create_format_instructions(self):
+        sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([])
+        ret_val = get_return_value(UnionNode(
+            [
+                FilterNode(TrueNode(), [
+                    RecurseDefNode(2),
+                    FormatLNode("abc"),
+                    FormatDNode({"a": "{a}", "b": "{b}"})
+                ]),
+            ]))
+
+        res = evaluator.eval(context, ret_val)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION)
+
+        instructions = res.body.instructions
+        assert instructions.recursive_props == {"children": 2}
+        assert instructions.format_l == {'core.sheerka.ExecutionContext.ExecutionContext': 'abc'}
+        assert instructions.format_d == [FormatDetailDesc(
+            IsaNode(ExecutionContext),
+            FormatDetailType.Props_In_Line,
+            {"a": "{a}", "b": "{b}"})]
+
+    def test_i_correctly_create_format_instructions_with_filtering(self):
+        sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([])
+
+        ret_val = get_return_value(UnionNode(
+            [
+                FilterNode(TrueNode()),
+                FilterNode(PropertyEqualsNode("id", "1"), [RecurseDefNode(2), FormatLNode("abc")]),
+            ]))
+        res = evaluator.eval(context, ret_val)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.EXPLANATION)
+
+        instructions = res.body.instructions
+        assert instructions.format_l == {'core.sheerka.ExecutionContext.ExecutionContext': 'abc'}
+        assert instructions.recursive_props == {"children": 2}
+
+    def test_i_can_have_different_instructions_for_different_filtering(self):
+        sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([])
+        ret_val = get_return_value(UnionNode(
+            [
+                FilterNode(TrueNode()),
+                FilterNode(PropertyEqualsNode("id", "1"), [RecurseDefNode(2)]),
+                FilterNode(PropertyEqualsNode("desc", "good sub"), [FormatLNode("abc")]),
+            ]))
+
+        res = evaluator.eval(context, ret_val)
+        assert res.status
+        assert len(res.body) == 2
+
+        assert res.body[0].instructions.recursive_props == {"children": 2}
+        assert res.body[1].instructions.format_l == {'core.sheerka.ExecutionContext.ExecutionContext': 'abc'}
+
+    def test_filtering_instructions_inherit_from_the_first_filtering_node(self):
+        sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([])
+        ret_val = get_return_value(UnionNode(
+            [
+                FilterNode(TrueNode(), [RecurseDefNode(2)]),
+                FilterNode(PropertyEqualsNode("id", "1"), [RecurseDefNode(1)]),
+                FilterNode(PropertyEqualsNode("desc", "good sub"), [FormatLNode("abc")]),
+            ]))
+
+        res = evaluator.eval(context, ret_val)
+        assert res.status
+        assert len(res.body) == 2
+
+        assert res.body[0].instructions.recursive_props == {"children": 1}  # overridden
+
+        assert res.body[1].instructions.format_l == {'core.sheerka.ExecutionContext.ExecutionContext': 'abc'}
+        assert res.body[1].instructions.recursive_props == {"children": 2}
+
+    def test_i_can_reuse_a_recorded_digest(self):
+        sheerka, context, evaluator, execution_results = self.init_evaluator_with_list([])
+        expr = UnionNode([FilterNode(TrueNode(), [RecurseDefNode(2)])])
+
+        # need a valid result to test this feature
+        event = Event("fake message")
+        execution_context = ExecutionContext("TestExplainEvaluator", event, sheerka)
+        sheerka.sdp.save_result(execution_context)
+
+        # save another result
+        event2 = Event("fake message")
+        execution_context = ExecutionContext("TestExplainEvaluator", event2, sheerka)
+        sheerka.sdp.save_result(execution_context)
+
+        # digest is recorded during the first call
+        explanation_node = ExplanationNode(event.get_digest(), "", expr=expr, record_digest=True)
+        ret_val = get_return_value(explanation_node)
+        evaluator.eval(context, ret_val)
+
+        # the next call to get_event_digest will load the recorded digest
+        explanation_node = ExplanationNode("", "", expr=expr, record_digest=False)  # digest is not provided
+        digest = evaluator.get_event_digest(sheerka, explanation_node)
+        assert digest == event.get_digest()
+
+        # test I can record another digest
+        explanation_node = ExplanationNode(event2.get_digest(), "", expr=expr, record_digest=True)
+        ret_val = get_return_value(explanation_node)
+        evaluator.eval(context, ret_val)
+
+        explanation_node = ExplanationNode("", "", expr=expr, record_digest=False)  # digest is not provided
+        digest = evaluator.get_event_digest(sheerka, explanation_node)
+        assert digest == event2.get_digest()
+
+        # test can now reset the recorded digest
+        # (a digest is provided, but record_digest is set to False)
+        explanation_node = ExplanationNode(event.get_digest(), "", expr=expr, record_digest=False)
+        ret_val = get_return_value(explanation_node)
+        evaluator.eval(context, ret_val)
+
+        explanation_node = ExplanationNode("", "", expr=expr, record_digest=False)  # digest is not provided
+        digest = evaluator.get_event_digest(sheerka, explanation_node)
+        assert digest is None

tests/parsers/test_BaseParser.py

@@ -1,7 +1,7 @@
 import pytest
 
-from core.tokenizer import Tokenizer, TokenKind
-from parsers.BaseParser import BaseParser
+from core.tokenizer import Tokenizer, TokenKind, Token
+from parsers.BaseParser import BaseParser, BaseSplitIterParser
 
 
 @pytest.mark.parametrize("text, expected_text", [
@@ -23,3 +23,45 @@ def test_i_can_get_text_from_tokens(text, expected_text):
 def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
     tokens = list(Tokenizer(text))
     assert BaseParser.get_text_from_tokens(tokens, custom) == expected_text
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("", ["<eof>"]),
+    ("one two -f --file", ["one", "two", "-f", "--file", "<eof>"]),
+    ("one 'two three'", ["one", "two three", "<eof>"]),
+    ('one "two three"', ["one", "two three", "<eof>"]),
+    ('one\\ two three"', ["one two", "three", "<eof>"]),
+    ("one 'two\\' three'", ["one", "two' three", "<eof>"]),
+    ("one\\\\two three", ["one\\two", "three", "<eof>"]),
+    ("one\ntwo three", ["one", "two", "three", "<eof>"]),
+    ("one \n two three", ["one", "two", "three", "<eof>"]),
+    ("'one \n two' three", ["one \n two", "three", "<eof>"]),
+    ("a=b", ["a", "=", "b", "<eof>"]),
+    ("a = b", ["a", "=", "b", "<eof>"]),
+    ("a==b", ["a", "==", "b", "<eof>"]),
+    ("a == b", ["a", "==", "b", "<eof>"]),
+])
+def test_i_can_split_using_base_split_iterparser_class(text, expected):
+    parser = BaseSplitIterParser("BaseSplitIterParser", 0)
+    parser.reset_parser(None, text)
+    res = [t.value for t in parser.split()]
+
+    assert res == expected
+
+
+def test_i_can_test_split_iter_parser_indexes():
+    parser = BaseSplitIterParser("BaseSplitIterParser", 0)
+    text = "one two \n three = ==(),"
+    parser.reset_parser(None, text)
+    res = []
+    while parser.next_token():
+        res.append(parser.get_token())
+
+    assert res[0] == Token(TokenKind.WORD, "one", 0, 1, 1)
+    assert res[1] == Token(TokenKind.WORD, "two", 4, 1, 5)
+    assert res[2] == Token(TokenKind.WORD, "three", 10, 2, 2)
+    assert res[3] == Token(TokenKind.EQUALS, "=", 16, 2, 8)
+    assert res[4] == Token(TokenKind.EQUALSEQUALS, "==", 18, 2, 10)
+    assert res[5] == Token(TokenKind.LPAR, "(", 20, 2, 12)
+    assert res[6] == Token(TokenKind.RPAR, ")", 21, 2, 13)
+    assert res[7] == Token(TokenKind.COMMA, ",", 22, 2, 14)

tests/parsers/test_BnfNodeParser.py

@@ -73,7 +73,7 @@ def cprop(concept, prop_name):
     return concept.compiled[prop_name]
 
 
-class TestBnfConceptLexerParser(TestUsingMemoryBasedSheerka):
+class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
     def init(self, concepts, grammar):
         sheerka = self.get_sheerka(singleton=True)

tests/parsers/test_ExplainParser.py

@@ -0,0 +1,205 @@
+import pytest
+from core.builtin_concepts import BuiltinConcepts
+from parsers.BaseParser import UnexpectedTokenErrorNode, UnexpectedEof
+from parsers.ExplainParser import ExplainParser, ExplanationNode, MultipleDigestError, ValueErrorNode, \
+    RecurseDefNode, FormatLNode, UnionNode, FilterNode, FormatDNode
+from parsers.ExpressionParser import PropertyContainsNode, PropertyEqualsNode, TrueNode, AndNode, OrNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestExplainParser(TestUsingMemoryBasedSheerka):
+    def init_parser(self, **kwargs):
+        sheerka = self.get_sheerka(singleton=True, **kwargs)
+        context = self.get_context(sheerka)
+        parser = ExplainParser()
+        return sheerka, context, parser
+
+    def test_i_cannot_parse_empty_string(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, "")
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+
+    def test_i_cannot_parse_if_not_for_me(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "foo"
+        res = parser.parse(context, text)
+        not_for_me = res.body
+
+        assert not res.status
+        assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME)
+        assert not_for_me.body == text
+        assert isinstance(not_for_me.reason[0], UnexpectedTokenErrorNode)
+
+    @pytest.mark.parametrize("text, digest, command, directives", [
+        # ("explain", "", "explain", []),
+        ("explain digest", "digest", "explain digest", []),
+        ("explain -r 3", "", "explain -r 3", [RecurseDefNode(3)]),
+        ("explain digest -r 3", "digest", "explain digest -r 3", [RecurseDefNode(3)]),
+    ])
+    def test_i_can_parse_explain_without_filter(self, text, digest, command, directives):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, text)
+        parser_result = res.body
+        explanation_node = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert parser_result.parser.name == "parsers.Explain"
+        assert parser_result.source == text
+
+        assert explanation_node.digest == digest
+        assert explanation_node.command == command
+        assert explanation_node.expr == UnionNode([FilterNode(TrueNode(), directives)])
+
+    def test_i_can_parse_using_filter(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "explain -f a=b"
+        res = parser.parse(context, text)
+        parser_result = res.body
+        explanation_node = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert parser_result.parser.name == "parsers.Explain"
+        assert parser_result.source == text
+
+        assert explanation_node.expr == UnionNode([
+            FilterNode(TrueNode()),
+            FilterNode(PropertyContainsNode("a", "b"))])
+
+    @pytest.mark.parametrize("text, expected", [
+        ("-f a==b", PropertyEqualsNode("a", "b")),
+        ("--filter a==b", PropertyEqualsNode("a", "b")),
+        ("-f a==b and c=d", AndNode(PropertyEqualsNode("a", "b"), PropertyContainsNode("c", "d"))),
+        ("-f a==b or c=d", OrNode(PropertyEqualsNode("a", "b"), PropertyContainsNode("c", "d"))),
+        ("-f a==b or c==d and e==f", OrNode(
+            PropertyEqualsNode("a", "b"),
+            AndNode(PropertyEqualsNode("c", "d"), PropertyEqualsNode("e", "f")))),
+        ("-f a==b and c==d or e==f", OrNode(
+            AndNode(PropertyEqualsNode("a", "b"), PropertyEqualsNode("c", "d")),
+            PropertyEqualsNode("e", "f"))),
+        ("-f (a==b or c==d) and e==f", AndNode(
+            OrNode(PropertyEqualsNode("a", "b"), PropertyEqualsNode("c", "d")),
+            PropertyEqualsNode("e", "f"))),
+    ])
+    def test_i_can_parse_filter_expressions(self, text, expected):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, "explain " + text)
+        parser_result = res.body
+        explanation_node = res.body.body
+        expr_node = explanation_node.expr.filters[-1].expr
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(explanation_node, ExplanationNode)
+
+        assert expr_node == expected
+
+    @pytest.mark.parametrize("text, expected", [
+        ("-r 2", [
+            FilterNode(TrueNode(), [RecurseDefNode(2)])
+        ]),
+        ("--format_l 'abc'", [
+            FilterNode(TrueNode(), [FormatLNode('abc')])
+        ]),
+        ("--format_d 'abc'", [
+            FilterNode(TrueNode(), [FormatDNode({"abc": "{abc}"})])
+        ]),
+        ("--format_d a,b,c", [
+            FilterNode(TrueNode(), [FormatDNode({"a": "{a}", "b": "{b}", "c": "{c}"})])
+        ]),
+        ("--format_d a , b , c", [
+            FilterNode(TrueNode(), [FormatDNode({"a": "{a}", "b": "{b}", "c": "{c}"})])
+        ]),
+        ("-r 2 --format_l 'abc'", [
+            FilterNode(TrueNode(), [RecurseDefNode(2), FormatLNode('abc')])
+        ]),
+        ("--format_d a, b -r 2", [
+            FilterNode(TrueNode(), [FormatDNode({"a": "{a}", "b": "{b}"}), RecurseDefNode(2)])
+        ]),
+        ("-f a==b -r 3", [
+            FilterNode(TrueNode()),
+            FilterNode(PropertyEqualsNode("a", "b"), [RecurseDefNode(3)]),
+        ]),
+        ("-f a==b --format_l 'abc'", [
+            FilterNode(TrueNode()),
+            FilterNode(PropertyEqualsNode("a", "b"), [FormatLNode("abc")]),
+        ]),
+        ("-r 3 -f a==b", [
+            FilterNode(TrueNode(), [RecurseDefNode(3)]),
+            FilterNode(PropertyEqualsNode("a", "b"), []),
+        ]),
+        ("--format_l 'abc' -f a==b", [
+            FilterNode(TrueNode(), [FormatLNode("abc")]),
+            FilterNode(PropertyEqualsNode("a", "b"), []),
+        ]),
+        ("-f a==b -f c==d", [
+            FilterNode(TrueNode()),
+            FilterNode(PropertyEqualsNode("a", "b")),
+            FilterNode(PropertyEqualsNode("c", "d"))
+        ]),
+        ("-r 1 -f a==b -r 2 -f c==d -r 3", [
+            FilterNode(TrueNode(), [RecurseDefNode(1)]),
+            FilterNode(PropertyEqualsNode("a", "b"), [RecurseDefNode(2)]),
+            FilterNode(PropertyEqualsNode("c", "d"), [RecurseDefNode(3)])
+        ]),
+    ])
+    def test_i_can_parse_other_directives(self, text, expected):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, "explain " + text)
+        parser_result = res.body
+        explanation_node = res.body.body
+        expr_node = explanation_node.expr
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(explanation_node, ExplanationNode)
+
+        assert expr_node.filters == expected
+
+    @pytest.mark.parametrize("text, expected", [
+        ("explain -d digest", "digest"),
+        ("explain -d", ""),
+        ("explain -d -f a=b", "")
+    ])
+    def test_i_can_parse_record_digest(self, text, expected):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, text)
+        explanation_node = res.body.body
+
+        assert explanation_node.digest == expected
+        assert explanation_node.record_digest
+
+    @pytest.mark.parametrize("text, expected_error_type", [
+        ("explain digest1 digest2", MultipleDigestError),
+        ("explain -r", UnexpectedEof),
+        ("explain -r foo", ValueErrorNode),
+        ("explain -r 1.2", ValueErrorNode),
+        ("explain -f -r 1.2", UnexpectedTokenErrorNode),
+        ("explain -f", UnexpectedEof),
+        ("explain --format_d", UnexpectedEof),
+        ("explain --format_l", UnexpectedEof),
+        ("explain --format_l -r foo", UnexpectedTokenErrorNode),
+        ("explain --format_d -r foo", UnexpectedTokenErrorNode),
+    ])
+    def test_i_cannot_parse(self, text, expected_error_type):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, text)
+        error = res.body
+        errors = res.body.body
+
+        assert not res.status
+        assert sheerka.isinstance(error, BuiltinConcepts.ERROR)
+        assert len(errors) == 1
+        assert isinstance(errors[0], expected_error_type)

tests/parsers/test_ExpressionParser.py

@@ -0,0 +1,103 @@
+from dataclasses import dataclass
+
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
+from core.concept import Concept
+from parsers.ExpressionParser import PropertyEqualsNode, PropertyEqualsSequenceNode, PropertyContainsNode, AndNode, \
+    OrNode, NotNode, LambdaNode, IsaNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+@dataclass
+class Obj:
+    prop_a: object
+    prop_b: object = None
+    prop_c: object = None
+    parent: object = None
+
+
+class TestExpressionParser(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_test_property_equals(self):
+        node = PropertyEqualsNode("prop_a", "good value")
+
+        assert node.eval(Obj(prop_a="good value"))
+        assert not node.eval(Obj(prop_a="other value"))
+
+    def test_i_can_test_property_equals_for_int(self):
+        node = PropertyEqualsNode("prop_a", "1")
+
+        assert node.eval(Obj(prop_a=1))
+        assert node.eval(Obj(prop_a="1"))
+
+    def test_i_can_test_property_equals_sequence(self):
+        node = PropertyEqualsSequenceNode(["prop_b", "prop_a"], ["good parent", "good child"])
+
+        assert node.eval(Obj(prop_a="good child", parent=Obj(prop_a="Don't care", prop_b="good parent")))
+        assert not node.eval(Obj(prop_a="good child", parent=Obj(prop_a="Don't care", prop_b="wrong parent")))
+        assert not node.eval(Obj(prop_a="good child"))
+        assert not node.eval(Obj(prop_a="wrong child", parent=Obj(prop_a="Don't care", prop_b="good parent")))
+
+    def test_i_can_test_property_contains(self):
+        node = PropertyContainsNode("prop_a", "substring")
+
+        assert node.eval(Obj(prop_a="it contains substring in it"))
+        assert not node.eval(Obj(prop_a="it does not"))
+
+    def test_i_can_test_property_contains_for_int(self):
+        node = PropertyContainsNode("prop_a", "44")
+
+        assert node.eval(Obj(prop_a=123445))
+        assert not node.eval(Obj(prop_a=12435))
+
+    def test_i_can_test_and(self):
+        left = PropertyEqualsNode("prop_a", "good a")
+        right = PropertyEqualsNode("prop_b", "good b")
+        other = PropertyEqualsNode("prop_c", "good c")
+        and_node = AndNode(left, right, other)
+
+        assert and_node.eval(Obj("good a", "good b", "good c"))
+        assert not and_node.eval(Obj("wrong a", "good b", "good c"))
+        assert not and_node.eval(Obj("good a", "wrong b", "good c"))
+        assert not and_node.eval(Obj("good a", "good b", "wrong c"))
+
+    def test_i_can_test_or(self):
+        left = PropertyEqualsNode("prop_a", "good a")
+        right = PropertyEqualsNode("prop_b", "good b")
+        other = PropertyEqualsNode("prop_c", "good c")
+        or_node = OrNode(left, right, other)
+
+        assert or_node.eval(Obj("wrong a", "good b", "good c"))
+        assert or_node.eval(Obj("good a", "wrong b", "good c"))
+        assert or_node.eval(Obj("good a", "good b", "wrong c"))
+        assert not or_node.eval(Obj("wrong a", "wrong b", "wrong c"))
+
+    def test_i_can_test_not(self):
+        node = PropertyEqualsNode("prop_a", "good value")
+        not_node = NotNode(node)
+
+        assert not not_node.eval(Obj(prop_a="good value"))
+        assert not_node.eval(Obj(prop_a="wrong value"))
+
+    def test_i_can_test_lambda_node(self):
+        node = LambdaNode(lambda o: o.prop_a + o.prop_b == "ab")
+
+        assert node.eval(Obj(prop_a="a", prop_b="b"))
+        assert not node.eval(Obj(prop_a="wrong value", prop_b="wrong value"))
+        assert not node.eval(Obj(prop_a="wrong value"))  # exception is caught
+
+    def test_i_can_test_isa_node(self):
+        class_node = IsaNode(Obj)
+        assert class_node.eval(Obj(prop_a="value"))
+        assert not class_node.eval(TestExpressionParser())
+
+        concept_node = IsaNode(BuiltinConcepts.RETURN_VALUE)
+        assert concept_node.eval(ReturnValueConcept())
+        assert concept_node.eval(Concept(name="foo", key=BuiltinConcepts.RETURN_VALUE))
+        assert not concept_node.eval(Obj)
+        assert not concept_node.eval(Concept())
+
+        concept_node2 = IsaNode("foo")
+        assert concept_node2.eval(Concept("foo").init_key())
+        assert not concept_node2.eval(Obj)
+        assert not concept_node2.eval(Concept())

tests/parsers/test_MultipleConceptsParser.py

@@ -191,7 +191,7 @@ class TestMultipleConceptsParser(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("text, expected_source, expected_end", [
         ("True", "True", 0),
-        ("1 == 1", "1 == 1", 5),
+        ("1 == 1", "1 == 1", 4),
         ("1!xdf", "1", 0),
         ("1", "1", 0),
     ])

tests/sdp/test_sheerkaDataProvider.py

@@ -1,16 +1,15 @@
 import hashlib
-
-import pytest
+import json
 import os
+import shutil
+from datetime import date, datetime
 from os import path
+
+import core.utils
+import pytest
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderError, \
     SheerkaDataProviderDuplicateKeyError, SheerkaDataProviderResult, SheerkaDataProviderRef
-from datetime import date, datetime
-import shutil
-import json
-
 from sdp.sheerkaSerializer import JsonSerializer, Serializer, PickleSerializer
-import core.utils
 
 tests_root = path.abspath("../../build/tests")
 evt_digest = "3a571cb6034ef6fc8d7fe91948d0d29728eed74de02bac7968b0e9facca2c2d7"
@@ -1021,14 +1020,14 @@ def test_i_can_set_a_reference(root):
 def test_i_cannot_set_using_use_ref_and_is_ref():
     sdp = SheerkaDataProvider("mem://")
 
-    with pytest.raises(SheerkaDataProviderError) as error:
+    with pytest.raises(SheerkaDataProviderError):
         sdp.set(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), use_ref=True, is_ref=True)
 
 
 def test_i_cannot_set_using_is_ref_if_obj_is_not_a_dictionary():
     sdp = SheerkaDataProvider("mem://")
 
-    with pytest.raises(SheerkaDataProviderError) as error:
+    with pytest.raises(SheerkaDataProviderError):
         sdp.set(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), is_ref=True)
 
 
@@ -1435,8 +1434,17 @@ def test_i_can_remove_when_only_one_element(root):
 def test_i_cannot_remove_if_entry_does_not_exist(root):
     sdp = SheerkaDataProvider(root)
     with pytest.raises(IndexError) as e:
-        sdp.remove(evt_digest, "entry")
-        assert str(e) == "entry"
+        sdp.remove(evt_digest, "entry", silent_remove=False)
+    assert str(e.value) == "entry"
+
+
+@pytest.mark.parametrize("root", [
+    ".sheerka",
+    "mem://"
+])
+def test_no_exception_is_raise_when_remove_in_silent_mode(root):
+    sdp = SheerkaDataProvider(root)
+    sdp.remove(evt_digest, "entry", silent_remove=True)  # default
 
 
 @pytest.mark.parametrize("root", [
@@ -1773,7 +1781,7 @@ def test_i_cannot_modify_a_key_that_does_not_exist(root):
 
     with pytest.raises(IndexError) as e:
         sdp.modify(evt_digest, "entry1", "2", "bar")
-        assert str(e) == "entry1.2"
+    assert str(e.value) == "entry1.2"
 
 
 @pytest.mark.parametrize("root", [
@@ -1938,7 +1946,7 @@ def test_i_cannot_get_an_entry_that_does_not_exist(root):
     assert sdp.get_safe("entry") is None
     with pytest.raises(IndexError) as e:
         sdp.get("entry")
-        assert str(e) == "entry"
+    assert str(e.value) == "entry"
 
 
 @pytest.mark.parametrize("root", [
@@ -1952,7 +1960,7 @@ def test_i_cannot_get_a_key_that_does_not_exist(root):
     assert sdp.get_safe("entry1", "2") is None
     with pytest.raises(IndexError) as e:
         sdp.get("entry1", "2")
-        assert str(e) == "entry.1"
+    assert str(e.value) == "entry1.2"
 
 
 @pytest.mark.parametrize("root", [
@@ -2257,3 +2265,50 @@ def test_i_get_safe_object_without_origin(root):
     from_db_no_origin = sdp.get_safe(result.entry, result.key, load_origin=False)
     assert from_db_no_origin == obj
     assert not hasattr(from_db_no_origin, Serializer.ORIGIN)
+
+
+def test_i_can_get_ref():
+    sdp = SheerkaDataProvider("mem://")
+    obj = ObjDumpJson("my_key", "value1")
+
+    obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
+    sdp.serializer.register(obj_serializer)
+
+    result = sdp.add(evt_digest, "entry", obj, use_ref=True)
+
+    ref = sdp.get_ref(result.entry, result.key)
+    assert ref == "076f0df0f110c304982242a88088efacce71f361e49f065db75919a7f72c2821"
+
+
+def test_i_can_get_ref_when_list():
+    sdp = SheerkaDataProvider("mem://")
+
+    obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(ObjDumpJson))
+    sdp.serializer.register(obj_serializer)
+
+    sdp.add(evt_digest, "entry", ObjDumpJson("my_key", "value1"), use_ref=True)
+    result = sdp.add(evt_digest, "entry", ObjDumpJson("my_key", "value2"), use_ref=True)
+
+    ref = sdp.get_ref(result.entry, result.key)
+    assert ref == [
+        "076f0df0f110c304982242a88088efacce71f361e49f065db75919a7f72c2821",
+        "e6bf5b56428cfce0f08c94f2c3625dc3b3a8180d7229eaa9f8aa967fb16e5256"
+    ]
+
+
+def test_i_cannot_get_ref_if_the_saved_item_is_not_a_ref():
+    sdp = SheerkaDataProvider("mem://")
+    obj = ObjDumpJson("my_key", "value1")
+    result = sdp.add(evt_digest, "entry", obj, use_ref=False)
+
+    with pytest.raises(SheerkaDataProviderError) as e:
+        sdp.get_ref(result.entry, result.key)
+
+    assert e.value.args[0] == "Not a reference"
+    assert e.value.obj == f"{result.entry}.{result.key}"
+
+
+def test_i_cannot_get_ref_if_the_item_does_not_exist():
+    sdp = SheerkaDataProvider("mem://")
+    with pytest.raises(IndexError):
+        sdp.get_ref("fake", "fake")