Refactored ExecutionContext serialization (added sheerkapickle) and added History management

tests/sheerkapickle/test_SheerkaPickler.py

@@ -0,0 +1,172 @@
+import logging
+
+import pytest
+from core.concept import Concept, ConceptParts
+from sheerkapickle import tags
+from sheerkapickle.SheerkaPickler import SheerkaPickler
+from sheerkapickle.SheerkaUnpickler import SheerkaUnpickler
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class Obj:
+    def __init__(self, a, b, c):
+        self.a = a
+        self.b = b
+        self.c = c
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, Obj):
+            return False
+
+        return self.a == other.a and self.b == other.b and self.c == other.c
+
+    def __hash__(self):
+        return hash((self.a, self.b, self.c))
+
+
+class TestSheerkaPickler(TestUsingFileBasedSheerka):
+
+    @pytest.mark.parametrize("obj, expected", [
+        (1, 1),
+        (3.14, 3.14),
+        ("a string", "a string"),
+        (True, True),
+        (None, None),
+        ([1, 3.14, "a string"], [1, 3.14, "a string"]),
+        ((1, 3.14, "a string"), {tags.TUPLE: [1, 3.14, "a string"]}),
+        ({1}, {tags.SET: [1]}),
+        ({"a": "a", "b": 3.14, "c": True}, {"a": "a", "b": 3.14, "c": True}),
+        ({1: "a", 2: 3.14, 3: True}, {1: "a", 2: 3.14, 3: True}),
+        ([1, [3.14, "a string"]], [1, [3.14, "a string"]]),
+        ([1, (3.14, "a string")], [1, {tags.TUPLE: [3.14, "a string"]}]),
+        ([], []),
+        (ConceptParts.BODY, {tags.ENUM: "core.concept.ConceptParts.BODY"}),
+    ])
+    def test_i_can_flatten_and_restore_primitives(self, obj, expected):
+        sheerka = self.get_sheerka()
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == expected
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_can_flatten_and_restore_instances(self):
+        sheerka = self.get_sheerka()
+
+        obj1 = Obj(1, "b", True)
+        obj2 = Obj(3.14, ("a", "b"), obj1)
+
+        flatten = SheerkaPickler(sheerka).flatten(obj2)
+        assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                           'a': 3.14,
+                           'b': {'_sheerka/tuple': ['a', 'b']},
+                           'c': {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                                 'a': 1,
+                                 'b': 'b',
+                                 'c': True}}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj2
+
+    def test_i_can_manage_circular_reference(self):
+        sheerka = self.get_sheerka()
+
+        obj1 = Obj(1, "b", True)
+        obj1.c = obj1
+
+        flatten = SheerkaPickler(sheerka).flatten(obj1)
+        assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                           'a': 1,
+                           'b': 'b',
+                           'c': {'_sheerka/id': 0}}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded.a == obj1.a
+        assert decoded.b == obj1.b
+        assert decoded.c == decoded
+
+    def test_i_can_flatten_obj_with_new_props(self):
+        sheerka = self.get_sheerka()
+
+        obj = Obj(1, "b", True)
+        obj.z = "new prop"
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                           'a': 1,
+                           'b': 'b',
+                           'c': True,
+                           'z': "new prop"}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    @pytest.mark.parametrize("obj, expected", [
+        ({None: "a"}, {'null': "a"}),
+        ({ConceptParts.BODY: "a"}, {'core.concept.ConceptParts.BODY': 'a'}),
+        ({(1, 2): "a"}, {(1, 2): "a"}),
+    ])
+    def test_i_can_manage_specific_keys_in_dictionaries(self, obj, expected):
+        sheerka = self.get_sheerka()
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == expected
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_can_use_concept_as_dictionary_key(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo").init_key()
+        sheerka.set_id_if_needed(concept, False)
+        sheerka.add_in_cache(concept)
+        obj = {concept: "a"}
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == {':c:foo:1001:': 'a'}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_can_manage_references(self):
+        sheerka = self.get_sheerka()
+
+        foo = Obj("foo", "bar", "baz")
+        obj = [ConceptParts.BODY, foo, ConceptParts.WHERE, ConceptParts.BODY, foo]
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+
+        assert flatten == [{'_sheerka/enum': 'core.concept.ConceptParts.BODY'},
+                           {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                            'a': 'foo',
+                            'b': 'bar',
+                            'c': 'baz'},
+                           {'_sheerka/enum': 'core.concept.ConceptParts.WHERE'},
+                           {'_sheerka/id': 0},
+                           {'_sheerka/id': 1}]
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_serialize_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("doo")
+        flatten = SheerkaPickler(sheerka).flatten(foo)
+        restored = SheerkaUnpickler(sheerka).restore(flatten)
+
+        assert restored == foo
+
+    def test_i_do_not_encode_logger(self):
+        sheerka = self.get_sheerka()
+
+        logger = logging.getLogger("log_name")
+        logger2 = logging.getLogger("log_name2")
+        obj = Obj("foo", logger, {"a": logger, "b": logger2})
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == Obj("foo", None, {"a": None, "b": None})

tests/sheerkapickle/test_sheerka_handlers.py

@@ -0,0 +1,285 @@
+import sheerkapickle
+from core.builtin_concepts import BuiltinConcepts, UserInputConcept, ReturnValueConcept
+from core.concept import Concept, ConceptParts
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.tokenizer import Tokenizer
+from evaluators.ConceptEvaluator import ConceptEvaluator
+from parsers.DefaultParser import DefaultParser
+from sdp.sheerkaDataProvider import Event
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_encode_decode_unknown_concept_metadata(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept(name="foo", key="my_key")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.key": "my_key"}'
+        assert decoded == concept
+
+        concept = Concept("foo", is_builtin=True, is_unique=True)
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.is_builtin": true, "meta.is_unique": true}'
+
+        concept = Concept("foo", body="my_body")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "my_body"}'
+
+        concept = Concept("foo", pre="my_pre")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.pre": "my_pre"}'
+
+        concept = Concept("foo", post="my_post")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.post": "my_post"}'
+
+        concept = Concept("foo", where="my_where")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.where": "my_where"}'
+
+        concept = Concept("foo").def_prop("a", "value_a").def_prop("b", "value_b")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+
+    def test_i_can_encode_decode_unknown_concept_values(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.PRE] = 10  # an int
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "pre": 10}'
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.POST] = 'a string'  # an int
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "post": "a string"}'
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.WHERE] = ['a string', 3.14]  # a list
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": ["a string", 3.14]}'
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.WHERE] = ('a string', 3.14)  # a tuple
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": {"_sheerka/tuple": ["a string", 3.14]}}'
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.BODY] = Concept("foo", body="foo_body")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "foo_body"}}'
+
+    def test_i_can_encode_decode_unknown_concept_properties(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo")
+        concept.set_prop("a", "value_a")  # string
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "value_a"]]}'
+
+        concept = Concept("foo")
+        concept.set_prop("a", 10)  # int
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", 10]]}'
+
+        concept = Concept("foo")
+        concept.set_prop("a", Concept("bar"))  # another concept
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}'
+
+        concept = Concept("foo")
+        concept.set_prop("a", "a").set_prop("b", "b")  # at least two props
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "a"], ["b", "b"]]}'
+
+    def test_i_can_encode_decode_known_concepts(self):
+        sheerka = self.get_sheerka()
+
+        ref_concept = Concept("my_name", True, True, "my_key", "my_body", "my_where", "my_pre", "my_post", "my_def")
+        ref_concept.def_prop("a", "value_a").def_prop("b", "value_b")
+
+        sheerka.create_new_concept(self.get_context(sheerka), ref_concept)
+
+        to_string = sheerkapickle.encode(sheerka, ref_concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ref_concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"]}'
+
+        concept = Concept().update_from(sheerka.get_by_id(ref_concept.id))
+        concept.set_metadata_value(ConceptParts.BODY, Concept("bar"))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"], "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}}'
+
+    def test_i_can_manage_reference_of_the_same_object(self):
+        sheerka = self.get_sheerka()
+
+        concept_ref = Concept("foo")
+
+        concept = Concept("bar")
+        concept.set_metadata_value(ConceptParts.PRE, concept_ref)
+        concept.set_metadata_value(ConceptParts.BODY, concept_ref)
+
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "bar", "pre": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo"}, "body": {"_sheerka/id": 1}}'
+
+    def test_i_can_encode_decode_user_input(self):
+        sheerka = self.get_sheerka()
+
+        user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body="my_text", user_name="my_user_name")
+
+        to_string = sheerkapickle.encode(sheerka, user_input)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == user_input
+        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "my_text"}'
+
+    def test_i_can_encode_decode_user_input_when_tokens(self):
+        sheerka = self.get_sheerka()
+
+        text = "I have 'a complicated' 10 text"
+        tokens = list(Tokenizer(text))
+        user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens, user_name="my_user_name")
+
+        to_string = sheerkapickle.encode(sheerka, user_input)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == UserInputConcept(text, "my_user_name")
+        assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "{text}"' + '}'
+
+    def test_i_can_encode_decode_return_value(self):
+        sheerka = self.get_sheerka()
+
+        ret_val = sheerka.ret("who", True, 10)
+
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ret_val
+        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10}'
+
+    def test_i_can_encode_decode_return_value_with_parent(self):
+        sheerka = self.get_sheerka()
+
+        ret_val = sheerka.ret("who", True, 10)
+        ret_val_parent = sheerka.ret("parent_who", True, "10")
+        ret_val.parents = [ret_val_parent, ret_val_parent]
+
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ret_val
+        assert decoded.parents == ret_val.parents
+        parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]'
+        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}'
+
+    def test_i_can_encode_decode_return_values_with_complex_body(self):
+        sheerka = self.get_sheerka()
+
+        ret_val = sheerka.ret("who", True, Concept("foo", body="bar"))
+
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ret_val
+
+    def test_i_can_encode_decode_return_values_from_concepts_parsers_or_evaluators(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+        ret_val = sheerka.ret(foo, True, 10)
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.ret("c:1001:", True, 10)
+
+        ret_val = sheerka.ret(DefaultParser(), True, 10)
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.ret("parsers.Default", True, 10)
+
+        ret_val = sheerka.ret(ConceptEvaluator(), True, 10)
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.ret("evaluators.Concept", True, 10)
+
+    def test_i_can_encode_decode_execution_context(self):
+        sheerka = self.get_sheerka()
+
+        context = ExecutionContext("who", Event("xxx"), sheerka, "my desc")
+        input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)]
+        context.inputs = {"a": input_list, "b": Concept("foo")}
+        context.values = {"c": input_list, "d": Concept("bar")}
+        context.obj = Concept("baz")
+        context.push("who3", "sub_child1")
+        context.push("who4", "sub_child2")
+
+        to_string = sheerkapickle.encode(sheerka, context)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == context
+
+    def test_complicated_execution_context(self):
+        sheerka = self.get_sheerka(skip_builtins_in_db=False)
+
+        text = "def concept one as 1"
+        execution_context = ExecutionContext("s", Event(), sheerka, f"Evaluating '{text}'")
+        user_input = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="n"))
+        reduce_requested = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+
+        steps = [
+            BuiltinConcepts.BEFORE_PARSING,
+            BuiltinConcepts.PARSING,
+            BuiltinConcepts.AFTER_PARSING,
+            BuiltinConcepts.BEFORE_EVALUATION,
+            BuiltinConcepts.EVALUATION,
+            BuiltinConcepts.AFTER_EVALUATION
+        ]
+
+        ret = sheerka.execute(execution_context, [user_input, reduce_requested], steps)
+        execution_context.add_values(return_values=ret)
+
+        to_string = sheerkapickle.encode(sheerka, execution_context)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+
+        return_value = decoded.values["return_values"][0].value
+        assert sheerka.isinstance(return_value, BuiltinConcepts.NEW_CONCEPT)
+
+    def test_encode_simple_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo")
+        to_string = sheerkapickle.encode(sheerka, foo)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+
+        assert decoded == foo