import sheerkapickle from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept from core.concept import Concept, ConceptParts from core.sheerka.ExecutionContext import ExecutionContext from core.tokenizer import Tokenizer from evaluators.ConceptEvaluator import ConceptEvaluator from parsers.DefaultParser import DefaultParser from sdp.sheerkaDataProvider import Event from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka def set_full_serialization(concept): concept.metadata.full_serialization = True return concept class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): def test_i_can_encode_decode_unknown_concept_metadata(self): sheerka = self.get_sheerka() concept = set_full_serialization(Concept(name="foo", key="my_key")) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.key": "my_key"}' assert decoded == concept concept = set_full_serialization(Concept("foo", is_builtin=True, is_unique=True)) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.is_builtin": true, "meta.is_unique": true}' concept = set_full_serialization(Concept("foo", body="my_body")) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "my_body"}' concept = set_full_serialization(Concept("foo", pre="my_pre")) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.pre": "my_pre"}' concept = set_full_serialization(Concept("foo", post="my_post")) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.post": "my_post"}' concept = set_full_serialization(Concept("foo", where="my_where")) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.where": "my_where"}' concept = set_full_serialization(Concept("foo").def_var("a", "value_a").def_var("b", "value_b")) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.variables": [["a", "value_a"], ["b", "value_b"]], "values": [["a", null], ["b", null]]}' concept = Concept("foo").init_key() sheerka.create_new_concept(self.get_context(sheerka), concept) concept.metadata.full_serialization = True to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.key": "foo", "meta.id": "1001"}' def test_i_can_encode_decode_unknown_concept_values(self): sheerka = self.get_sheerka() concept = set_full_serialization(Concept("foo")) concept.set_value(ConceptParts.PRE, 10) # an int to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.pre", 10]]}' concept = set_full_serialization(Concept("foo")) concept.set_value(ConceptParts.POST, 'a string') # an string to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.post", "a string"]]}' concept = set_full_serialization(Concept("foo")) concept.set_value(ConceptParts.WHERE, ['a string', 3.14]) # a list to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.where", ["a string", 3.14]]]}' concept = set_full_serialization(Concept("foo")) concept.set_value(ConceptParts.WHERE, ('a string', 3.14)) # a tuple to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.where", {"_sheerka/tuple": ["a string", 3.14]}]]}' concept = set_full_serialization(Concept("foo")) concept.set_value(ConceptParts.BODY, set_full_serialization(Concept("foo", body="foo_body"))) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["cParts.body", {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "foo_body"}]]}' def test_i_can_encode_decode_unknown_concept_variables(self): sheerka = self.get_sheerka() concept = set_full_serialization(Concept("foo")) concept.set_value("a", "value_a") # string to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["a", "value_a"]]}' concept = set_full_serialization(Concept("foo")) concept.set_value("a", 10) # int to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["a", 10]]}' concept = set_full_serialization(Concept("foo")) concept.set_value("a", set_full_serialization(Concept("bar"))) # another concept to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["a", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}' concept = set_full_serialization(Concept("foo")) concept.set_value("a", "a").set_value("b", "b") # at least two variables to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["a", "a"], ["b", "b"]]}' def test_i_can_encode_decode_known_concepts(self): sheerka = self.get_sheerka() ref_concept = Concept("my_name", True, True, "my_key", "my_body", "my_where", "my_pre", "my_post", "my_def") ref_concept.def_var("a", "value_a").def_var("b", "value_b") sheerka.create_new_concept(self.get_context(sheerka), ref_concept) to_string = sheerkapickle.encode(sheerka, ref_concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ref_concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"]}' # same test, modify a value and check if this modification is correctly saved concept = Concept().update_from(sheerka.get_by_id(ref_concept.id)) concept.set_value(ConceptParts.BODY, set_full_serialization(Concept("bar"))) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"], "values": [["cParts.body", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}' def test_i_can_encode_decode_when_variable_is_a_concept(self): sheerka = self.get_sheerka() foo = Concept("foo") sheerka.create_new_concept(self.get_context(sheerka), foo) concept = Concept("my_name") sheerka.create_new_concept(self.get_context(sheerka), concept) concept.def_var(foo, "a value") concept.set_value(foo, "another value") concept.metadata.full_serialization = True to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "my_name", "meta.key": "my_name", ' + \ '"meta.variables": [[{"_sheerka/obj": "core.concept.Concept", "concept/id": ["foo", "1001"]}, "a value"]], ' + \ '"meta.id": "1002", "values": [[{"_sheerka/id": 1}, "another value"]]}' def test_i_can_manage_reference_of_the_same_object(self): sheerka = self.get_sheerka() concept_ref = set_full_serialization(Concept("foo")) concept = set_full_serialization(Concept("bar")) concept.set_value(ConceptParts.PRE, concept_ref) concept.set_value(ConceptParts.BODY, concept_ref) to_string = sheerkapickle.encode(sheerka, concept) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == concept assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "bar", "values": [["cParts.pre", {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo"}], ["cParts.body", {"_sheerka/id": 1}]]}' def test_i_can_encode_decode_user_input(self): sheerka = self.get_sheerka() user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body="my_text", user_name="my_user_name") to_string = sheerkapickle.encode(sheerka, user_input) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == user_input assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "11"], "user_name": "my_user_name", "text": "my_text"}' def test_i_can_encode_decode_user_input_when_tokens(self): sheerka = self.get_sheerka() text = "I have 'a complicated' 10 text" tokens = list(Tokenizer(text)) user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens, user_name="my_user_name") to_string = sheerkapickle.encode(sheerka, user_input) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="my_user_name") assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "11"], "user_name": "my_user_name", "text": "{text}"' + '}' def test_i_can_encode_decode_return_value(self): sheerka = self.get_sheerka() ret_val = sheerka.ret("who", True, 10) to_string = sheerkapickle.encode(sheerka, ret_val) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ret_val assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "16"], "who": "who", "status": true, "value": 10}' def test_i_can_encode_decode_return_value_with_parent(self): sheerka = self.get_sheerka() ret_val = sheerka.ret("who", True, 10) ret_val_parent = sheerka.ret("parent_who", True, "10") ret_val.parents = [ret_val_parent, ret_val_parent] to_string = sheerkapickle.encode(sheerka, ret_val) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ret_val assert decoded.parents == ret_val.parents id_str = ', "concept/id": ["__RETURN_VALUE", "16"]' parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]' assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}' def test_i_can_encode_decode_return_values_with_complex_body(self): sheerka = self.get_sheerka() ret_val = sheerka.ret("who", True, set_full_serialization(Concept("foo", body="bar"))) to_string = sheerkapickle.encode(sheerka, ret_val) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ret_val def test_i_can_encode_decode_return_values_from_concepts_parsers_or_evaluators(self): sheerka = self.get_sheerka() foo = Concept("foo") sheerka.set_id_if_needed(foo, False) ret_val = sheerka.ret(foo, True, 10) to_string = sheerkapickle.encode(sheerka, ret_val) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == sheerka.ret("c:1001:", True, 10) ret_val = sheerka.ret(DefaultParser(), True, 10) to_string = sheerkapickle.encode(sheerka, ret_val) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == sheerka.ret("parsers.Default", True, 10) ret_val = sheerka.ret(ConceptEvaluator(), True, 10) to_string = sheerkapickle.encode(sheerka, ret_val) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == sheerka.ret("evaluators.Concept", True, 10) def test_i_can_encode_decode_execution_context(self): sheerka = self.get_sheerka() context = ExecutionContext("who", Event("xxx"), sheerka, "my desc") input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)] context.inputs = {"a": input_list, "b": set_full_serialization(Concept("foo"))} context.values = {"c": input_list, "d": set_full_serialization(Concept("bar"))} context.obj = set_full_serialization(Concept("baz")) context.push("who3", "sub_child1") context.push("who4", "sub_child2") to_string = sheerkapickle.encode(sheerka, context) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == context def test_complicated_execution_context(self): sheerka = self.get_sheerka(skip_builtins_in_db=False) text = "def concept one as 1" execution_context = ExecutionContext("s", Event(), sheerka, f"Evaluating '{text}'") user_input = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="n")) reduce_requested = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) steps = [ BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING, BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION ] ret = sheerka.execute(execution_context, [user_input, reduce_requested], steps) execution_context.add_values(return_values=ret) to_string = sheerkapickle.encode(sheerka, execution_context) decoded = sheerkapickle.decode(sheerka, to_string) return_value = decoded.values["return_values"][0].value assert sheerka.isinstance(return_value, BuiltinConcepts.NEW_CONCEPT) def test_encode_simple_concept(self): sheerka = self.get_sheerka() foo = set_full_serialization(Concept("foo")) to_string = sheerkapickle.encode(sheerka, foo) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == foo