Refactored ExecutionContext serialization (added sheerkapickle) and added History management
This commit is contained in:
@@ -2,7 +2,7 @@ import pytest
|
||||
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from core.sheerka.Sheerka import ExecutionContext
|
||||
from core.sheerka.ExecutionContext import ExecutionContext
|
||||
from sdp.sheerkaDataProvider import Event
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
from core.sheerka.Services.SheerkaHistoryManager import hist
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
|
||||
class TestSheerkaHistoryManager(TestUsingMemoryBasedSheerka):
|
||||
def test_i_can_retrieve_history(self):
|
||||
sheerka = self.get_sheerka(skip_builtins_in_db=False)
|
||||
|
||||
sheerka.evaluate_user_input("def concept one as 1")
|
||||
sheerka.evaluate_user_input("one")
|
||||
sheerka.evaluate_user_input("xxx")
|
||||
sheerka.evaluate_user_input("def concept two as 2")
|
||||
sheerka.evaluate_user_input("two")
|
||||
sheerka.evaluate_user_input("def concept three as 3")
|
||||
sheerka.evaluate_user_input("three")
|
||||
sheerka.evaluate_user_input("def concept four as 4")
|
||||
sheerka.evaluate_user_input("four")
|
||||
sheerka.evaluate_user_input("def concept five as 5")
|
||||
sheerka.evaluate_user_input("five")
|
||||
|
||||
h = list(sheerka.history(-1)) # all
|
||||
assert h == [
|
||||
hist("five", True),
|
||||
hist("def concept five as 5", True),
|
||||
hist("four", True),
|
||||
hist("def concept four as 4", True),
|
||||
hist("three", True),
|
||||
hist("def concept three as 3", True),
|
||||
hist("two", True),
|
||||
hist("def concept two as 2", True),
|
||||
hist("xxx", False),
|
||||
hist("one", True),
|
||||
hist("def concept one as 1", True),
|
||||
hist("Initializing Sheerka.", True)]
|
||||
|
||||
h = list(sheerka.history(2))
|
||||
assert h == [
|
||||
hist("two", True),
|
||||
hist("def concept two as 2", True)
|
||||
]
|
||||
|
||||
h = list(sheerka.history(2, 2))
|
||||
assert h == [
|
||||
hist("xxx", False),
|
||||
hist("one", True),
|
||||
]
|
||||
|
||||
h = list(sheerka.history(-1))
|
||||
assert h == [
|
||||
hist("xxx", False),
|
||||
hist("one", True),
|
||||
]
|
||||
@@ -1,318 +0,0 @@
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.concept import Concept, ConceptParts
|
||||
from core.sheerka.ExecutionContext import ExecutionContext
|
||||
from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY
|
||||
from sdp.sheerkaDataProvider import Event
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
|
||||
class TestSheerkaTransform(TestUsingMemoryBasedSheerka):
|
||||
def test_i_can_transform_an_unknown_concept(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
foo = Concept("foo", body="body")
|
||||
concept_with_sub = Concept("concept_with_sub", body=foo)
|
||||
|
||||
concept = Concept(
|
||||
name="concept_name",
|
||||
is_builtin=True,
|
||||
is_unique=True,
|
||||
key="concept_key",
|
||||
body=concept_with_sub,
|
||||
where=[foo, 1, "1", True, 1.0],
|
||||
pre=foo,
|
||||
post=None, # will not appear
|
||||
definition="it is a definition",
|
||||
definition_type="def type",
|
||||
desc="this this the desc"
|
||||
).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub")
|
||||
|
||||
# add values and props
|
||||
concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init()
|
||||
concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0]
|
||||
concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init()
|
||||
concept.values[ConceptParts.POST] = "a value for POST"
|
||||
concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub)
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(concept)
|
||||
|
||||
assert to_dict == {
|
||||
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
|
||||
OBJ_ID_KEY: 0,
|
||||
'meta.name': 'concept_name',
|
||||
'meta.key': 'concept_key',
|
||||
'meta.is_builtin': True,
|
||||
'meta.is_unique': True,
|
||||
'meta.definition': 'it is a definition',
|
||||
'meta.definition_type': 'def type',
|
||||
'meta.desc': 'this this the desc',
|
||||
'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept,
|
||||
OBJ_ID_KEY: 1,
|
||||
'meta.body': 'body',
|
||||
'meta.name': 'foo'}, 1, '1', True, 1.0],
|
||||
'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1},
|
||||
'meta.body': {
|
||||
'__type__': SheerkaTransformType.Concept,
|
||||
'__id__': 2,
|
||||
'meta.name': 'concept_with_sub',
|
||||
'meta.body': {
|
||||
'__type__': SheerkaTransformType.Reference,
|
||||
'__id__': 1}},
|
||||
'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']],
|
||||
'pre': {'__type__': SheerkaTransformType.Concept,
|
||||
'__id__': 4,
|
||||
'meta.body': 'body',
|
||||
'meta.name': 'foo',
|
||||
'body': 'body'},
|
||||
'post': "a value for POST",
|
||||
'body': {'__type__': SheerkaTransformType.Concept,
|
||||
'__id__': 3,
|
||||
'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference},
|
||||
'meta.name': 'concept_with_sub',
|
||||
'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}},
|
||||
'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0],
|
||||
'props': [('a', 10),
|
||||
('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}),
|
||||
('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})],
|
||||
}
|
||||
|
||||
def test_i_can_transform_unknown_concept_with_almost_same_value(self):
|
||||
sheerka = self.get_sheerka()
|
||||
concept = Concept("foo")
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(concept)
|
||||
|
||||
assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'}
|
||||
|
||||
def test_i_can_transform_known_concept_when_the_values_are_the_same(self):
|
||||
"""
|
||||
Values are the same means that we are serializing a concept which has kept all its default values
|
||||
There is not diff between the concept to serialize and the one which was registered with create_new_concept()
|
||||
We serialize only the id of the concept
|
||||
:return:
|
||||
"""
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
concept = Concept(
|
||||
name="concept_name",
|
||||
is_builtin=True,
|
||||
is_unique=False,
|
||||
key="concept_key",
|
||||
body="body definition",
|
||||
where="where definition",
|
||||
pre="pre definition",
|
||||
post="post definition",
|
||||
definition="it is a definition",
|
||||
definition_type="def type",
|
||||
desc="this this the desc"
|
||||
).def_prop("a").def_prop("b")
|
||||
sheerka.create_new_concept(self.get_context(sheerka), concept)
|
||||
|
||||
new_concept = sheerka.new(concept.key)
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(new_concept)
|
||||
|
||||
assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"}
|
||||
|
||||
def test_i_can_transform_known_concept_when_the_values_are_different(self):
|
||||
"""
|
||||
Values are the different means the concept was modified.
|
||||
It's different from the one which was registered with create_new_concept()
|
||||
We serialize only the differences
|
||||
:return:
|
||||
"""
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
concept = Concept(
|
||||
name="concept_name",
|
||||
is_builtin=True,
|
||||
is_unique=False,
|
||||
key="concept_key",
|
||||
body="body definition",
|
||||
where="where definition",
|
||||
pre="pre definition",
|
||||
post="post definition",
|
||||
definition="it is a definition",
|
||||
definition_type="def type",
|
||||
desc="this this the desc"
|
||||
).def_prop("a").def_prop("b")
|
||||
sheerka.create_new_concept(self.get_context(sheerka), concept)
|
||||
|
||||
new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre")
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(new_concept)
|
||||
|
||||
assert to_dict == {
|
||||
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
|
||||
OBJ_ID_KEY: 0,
|
||||
"id": "1001",
|
||||
'pre': 'another pre',
|
||||
"body": "another",
|
||||
'props': [('a', 10)]
|
||||
}
|
||||
|
||||
def test_i_can_transform_concept_with_circular_reference(self):
|
||||
sheerka = self.get_sheerka()
|
||||
foo = Concept("foo")
|
||||
bar = Concept("bar", body=foo)
|
||||
foo.metadata.body = bar
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(foo)
|
||||
|
||||
assert to_dict == {
|
||||
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
|
||||
OBJ_ID_KEY: 0,
|
||||
'meta.name': 'foo',
|
||||
'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
|
||||
OBJ_ID_KEY: 1,
|
||||
'meta.name': 'bar',
|
||||
'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
|
||||
OBJ_ID_KEY: 0},
|
||||
},
|
||||
}
|
||||
|
||||
def test_i_can_transform_concept_with_circular_reference_2(self):
|
||||
sheerka = self.get_sheerka()
|
||||
foo = Concept("foo")
|
||||
bar = Concept("foo", body=foo)
|
||||
foo.metadata.body = bar
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(foo)
|
||||
|
||||
assert to_dict == {
|
||||
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
|
||||
OBJ_ID_KEY: 0,
|
||||
'meta.name': 'foo',
|
||||
'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
|
||||
OBJ_ID_KEY: 1,
|
||||
'meta.name': 'foo',
|
||||
'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
|
||||
OBJ_ID_KEY: 0},
|
||||
},
|
||||
}
|
||||
|
||||
def test_i_can_transform_the_unknown_concept(self):
|
||||
sheerka = self.get_sheerka(False)
|
||||
|
||||
unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(unknown)
|
||||
|
||||
assert len(to_dict) == 3
|
||||
assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept
|
||||
assert to_dict[OBJ_ID_KEY] == 0
|
||||
assert "id" in to_dict
|
||||
|
||||
def test_i_can_transform_simple_execution_context(self):
|
||||
sheerka = self.get_sheerka()
|
||||
ExecutionContext.ids = {}
|
||||
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(context)
|
||||
|
||||
assert to_dict == {
|
||||
OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
|
||||
OBJ_ID_KEY: 0,
|
||||
'_parent': None,
|
||||
'_id': 0,
|
||||
'_tab': '',
|
||||
'_bag': {},
|
||||
'_start': 0,
|
||||
'_stop': 0,
|
||||
'who': 'requester',
|
||||
'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'},
|
||||
'desc': 'this is the desc',
|
||||
'children': [],
|
||||
'preprocess': None,
|
||||
'inputs': {},
|
||||
'values': {},
|
||||
'obj': None,
|
||||
'concepts': {}
|
||||
}
|
||||
|
||||
def test_i_can_transform_list(self):
|
||||
sheerka = self.get_sheerka()
|
||||
ExecutionContext.ids = {}
|
||||
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict([context])
|
||||
|
||||
assert len(to_dict) == 1
|
||||
assert isinstance(to_dict, list)
|
||||
assert to_dict[0]["who"] == "requester"
|
||||
assert to_dict[0]["desc"] == "this is the desc"
|
||||
|
||||
def test_i_can_transform_set(self):
|
||||
sheerka = self.get_sheerka()
|
||||
ExecutionContext.ids = {}
|
||||
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict({context})
|
||||
|
||||
assert len(to_dict) == 1
|
||||
assert isinstance(to_dict, list)
|
||||
assert to_dict[0]["who"] == "requester"
|
||||
assert to_dict[0]["desc"] == "this is the desc"
|
||||
|
||||
def test_i_can_transform_dict(self):
|
||||
sheerka = self.get_sheerka()
|
||||
ExecutionContext.ids = {}
|
||||
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
|
||||
known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key()
|
||||
sheerka.create_new_concept(self.get_context(sheerka), known_concept)
|
||||
unknown_concept = Concept("bar")
|
||||
known = sheerka.new("foo")
|
||||
|
||||
bag = {
|
||||
"context": context,
|
||||
"known_concept": known_concept,
|
||||
"unknown_concept": unknown_concept,
|
||||
"True": True,
|
||||
"Number": 1.1,
|
||||
"String": "a string value",
|
||||
"None": None,
|
||||
unknown_concept: "hello",
|
||||
known: "world"
|
||||
}
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(bag)
|
||||
|
||||
assert isinstance(to_dict, dict)
|
||||
assert to_dict['Number'] == 1.1
|
||||
assert to_dict['String'] == 'a string value'
|
||||
assert to_dict['True']
|
||||
assert to_dict['None'] is None
|
||||
assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
|
||||
assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
|
||||
assert to_dict["known_concept"]["id"] == '1001'
|
||||
assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
|
||||
assert to_dict["(None)bar"] == "hello"
|
||||
assert to_dict["(1001)foo"] == "world"
|
||||
|
||||
def test_i_can_transform_when_circular_references(self):
|
||||
sheerka = self.get_sheerka()
|
||||
ExecutionContext.ids = {}
|
||||
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
|
||||
context.push("another requester", "another desc")
|
||||
|
||||
st = SheerkaTransform(sheerka)
|
||||
to_dict = st.to_dict(context)
|
||||
|
||||
assert isinstance(to_dict, dict)
|
||||
assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
|
||||
assert len(to_dict["children"]) == 1
|
||||
assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
|
||||
assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
|
||||
assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0
|
||||
assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
|
||||
assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1
|
||||
@@ -1,5 +1,6 @@
|
||||
import core.utils
|
||||
import pytest
|
||||
from core.concept import ConceptParts
|
||||
|
||||
from core.tokenizer import Token, TokenKind
|
||||
|
||||
@@ -130,6 +131,37 @@ def test_i_can_escape():
|
||||
assert actual == "hello \\'world\\' my friend"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, expected_key, expected_id", [
|
||||
(None, None, None),
|
||||
(10, None, None),
|
||||
("", None, None),
|
||||
("xxx", None, None),
|
||||
(":c:", None, None),
|
||||
(":c:key", None, None),
|
||||
(":c:key:", "key", None),
|
||||
(":c:key:id", None, None),
|
||||
(":c:key:id:", "key", "id"),
|
||||
])
|
||||
def test_i_can_decode_concept_repr(text, expected_key, expected_id):
|
||||
k, i = core.utils.decode_concept(text)
|
||||
assert k == expected_key
|
||||
assert i == expected_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
(None, None),
|
||||
(10, None),
|
||||
("", None),
|
||||
("xxx", None),
|
||||
("xxx.", None),
|
||||
("xxx.yyy", None),
|
||||
("core.concept.ConceptParts.BODY", ConceptParts.BODY),
|
||||
])
|
||||
def test_i_can_decode_enum(text, expected):
|
||||
actual = core.utils.decode_enum(text)
|
||||
assert actual == expected
|
||||
|
||||
|
||||
def get_tokens(lst):
|
||||
res = []
|
||||
for e in lst:
|
||||
|
||||
@@ -257,24 +257,27 @@ def test_i_can_load_events(root):
|
||||
for i in range(15):
|
||||
sdp.save_event(Event(f"Hello {i}"))
|
||||
|
||||
events = list(sdp.load_events(10))
|
||||
events = list(sdp.load_events(10)) # first ten
|
||||
assert len(events) == 10
|
||||
assert events[0].message == "Hello 14"
|
||||
assert events[9].message == "Hello 5"
|
||||
|
||||
events = list(sdp.load_events(10, 5))
|
||||
events = list(sdp.load_events(10, 5)) # skip first 5, then take 10
|
||||
assert len(events) == 10
|
||||
assert events[0].message == "Hello 9"
|
||||
assert events[9].message == "Hello 0"
|
||||
|
||||
events = list(sdp.load_events(20, 10))
|
||||
events = list(sdp.load_events(20, 10)) # skip first 10, take 20,(but only 5 remaining)
|
||||
assert len(events) == 5
|
||||
assert events[0].message == "Hello 4"
|
||||
assert events[4].message == "Hello 0"
|
||||
|
||||
events = list(sdp.load_events(1, 20))
|
||||
events = list(sdp.load_events(1, 20)) # skip first 20, take one
|
||||
assert len(events) == 0
|
||||
|
||||
events = list(sdp.load_events(0)) # all
|
||||
assert len(events) == 15
|
||||
|
||||
|
||||
@pytest.mark.parametrize("root", [
|
||||
".sheerka",
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
from core.concept import Concept, ConceptParts
|
||||
from sheerkapickle import tags
|
||||
from sheerkapickle.SheerkaPickler import SheerkaPickler
|
||||
from sheerkapickle.SheerkaUnpickler import SheerkaUnpickler
|
||||
|
||||
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
|
||||
|
||||
|
||||
class Obj:
|
||||
def __init__(self, a, b, c):
|
||||
self.a = a
|
||||
self.b = b
|
||||
self.c = c
|
||||
|
||||
def __eq__(self, other):
|
||||
if id(self) == id(other):
|
||||
return True
|
||||
|
||||
if not isinstance(other, Obj):
|
||||
return False
|
||||
|
||||
return self.a == other.a and self.b == other.b and self.c == other.c
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.a, self.b, self.c))
|
||||
|
||||
|
||||
class TestSheerkaPickler(TestUsingFileBasedSheerka):
|
||||
|
||||
@pytest.mark.parametrize("obj, expected", [
|
||||
(1, 1),
|
||||
(3.14, 3.14),
|
||||
("a string", "a string"),
|
||||
(True, True),
|
||||
(None, None),
|
||||
([1, 3.14, "a string"], [1, 3.14, "a string"]),
|
||||
((1, 3.14, "a string"), {tags.TUPLE: [1, 3.14, "a string"]}),
|
||||
({1}, {tags.SET: [1]}),
|
||||
({"a": "a", "b": 3.14, "c": True}, {"a": "a", "b": 3.14, "c": True}),
|
||||
({1: "a", 2: 3.14, 3: True}, {1: "a", 2: 3.14, 3: True}),
|
||||
([1, [3.14, "a string"]], [1, [3.14, "a string"]]),
|
||||
([1, (3.14, "a string")], [1, {tags.TUPLE: [3.14, "a string"]}]),
|
||||
([], []),
|
||||
(ConceptParts.BODY, {tags.ENUM: "core.concept.ConceptParts.BODY"}),
|
||||
])
|
||||
def test_i_can_flatten_and_restore_primitives(self, obj, expected):
|
||||
sheerka = self.get_sheerka()
|
||||
flatten = SheerkaPickler(sheerka).flatten(obj)
|
||||
assert flatten == expected
|
||||
|
||||
decoded = SheerkaUnpickler(sheerka).restore(flatten)
|
||||
assert decoded == obj
|
||||
|
||||
def test_i_can_flatten_and_restore_instances(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
obj1 = Obj(1, "b", True)
|
||||
obj2 = Obj(3.14, ("a", "b"), obj1)
|
||||
|
||||
flatten = SheerkaPickler(sheerka).flatten(obj2)
|
||||
assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
|
||||
'a': 3.14,
|
||||
'b': {'_sheerka/tuple': ['a', 'b']},
|
||||
'c': {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
|
||||
'a': 1,
|
||||
'b': 'b',
|
||||
'c': True}}
|
||||
|
||||
decoded = SheerkaUnpickler(sheerka).restore(flatten)
|
||||
assert decoded == obj2
|
||||
|
||||
def test_i_can_manage_circular_reference(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
obj1 = Obj(1, "b", True)
|
||||
obj1.c = obj1
|
||||
|
||||
flatten = SheerkaPickler(sheerka).flatten(obj1)
|
||||
assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
|
||||
'a': 1,
|
||||
'b': 'b',
|
||||
'c': {'_sheerka/id': 0}}
|
||||
|
||||
decoded = SheerkaUnpickler(sheerka).restore(flatten)
|
||||
assert decoded.a == obj1.a
|
||||
assert decoded.b == obj1.b
|
||||
assert decoded.c == decoded
|
||||
|
||||
def test_i_can_flatten_obj_with_new_props(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
obj = Obj(1, "b", True)
|
||||
obj.z = "new prop"
|
||||
|
||||
flatten = SheerkaPickler(sheerka).flatten(obj)
|
||||
assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
|
||||
'a': 1,
|
||||
'b': 'b',
|
||||
'c': True,
|
||||
'z': "new prop"}
|
||||
|
||||
decoded = SheerkaUnpickler(sheerka).restore(flatten)
|
||||
assert decoded == obj
|
||||
|
||||
@pytest.mark.parametrize("obj, expected", [
|
||||
({None: "a"}, {'null': "a"}),
|
||||
({ConceptParts.BODY: "a"}, {'core.concept.ConceptParts.BODY': 'a'}),
|
||||
({(1, 2): "a"}, {(1, 2): "a"}),
|
||||
])
|
||||
def test_i_can_manage_specific_keys_in_dictionaries(self, obj, expected):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
flatten = SheerkaPickler(sheerka).flatten(obj)
|
||||
assert flatten == expected
|
||||
|
||||
decoded = SheerkaUnpickler(sheerka).restore(flatten)
|
||||
assert decoded == obj
|
||||
|
||||
def test_i_can_use_concept_as_dictionary_key(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
concept = Concept("foo").init_key()
|
||||
sheerka.set_id_if_needed(concept, False)
|
||||
sheerka.add_in_cache(concept)
|
||||
obj = {concept: "a"}
|
||||
flatten = SheerkaPickler(sheerka).flatten(obj)
|
||||
assert flatten == {':c:foo:1001:': 'a'}
|
||||
|
||||
decoded = SheerkaUnpickler(sheerka).restore(flatten)
|
||||
assert decoded == obj
|
||||
|
||||
def test_i_can_manage_references(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
foo = Obj("foo", "bar", "baz")
|
||||
obj = [ConceptParts.BODY, foo, ConceptParts.WHERE, ConceptParts.BODY, foo]
|
||||
flatten = SheerkaPickler(sheerka).flatten(obj)
|
||||
|
||||
assert flatten == [{'_sheerka/enum': 'core.concept.ConceptParts.BODY'},
|
||||
{'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
|
||||
'a': 'foo',
|
||||
'b': 'bar',
|
||||
'c': 'baz'},
|
||||
{'_sheerka/enum': 'core.concept.ConceptParts.WHERE'},
|
||||
{'_sheerka/id': 0},
|
||||
{'_sheerka/id': 1}]
|
||||
|
||||
decoded = SheerkaUnpickler(sheerka).restore(flatten)
|
||||
assert decoded == obj
|
||||
|
||||
def test_serialize_concept(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
foo = Concept("doo")
|
||||
flatten = SheerkaPickler(sheerka).flatten(foo)
|
||||
restored = SheerkaUnpickler(sheerka).restore(flatten)
|
||||
|
||||
assert restored == foo
|
||||
|
||||
def test_i_do_not_encode_logger(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
logger = logging.getLogger("log_name")
|
||||
logger2 = logging.getLogger("log_name2")
|
||||
obj = Obj("foo", logger, {"a": logger, "b": logger2})
|
||||
|
||||
flatten = SheerkaPickler(sheerka).flatten(obj)
|
||||
decoded = SheerkaUnpickler(sheerka).restore(flatten)
|
||||
assert decoded == Obj("foo", None, {"a": None, "b": None})
|
||||
@@ -0,0 +1,285 @@
|
||||
import sheerkapickle
|
||||
from core.builtin_concepts import BuiltinConcepts, UserInputConcept, ReturnValueConcept
|
||||
from core.concept import Concept, ConceptParts
|
||||
from core.sheerka.ExecutionContext import ExecutionContext
|
||||
from core.tokenizer import Tokenizer
|
||||
from evaluators.ConceptEvaluator import ConceptEvaluator
|
||||
from parsers.DefaultParser import DefaultParser
|
||||
from sdp.sheerkaDataProvider import Event
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
|
||||
class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
|
||||
|
||||
def test_i_can_encode_decode_unknown_concept_metadata(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
concept = Concept(name="foo", key="my_key")
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.key": "my_key"}'
|
||||
assert decoded == concept
|
||||
|
||||
concept = Concept("foo", is_builtin=True, is_unique=True)
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.is_builtin": true, "meta.is_unique": true}'
|
||||
|
||||
concept = Concept("foo", body="my_body")
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "my_body"}'
|
||||
|
||||
concept = Concept("foo", pre="my_pre")
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.pre": "my_pre"}'
|
||||
|
||||
concept = Concept("foo", post="my_post")
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.post": "my_post"}'
|
||||
|
||||
concept = Concept("foo", where="my_where")
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.where": "my_where"}'
|
||||
|
||||
concept = Concept("foo").def_prop("a", "value_a").def_prop("b", "value_b")
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
|
||||
def test_i_can_encode_decode_unknown_concept_values(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
concept = Concept("foo")
|
||||
concept.values[ConceptParts.PRE] = 10 # an int
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "pre": 10}'
|
||||
|
||||
concept = Concept("foo")
|
||||
concept.values[ConceptParts.POST] = 'a string' # an int
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "post": "a string"}'
|
||||
|
||||
concept = Concept("foo")
|
||||
concept.values[ConceptParts.WHERE] = ['a string', 3.14] # a list
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": ["a string", 3.14]}'
|
||||
|
||||
concept = Concept("foo")
|
||||
concept.values[ConceptParts.WHERE] = ('a string', 3.14) # a tuple
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": {"_sheerka/tuple": ["a string", 3.14]}}'
|
||||
|
||||
concept = Concept("foo")
|
||||
concept.values[ConceptParts.BODY] = Concept("foo", body="foo_body")
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "foo_body"}}'
|
||||
|
||||
def test_i_can_encode_decode_unknown_concept_properties(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
concept = Concept("foo")
|
||||
concept.set_prop("a", "value_a") # string
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "value_a"]]}'
|
||||
|
||||
concept = Concept("foo")
|
||||
concept.set_prop("a", 10) # int
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", 10]]}'
|
||||
|
||||
concept = Concept("foo")
|
||||
concept.set_prop("a", Concept("bar")) # another concept
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}'
|
||||
|
||||
concept = Concept("foo")
|
||||
concept.set_prop("a", "a").set_prop("b", "b") # at least two props
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "a"], ["b", "b"]]}'
|
||||
|
||||
def test_i_can_encode_decode_known_concepts(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
ref_concept = Concept("my_name", True, True, "my_key", "my_body", "my_where", "my_pre", "my_post", "my_def")
|
||||
ref_concept.def_prop("a", "value_a").def_prop("b", "value_b")
|
||||
|
||||
sheerka.create_new_concept(self.get_context(sheerka), ref_concept)
|
||||
|
||||
to_string = sheerkapickle.encode(sheerka, ref_concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == ref_concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"]}'
|
||||
|
||||
concept = Concept().update_from(sheerka.get_by_id(ref_concept.id))
|
||||
concept.set_metadata_value(ConceptParts.BODY, Concept("bar"))
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"], "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}}'
|
||||
|
||||
def test_i_can_manage_reference_of_the_same_object(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
concept_ref = Concept("foo")
|
||||
|
||||
concept = Concept("bar")
|
||||
concept.set_metadata_value(ConceptParts.PRE, concept_ref)
|
||||
concept.set_metadata_value(ConceptParts.BODY, concept_ref)
|
||||
|
||||
to_string = sheerkapickle.encode(sheerka, concept)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == concept
|
||||
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "bar", "pre": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo"}, "body": {"_sheerka/id": 1}}'
|
||||
|
||||
def test_i_can_encode_decode_user_input(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body="my_text", user_name="my_user_name")
|
||||
|
||||
to_string = sheerkapickle.encode(sheerka, user_input)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == user_input
|
||||
assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "my_text"}'
|
||||
|
||||
def test_i_can_encode_decode_user_input_when_tokens(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
text = "I have 'a complicated' 10 text"
|
||||
tokens = list(Tokenizer(text))
|
||||
user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens, user_name="my_user_name")
|
||||
|
||||
to_string = sheerkapickle.encode(sheerka, user_input)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == UserInputConcept(text, "my_user_name")
|
||||
assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "{text}"' + '}'
|
||||
|
||||
def test_i_can_encode_decode_return_value(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
ret_val = sheerka.ret("who", True, 10)
|
||||
|
||||
to_string = sheerkapickle.encode(sheerka, ret_val)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == ret_val
|
||||
assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10}'
|
||||
|
||||
def test_i_can_encode_decode_return_value_with_parent(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
ret_val = sheerka.ret("who", True, 10)
|
||||
ret_val_parent = sheerka.ret("parent_who", True, "10")
|
||||
ret_val.parents = [ret_val_parent, ret_val_parent]
|
||||
|
||||
to_string = sheerkapickle.encode(sheerka, ret_val)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == ret_val
|
||||
assert decoded.parents == ret_val.parents
|
||||
parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]'
|
||||
assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}'
|
||||
|
||||
def test_i_can_encode_decode_return_values_with_complex_body(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
ret_val = sheerka.ret("who", True, Concept("foo", body="bar"))
|
||||
|
||||
to_string = sheerkapickle.encode(sheerka, ret_val)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == ret_val
|
||||
|
||||
def test_i_can_encode_decode_return_values_from_concepts_parsers_or_evaluators(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
foo = Concept("foo")
|
||||
sheerka.set_id_if_needed(foo, False)
|
||||
ret_val = sheerka.ret(foo, True, 10)
|
||||
to_string = sheerkapickle.encode(sheerka, ret_val)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == sheerka.ret("c:1001:", True, 10)
|
||||
|
||||
ret_val = sheerka.ret(DefaultParser(), True, 10)
|
||||
to_string = sheerkapickle.encode(sheerka, ret_val)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == sheerka.ret("parsers.Default", True, 10)
|
||||
|
||||
ret_val = sheerka.ret(ConceptEvaluator(), True, 10)
|
||||
to_string = sheerkapickle.encode(sheerka, ret_val)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == sheerka.ret("evaluators.Concept", True, 10)
|
||||
|
||||
def test_i_can_encode_decode_execution_context(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
context = ExecutionContext("who", Event("xxx"), sheerka, "my desc")
|
||||
input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)]
|
||||
context.inputs = {"a": input_list, "b": Concept("foo")}
|
||||
context.values = {"c": input_list, "d": Concept("bar")}
|
||||
context.obj = Concept("baz")
|
||||
context.push("who3", "sub_child1")
|
||||
context.push("who4", "sub_child2")
|
||||
|
||||
to_string = sheerkapickle.encode(sheerka, context)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
assert decoded == context
|
||||
|
||||
def test_complicated_execution_context(self):
|
||||
sheerka = self.get_sheerka(skip_builtins_in_db=False)
|
||||
|
||||
text = "def concept one as 1"
|
||||
execution_context = ExecutionContext("s", Event(), sheerka, f"Evaluating '{text}'")
|
||||
user_input = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="n"))
|
||||
reduce_requested = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
|
||||
|
||||
steps = [
|
||||
BuiltinConcepts.BEFORE_PARSING,
|
||||
BuiltinConcepts.PARSING,
|
||||
BuiltinConcepts.AFTER_PARSING,
|
||||
BuiltinConcepts.BEFORE_EVALUATION,
|
||||
BuiltinConcepts.EVALUATION,
|
||||
BuiltinConcepts.AFTER_EVALUATION
|
||||
]
|
||||
|
||||
ret = sheerka.execute(execution_context, [user_input, reduce_requested], steps)
|
||||
execution_context.add_values(return_values=ret)
|
||||
|
||||
to_string = sheerkapickle.encode(sheerka, execution_context)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
|
||||
return_value = decoded.values["return_values"][0].value
|
||||
assert sheerka.isinstance(return_value, BuiltinConcepts.NEW_CONCEPT)
|
||||
|
||||
def test_encode_simple_concept(self):
|
||||
sheerka = self.get_sheerka()
|
||||
|
||||
foo = Concept("foo")
|
||||
to_string = sheerkapickle.encode(sheerka, foo)
|
||||
decoded = sheerkapickle.decode(sheerka, to_string)
|
||||
|
||||
assert decoded == foo
|
||||
Reference in New Issue
Block a user