kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored ExecutionContext serialization (added sheerkapickle) and added History management

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-01-31 18:58:03 +01:00
parent fed0735eb9
commit b9afcba61f
31 changed files with 1546 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_concepts.txt
+10
View File
@@ -0,0 +1,10 @@
def concept one as 1
def concept two as 2
def concept three as 3
def concept one as 1
def concept two as 2
def concept three as 3
def concept four as 4
src/__init__.py
View File
src/core/concept.py
+2
View File
@@ -343,6 +343,8 @@ class Concept:
:param metadata:
:return:
"""
if metadata not in self.values:
return None
return self.values[metadata]
def auto_init(self):
src/core/sheerka/ExecutionContext.py
+34 -5
View File
@@ -4,9 +4,22 @@ import time
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from sdp.sheerkaDataProvider import Event
from sheerkapickle.SheerkaPickler import SheerkaPickler
DEBUG_TAB_SIZE = 4
PROPERTIES_TO_SERIALIZE = ("_id",
"_bag",
"_start",
"_stop",
"who",
"desc",
"children",
"inputs",
"values",
"obj",
"concepts")
class ExecutionContext:
"""
@@ -31,7 +44,7 @@ class ExecutionContext:
**kwargs):
self._parent = None
self._id = ExecutionContext.get_id(event.get_digest())
self._id = ExecutionContext.get_id(event.get_digest()) if event else None
self._tab = ""
self._bag = {} # other variables
self._start = 0
@@ -90,6 +103,26 @@ class ExecutionContext:
msg += ")"
return msg
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, ExecutionContext):
return False
for prop in PROPERTIES_TO_SERIALIZE:
if prop == "who":
value = str(getattr(self, prop))
other_value = str(getattr(other, prop))
else:
value = getattr(self, prop)
other_value = getattr(other, prop)
if value != other_value:
return False
return True
def add_preprocess(self, name, **kwargs):
preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
preprocess.set_prop("name", name)
@@ -189,10 +222,6 @@ class ExecutionContext:
to_str = self.return_value_to_str(r)
logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
def to_dict(self):
from core.sheerka_transform import SheerkaTransform
st = SheerkaTransform(self.sheerka)
return st.to_dict(self)
@staticmethod
def return_value_to_str(r):
src/core/sheerka/SheerkaCreateNewConcept.py → src/core/sheerka/Services/SheerkaCreateNewConcept.py
View File
src/core/sheerka/SheerkaDump.py → src/core/sheerka/Services/SheerkaDump.py
+3 -2
View File
@@ -39,7 +39,8 @@ class SheerkaDump:
self.sheerka.log.info(f"name : {c.name}")
self.sheerka.log.info(f"bnf : {c.metadata.definition}")
self.sheerka.log.info(f"key : {c.key}")
self.sheerka.log.info(f"body : {c.body}")
self.sheerka.log.info(f"body : {c.metadata.body}")
self.sheerka.log.info(f"value : {c.body}")
self.sheerka.log.info(f"digest : {c.get_digest()}")
first = False
@@ -57,7 +58,7 @@ class SheerkaDump:
while True:
try:
if h.user != self.sheerka.name:
if h.event.user != self.sheerka.name:
self.sheerka.log.info(h)
count += 1
h = next(history)
src/core/sheerka/SheerkaEvaluateConcept.py → src/core/sheerka/Services/SheerkaEvaluateConcept.py
View File
src/core/sheerka/SheerkaExecute.py → src/core/sheerka/Services/SheerkaExecute.py
+1 -1
View File
@@ -218,7 +218,7 @@ class SheerkaExecute:
for step in execution_steps:
copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values]
with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context:
with execution_context.push(step=step, iteration=0, desc=f"{step=}") as sub_context:
sub_context.log(logger or self.sheerka.log, f"{step=}, context='{sub_context}'")
if step == BuiltinConcepts.PARSING:
src/core/sheerka/Services/SheerkaHistoryManager.py
+74
View File
@@ -0,0 +1,74 @@
from collections import namedtuple
from sdp.sheerkaDataProvider import Event
hist = namedtuple("History", "text status") # tests purposes only
class History:
def __init__(self, event: Event, result):
self.event = event
self.result = result
self._status = None
def __str__(self):
msg = f"{self.event.get_digest()} {self.event.date.strftime('%d/%m/%Y %H:%M:%S')} : {self.event.message}"
status = self.status
if status is not None:
msg += f" => {status}"
return msg
def __repr__(self):
return f"event={self.event!r}, status={self.status}, result={self.result}"
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, hist):
return self.event.message == other.text and self.status == other.status
if not isinstance(other, History):
return False
return self.event == other.event and self.result == other.result
@property
def status(self):
if self._status:
return self._status
if not self.result or "return_values" not in self.result.values:
return
if hasattr(self.result.values["return_values"], "__iter__"):
if len(self.result.values["return_values"]) != 1:
self._status = False
return self._status
else:
self._status = self.result.values["return_values"][0].status
return self._status
else:
self._status = self.result.values["return_values"].status
return self._status
class SheerkaHistoryManager:
def __init__(self, sheerka):
self.sheerka = sheerka
def history(self, depth_or_digest, start):
"""
Load history
:param depth_or_digest: number of items or digest
:param start:
:return:
"""
events = list(self.sheerka.sdp.load_events(depth_or_digest, start))
for event in events:
try:
result = self.sheerka.sdp.load_result(self.sheerka, event.get_digest())
except (IOError, KeyError):
result = None
yield History(event, result)
src/core/sheerka/SheerkaSetsManager.py → src/core/sheerka/Services/SheerkaSetsManager.py
View File
src/core/sheerka/Services/__init__.py
View File
src/core/sheerka/Sheerka.py
+36 -10
View File
@@ -2,11 +2,12 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConc
UnknownConcept
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.SheerkaCreateNewConcept import SheerkaCreateNewConcept
from core.sheerka.SheerkaDump import SheerkaDump
from core.sheerka.SheerkaEvaluateConcept import SheerkaEvaluateConcept
from core.sheerka.SheerkaExecute import SheerkaExecute
from core.sheerka.SheerkaSetsManager import SheerkaSetsManager
from core.sheerka.Services.SheerkaCreateNewConcept import SheerkaCreateNewConcept
from core.sheerka.Services.SheerkaDump import SheerkaDump
from core.sheerka.Services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
from core.sheerka.Services.SheerkaExecute import SheerkaExecute
from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager
from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
import core.utils
import core.builtin_helpers
@@ -21,7 +22,7 @@ import logging
# BuiltinConcepts.AFTER_EVALUATION]
CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
CONCEPTS_FILE = "_concepts.txt"
class Sheerka(Concept):
"""
@@ -81,6 +82,7 @@ class Sheerka(Concept):
self.dump_handler = SheerkaDump(self)
self.sets_handler = SheerkaSetsManager(self)
self.evaluate_concept_handler = SheerkaEvaluateConcept(self)
self.history_handler = SheerkaHistoryManager(self)
def initialize(self, root_folder: str = None):
"""
@@ -92,6 +94,9 @@ class Sheerka(Concept):
"""
try:
from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
initialize_pickle_handlers()
self.sdp = SheerkaDataProvider(root_folder)
if self.sdp.first_time:
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
@@ -104,11 +109,16 @@ class Sheerka(Concept):
self.initialize_builtin_parsers()
self.initialize_builtin_evaluators()
self.initialize_concepts_definitions(exec_context)
res = ReturnValueConcept(self, True, self)
exec_context.add_values(return_values=res)
if not self.skip_builtins_in_db:
self.sdp.save_result(self, exec_context)
except IOError as e:
return ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
return ReturnValueConcept(self, True, self)
return res
def initialize_builtin_concepts(self):
"""
@@ -232,7 +242,12 @@ class Sheerka(Concept):
execution_context.add_values(return_values=ret)
if not self.skip_builtins_in_db:
self.sdp.save_result(execution_context)
self.sdp.save_result(self, execution_context)
#hack to save valid concept definition
if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
with open(CONCEPTS_FILE, "a") as f:
f.write(text + "\n")
return ret
def execute(self, execution_context, return_values, execution_steps, logger=None):
@@ -549,8 +564,19 @@ class Sheerka(Concept):
def history(self, page=10, start=0):
"""Gets the history of all commands"""
return self.sdp.load_events(page, start)
return self.history_handler.history(page, start)
def restore(self):
"""
Restore the state with all previous valid concept definitions
:return:
"""
try:
with open(CONCEPTS_FILE, "r") as f:
for line in f.readlines():
self.evaluate_user_input(line)
except IOError:
pass
def test(self):
return f"I have access to Sheerka !"
src/core/sheerka_transform.py
-161
View File
@@ -1,161 +0,0 @@
import dataclasses
from enum import Enum
from core.concept import Concept, PROPERTIES_TO_SERIALIZE
from core.sheerka.Sheerka import ExecutionContext
from core.tokenizer import Token
from evaluators.BaseEvaluator import BaseEvaluator
from parsers.BaseParser import BaseParser, Node
from parsers.BnfParser import BnfParser
from parsers.ConceptLexerParser import UnrecognizedTokensNode, ParsingExpression
from parsers.PythonParser import PythonNode
from sdp.sheerkaDataProvider import Event
OBJ_TYPE_KEY = "__type__"
OBJ_ID_KEY = "__id__"
OBJ_NAME_KEY = "__name__"
default_concept = Concept()
class SheerkaTransformType(Enum):
Concept = 1
Reference = 2
ExecutionContext = 3
Event = 4
Node = 5
Exception = 6
def __repr__(self):
return self.__class__.__name__ + "." + self.name
class SheerkaTransform:
def __init__(self, sheerka):
self.ids = {}
self.sheerka = sheerka
self.id_count = -1
def to_dict(self, obj):
if isinstance(obj, (Concept, ExecutionContext, Event)):
exists, _id = self.exist(obj)
if exists:
return {
OBJ_TYPE_KEY: SheerkaTransformType.Reference,
OBJ_ID_KEY: _id
}
else:
self.id_count = self.id_count + 1
self.ids[obj] = self.id_count
if isinstance(obj, Concept):
return self.concept_to_dict(obj)
elif isinstance(obj, ExecutionContext):
return self.execution_context_to_dict(obj)
elif isinstance(obj, Event):
return {
OBJ_TYPE_KEY: SheerkaTransformType.Event,
OBJ_ID_KEY: self.id_count,
'digest': obj.get_digest()}
elif isinstance(obj, (BaseParser, BaseEvaluator, BnfParser)):
return obj.name
elif isinstance(obj, Token):
return obj.__dict__
elif isinstance(obj, PythonNode):
return {
OBJ_TYPE_KEY: SheerkaTransformType.Node,
OBJ_NAME_KEY: "PythonNode",
'source': obj.source,
'ast_': obj.get_dump(obj.ast_)
}
elif isinstance(obj, Node):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.Node,
OBJ_NAME_KEY: obj.__class__.__name__,
}
for k, v in obj.__dict__.items():
to_dict[k] = self.to_dict(v)
return to_dict
elif isinstance(obj, Exception):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.Exception,
OBJ_NAME_KEY: obj.__class__.__name__,
}
for k, v in obj.__dict__.items():
to_dict[k] = self.to_dict(v)
return to_dict
elif isinstance(obj, ParsingExpression):
return obj.__repr__()
elif isinstance(obj, dict):
return dict((str(k) if isinstance(k, Concept) else k, self.to_dict(v)) for k, v in obj.items())
elif hasattr(obj, "__iter__") and not isinstance(obj, str):
return list(self.to_dict(o) for o in obj)
else:
return obj
def concept_to_dict(self, obj: Concept):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: self.id_count,
}
if obj.id:
ref = self.sheerka.get(obj.key, obj.id)
to_dict["id"] = obj.id
else:
ref = default_concept
# transform metadata
for prop in PROPERTIES_TO_SERIALIZE:
value = getattr(obj.metadata, prop)
ref_value = getattr(ref.metadata, prop)
if value != ref_value:
to_dict["meta." + prop] = self.to_dict(value)
# transform value
for metadata, value in obj.values.items():
ref_value = ref.values[metadata] if metadata in ref.values else None
if value != ref_value:
to_dict[metadata.value] = self.to_dict(value)
# transform properties
for prop in obj.props:
value = obj.props[prop].value
if prop not in ref.props or value != ref.props[prop].value:
if "props" not in to_dict:
to_dict["props"] = []
to_dict["props"].append((prop, self.to_dict(value)))
return to_dict
def execution_context_to_dict(self, obj: ExecutionContext):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
OBJ_ID_KEY: self.id_count
}
for property_name in obj.__dict__:
if property_name == "sheerka":
continue
to_dict[property_name] = self.to_dict(getattr(obj, property_name))
return to_dict
def exist(self, obj):
for k, v in self.ids.items():
if id(k) == id(obj) or k == obj:
return True, v
return False, None
src/core/utils.py
+64
View File
@@ -228,6 +228,7 @@ def escape_char(text, to_escape):
return res
def pp(items):
if not hasattr(items, "__iter__"):
return str(items)
@@ -236,3 +237,66 @@ def pp(items):
return str(items)
return " \n" + " \n".join(str(item) for item in items)
def decode_concept(concept_repr):
"""
if concept_repr is like :c:key:id:
return the key and the id
:param concept_repr:
:return:
"""
if not (concept_repr and isinstance(concept_repr, str) and concept_repr.startswith(":c:")):
return None, None
i = 3
length = len(concept_repr)
key = ""
while i < length:
if concept_repr[i] == ":":
break
key += concept_repr[i]
i += 1
else:
return None, None
i += 1
if i >= length:
return key, None
id = ""
while i < length:
if concept_repr[i] == ":":
break
id += concept_repr[i]
i += 1
else:
return None, None
return key, id
def decode_enum(enum_repr: str):
"""
Tries to transform ClassName.Name into an enum
:param enum_repr:
:return:
"""
if not (enum_repr and isinstance(enum_repr, str)):
return None
try:
idx = enum_repr.rindex(".")
if idx == len(enum_repr):
return None
cls_name = enum_repr[:idx]
cls = get_class(cls_name)
name = enum_repr[idx + 1:]
return cls[name]
except ValueError:
return None
except TypeError:
return None
src/sdp/sheerkaDataProvider.py
+13 -6
View File
@@ -36,6 +36,9 @@ class Event(object):
def __str__(self):
return f"{self.date.strftime('%d/%m/%Y %H:%M:%S')} {self.message}"
def __repr__(self):
return f"{self.get_digest()[:12]} {self.message}"
def get_digest(self):
"""
Returns the digest of the event
@@ -64,6 +67,7 @@ class Event(object):
self.date = datetime.fromisoformat(as_dict["date"])
self.message = as_dict["message"]
self.parents = as_dict["parents"]
self._digest = as_dict["_digest"] # freeze the digest
class ObjToUpdate:
@@ -667,9 +671,9 @@ class SheerkaDataProvider:
:param event:
:return: digest of the event
"""
digest = event.get_digest()
parent = self.get_snapshot(SheerkaDataProvider.LastEventFile)
event.parents = [parent] if parent else None
digest = event.get_digest() # must be call after setting the parents
target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest)
if self.io.exists(target_path):
@@ -712,7 +716,7 @@ class SheerkaDataProvider:
digest = event.parents[0]
count = 0
while count < page_size:
while count < page_size or page_size <= 0:
event = self.load_event(digest)
if event is None:
return
@@ -725,12 +729,13 @@ class SheerkaDataProvider:
digest = event.parents[0]
count += 1
def save_result(self, execution_context):
def save_result(self, sheerka, execution_context):
"""
Save the execution context associated with an event
To make a long story short,
for every single user input, there is an event (which is the first thing that is created)
and a result (the ExecutionContext created by sheerka.evaluate_user_input()
:param sheerka:
:param execution_context:
:return:
"""
@@ -740,14 +745,16 @@ class SheerkaDataProvider:
if self.io.exists(target_path):
return digest
self.io.write_binary(target_path, self.serializer.serialize(execution_context, None).read())
context = SerializerContext(sheerka=sheerka)
self.io.write_binary(target_path, self.serializer.serialize(execution_context, context).read())
return digest
def load_result(self, digest):
def load_result(self, sheerka, digest):
target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + "_result"
with self.io.open(target_path, "rb") as f:
return self.serializer.deserialize(f, None)
context = SerializerContext(sheerka=sheerka)
return self.serializer.deserialize(f, context)
def save_state(self, state: State):
digest = state.get_digest()
src/sdp/sheerkaSerializer.py
+11 -10
View File
@@ -1,18 +1,17 @@
import dataclasses
import json
import pickle
import datetime
import pickle
import struct
import io
from dataclasses import dataclass
import sheerkapickle
from core.sheerka_logger import get_logger
from enum import Enum
import core.utils
from core.concept import Concept
from core.tokenizer import Token
from parsers.BaseParser import Node
def json_default_converter(o):
@@ -40,6 +39,7 @@ def json_default_converter(o):
class SerializerContext:
user_name: str = None
origin: str = None
sheerka: object = None
class Serializer:
@@ -85,6 +85,7 @@ class Serializer:
raise TypeError(f"Don't know how to serialize {type(obj)}")
serializer = serializers[0]
self.log.debug(f"Serializing '{obj}' using '{serializer.name}'")
stream = io.BytesIO()
header = struct.pack(Serializer.HEADER_FORMAT, bytes(serializer.name, "utf-8"), serializer.version)
@@ -248,23 +249,23 @@ class DictionarySerializer(PickleSerializer):
class ExecutionContextSerializer(BaseSerializer):
CLASS_NAME = "core.sheerka.ExecutionContext.ExecutionContext"
def __init__(self):
BaseSerializer.__init__(self, "R", 1)
def matches(self, obj):
return core.utils.get_full_qualified_name(obj) == "core.sheerka.ExecutionContext.ExecutionContext"
return core.utils.get_full_qualified_name(obj) == self.CLASS_NAME
def dump(self, stream, obj, context):
as_json = obj.to_dict()
stream.write(json.dumps(as_json, default=json_default_converter).encode("utf-8"))
stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8"))
stream.seek(0)
return stream
def load(self, stream, context):
json_stream = stream.read().decode("utf-8")
json_message = json.loads(json_stream)
obj = core.utils.get_class("core.sheerka.ExecutionContext")()
obj.from_dict(json_message)
obj = sheerkapickle.decode(context.sheerka, json_stream)
#json_message = json.loads(json_stream)
return obj
#
src/sheerkapickle/SheerkaPickler.py
+132
View File
@@ -0,0 +1,132 @@
import json
from logging import Logger
import core.utils
from core.concept import Concept
from sheerkapickle import utils, tags, handlers
def encode(sheerka, obj):
pickler = SheerkaPickler(sheerka)
data = pickler.flatten(obj)
return json.dumps(data)
class ToReduce:
def __init__(self, predicate, get_value):
self.predicate = predicate
self.get_value = get_value
class SheerkaPickler:
"""
Json sheerkapickle
Inspired by jsonpickle (https://github.com/jsonpickle/jsonpickle)
which failed to work in my environment
"""
def __init__(self, sheerka):
self.ids = {}
self.objs = []
self.id_count = -1
self.sheerka = sheerka
self.to_reduce = []
self.to_reduce.append(ToReduce(lambda o: isinstance(o, Logger), lambda o: None))
from parsers.BaseParser import BaseParser
from evaluators.BaseEvaluator import BaseEvaluator
self.to_reduce.append(ToReduce(lambda o: isinstance(o, (BaseParser, BaseEvaluator)), lambda o: o.name))
def flatten(self, obj):
if utils.is_primitive(obj):
return obj
if utils.is_tuple(obj):
return {tags.TUPLE: [self.flatten(v) for v in obj]}
if utils.is_set(obj):
return {tags.SET: [self.flatten(v) for v in obj]}
if utils.is_list(obj):
return [self.flatten(v) for v in obj]
if utils.is_dictionary(obj):
return self._flatten_dict(obj)
if utils.is_enum(obj):
return self._flatten_enum(obj)
if utils.is_object(obj):
return self._flatten_obj_instance(obj)
raise Exception(f"Cannot flatten '{obj}'")
def _flatten_dict(self, obj):
data = {}
for k, v in obj.items():
if k is None:
k_str = "null"
elif utils.is_enum(k):
k_str = core.utils.get_full_qualified_name(k) + "." + k.name
elif isinstance(k, Concept):
k_str = f":c:{k.key}:{k.id}:"
else:
k_str = k
data[k_str] = self.flatten(v)
return data
def _flatten_enum(self, obj):
# check if the object was already seen
exists, _id = self.exist(obj)
if exists:
return {tags.ID: _id}
else:
self.id_count = self.id_count + 1
self.ids[id(obj)] = self.id_count
self.objs.append(obj)
data = {}
class_name = core.utils.get_full_qualified_name(obj)
data[tags.ENUM] = class_name + "." + obj.name
return data
def _flatten_obj_instance(self, obj):
for reduce in self.to_reduce:
if reduce.predicate(obj):
return reduce.get_value(obj)
# check if the object was already seen
exists, _id = self.exist(obj)
if exists:
return {tags.ID: _id}
else:
self.id_count = self.id_count + 1
self.ids[id(obj)] = self.id_count
self.objs.append(obj)
# flatten
data = {}
cls = obj.__class__ if hasattr(obj, '__class__') else type(obj)
class_name = utils.importable_name(cls)
data[tags.OBJECT] = class_name
handler = handlers.get(class_name)
if handler is not None:
return handler(self.sheerka, self).flatten(obj, data)
if hasattr(obj, "__dict__"):
for k, v in obj.__dict__.items():
data[k] = self.flatten(v)
return data
return None
def exist(self, obj):
for k, v in self.ids.items():
if k == id(obj):
return True, v
return False, None
src/sheerkapickle/SheerkaUnpickler.py
+105
View File
@@ -0,0 +1,105 @@
import json
import core.utils
from sheerkapickle import tags, utils, handlers
def decode(sheerka, obj):
return SheerkaUnpickler(sheerka).restore(json.loads(obj))
class SheerkaUnpickler:
def __init__(self, sheerka):
self.sheerka = sheerka
self.objs = []
def restore(self, obj):
if has_tag(obj, tags.ID):
return self._restore_id(obj)
if has_tag(obj, tags.TUPLE):
return self._restore_tuple(obj)
if has_tag(obj, tags.SET):
return self._restore_set(obj)
if has_tag(obj, tags.ENUM):
return self._restore_enum(obj)
if has_tag(obj, tags.OBJECT):
return self._restore_obj(obj)
if utils.is_list(obj):
return self._restore_list(obj)
if utils.is_dictionary(obj):
return self._restore_dict(obj)
return obj
def _restore_list(self, obj):
return [self.restore(v) for v in obj]
def _restore_tuple(self, obj):
return tuple([self.restore(v) for v in obj[tags.TUPLE]])
def _restore_set(self, obj):
return set([self.restore(v) for v in obj[tags.SET]])
def _restore_enum(self, obj):
instance = core.utils.decode_enum(obj[tags.ENUM])
self.objs.append(instance)
return instance
def _restore_dict(self, obj):
data = {}
for k, v in obj.items():
resolved_key = self._resolve_key(k)
data[resolved_key] = self.restore(v)
return data
def _restore_id(self, obj):
try:
return self.objs[obj[tags.ID]]
except IndexError:
pass
def _restore_obj(self, obj):
handler = handlers.get(obj[tags.OBJECT])
if handler:
handler = handler(self.sheerka, self)
instance = handler.new(obj)
self.objs.append(instance)
instance = handler.restore(obj, instance)
else:
cls = core.utils.get_class(obj[tags.OBJECT])
instance = cls.__new__(cls)
self.objs.append(instance)
for k, v in obj.items():
if k == tags.OBJECT:
continue
value = self.restore(v)
setattr(instance, k, value)
return instance
def _resolve_key(self, key):
if key == "null":
return None
concept_key, concept_id = core.utils.decode_concept(key)
if concept_key is not None:
return self.sheerka.new((concept_key, concept_id)) if concept_id else self.sheerka.new(concept_key)
as_enum = core.utils.decode_enum(key)
if as_enum is not None:
return as_enum
return key
def has_tag(obj, tag):
return type(obj) is dict and tag in obj
src/sheerkapickle/__init__.py
+7
View File
@@ -0,0 +1,7 @@
from .SheerkaPickler import encode
from .SheerkaUnpickler import decode
__all__ = ('encode', 'decode')
# register built-in handlers
__import__('sheerkapickle.handlers', level=0)
src/sheerkapickle/handlers.py
+233
View File
@@ -0,0 +1,233 @@
import datetime
import re
import threading
import uuid
from sheerkapickle import utils
class ToReduce:
def __init__(self, predicate, get_value):
self.predicate = predicate
self.get_value = get_value
class SheerkaRegistry(object):
def __init__(self):
self._handlers = {}
self._base_handlers = {}
def get(self, cls_or_name, default=None):
"""
:param cls_or_name: the type or its fully qualified name
:param default: default value, if a matching handler is not found
Looks up a handler by type reference or its fully
qualified name. If a direct match
is not found, the search is performed over all
handlers registered with base=True.
"""
handler = self._handlers.get(cls_or_name)
# attempt to find a base class
if handler is None and utils.is_type(cls_or_name):
for cls, base_handler in self._base_handlers.items():
if issubclass(cls_or_name, cls):
return base_handler
return default if handler is None else handler
def register(self, cls, handler=None, base=False):
"""Register the a custom handler for a class
:param cls: The custom object class to handle
:param handler: The custom handler class (if
None, a decorator wrapper is returned)
:param base: Indicates whether the handler should
be registered for all subclasses
This function can be also used as a decorator
by omitting the `handler` argument::
@jsonpickle.handlers.register(Foo, base=True)
class FooHandler(jsonpickle.handlers.BaseHandler):
pass
"""
if handler is None:
def _register(handler_cls):
self.register(cls, handler=handler_cls, base=base)
return handler_cls
return _register
if not utils.is_type(cls):
raise TypeError('{!r} is not a class/type'.format(cls))
# store both the name and the actual type for the ugly cases like
# _sre.SRE_Pattern that cannot be loaded back directly
self._handlers[utils.importable_name(cls)] = \
self._handlers[cls] = handler
if base:
# only store the actual type for subclass checking
self._base_handlers[cls] = handler
def unregister(self, cls):
self._handlers.pop(cls, None)
self._handlers.pop(utils.importable_name(cls), None)
self._base_handlers.pop(cls, None)
registry = SheerkaRegistry()
register = registry.register
unregister = registry.unregister
get = registry.get
class BaseHandler(object):
def __init__(self, sheerka, context):
"""
Initialize a new handler to handle a registered type.
:Parameters:
- `context`: reference to pickler/unpickler
"""
self.sheerka = sheerka
self.context = context
def __call__(self, sheerka, context):
"""This permits registering either Handler instances or classes
:Parameters:
- `context`: reference to pickler/unpickler
"""
self.sheerka = sheerka
self.context = context
return self
def flatten(self, obj, data):
"""
Flatten `obj` into a json-friendly form and write result to `data`.
:param object obj: The object to be serialized.
:param dict data: A partially filled dictionary which will contain the
json-friendly representation of `obj` once this method has
finished.
"""
raise NotImplementedError('You must implement flatten() in %s' %
self.__class__)
def new(self, data):
raise NotImplementedError('You must implement new() in %s' %
self.__class__)
def restore(self, data, instance):
"""
Restore an object of the registered type from the json-friendly
representation `obj` and return it.
"""
raise NotImplementedError('You must implement restore() in %s' %
self.__class__)
@classmethod
def handles(self, cls):
"""
Register this handler for the given class. Suitable as a decorator,
e.g.::
@MyCustomHandler.handles
class MyCustomClass:
def __reduce__(self):
...
"""
registry.register(cls, self)
return cls
# class DatetimeHandler(BaseHandler):
# """Custom handler for datetime objects
#
# Datetime objects use __reduce__, and they generate binary strings encoding
# the payload. This handler encodes that payload to reconstruct the
# object.
#
# """
#
# def flatten(self, obj, data):
# pickler = self.context
# if not pickler.unpicklable:
# return str(obj)
# cls, args = obj.__reduce__()
# flatten = pickler.flatten
# payload = utils.b64encode(args[0])
# args = [payload] + [flatten(i, reset=False) for i in args[1:]]
# data['__reduce__'] = (flatten(cls, reset=False), args)
# return data
#
# def restore(self, data):
# cls, args = data['__reduce__']
# unpickler = self.context
# restore = unpickler.restore
# cls = restore(cls, reset=False)
# value = utils.b64decode(args[0])
# params = (value,) + tuple([restore(i, reset=False) for i in args[1:]])
# return cls.__new__(cls, *params)
#
#
# DatetimeHandler.handles(datetime.datetime)
# DatetimeHandler.handles(datetime.date)
# DatetimeHandler.handles(datetime.time)
class RegexHandler(BaseHandler):
"""Flatten _sre.SRE_Pattern (compiled regex) objects"""
def flatten(self, obj, data):
data['pattern'] = obj.pattern
return data
def new(self, data):
return re.compile(data['pattern'])
def restore(self, data, instance):
return instance
RegexHandler.handles(type(re.compile('')))
class UUIDHandler(BaseHandler):
"""Serialize uuid.UUID objects"""
def flatten(self, obj, data):
data['hex'] = obj.hex
return data
def new(self, data):
return uuid.UUID(data['hex'])
def restore(self, data, instance):
return instance
UUIDHandler.handles(uuid.UUID)
class LockHandler(BaseHandler):
"""Serialize threading.Lock objects"""
def flatten(self, obj, data):
data['locked'] = obj.locked()
return data
def new(self, data):
lock = threading.Lock()
if data.get('locked', False):
lock.acquire()
return lock
def restore(self, data, instance):
return instance
_lock = threading.Lock()
LockHandler.handles(_lock.__class__)
src/sheerkapickle/sheerka_handlers.py
+182
View File
@@ -0,0 +1,182 @@
from core.builtin_concepts import UserInputConcept, ReturnValueConcept, BuiltinConcepts
from core.sheerka.Sheerka import Sheerka
from evaluators.BaseEvaluator import BaseEvaluator
from parsers.BaseParser import BaseParser
from sheerkapickle.handlers import BaseHandler, registry
from core.concept import Concept, PROPERTIES_TO_SERIALIZE as CONCEPT_PROPERTIES_TO_SERIALIZE, ConceptParts
from core.sheerka.ExecutionContext import ExecutionContext, PROPERTIES_TO_SERIALIZE as CONTEXT_PROPERTIES_TO_SERIALIZE
default_concept = Concept()
CONCEPT_ID = "concept/id"
class ConceptHandler(BaseHandler):
def flatten(self, obj: Concept, data):
pickler = self.context
sheerka = self.sheerka
if obj.id:
ref = sheerka.get_by_id(obj.id)
data[CONCEPT_ID] = (obj.key, obj.id)
else:
ref = default_concept
# transform metadata
for prop in CONCEPT_PROPERTIES_TO_SERIALIZE:
value = getattr(obj.metadata, prop)
ref_value = getattr(ref.metadata, prop)
if value != ref_value:
data["meta." + prop] = pickler.flatten(value)
# transform value
for metadata, value in obj.values.items():
ref_value = ref.values[metadata] if metadata in ref.values else None
if value != ref_value:
data[metadata.value] = pickler.flatten(value)
# transform properties
for prop in obj.props:
value = obj.props[prop].value
if prop not in ref.props or value != ref.props[prop].value:
if "props" not in data:
data["props"] = []
data["props"].append((prop, pickler.flatten(value)))
return data
def new(self, data):
sheerka = self.sheerka
return sheerka.new(tuple(data[CONCEPT_ID])) if CONCEPT_ID in data else Concept()
def restore(self, data, instance):
pickler = self.context
for key, value in data.items():
if key.startswith("_sheerka/") or key == CONCEPT_ID:
continue
resolved_value = pickler.restore(data[key])
if key.startswith("meta."):
# get metadata
resolved_prop = key[5:]
if resolved_prop == "props":
for prop_name, prop_value in resolved_value:
instance.def_prop(prop_name, prop_value)
else:
setattr(instance.metadata, resolved_prop, resolved_value)
elif key == "props":
# get properties
for prop_name, prop_value in resolved_value:
instance.set_prop(prop_name, prop_value)
else:
# get value
instance.set_metadata_value(ConceptParts(key), resolved_value)
return instance
class UserInputHandler(BaseHandler):
def flatten(self, obj: UserInputConcept, data):
data[CONCEPT_ID] = (obj.key, obj.id)
data["user_name"] = obj.user_name
data["text"] = BaseParser.get_text_from_tokens(obj.text) if isinstance(obj.text, list) else obj.text
return data
def new(self, data):
sheerka = self.sheerka
instance = sheerka.new(tuple(data[CONCEPT_ID]), body=data["text"], user_name=data["user_name"])
return instance
def restore(self, data, instance):
return instance
class ReturnValueHandler(BaseHandler):
def flatten(self, obj: ReturnValueConcept, data):
pickler = self.context
data["who"] = f"c:{obj.who.id}:" if isinstance(obj.who, Concept) else \
obj.who.name if isinstance(obj.who, (BaseParser, BaseEvaluator)) else \
obj.who
data["status"] = obj.status
data["value"] = pickler.flatten(obj.value)
if obj.parents:
data["parents"] = pickler.flatten(obj.parents)
return data
def new(self, data):
sheerka = self.sheerka
instance = sheerka.ret(data["who"], data["status"], None)
return instance
def restore(self, data, instance):
pickler = self.context
instance.value = pickler.restore(data["value"])
if "parents" in data:
instance.parents = pickler.restore(data["parents"])
return instance
# class BuiltinConceptsHandler(BaseHandler):
#
# def flatten(self, obj: BuiltinConcepts, data):
# return data
#
# def restore(self, obj):
# pass
class SheerkaHandler(BaseHandler):
def flatten(self, obj: BuiltinConcepts, data):
return data
def new(self, data):
return self.sheerka
def restore(self, data, instance):
return instance
class ExecutionContextHandler(BaseHandler):
def flatten(self, obj, data):
pickler = self.context
for prop in CONTEXT_PROPERTIES_TO_SERIALIZE:
if prop == "who":
value = str(getattr(obj, prop))
else:
value = getattr(obj, prop)
if value is not None:
data[prop] = pickler.flatten(value)
return data
def new(self, data):
return ExecutionContext(data["who"], None, None)
def restore(self, data, instance):
pickler = self.context
for prop in CONTEXT_PROPERTIES_TO_SERIALIZE:
if prop not in data or prop == "who":
continue
setattr(instance, prop, pickler.restore(data[prop]))
return instance
def initialize_pickle_handlers():
registry.register(Concept, ConceptHandler, True)
registry.register(UserInputConcept, UserInputHandler, True)
registry.register(ReturnValueConcept, ReturnValueHandler, True)
registry.register(Sheerka, SheerkaHandler, True)
registry.register(ExecutionContext, ExecutionContextHandler, True)
src/sheerkapickle/tags.py
+5
View File
@@ -0,0 +1,5 @@
ID = "_sheerka/id"
TUPLE = "_sheerka/tuple"
SET = "_sheerka/set"
OBJECT = "_sheerka/obj"
ENUM = "_sheerka/enum"
src/sheerkapickle/utils.py
+85
View File
@@ -0,0 +1,85 @@
import base64
import types
from enum import Enum
class_types = (type,)
PRIMITIVES = (str, bool, type(None), int, float)
def is_type(obj):
"""Returns True is obj is a reference to a type.
"""
# use "isinstance" and not "is" to allow for metaclasses
return isinstance(obj, class_types)
def is_enum(obj):
return isinstance(obj, Enum)
def is_object(obj):
"""Returns True is obj is a reference to an object instance."""
return (isinstance(obj, object) and
not isinstance(obj, (type, types.FunctionType,
types.BuiltinFunctionType)))
def is_primitive(obj):
return type(obj) in PRIMITIVES
def is_dictionary(obj):
return type(obj) is dict
def is_list(obj):
return type(obj) is list
def is_set(obj):
return type(obj) is set
def is_bytes(obj):
return type(obj) is bytes
def is_tuple(obj):
return type(obj) is tuple
def b64encode(data):
"""
Encode binary data to ascii text in base64. Data must be bytes.
"""
return base64.b64encode(data).decode('ascii')
def translate_module_name(module):
"""Rename builtin modules to a consistent module name.
Prefer the more modern naming.
This is used so that references to Python's `builtins` module can
be loaded in both Python 2 and 3. We remap to the "__builtin__"
name and unmap it when importing.
Map the Python2 `exceptions` module to `builtins` because
`builtins` is a superset and contains everything that is
available in `exceptions`, which makes the translation simpler.
See untranslate_module_name() for the reverse operation.
"""
lookup = dict(__builtin__='builtins', exceptions='builtins')
return lookup.get(module, module)
def importable_name(cls):
"""
Fully qualified name (prefixed by builtin when needed)
"""
# Use the fully-qualified name if available (Python >= 3.3)
name = getattr(cls, '__qualname__', cls.__name__)
module = translate_module_name(cls.__module__)
return '{}.{}'.format(module, name)
tests/core/test_ExecutionContext.py
+1 -1
View File
@@ -2,7 +2,7 @@ import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.Sheerka import ExecutionContext
from core.sheerka.ExecutionContext import ExecutionContext
from sdp.sheerkaDataProvider import Event
tests/core/test_SheerkaHistoryManager.py
+52
View File
@@ -0,0 +1,52 @@
from core.sheerka.Services.SheerkaHistoryManager import hist
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class TestSheerkaHistoryManager(TestUsingMemoryBasedSheerka):
def test_i_can_retrieve_history(self):
sheerka = self.get_sheerka(skip_builtins_in_db=False)
sheerka.evaluate_user_input("def concept one as 1")
sheerka.evaluate_user_input("one")
sheerka.evaluate_user_input("xxx")
sheerka.evaluate_user_input("def concept two as 2")
sheerka.evaluate_user_input("two")
sheerka.evaluate_user_input("def concept three as 3")
sheerka.evaluate_user_input("three")
sheerka.evaluate_user_input("def concept four as 4")
sheerka.evaluate_user_input("four")
sheerka.evaluate_user_input("def concept five as 5")
sheerka.evaluate_user_input("five")
h = list(sheerka.history(-1)) # all
assert h == [
hist("five", True),
hist("def concept five as 5", True),
hist("four", True),
hist("def concept four as 4", True),
hist("three", True),
hist("def concept three as 3", True),
hist("two", True),
hist("def concept two as 2", True),
hist("xxx", False),
hist("one", True),
hist("def concept one as 1", True),
hist("Initializing Sheerka.", True)]
h = list(sheerka.history(2))
assert h == [
hist("two", True),
hist("def concept two as 2", True)
]
h = list(sheerka.history(2, 2))
assert h == [
hist("xxx", False),
hist("one", True),
]
h = list(sheerka.history(-1))
assert h == [
hist("xxx", False),
hist("one", True),
]
tests/core/test_sheerka_transform.py
-318
View File
@@ -1,318 +0,0 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY
from sdp.sheerkaDataProvider import Event
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class TestSheerkaTransform(TestUsingMemoryBasedSheerka):
def test_i_can_transform_an_unknown_concept(self):
sheerka = self.get_sheerka()
foo = Concept("foo", body="body")
concept_with_sub = Concept("concept_with_sub", body=foo)
concept = Concept(
name="concept_name",
is_builtin=True,
is_unique=True,
key="concept_key",
body=concept_with_sub,
where=[foo, 1, "1", True, 1.0],
pre=foo,
post=None, # will not appear
definition="it is a definition",
definition_type="def type",
desc="this this the desc"
).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub")
# add values and props
concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init()
concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0]
concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init()
concept.values[ConceptParts.POST] = "a value for POST"
concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub)
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(concept)
assert to_dict == {
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: 0,
'meta.name': 'concept_name',
'meta.key': 'concept_key',
'meta.is_builtin': True,
'meta.is_unique': True,
'meta.definition': 'it is a definition',
'meta.definition_type': 'def type',
'meta.desc': 'this this the desc',
'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: 1,
'meta.body': 'body',
'meta.name': 'foo'}, 1, '1', True, 1.0],
'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1},
'meta.body': {
'__type__': SheerkaTransformType.Concept,
'__id__': 2,
'meta.name': 'concept_with_sub',
'meta.body': {
'__type__': SheerkaTransformType.Reference,
'__id__': 1}},
'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']],
'pre': {'__type__': SheerkaTransformType.Concept,
'__id__': 4,
'meta.body': 'body',
'meta.name': 'foo',
'body': 'body'},
'post': "a value for POST",
'body': {'__type__': SheerkaTransformType.Concept,
'__id__': 3,
'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference},
'meta.name': 'concept_with_sub',
'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}},
'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0],
'props': [('a', 10),
('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}),
('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})],
}
def test_i_can_transform_unknown_concept_with_almost_same_value(self):
sheerka = self.get_sheerka()
concept = Concept("foo")
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(concept)
assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'}
def test_i_can_transform_known_concept_when_the_values_are_the_same(self):
"""
Values are the same means that we are serializing a concept which has kept all its default values
There is not diff between the concept to serialize and the one which was registered with create_new_concept()
We serialize only the id of the concept
:return:
"""
sheerka = self.get_sheerka()
concept = Concept(
name="concept_name",
is_builtin=True,
is_unique=False,
key="concept_key",
body="body definition",
where="where definition",
pre="pre definition",
post="post definition",
definition="it is a definition",
definition_type="def type",
desc="this this the desc"
).def_prop("a").def_prop("b")
sheerka.create_new_concept(self.get_context(sheerka), concept)
new_concept = sheerka.new(concept.key)
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(new_concept)
assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"}
def test_i_can_transform_known_concept_when_the_values_are_different(self):
"""
Values are the different means the concept was modified.
It's different from the one which was registered with create_new_concept()
We serialize only the differences
:return:
"""
sheerka = self.get_sheerka()
concept = Concept(
name="concept_name",
is_builtin=True,
is_unique=False,
key="concept_key",
body="body definition",
where="where definition",
pre="pre definition",
post="post definition",
definition="it is a definition",
definition_type="def type",
desc="this this the desc"
).def_prop("a").def_prop("b")
sheerka.create_new_concept(self.get_context(sheerka), concept)
new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre")
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(new_concept)
assert to_dict == {
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: 0,
"id": "1001",
'pre': 'another pre',
"body": "another",
'props': [('a', 10)]
}
def test_i_can_transform_concept_with_circular_reference(self):
sheerka = self.get_sheerka()
foo = Concept("foo")
bar = Concept("bar", body=foo)
foo.metadata.body = bar
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(foo)
assert to_dict == {
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: 0,
'meta.name': 'foo',
'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: 1,
'meta.name': 'bar',
'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
OBJ_ID_KEY: 0},
},
}
def test_i_can_transform_concept_with_circular_reference_2(self):
sheerka = self.get_sheerka()
foo = Concept("foo")
bar = Concept("foo", body=foo)
foo.metadata.body = bar
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(foo)
assert to_dict == {
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: 0,
'meta.name': 'foo',
'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: 1,
'meta.name': 'foo',
'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
OBJ_ID_KEY: 0},
},
}
def test_i_can_transform_the_unknown_concept(self):
sheerka = self.get_sheerka(False)
unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(unknown)
assert len(to_dict) == 3
assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept
assert to_dict[OBJ_ID_KEY] == 0
assert "id" in to_dict
def test_i_can_transform_simple_execution_context(self):
sheerka = self.get_sheerka()
ExecutionContext.ids = {}
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(context)
assert to_dict == {
OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
OBJ_ID_KEY: 0,
'_parent': None,
'_id': 0,
'_tab': '',
'_bag': {},
'_start': 0,
'_stop': 0,
'who': 'requester',
'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'},
'desc': 'this is the desc',
'children': [],
'preprocess': None,
'inputs': {},
'values': {},
'obj': None,
'concepts': {}
}
def test_i_can_transform_list(self):
sheerka = self.get_sheerka()
ExecutionContext.ids = {}
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
st = SheerkaTransform(sheerka)
to_dict = st.to_dict([context])
assert len(to_dict) == 1
assert isinstance(to_dict, list)
assert to_dict[0]["who"] == "requester"
assert to_dict[0]["desc"] == "this is the desc"
def test_i_can_transform_set(self):
sheerka = self.get_sheerka()
ExecutionContext.ids = {}
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
st = SheerkaTransform(sheerka)
to_dict = st.to_dict({context})
assert len(to_dict) == 1
assert isinstance(to_dict, list)
assert to_dict[0]["who"] == "requester"
assert to_dict[0]["desc"] == "this is the desc"
def test_i_can_transform_dict(self):
sheerka = self.get_sheerka()
ExecutionContext.ids = {}
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key()
sheerka.create_new_concept(self.get_context(sheerka), known_concept)
unknown_concept = Concept("bar")
known = sheerka.new("foo")
bag = {
"context": context,
"known_concept": known_concept,
"unknown_concept": unknown_concept,
"True": True,
"Number": 1.1,
"String": "a string value",
"None": None,
unknown_concept: "hello",
known: "world"
}
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(bag)
assert isinstance(to_dict, dict)
assert to_dict['Number'] == 1.1
assert to_dict['String'] == 'a string value'
assert to_dict['True']
assert to_dict['None'] is None
assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
assert to_dict["known_concept"]["id"] == '1001'
assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
assert to_dict["(None)bar"] == "hello"
assert to_dict["(1001)foo"] == "world"
def test_i_can_transform_when_circular_references(self):
sheerka = self.get_sheerka()
ExecutionContext.ids = {}
context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
context.push("another requester", "another desc")
st = SheerkaTransform(sheerka)
to_dict = st.to_dict(context)
assert isinstance(to_dict, dict)
assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
assert len(to_dict["children"]) == 1
assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0
assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1
tests/core/test_utils.py
+32
View File
@@ -1,5 +1,6 @@
import core.utils
import pytest
from core.concept import ConceptParts
from core.tokenizer import Token, TokenKind
@@ -130,6 +131,37 @@ def test_i_can_escape():
assert actual == "hello \\'world\\' my friend"
@pytest.mark.parametrize("text, expected_key, expected_id", [
(None, None, None),
(10, None, None),
("", None, None),
("xxx", None, None),
(":c:", None, None),
(":c:key", None, None),
(":c:key:", "key", None),
(":c:key:id", None, None),
(":c:key:id:", "key", "id"),
])
def test_i_can_decode_concept_repr(text, expected_key, expected_id):
k, i = core.utils.decode_concept(text)
assert k == expected_key
assert i == expected_id
@pytest.mark.parametrize("text, expected", [
(None, None),
(10, None),
("", None),
("xxx", None),
("xxx.", None),
("xxx.yyy", None),
("core.concept.ConceptParts.BODY", ConceptParts.BODY),
])
def test_i_can_decode_enum(text, expected):
actual = core.utils.decode_enum(text)
assert actual == expected
def get_tokens(lst):
res = []
for e in lst:
tests/sdp/test_sheerkaDataProvider.py
+7 -4
View File
@@ -257,24 +257,27 @@ def test_i_can_load_events(root):
for i in range(15):
sdp.save_event(Event(f"Hello {i}"))
events = list(sdp.load_events(10))
events = list(sdp.load_events(10)) # first ten
assert len(events) == 10
assert events[0].message == "Hello 14"
assert events[9].message == "Hello 5"
events = list(sdp.load_events(10, 5))
events = list(sdp.load_events(10, 5)) # skip first 5, then take 10
assert len(events) == 10
assert events[0].message == "Hello 9"
assert events[9].message == "Hello 0"
events = list(sdp.load_events(20, 10))
events = list(sdp.load_events(20, 10)) # skip first 10, take 20,(but only 5 remaining)
assert len(events) == 5
assert events[0].message == "Hello 4"
assert events[4].message == "Hello 0"
events = list(sdp.load_events(1, 20))
events = list(sdp.load_events(1, 20)) # skip first 20, take one
assert len(events) == 0
events = list(sdp.load_events(0)) # all
assert len(events) == 15
@pytest.mark.parametrize("root", [
".sheerka",
tests/sheerkapickle/__init__.py
View File
tests/sheerkapickle/test_SheerkaPickler.py
+172
View File
@@ -0,0 +1,172 @@
import logging
import pytest
from core.concept import Concept, ConceptParts
from sheerkapickle import tags
from sheerkapickle.SheerkaPickler import SheerkaPickler
from sheerkapickle.SheerkaUnpickler import SheerkaUnpickler
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
class Obj:
def __init__(self, a, b, c):
self.a = a
self.b = b
self.c = c
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, Obj):
return False
return self.a == other.a and self.b == other.b and self.c == other.c
def __hash__(self):
return hash((self.a, self.b, self.c))
class TestSheerkaPickler(TestUsingFileBasedSheerka):
@pytest.mark.parametrize("obj, expected", [
(1, 1),
(3.14, 3.14),
("a string", "a string"),
(True, True),
(None, None),
([1, 3.14, "a string"], [1, 3.14, "a string"]),
((1, 3.14, "a string"), {tags.TUPLE: [1, 3.14, "a string"]}),
({1}, {tags.SET: [1]}),
({"a": "a", "b": 3.14, "c": True}, {"a": "a", "b": 3.14, "c": True}),
({1: "a", 2: 3.14, 3: True}, {1: "a", 2: 3.14, 3: True}),
([1, [3.14, "a string"]], [1, [3.14, "a string"]]),
([1, (3.14, "a string")], [1, {tags.TUPLE: [3.14, "a string"]}]),
([], []),
(ConceptParts.BODY, {tags.ENUM: "core.concept.ConceptParts.BODY"}),
])
def test_i_can_flatten_and_restore_primitives(self, obj, expected):
sheerka = self.get_sheerka()
flatten = SheerkaPickler(sheerka).flatten(obj)
assert flatten == expected
decoded = SheerkaUnpickler(sheerka).restore(flatten)
assert decoded == obj
def test_i_can_flatten_and_restore_instances(self):
sheerka = self.get_sheerka()
obj1 = Obj(1, "b", True)
obj2 = Obj(3.14, ("a", "b"), obj1)
flatten = SheerkaPickler(sheerka).flatten(obj2)
assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
'a': 3.14,
'b': {'_sheerka/tuple': ['a', 'b']},
'c': {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
'a': 1,
'b': 'b',
'c': True}}
decoded = SheerkaUnpickler(sheerka).restore(flatten)
assert decoded == obj2
def test_i_can_manage_circular_reference(self):
sheerka = self.get_sheerka()
obj1 = Obj(1, "b", True)
obj1.c = obj1
flatten = SheerkaPickler(sheerka).flatten(obj1)
assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
'a': 1,
'b': 'b',
'c': {'_sheerka/id': 0}}
decoded = SheerkaUnpickler(sheerka).restore(flatten)
assert decoded.a == obj1.a
assert decoded.b == obj1.b
assert decoded.c == decoded
def test_i_can_flatten_obj_with_new_props(self):
sheerka = self.get_sheerka()
obj = Obj(1, "b", True)
obj.z = "new prop"
flatten = SheerkaPickler(sheerka).flatten(obj)
assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
'a': 1,
'b': 'b',
'c': True,
'z': "new prop"}
decoded = SheerkaUnpickler(sheerka).restore(flatten)
assert decoded == obj
@pytest.mark.parametrize("obj, expected", [
({None: "a"}, {'null': "a"}),
({ConceptParts.BODY: "a"}, {'core.concept.ConceptParts.BODY': 'a'}),
({(1, 2): "a"}, {(1, 2): "a"}),
])
def test_i_can_manage_specific_keys_in_dictionaries(self, obj, expected):
sheerka = self.get_sheerka()
flatten = SheerkaPickler(sheerka).flatten(obj)
assert flatten == expected
decoded = SheerkaUnpickler(sheerka).restore(flatten)
assert decoded == obj
def test_i_can_use_concept_as_dictionary_key(self):
sheerka = self.get_sheerka()
concept = Concept("foo").init_key()
sheerka.set_id_if_needed(concept, False)
sheerka.add_in_cache(concept)
obj = {concept: "a"}
flatten = SheerkaPickler(sheerka).flatten(obj)
assert flatten == {':c:foo:1001:': 'a'}
decoded = SheerkaUnpickler(sheerka).restore(flatten)
assert decoded == obj
def test_i_can_manage_references(self):
sheerka = self.get_sheerka()
foo = Obj("foo", "bar", "baz")
obj = [ConceptParts.BODY, foo, ConceptParts.WHERE, ConceptParts.BODY, foo]
flatten = SheerkaPickler(sheerka).flatten(obj)
assert flatten == [{'_sheerka/enum': 'core.concept.ConceptParts.BODY'},
{'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
'a': 'foo',
'b': 'bar',
'c': 'baz'},
{'_sheerka/enum': 'core.concept.ConceptParts.WHERE'},
{'_sheerka/id': 0},
{'_sheerka/id': 1}]
decoded = SheerkaUnpickler(sheerka).restore(flatten)
assert decoded == obj
def test_serialize_concept(self):
sheerka = self.get_sheerka()
foo = Concept("doo")
flatten = SheerkaPickler(sheerka).flatten(foo)
restored = SheerkaUnpickler(sheerka).restore(flatten)
assert restored == foo
def test_i_do_not_encode_logger(self):
sheerka = self.get_sheerka()
logger = logging.getLogger("log_name")
logger2 = logging.getLogger("log_name2")
obj = Obj("foo", logger, {"a": logger, "b": logger2})
flatten = SheerkaPickler(sheerka).flatten(obj)
decoded = SheerkaUnpickler(sheerka).restore(flatten)
assert decoded == Obj("foo", None, {"a": None, "b": None})
tests/sheerkapickle/test_sheerka_handlers.py
+285
View File
@@ -0,0 +1,285 @@
import sheerkapickle
from core.builtin_concepts import BuiltinConcepts, UserInputConcept, ReturnValueConcept
from core.concept import Concept, ConceptParts
from core.sheerka.ExecutionContext import ExecutionContext
from core.tokenizer import Tokenizer
from evaluators.ConceptEvaluator import ConceptEvaluator
from parsers.DefaultParser import DefaultParser
from sdp.sheerkaDataProvider import Event
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
def test_i_can_encode_decode_unknown_concept_metadata(self):
sheerka = self.get_sheerka()
concept = Concept(name="foo", key="my_key")
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.key": "my_key"}'
assert decoded == concept
concept = Concept("foo", is_builtin=True, is_unique=True)
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.is_builtin": true, "meta.is_unique": true}'
concept = Concept("foo", body="my_body")
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "my_body"}'
concept = Concept("foo", pre="my_pre")
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.pre": "my_pre"}'
concept = Concept("foo", post="my_post")
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.post": "my_post"}'
concept = Concept("foo", where="my_where")
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.where": "my_where"}'
concept = Concept("foo").def_prop("a", "value_a").def_prop("b", "value_b")
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
def test_i_can_encode_decode_unknown_concept_values(self):
sheerka = self.get_sheerka()
concept = Concept("foo")
concept.values[ConceptParts.PRE] = 10 # an int
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "pre": 10}'
concept = Concept("foo")
concept.values[ConceptParts.POST] = 'a string' # an int
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "post": "a string"}'
concept = Concept("foo")
concept.values[ConceptParts.WHERE] = ['a string', 3.14] # a list
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": ["a string", 3.14]}'
concept = Concept("foo")
concept.values[ConceptParts.WHERE] = ('a string', 3.14) # a tuple
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": {"_sheerka/tuple": ["a string", 3.14]}}'
concept = Concept("foo")
concept.values[ConceptParts.BODY] = Concept("foo", body="foo_body")
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "foo_body"}}'
def test_i_can_encode_decode_unknown_concept_properties(self):
sheerka = self.get_sheerka()
concept = Concept("foo")
concept.set_prop("a", "value_a") # string
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "value_a"]]}'
concept = Concept("foo")
concept.set_prop("a", 10) # int
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", 10]]}'
concept = Concept("foo")
concept.set_prop("a", Concept("bar")) # another concept
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}'
concept = Concept("foo")
concept.set_prop("a", "a").set_prop("b", "b") # at least two props
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "a"], ["b", "b"]]}'
def test_i_can_encode_decode_known_concepts(self):
sheerka = self.get_sheerka()
ref_concept = Concept("my_name", True, True, "my_key", "my_body", "my_where", "my_pre", "my_post", "my_def")
ref_concept.def_prop("a", "value_a").def_prop("b", "value_b")
sheerka.create_new_concept(self.get_context(sheerka), ref_concept)
to_string = sheerkapickle.encode(sheerka, ref_concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == ref_concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"]}'
concept = Concept().update_from(sheerka.get_by_id(ref_concept.id))
concept.set_metadata_value(ConceptParts.BODY, Concept("bar"))
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"], "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}}'
def test_i_can_manage_reference_of_the_same_object(self):
sheerka = self.get_sheerka()
concept_ref = Concept("foo")
concept = Concept("bar")
concept.set_metadata_value(ConceptParts.PRE, concept_ref)
concept.set_metadata_value(ConceptParts.BODY, concept_ref)
to_string = sheerkapickle.encode(sheerka, concept)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == concept
assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "bar", "pre": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo"}, "body": {"_sheerka/id": 1}}'
def test_i_can_encode_decode_user_input(self):
sheerka = self.get_sheerka()
user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body="my_text", user_name="my_user_name")
to_string = sheerkapickle.encode(sheerka, user_input)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == user_input
assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "my_text"}'
def test_i_can_encode_decode_user_input_when_tokens(self):
sheerka = self.get_sheerka()
text = "I have 'a complicated' 10 text"
tokens = list(Tokenizer(text))
user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens, user_name="my_user_name")
to_string = sheerkapickle.encode(sheerka, user_input)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == UserInputConcept(text, "my_user_name")
assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "{text}"' + '}'
def test_i_can_encode_decode_return_value(self):
sheerka = self.get_sheerka()
ret_val = sheerka.ret("who", True, 10)
to_string = sheerkapickle.encode(sheerka, ret_val)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == ret_val
assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10}'
def test_i_can_encode_decode_return_value_with_parent(self):
sheerka = self.get_sheerka()
ret_val = sheerka.ret("who", True, 10)
ret_val_parent = sheerka.ret("parent_who", True, "10")
ret_val.parents = [ret_val_parent, ret_val_parent]
to_string = sheerkapickle.encode(sheerka, ret_val)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == ret_val
assert decoded.parents == ret_val.parents
parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]'
assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}'
def test_i_can_encode_decode_return_values_with_complex_body(self):
sheerka = self.get_sheerka()
ret_val = sheerka.ret("who", True, Concept("foo", body="bar"))
to_string = sheerkapickle.encode(sheerka, ret_val)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == ret_val
def test_i_can_encode_decode_return_values_from_concepts_parsers_or_evaluators(self):
sheerka = self.get_sheerka()
foo = Concept("foo")
sheerka.set_id_if_needed(foo, False)
ret_val = sheerka.ret(foo, True, 10)
to_string = sheerkapickle.encode(sheerka, ret_val)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == sheerka.ret("c:1001:", True, 10)
ret_val = sheerka.ret(DefaultParser(), True, 10)
to_string = sheerkapickle.encode(sheerka, ret_val)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == sheerka.ret("parsers.Default", True, 10)
ret_val = sheerka.ret(ConceptEvaluator(), True, 10)
to_string = sheerkapickle.encode(sheerka, ret_val)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == sheerka.ret("evaluators.Concept", True, 10)
def test_i_can_encode_decode_execution_context(self):
sheerka = self.get_sheerka()
context = ExecutionContext("who", Event("xxx"), sheerka, "my desc")
input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)]
context.inputs = {"a": input_list, "b": Concept("foo")}
context.values = {"c": input_list, "d": Concept("bar")}
context.obj = Concept("baz")
context.push("who3", "sub_child1")
context.push("who4", "sub_child2")
to_string = sheerkapickle.encode(sheerka, context)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == context
def test_complicated_execution_context(self):
sheerka = self.get_sheerka(skip_builtins_in_db=False)
text = "def concept one as 1"
execution_context = ExecutionContext("s", Event(), sheerka, f"Evaluating '{text}'")
user_input = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="n"))
reduce_requested = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
steps = [
BuiltinConcepts.BEFORE_PARSING,
BuiltinConcepts.PARSING,
BuiltinConcepts.AFTER_PARSING,
BuiltinConcepts.BEFORE_EVALUATION,
BuiltinConcepts.EVALUATION,
BuiltinConcepts.AFTER_EVALUATION
]
ret = sheerka.execute(execution_context, [user_input, reduce_requested], steps)
execution_context.add_values(return_values=ret)
to_string = sheerkapickle.encode(sheerka, execution_context)
decoded = sheerkapickle.decode(sheerka, to_string)
return_value = decoded.values["return_values"][0].value
assert sheerka.isinstance(return_value, BuiltinConcepts.NEW_CONCEPT)
def test_encode_simple_concept(self):
sheerka = self.get_sheerka()
foo = Concept("foo")
to_string = sheerkapickle.encode(sheerka, foo)
decoded = sheerkapickle.decode(sheerka, to_string)
assert decoded == foo