kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored ExecutionContext serialization (added sheerkapickle) and added History management

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-01-31 18:58:03 +01:00
parent fed0735eb9
commit b9afcba61f
31 changed files with 1546 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/__init__.py
View File
src/core/concept.py
+2
View File
@@ -343,6 +343,8 @@ class Concept:
:param metadata:
:return:
"""
if metadata not in self.values:
return None
return self.values[metadata]
def auto_init(self):
src/core/sheerka/ExecutionContext.py
+34 -5
View File
@@ -4,9 +4,22 @@ import time
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from sdp.sheerkaDataProvider import Event
from sheerkapickle.SheerkaPickler import SheerkaPickler
DEBUG_TAB_SIZE = 4
PROPERTIES_TO_SERIALIZE = ("_id",
"_bag",
"_start",
"_stop",
"who",
"desc",
"children",
"inputs",
"values",
"obj",
"concepts")
class ExecutionContext:
"""
@@ -31,7 +44,7 @@ class ExecutionContext:
**kwargs):
self._parent = None
self._id = ExecutionContext.get_id(event.get_digest())
self._id = ExecutionContext.get_id(event.get_digest()) if event else None
self._tab = ""
self._bag = {} # other variables
self._start = 0
@@ -90,6 +103,26 @@ class ExecutionContext:
msg += ")"
return msg
def __eq__(self, other):
if id(self) == id(other):
return True
if not isinstance(other, ExecutionContext):
return False
for prop in PROPERTIES_TO_SERIALIZE:
if prop == "who":
value = str(getattr(self, prop))
other_value = str(getattr(other, prop))
else:
value = getattr(self, prop)
other_value = getattr(other, prop)
if value != other_value:
return False
return True
def add_preprocess(self, name, **kwargs):
preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
preprocess.set_prop("name", name)
@@ -189,10 +222,6 @@ class ExecutionContext:
to_str = self.return_value_to_str(r)
logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
def to_dict(self):
from core.sheerka_transform import SheerkaTransform
st = SheerkaTransform(self.sheerka)
return st.to_dict(self)
@staticmethod
def return_value_to_str(r):
src/core/sheerka/SheerkaCreateNewConcept.py → src/core/sheerka/Services/SheerkaCreateNewConcept.py
View File
src/core/sheerka/SheerkaDump.py → src/core/sheerka/Services/SheerkaDump.py
+3 -2
View File
@@ -39,7 +39,8 @@ class SheerkaDump:
self.sheerka.log.info(f"name : {c.name}")
self.sheerka.log.info(f"bnf : {c.metadata.definition}")
self.sheerka.log.info(f"key : {c.key}")
self.sheerka.log.info(f"body : {c.body}")
self.sheerka.log.info(f"body : {c.metadata.body}")
self.sheerka.log.info(f"value : {c.body}")
self.sheerka.log.info(f"digest : {c.get_digest()}")
first = False
@@ -57,7 +58,7 @@ class SheerkaDump:
while True:
try:
if h.user != self.sheerka.name:
if h.event.user != self.sheerka.name:
self.sheerka.log.info(h)
count += 1
h = next(history)
src/core/sheerka/SheerkaEvaluateConcept.py → src/core/sheerka/Services/SheerkaEvaluateConcept.py
View File
src/core/sheerka/SheerkaExecute.py → src/core/sheerka/Services/SheerkaExecute.py
+1 -1
View File
@@ -218,7 +218,7 @@ class SheerkaExecute:
for step in execution_steps:
copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values]
with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context:
with execution_context.push(step=step, iteration=0, desc=f"{step=}") as sub_context:
sub_context.log(logger or self.sheerka.log, f"{step=}, context='{sub_context}'")
if step == BuiltinConcepts.PARSING:
src/core/sheerka/Services/SheerkaHistoryManager.py
+74
View File
@@ -0,0 +1,74 @@
from collections import namedtuple
from sdp.sheerkaDataProvider import Event
hist = namedtuple("History", "text status") # tests purposes only
class History:
def __init__(self, event: Event, result):
self.event = event
self.result = result
self._status = None
def __str__(self):
msg = f"{self.event.get_digest()} {self.event.date.strftime('%d/%m/%Y %H:%M:%S')} : {self.event.message}"
status = self.status
if status is not None:
msg += f" => {status}"
return msg
def __repr__(self):
return f"event={self.event!r}, status={self.status}, result={self.result}"
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, hist):
return self.event.message == other.text and self.status == other.status
if not isinstance(other, History):
return False
return self.event == other.event and self.result == other.result
@property
def status(self):
if self._status:
return self._status
if not self.result or "return_values" not in self.result.values:
return
if hasattr(self.result.values["return_values"], "__iter__"):
if len(self.result.values["return_values"]) != 1:
self._status = False
return self._status
else:
self._status = self.result.values["return_values"][0].status
return self._status
else:
self._status = self.result.values["return_values"].status
return self._status
class SheerkaHistoryManager:
def __init__(self, sheerka):
self.sheerka = sheerka
def history(self, depth_or_digest, start):
"""
Load history
:param depth_or_digest: number of items or digest
:param start:
:return:
"""
events = list(self.sheerka.sdp.load_events(depth_or_digest, start))
for event in events:
try:
result = self.sheerka.sdp.load_result(self.sheerka, event.get_digest())
except (IOError, KeyError):
result = None
yield History(event, result)
src/core/sheerka/SheerkaSetsManager.py → src/core/sheerka/Services/SheerkaSetsManager.py
View File
src/core/sheerka/Services/__init__.py
View File
src/core/sheerka/Sheerka.py
+36 -10
View File
@@ -2,11 +2,12 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConc
UnknownConcept
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.SheerkaCreateNewConcept import SheerkaCreateNewConcept
from core.sheerka.SheerkaDump import SheerkaDump
from core.sheerka.SheerkaEvaluateConcept import SheerkaEvaluateConcept
from core.sheerka.SheerkaExecute import SheerkaExecute
from core.sheerka.SheerkaSetsManager import SheerkaSetsManager
from core.sheerka.Services.SheerkaCreateNewConcept import SheerkaCreateNewConcept
from core.sheerka.Services.SheerkaDump import SheerkaDump
from core.sheerka.Services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
from core.sheerka.Services.SheerkaExecute import SheerkaExecute
from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager
from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
import core.utils
import core.builtin_helpers
@@ -21,7 +22,7 @@ import logging
# BuiltinConcepts.AFTER_EVALUATION]
CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
CONCEPTS_FILE = "_concepts.txt"
class Sheerka(Concept):
"""
@@ -81,6 +82,7 @@ class Sheerka(Concept):
self.dump_handler = SheerkaDump(self)
self.sets_handler = SheerkaSetsManager(self)
self.evaluate_concept_handler = SheerkaEvaluateConcept(self)
self.history_handler = SheerkaHistoryManager(self)
def initialize(self, root_folder: str = None):
"""
@@ -92,6 +94,9 @@ class Sheerka(Concept):
"""
try:
from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
initialize_pickle_handlers()
self.sdp = SheerkaDataProvider(root_folder)
if self.sdp.first_time:
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
@@ -104,11 +109,16 @@ class Sheerka(Concept):
self.initialize_builtin_parsers()
self.initialize_builtin_evaluators()
self.initialize_concepts_definitions(exec_context)
res = ReturnValueConcept(self, True, self)
exec_context.add_values(return_values=res)
if not self.skip_builtins_in_db:
self.sdp.save_result(self, exec_context)
except IOError as e:
return ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
return ReturnValueConcept(self, True, self)
return res
def initialize_builtin_concepts(self):
"""
@@ -232,7 +242,12 @@ class Sheerka(Concept):
execution_context.add_values(return_values=ret)
if not self.skip_builtins_in_db:
self.sdp.save_result(execution_context)
self.sdp.save_result(self, execution_context)
#hack to save valid concept definition
if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
with open(CONCEPTS_FILE, "a") as f:
f.write(text + "\n")
return ret
def execute(self, execution_context, return_values, execution_steps, logger=None):
@@ -549,8 +564,19 @@ class Sheerka(Concept):
def history(self, page=10, start=0):
"""Gets the history of all commands"""
return self.sdp.load_events(page, start)
return self.history_handler.history(page, start)
def restore(self):
"""
Restore the state with all previous valid concept definitions
:return:
"""
try:
with open(CONCEPTS_FILE, "r") as f:
for line in f.readlines():
self.evaluate_user_input(line)
except IOError:
pass
def test(self):
return f"I have access to Sheerka !"
src/core/sheerka_transform.py
-161
View File
@@ -1,161 +0,0 @@
import dataclasses
from enum import Enum
from core.concept import Concept, PROPERTIES_TO_SERIALIZE
from core.sheerka.Sheerka import ExecutionContext
from core.tokenizer import Token
from evaluators.BaseEvaluator import BaseEvaluator
from parsers.BaseParser import BaseParser, Node
from parsers.BnfParser import BnfParser
from parsers.ConceptLexerParser import UnrecognizedTokensNode, ParsingExpression
from parsers.PythonParser import PythonNode
from sdp.sheerkaDataProvider import Event
OBJ_TYPE_KEY = "__type__"
OBJ_ID_KEY = "__id__"
OBJ_NAME_KEY = "__name__"
default_concept = Concept()
class SheerkaTransformType(Enum):
Concept = 1
Reference = 2
ExecutionContext = 3
Event = 4
Node = 5
Exception = 6
def __repr__(self):
return self.__class__.__name__ + "." + self.name
class SheerkaTransform:
def __init__(self, sheerka):
self.ids = {}
self.sheerka = sheerka
self.id_count = -1
def to_dict(self, obj):
if isinstance(obj, (Concept, ExecutionContext, Event)):
exists, _id = self.exist(obj)
if exists:
return {
OBJ_TYPE_KEY: SheerkaTransformType.Reference,
OBJ_ID_KEY: _id
}
else:
self.id_count = self.id_count + 1
self.ids[obj] = self.id_count
if isinstance(obj, Concept):
return self.concept_to_dict(obj)
elif isinstance(obj, ExecutionContext):
return self.execution_context_to_dict(obj)
elif isinstance(obj, Event):
return {
OBJ_TYPE_KEY: SheerkaTransformType.Event,
OBJ_ID_KEY: self.id_count,
'digest': obj.get_digest()}
elif isinstance(obj, (BaseParser, BaseEvaluator, BnfParser)):
return obj.name
elif isinstance(obj, Token):
return obj.__dict__
elif isinstance(obj, PythonNode):
return {
OBJ_TYPE_KEY: SheerkaTransformType.Node,
OBJ_NAME_KEY: "PythonNode",
'source': obj.source,
'ast_': obj.get_dump(obj.ast_)
}
elif isinstance(obj, Node):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.Node,
OBJ_NAME_KEY: obj.__class__.__name__,
}
for k, v in obj.__dict__.items():
to_dict[k] = self.to_dict(v)
return to_dict
elif isinstance(obj, Exception):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.Exception,
OBJ_NAME_KEY: obj.__class__.__name__,
}
for k, v in obj.__dict__.items():
to_dict[k] = self.to_dict(v)
return to_dict
elif isinstance(obj, ParsingExpression):
return obj.__repr__()
elif isinstance(obj, dict):
return dict((str(k) if isinstance(k, Concept) else k, self.to_dict(v)) for k, v in obj.items())
elif hasattr(obj, "__iter__") and not isinstance(obj, str):
return list(self.to_dict(o) for o in obj)
else:
return obj
def concept_to_dict(self, obj: Concept):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.Concept,
OBJ_ID_KEY: self.id_count,
}
if obj.id:
ref = self.sheerka.get(obj.key, obj.id)
to_dict["id"] = obj.id
else:
ref = default_concept
# transform metadata
for prop in PROPERTIES_TO_SERIALIZE:
value = getattr(obj.metadata, prop)
ref_value = getattr(ref.metadata, prop)
if value != ref_value:
to_dict["meta." + prop] = self.to_dict(value)
# transform value
for metadata, value in obj.values.items():
ref_value = ref.values[metadata] if metadata in ref.values else None
if value != ref_value:
to_dict[metadata.value] = self.to_dict(value)
# transform properties
for prop in obj.props:
value = obj.props[prop].value
if prop not in ref.props or value != ref.props[prop].value:
if "props" not in to_dict:
to_dict["props"] = []
to_dict["props"].append((prop, self.to_dict(value)))
return to_dict
def execution_context_to_dict(self, obj: ExecutionContext):
to_dict = {
OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
OBJ_ID_KEY: self.id_count
}
for property_name in obj.__dict__:
if property_name == "sheerka":
continue
to_dict[property_name] = self.to_dict(getattr(obj, property_name))
return to_dict
def exist(self, obj):
for k, v in self.ids.items():
if id(k) == id(obj) or k == obj:
return True, v
return False, None
src/core/utils.py
+64
View File
@@ -228,6 +228,7 @@ def escape_char(text, to_escape):
return res
def pp(items):
if not hasattr(items, "__iter__"):
return str(items)
@@ -236,3 +237,66 @@ def pp(items):
return str(items)
return " \n" + " \n".join(str(item) for item in items)
def decode_concept(concept_repr):
"""
if concept_repr is like :c:key:id:
return the key and the id
:param concept_repr:
:return:
"""
if not (concept_repr and isinstance(concept_repr, str) and concept_repr.startswith(":c:")):
return None, None
i = 3
length = len(concept_repr)
key = ""
while i < length:
if concept_repr[i] == ":":
break
key += concept_repr[i]
i += 1
else:
return None, None
i += 1
if i >= length:
return key, None
id = ""
while i < length:
if concept_repr[i] == ":":
break
id += concept_repr[i]
i += 1
else:
return None, None
return key, id
def decode_enum(enum_repr: str):
"""
Tries to transform ClassName.Name into an enum
:param enum_repr:
:return:
"""
if not (enum_repr and isinstance(enum_repr, str)):
return None
try:
idx = enum_repr.rindex(".")
if idx == len(enum_repr):
return None
cls_name = enum_repr[:idx]
cls = get_class(cls_name)
name = enum_repr[idx + 1:]
return cls[name]
except ValueError:
return None
except TypeError:
return None
src/sdp/sheerkaDataProvider.py
+13 -6
View File
@@ -36,6 +36,9 @@ class Event(object):
def __str__(self):
return f"{self.date.strftime('%d/%m/%Y %H:%M:%S')} {self.message}"
def __repr__(self):
return f"{self.get_digest()[:12]} {self.message}"
def get_digest(self):
"""
Returns the digest of the event
@@ -64,6 +67,7 @@ class Event(object):
self.date = datetime.fromisoformat(as_dict["date"])
self.message = as_dict["message"]
self.parents = as_dict["parents"]
self._digest = as_dict["_digest"] # freeze the digest
class ObjToUpdate:
@@ -667,9 +671,9 @@ class SheerkaDataProvider:
:param event:
:return: digest of the event
"""
digest = event.get_digest()
parent = self.get_snapshot(SheerkaDataProvider.LastEventFile)
event.parents = [parent] if parent else None
digest = event.get_digest() # must be call after setting the parents
target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest)
if self.io.exists(target_path):
@@ -712,7 +716,7 @@ class SheerkaDataProvider:
digest = event.parents[0]
count = 0
while count < page_size:
while count < page_size or page_size <= 0:
event = self.load_event(digest)
if event is None:
return
@@ -725,12 +729,13 @@ class SheerkaDataProvider:
digest = event.parents[0]
count += 1
def save_result(self, execution_context):
def save_result(self, sheerka, execution_context):
"""
Save the execution context associated with an event
To make a long story short,
for every single user input, there is an event (which is the first thing that is created)
and a result (the ExecutionContext created by sheerka.evaluate_user_input()
:param sheerka:
:param execution_context:
:return:
"""
@@ -740,14 +745,16 @@ class SheerkaDataProvider:
if self.io.exists(target_path):
return digest
self.io.write_binary(target_path, self.serializer.serialize(execution_context, None).read())
context = SerializerContext(sheerka=sheerka)
self.io.write_binary(target_path, self.serializer.serialize(execution_context, context).read())
return digest
def load_result(self, digest):
def load_result(self, sheerka, digest):
target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + "_result"
with self.io.open(target_path, "rb") as f:
return self.serializer.deserialize(f, None)
context = SerializerContext(sheerka=sheerka)
return self.serializer.deserialize(f, context)
def save_state(self, state: State):
digest = state.get_digest()
src/sdp/sheerkaSerializer.py
+11 -10
View File
@@ -1,18 +1,17 @@
import dataclasses
import json
import pickle
import datetime
import pickle
import struct
import io
from dataclasses import dataclass
import sheerkapickle
from core.sheerka_logger import get_logger
from enum import Enum
import core.utils
from core.concept import Concept
from core.tokenizer import Token
from parsers.BaseParser import Node
def json_default_converter(o):
@@ -40,6 +39,7 @@ def json_default_converter(o):
class SerializerContext:
user_name: str = None
origin: str = None
sheerka: object = None
class Serializer:
@@ -85,6 +85,7 @@ class Serializer:
raise TypeError(f"Don't know how to serialize {type(obj)}")
serializer = serializers[0]
self.log.debug(f"Serializing '{obj}' using '{serializer.name}'")
stream = io.BytesIO()
header = struct.pack(Serializer.HEADER_FORMAT, bytes(serializer.name, "utf-8"), serializer.version)
@@ -248,23 +249,23 @@ class DictionarySerializer(PickleSerializer):
class ExecutionContextSerializer(BaseSerializer):
CLASS_NAME = "core.sheerka.ExecutionContext.ExecutionContext"
def __init__(self):
BaseSerializer.__init__(self, "R", 1)
def matches(self, obj):
return core.utils.get_full_qualified_name(obj) == "core.sheerka.ExecutionContext.ExecutionContext"
return core.utils.get_full_qualified_name(obj) == self.CLASS_NAME
def dump(self, stream, obj, context):
as_json = obj.to_dict()
stream.write(json.dumps(as_json, default=json_default_converter).encode("utf-8"))
stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8"))
stream.seek(0)
return stream
def load(self, stream, context):
json_stream = stream.read().decode("utf-8")
json_message = json.loads(json_stream)
obj = core.utils.get_class("core.sheerka.ExecutionContext")()
obj.from_dict(json_message)
obj = sheerkapickle.decode(context.sheerka, json_stream)
#json_message = json.loads(json_stream)
return obj
#
src/sheerkapickle/SheerkaPickler.py
+132
View File
@@ -0,0 +1,132 @@
import json
from logging import Logger
import core.utils
from core.concept import Concept
from sheerkapickle import utils, tags, handlers
def encode(sheerka, obj):
pickler = SheerkaPickler(sheerka)
data = pickler.flatten(obj)
return json.dumps(data)
class ToReduce:
def __init__(self, predicate, get_value):
self.predicate = predicate
self.get_value = get_value
class SheerkaPickler:
"""
Json sheerkapickle
Inspired by jsonpickle (https://github.com/jsonpickle/jsonpickle)
which failed to work in my environment
"""
def __init__(self, sheerka):
self.ids = {}
self.objs = []
self.id_count = -1
self.sheerka = sheerka
self.to_reduce = []
self.to_reduce.append(ToReduce(lambda o: isinstance(o, Logger), lambda o: None))
from parsers.BaseParser import BaseParser
from evaluators.BaseEvaluator import BaseEvaluator
self.to_reduce.append(ToReduce(lambda o: isinstance(o, (BaseParser, BaseEvaluator)), lambda o: o.name))
def flatten(self, obj):
if utils.is_primitive(obj):
return obj
if utils.is_tuple(obj):
return {tags.TUPLE: [self.flatten(v) for v in obj]}
if utils.is_set(obj):
return {tags.SET: [self.flatten(v) for v in obj]}
if utils.is_list(obj):
return [self.flatten(v) for v in obj]
if utils.is_dictionary(obj):
return self._flatten_dict(obj)
if utils.is_enum(obj):
return self._flatten_enum(obj)
if utils.is_object(obj):
return self._flatten_obj_instance(obj)
raise Exception(f"Cannot flatten '{obj}'")
def _flatten_dict(self, obj):
data = {}
for k, v in obj.items():
if k is None:
k_str = "null"
elif utils.is_enum(k):
k_str = core.utils.get_full_qualified_name(k) + "." + k.name
elif isinstance(k, Concept):
k_str = f":c:{k.key}:{k.id}:"
else:
k_str = k
data[k_str] = self.flatten(v)
return data
def _flatten_enum(self, obj):
# check if the object was already seen
exists, _id = self.exist(obj)
if exists:
return {tags.ID: _id}
else:
self.id_count = self.id_count + 1
self.ids[id(obj)] = self.id_count
self.objs.append(obj)
data = {}
class_name = core.utils.get_full_qualified_name(obj)
data[tags.ENUM] = class_name + "." + obj.name
return data
def _flatten_obj_instance(self, obj):
for reduce in self.to_reduce:
if reduce.predicate(obj):
return reduce.get_value(obj)
# check if the object was already seen
exists, _id = self.exist(obj)
if exists:
return {tags.ID: _id}
else:
self.id_count = self.id_count + 1
self.ids[id(obj)] = self.id_count
self.objs.append(obj)
# flatten
data = {}
cls = obj.__class__ if hasattr(obj, '__class__') else type(obj)
class_name = utils.importable_name(cls)
data[tags.OBJECT] = class_name
handler = handlers.get(class_name)
if handler is not None:
return handler(self.sheerka, self).flatten(obj, data)
if hasattr(obj, "__dict__"):
for k, v in obj.__dict__.items():
data[k] = self.flatten(v)
return data
return None
def exist(self, obj):
for k, v in self.ids.items():
if k == id(obj):
return True, v
return False, None
src/sheerkapickle/SheerkaUnpickler.py
+105
View File
@@ -0,0 +1,105 @@
import json
import core.utils
from sheerkapickle import tags, utils, handlers
def decode(sheerka, obj):
return SheerkaUnpickler(sheerka).restore(json.loads(obj))
class SheerkaUnpickler:
def __init__(self, sheerka):
self.sheerka = sheerka
self.objs = []
def restore(self, obj):
if has_tag(obj, tags.ID):
return self._restore_id(obj)
if has_tag(obj, tags.TUPLE):
return self._restore_tuple(obj)
if has_tag(obj, tags.SET):
return self._restore_set(obj)
if has_tag(obj, tags.ENUM):
return self._restore_enum(obj)
if has_tag(obj, tags.OBJECT):
return self._restore_obj(obj)
if utils.is_list(obj):
return self._restore_list(obj)
if utils.is_dictionary(obj):
return self._restore_dict(obj)
return obj
def _restore_list(self, obj):
return [self.restore(v) for v in obj]
def _restore_tuple(self, obj):
return tuple([self.restore(v) for v in obj[tags.TUPLE]])
def _restore_set(self, obj):
return set([self.restore(v) for v in obj[tags.SET]])
def _restore_enum(self, obj):
instance = core.utils.decode_enum(obj[tags.ENUM])
self.objs.append(instance)
return instance
def _restore_dict(self, obj):
data = {}
for k, v in obj.items():
resolved_key = self._resolve_key(k)
data[resolved_key] = self.restore(v)
return data
def _restore_id(self, obj):
try:
return self.objs[obj[tags.ID]]
except IndexError:
pass
def _restore_obj(self, obj):
handler = handlers.get(obj[tags.OBJECT])
if handler:
handler = handler(self.sheerka, self)
instance = handler.new(obj)
self.objs.append(instance)
instance = handler.restore(obj, instance)
else:
cls = core.utils.get_class(obj[tags.OBJECT])
instance = cls.__new__(cls)
self.objs.append(instance)
for k, v in obj.items():
if k == tags.OBJECT:
continue
value = self.restore(v)
setattr(instance, k, value)
return instance
def _resolve_key(self, key):
if key == "null":
return None
concept_key, concept_id = core.utils.decode_concept(key)
if concept_key is not None:
return self.sheerka.new((concept_key, concept_id)) if concept_id else self.sheerka.new(concept_key)
as_enum = core.utils.decode_enum(key)
if as_enum is not None:
return as_enum
return key
def has_tag(obj, tag):
return type(obj) is dict and tag in obj
src/sheerkapickle/__init__.py
+7
View File
@@ -0,0 +1,7 @@
from .SheerkaPickler import encode
from .SheerkaUnpickler import decode
__all__ = ('encode', 'decode')
# register built-in handlers
__import__('sheerkapickle.handlers', level=0)
src/sheerkapickle/handlers.py
+233
View File
@@ -0,0 +1,233 @@
import datetime
import re
import threading
import uuid
from sheerkapickle import utils
class ToReduce:
def __init__(self, predicate, get_value):
self.predicate = predicate
self.get_value = get_value
class SheerkaRegistry(object):
def __init__(self):
self._handlers = {}
self._base_handlers = {}
def get(self, cls_or_name, default=None):
"""
:param cls_or_name: the type or its fully qualified name
:param default: default value, if a matching handler is not found
Looks up a handler by type reference or its fully
qualified name. If a direct match
is not found, the search is performed over all
handlers registered with base=True.
"""
handler = self._handlers.get(cls_or_name)
# attempt to find a base class
if handler is None and utils.is_type(cls_or_name):
for cls, base_handler in self._base_handlers.items():
if issubclass(cls_or_name, cls):
return base_handler
return default if handler is None else handler
def register(self, cls, handler=None, base=False):
"""Register the a custom handler for a class
:param cls: The custom object class to handle
:param handler: The custom handler class (if
None, a decorator wrapper is returned)
:param base: Indicates whether the handler should
be registered for all subclasses
This function can be also used as a decorator
by omitting the `handler` argument::
@jsonpickle.handlers.register(Foo, base=True)
class FooHandler(jsonpickle.handlers.BaseHandler):
pass
"""
if handler is None:
def _register(handler_cls):
self.register(cls, handler=handler_cls, base=base)
return handler_cls
return _register
if not utils.is_type(cls):
raise TypeError('{!r} is not a class/type'.format(cls))
# store both the name and the actual type for the ugly cases like
# _sre.SRE_Pattern that cannot be loaded back directly
self._handlers[utils.importable_name(cls)] = \
self._handlers[cls] = handler
if base:
# only store the actual type for subclass checking
self._base_handlers[cls] = handler
def unregister(self, cls):
self._handlers.pop(cls, None)
self._handlers.pop(utils.importable_name(cls), None)
self._base_handlers.pop(cls, None)
registry = SheerkaRegistry()
register = registry.register
unregister = registry.unregister
get = registry.get
class BaseHandler(object):
def __init__(self, sheerka, context):
"""
Initialize a new handler to handle a registered type.
:Parameters:
- `context`: reference to pickler/unpickler
"""
self.sheerka = sheerka
self.context = context
def __call__(self, sheerka, context):
"""This permits registering either Handler instances or classes
:Parameters:
- `context`: reference to pickler/unpickler
"""
self.sheerka = sheerka
self.context = context
return self
def flatten(self, obj, data):
"""
Flatten `obj` into a json-friendly form and write result to `data`.
:param object obj: The object to be serialized.
:param dict data: A partially filled dictionary which will contain the
json-friendly representation of `obj` once this method has
finished.
"""
raise NotImplementedError('You must implement flatten() in %s' %
self.__class__)
def new(self, data):
raise NotImplementedError('You must implement new() in %s' %
self.__class__)
def restore(self, data, instance):
"""
Restore an object of the registered type from the json-friendly
representation `obj` and return it.
"""
raise NotImplementedError('You must implement restore() in %s' %
self.__class__)
@classmethod
def handles(self, cls):
"""
Register this handler for the given class. Suitable as a decorator,
e.g.::
@MyCustomHandler.handles
class MyCustomClass:
def __reduce__(self):
...
"""
registry.register(cls, self)
return cls
# class DatetimeHandler(BaseHandler):
# """Custom handler for datetime objects
#
# Datetime objects use __reduce__, and they generate binary strings encoding
# the payload. This handler encodes that payload to reconstruct the
# object.
#
# """
#
# def flatten(self, obj, data):
# pickler = self.context
# if not pickler.unpicklable:
# return str(obj)
# cls, args = obj.__reduce__()
# flatten = pickler.flatten
# payload = utils.b64encode(args[0])
# args = [payload] + [flatten(i, reset=False) for i in args[1:]]
# data['__reduce__'] = (flatten(cls, reset=False), args)
# return data
#
# def restore(self, data):
# cls, args = data['__reduce__']
# unpickler = self.context
# restore = unpickler.restore
# cls = restore(cls, reset=False)
# value = utils.b64decode(args[0])
# params = (value,) + tuple([restore(i, reset=False) for i in args[1:]])
# return cls.__new__(cls, *params)
#
#
# DatetimeHandler.handles(datetime.datetime)
# DatetimeHandler.handles(datetime.date)
# DatetimeHandler.handles(datetime.time)
class RegexHandler(BaseHandler):
"""Flatten _sre.SRE_Pattern (compiled regex) objects"""
def flatten(self, obj, data):
data['pattern'] = obj.pattern
return data
def new(self, data):
return re.compile(data['pattern'])
def restore(self, data, instance):
return instance
RegexHandler.handles(type(re.compile('')))
class UUIDHandler(BaseHandler):
"""Serialize uuid.UUID objects"""
def flatten(self, obj, data):
data['hex'] = obj.hex
return data
def new(self, data):
return uuid.UUID(data['hex'])
def restore(self, data, instance):
return instance
UUIDHandler.handles(uuid.UUID)
class LockHandler(BaseHandler):
"""Serialize threading.Lock objects"""
def flatten(self, obj, data):
data['locked'] = obj.locked()
return data
def new(self, data):
lock = threading.Lock()
if data.get('locked', False):
lock.acquire()
return lock
def restore(self, data, instance):
return instance
_lock = threading.Lock()
LockHandler.handles(_lock.__class__)
src/sheerkapickle/sheerka_handlers.py
+182
View File
@@ -0,0 +1,182 @@
from core.builtin_concepts import UserInputConcept, ReturnValueConcept, BuiltinConcepts
from core.sheerka.Sheerka import Sheerka
from evaluators.BaseEvaluator import BaseEvaluator
from parsers.BaseParser import BaseParser
from sheerkapickle.handlers import BaseHandler, registry
from core.concept import Concept, PROPERTIES_TO_SERIALIZE as CONCEPT_PROPERTIES_TO_SERIALIZE, ConceptParts
from core.sheerka.ExecutionContext import ExecutionContext, PROPERTIES_TO_SERIALIZE as CONTEXT_PROPERTIES_TO_SERIALIZE
default_concept = Concept()
CONCEPT_ID = "concept/id"
class ConceptHandler(BaseHandler):
def flatten(self, obj: Concept, data):
pickler = self.context
sheerka = self.sheerka
if obj.id:
ref = sheerka.get_by_id(obj.id)
data[CONCEPT_ID] = (obj.key, obj.id)
else:
ref = default_concept
# transform metadata
for prop in CONCEPT_PROPERTIES_TO_SERIALIZE:
value = getattr(obj.metadata, prop)
ref_value = getattr(ref.metadata, prop)
if value != ref_value:
data["meta." + prop] = pickler.flatten(value)
# transform value
for metadata, value in obj.values.items():
ref_value = ref.values[metadata] if metadata in ref.values else None
if value != ref_value:
data[metadata.value] = pickler.flatten(value)
# transform properties
for prop in obj.props:
value = obj.props[prop].value
if prop not in ref.props or value != ref.props[prop].value:
if "props" not in data:
data["props"] = []
data["props"].append((prop, pickler.flatten(value)))
return data
def new(self, data):
sheerka = self.sheerka
return sheerka.new(tuple(data[CONCEPT_ID])) if CONCEPT_ID in data else Concept()
def restore(self, data, instance):
pickler = self.context
for key, value in data.items():
if key.startswith("_sheerka/") or key == CONCEPT_ID:
continue
resolved_value = pickler.restore(data[key])
if key.startswith("meta."):
# get metadata
resolved_prop = key[5:]
if resolved_prop == "props":
for prop_name, prop_value in resolved_value:
instance.def_prop(prop_name, prop_value)
else:
setattr(instance.metadata, resolved_prop, resolved_value)
elif key == "props":
# get properties
for prop_name, prop_value in resolved_value:
instance.set_prop(prop_name, prop_value)
else:
# get value
instance.set_metadata_value(ConceptParts(key), resolved_value)
return instance
class UserInputHandler(BaseHandler):
def flatten(self, obj: UserInputConcept, data):
data[CONCEPT_ID] = (obj.key, obj.id)
data["user_name"] = obj.user_name
data["text"] = BaseParser.get_text_from_tokens(obj.text) if isinstance(obj.text, list) else obj.text
return data
def new(self, data):
sheerka = self.sheerka
instance = sheerka.new(tuple(data[CONCEPT_ID]), body=data["text"], user_name=data["user_name"])
return instance
def restore(self, data, instance):
return instance
class ReturnValueHandler(BaseHandler):
def flatten(self, obj: ReturnValueConcept, data):
pickler = self.context
data["who"] = f"c:{obj.who.id}:" if isinstance(obj.who, Concept) else \
obj.who.name if isinstance(obj.who, (BaseParser, BaseEvaluator)) else \
obj.who
data["status"] = obj.status
data["value"] = pickler.flatten(obj.value)
if obj.parents:
data["parents"] = pickler.flatten(obj.parents)
return data
def new(self, data):
sheerka = self.sheerka
instance = sheerka.ret(data["who"], data["status"], None)
return instance
def restore(self, data, instance):
pickler = self.context
instance.value = pickler.restore(data["value"])
if "parents" in data:
instance.parents = pickler.restore(data["parents"])
return instance
# class BuiltinConceptsHandler(BaseHandler):
#
# def flatten(self, obj: BuiltinConcepts, data):
# return data
#
# def restore(self, obj):
# pass
class SheerkaHandler(BaseHandler):
def flatten(self, obj: BuiltinConcepts, data):
return data
def new(self, data):
return self.sheerka
def restore(self, data, instance):
return instance
class ExecutionContextHandler(BaseHandler):
def flatten(self, obj, data):
pickler = self.context
for prop in CONTEXT_PROPERTIES_TO_SERIALIZE:
if prop == "who":
value = str(getattr(obj, prop))
else:
value = getattr(obj, prop)
if value is not None:
data[prop] = pickler.flatten(value)
return data
def new(self, data):
return ExecutionContext(data["who"], None, None)
def restore(self, data, instance):
pickler = self.context
for prop in CONTEXT_PROPERTIES_TO_SERIALIZE:
if prop not in data or prop == "who":
continue
setattr(instance, prop, pickler.restore(data[prop]))
return instance
def initialize_pickle_handlers():
registry.register(Concept, ConceptHandler, True)
registry.register(UserInputConcept, UserInputHandler, True)
registry.register(ReturnValueConcept, ReturnValueHandler, True)
registry.register(Sheerka, SheerkaHandler, True)
registry.register(ExecutionContext, ExecutionContextHandler, True)
src/sheerkapickle/tags.py
+5
View File
@@ -0,0 +1,5 @@
ID = "_sheerka/id"
TUPLE = "_sheerka/tuple"
SET = "_sheerka/set"
OBJECT = "_sheerka/obj"
ENUM = "_sheerka/enum"
src/sheerkapickle/utils.py
+85
View File
@@ -0,0 +1,85 @@
import base64
import types
from enum import Enum
class_types = (type,)
PRIMITIVES = (str, bool, type(None), int, float)
def is_type(obj):
"""Returns True is obj is a reference to a type.
"""
# use "isinstance" and not "is" to allow for metaclasses
return isinstance(obj, class_types)
def is_enum(obj):
return isinstance(obj, Enum)
def is_object(obj):
"""Returns True is obj is a reference to an object instance."""
return (isinstance(obj, object) and
not isinstance(obj, (type, types.FunctionType,
types.BuiltinFunctionType)))
def is_primitive(obj):
return type(obj) in PRIMITIVES
def is_dictionary(obj):
return type(obj) is dict
def is_list(obj):
return type(obj) is list
def is_set(obj):
return type(obj) is set
def is_bytes(obj):
return type(obj) is bytes
def is_tuple(obj):
return type(obj) is tuple
def b64encode(data):
"""
Encode binary data to ascii text in base64. Data must be bytes.
"""
return base64.b64encode(data).decode('ascii')
def translate_module_name(module):
"""Rename builtin modules to a consistent module name.
Prefer the more modern naming.
This is used so that references to Python's `builtins` module can
be loaded in both Python 2 and 3. We remap to the "__builtin__"
name and unmap it when importing.
Map the Python2 `exceptions` module to `builtins` because
`builtins` is a superset and contains everything that is
available in `exceptions`, which makes the translation simpler.
See untranslate_module_name() for the reverse operation.
"""
lookup = dict(__builtin__='builtins', exceptions='builtins')
return lookup.get(module, module)
def importable_name(cls):
"""
Fully qualified name (prefixed by builtin when needed)
"""
# Use the fully-qualified name if available (Python >= 3.3)
name = getattr(cls, '__qualname__', cls.__name__)
module = translate_module_name(cls.__module__)
return '{}.{}'.format(module, name)