From b9afcba61f6b56b15386d7277b992a77cefa568d Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Fri, 31 Jan 2020 18:58:03 +0100 Subject: [PATCH] Refactored ExecutionContext serialization (added sheerkapickle) and added History management --- _concepts.txt | 10 + src/__init__.py | 0 src/core/concept.py | 2 + src/core/sheerka/ExecutionContext.py | 39 ++- .../{ => Services}/SheerkaCreateNewConcept.py | 0 .../sheerka/{ => Services}/SheerkaDump.py | 5 +- .../{ => Services}/SheerkaEvaluateConcept.py | 0 .../sheerka/{ => Services}/SheerkaExecute.py | 2 +- .../sheerka/Services/SheerkaHistoryManager.py | 74 ++++ .../{ => Services}/SheerkaSetsManager.py | 0 src/core/sheerka/Services/__init__.py | 0 src/core/sheerka/Sheerka.py | 46 ++- src/core/sheerka_transform.py | 161 --------- src/core/utils.py | 64 ++++ src/sdp/sheerkaDataProvider.py | 19 +- src/sdp/sheerkaSerializer.py | 21 +- src/sheerkapickle/SheerkaPickler.py | 132 ++++++++ src/sheerkapickle/SheerkaUnpickler.py | 105 ++++++ src/sheerkapickle/__init__.py | 7 + src/sheerkapickle/handlers.py | 233 +++++++++++++ src/sheerkapickle/sheerka_handlers.py | 182 ++++++++++ src/sheerkapickle/tags.py | 5 + src/sheerkapickle/utils.py | 85 +++++ tests/core/test_ExecutionContext.py | 2 +- tests/core/test_SheerkaHistoryManager.py | 52 +++ tests/core/test_sheerka_transform.py | 318 ------------------ tests/core/test_utils.py | 32 ++ tests/sdp/test_sheerkaDataProvider.py | 11 +- tests/sheerkapickle/__init__.py | 0 tests/sheerkapickle/test_SheerkaPickler.py | 172 ++++++++++ tests/sheerkapickle/test_sheerka_handlers.py | 285 ++++++++++++++++ 31 files changed, 1546 insertions(+), 518 deletions(-) create mode 100644 _concepts.txt create mode 100644 src/__init__.py rename src/core/sheerka/{ => Services}/SheerkaCreateNewConcept.py (100%) rename src/core/sheerka/{ => Services}/SheerkaDump.py (92%) rename src/core/sheerka/{ => Services}/SheerkaEvaluateConcept.py (100%) rename src/core/sheerka/{ => Services}/SheerkaExecute.py (99%) create mode 100644 src/core/sheerka/Services/SheerkaHistoryManager.py rename src/core/sheerka/{ => Services}/SheerkaSetsManager.py (100%) create mode 100644 src/core/sheerka/Services/__init__.py delete mode 100644 src/core/sheerka_transform.py create mode 100644 src/sheerkapickle/SheerkaPickler.py create mode 100644 src/sheerkapickle/SheerkaUnpickler.py create mode 100644 src/sheerkapickle/__init__.py create mode 100644 src/sheerkapickle/handlers.py create mode 100644 src/sheerkapickle/sheerka_handlers.py create mode 100644 src/sheerkapickle/tags.py create mode 100644 src/sheerkapickle/utils.py create mode 100644 tests/core/test_SheerkaHistoryManager.py delete mode 100644 tests/core/test_sheerka_transform.py create mode 100644 tests/sheerkapickle/__init__.py create mode 100644 tests/sheerkapickle/test_SheerkaPickler.py create mode 100644 tests/sheerkapickle/test_sheerka_handlers.py diff --git a/_concepts.txt b/_concepts.txt new file mode 100644 index 0000000..146a1a0 --- /dev/null +++ b/_concepts.txt @@ -0,0 +1,10 @@ +def concept one as 1 +def concept two as 2 +def concept three as 3 +def concept one as 1 + +def concept two as 2 + +def concept three as 3 + +def concept four as 4 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/concept.py b/src/core/concept.py index 920e79e..ceb3321 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -343,6 +343,8 @@ class Concept: :param metadata: :return: """ + if metadata not in self.values: + return None return self.values[metadata] def auto_init(self): diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index de2bcee..f2b4c7f 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -4,9 +4,22 @@ import time from core.builtin_concepts import BuiltinConcepts from core.concept import Concept from sdp.sheerkaDataProvider import Event +from sheerkapickle.SheerkaPickler import SheerkaPickler DEBUG_TAB_SIZE = 4 +PROPERTIES_TO_SERIALIZE = ("_id", + "_bag", + "_start", + "_stop", + "who", + "desc", + "children", + "inputs", + "values", + "obj", + "concepts") + class ExecutionContext: """ @@ -31,7 +44,7 @@ class ExecutionContext: **kwargs): self._parent = None - self._id = ExecutionContext.get_id(event.get_digest()) + self._id = ExecutionContext.get_id(event.get_digest()) if event else None self._tab = "" self._bag = {} # other variables self._start = 0 @@ -90,6 +103,26 @@ class ExecutionContext: msg += ")" return msg + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, ExecutionContext): + return False + + for prop in PROPERTIES_TO_SERIALIZE: + if prop == "who": + value = str(getattr(self, prop)) + other_value = str(getattr(other, prop)) + else: + value = getattr(self, prop) + other_value = getattr(other, prop) + + if value != other_value: + return False + + return True + def add_preprocess(self, name, **kwargs): preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS) preprocess.set_prop("name", name) @@ -189,10 +222,6 @@ class ExecutionContext: to_str = self.return_value_to_str(r) logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str) - def to_dict(self): - from core.sheerka_transform import SheerkaTransform - st = SheerkaTransform(self.sheerka) - return st.to_dict(self) @staticmethod def return_value_to_str(r): diff --git a/src/core/sheerka/SheerkaCreateNewConcept.py b/src/core/sheerka/Services/SheerkaCreateNewConcept.py similarity index 100% rename from src/core/sheerka/SheerkaCreateNewConcept.py rename to src/core/sheerka/Services/SheerkaCreateNewConcept.py diff --git a/src/core/sheerka/SheerkaDump.py b/src/core/sheerka/Services/SheerkaDump.py similarity index 92% rename from src/core/sheerka/SheerkaDump.py rename to src/core/sheerka/Services/SheerkaDump.py index 3a55ff3..c6aa73b 100644 --- a/src/core/sheerka/SheerkaDump.py +++ b/src/core/sheerka/Services/SheerkaDump.py @@ -39,7 +39,8 @@ class SheerkaDump: self.sheerka.log.info(f"name : {c.name}") self.sheerka.log.info(f"bnf : {c.metadata.definition}") self.sheerka.log.info(f"key : {c.key}") - self.sheerka.log.info(f"body : {c.body}") + self.sheerka.log.info(f"body : {c.metadata.body}") + self.sheerka.log.info(f"value : {c.body}") self.sheerka.log.info(f"digest : {c.get_digest()}") first = False @@ -57,7 +58,7 @@ class SheerkaDump: while True: try: - if h.user != self.sheerka.name: + if h.event.user != self.sheerka.name: self.sheerka.log.info(h) count += 1 h = next(history) diff --git a/src/core/sheerka/SheerkaEvaluateConcept.py b/src/core/sheerka/Services/SheerkaEvaluateConcept.py similarity index 100% rename from src/core/sheerka/SheerkaEvaluateConcept.py rename to src/core/sheerka/Services/SheerkaEvaluateConcept.py diff --git a/src/core/sheerka/SheerkaExecute.py b/src/core/sheerka/Services/SheerkaExecute.py similarity index 99% rename from src/core/sheerka/SheerkaExecute.py rename to src/core/sheerka/Services/SheerkaExecute.py index f3fc236..2027614 100644 --- a/src/core/sheerka/SheerkaExecute.py +++ b/src/core/sheerka/Services/SheerkaExecute.py @@ -218,7 +218,7 @@ class SheerkaExecute: for step in execution_steps: copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values] - with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context: + with execution_context.push(step=step, iteration=0, desc=f"{step=}") as sub_context: sub_context.log(logger or self.sheerka.log, f"{step=}, context='{sub_context}'") if step == BuiltinConcepts.PARSING: diff --git a/src/core/sheerka/Services/SheerkaHistoryManager.py b/src/core/sheerka/Services/SheerkaHistoryManager.py new file mode 100644 index 0000000..2004701 --- /dev/null +++ b/src/core/sheerka/Services/SheerkaHistoryManager.py @@ -0,0 +1,74 @@ +from collections import namedtuple + +from sdp.sheerkaDataProvider import Event + +hist = namedtuple("History", "text status") # tests purposes only + + +class History: + def __init__(self, event: Event, result): + self.event = event + self.result = result + self._status = None + + def __str__(self): + msg = f"{self.event.get_digest()} {self.event.date.strftime('%d/%m/%Y %H:%M:%S')} : {self.event.message}" + status = self.status + if status is not None: + msg += f" => {status}" + return msg + + def __repr__(self): + return f"event={self.event!r}, status={self.status}, result={self.result}" + + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, hist): + return self.event.message == other.text and self.status == other.status + + if not isinstance(other, History): + return False + + return self.event == other.event and self.result == other.result + + @property + def status(self): + if self._status: + return self._status + + if not self.result or "return_values" not in self.result.values: + return + + if hasattr(self.result.values["return_values"], "__iter__"): + if len(self.result.values["return_values"]) != 1: + self._status = False + return self._status + else: + self._status = self.result.values["return_values"][0].status + return self._status + else: + self._status = self.result.values["return_values"].status + return self._status + + +class SheerkaHistoryManager: + def __init__(self, sheerka): + self.sheerka = sheerka + + def history(self, depth_or_digest, start): + """ + Load history + :param depth_or_digest: number of items or digest + :param start: + :return: + """ + + events = list(self.sheerka.sdp.load_events(depth_or_digest, start)) + for event in events: + try: + result = self.sheerka.sdp.load_result(self.sheerka, event.get_digest()) + except (IOError, KeyError): + result = None + yield History(event, result) diff --git a/src/core/sheerka/SheerkaSetsManager.py b/src/core/sheerka/Services/SheerkaSetsManager.py similarity index 100% rename from src/core/sheerka/SheerkaSetsManager.py rename to src/core/sheerka/Services/SheerkaSetsManager.py diff --git a/src/core/sheerka/Services/__init__.py b/src/core/sheerka/Services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 175cbc2..7404573 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -2,11 +2,12 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConc UnknownConcept from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW from core.sheerka.ExecutionContext import ExecutionContext -from core.sheerka.SheerkaCreateNewConcept import SheerkaCreateNewConcept -from core.sheerka.SheerkaDump import SheerkaDump -from core.sheerka.SheerkaEvaluateConcept import SheerkaEvaluateConcept -from core.sheerka.SheerkaExecute import SheerkaExecute -from core.sheerka.SheerkaSetsManager import SheerkaSetsManager +from core.sheerka.Services.SheerkaCreateNewConcept import SheerkaCreateNewConcept +from core.sheerka.Services.SheerkaDump import SheerkaDump +from core.sheerka.Services.SheerkaEvaluateConcept import SheerkaEvaluateConcept +from core.sheerka.Services.SheerkaExecute import SheerkaExecute +from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager +from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager from sdp.sheerkaDataProvider import SheerkaDataProvider, Event import core.utils import core.builtin_helpers @@ -21,7 +22,7 @@ import logging # BuiltinConcepts.AFTER_EVALUATION] CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser" - +CONCEPTS_FILE = "_concepts.txt" class Sheerka(Concept): """ @@ -81,6 +82,7 @@ class Sheerka(Concept): self.dump_handler = SheerkaDump(self) self.sets_handler = SheerkaSetsManager(self) self.evaluate_concept_handler = SheerkaEvaluateConcept(self) + self.history_handler = SheerkaHistoryManager(self) def initialize(self, root_folder: str = None): """ @@ -92,6 +94,9 @@ class Sheerka(Concept): """ try: + from sheerkapickle.sheerka_handlers import initialize_pickle_handlers + initialize_pickle_handlers() + self.sdp = SheerkaDataProvider(root_folder) if self.sdp.first_time: self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000) @@ -104,11 +109,16 @@ class Sheerka(Concept): self.initialize_builtin_parsers() self.initialize_builtin_evaluators() self.initialize_concepts_definitions(exec_context) + res = ReturnValueConcept(self, True, self) + + exec_context.add_values(return_values=res) + if not self.skip_builtins_in_db: + self.sdp.save_result(self, exec_context) except IOError as e: - return ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e) + res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e) - return ReturnValueConcept(self, True, self) + return res def initialize_builtin_concepts(self): """ @@ -232,7 +242,12 @@ class Sheerka(Concept): execution_context.add_values(return_values=ret) if not self.skip_builtins_in_db: - self.sdp.save_result(execution_context) + self.sdp.save_result(self, execution_context) + + #hack to save valid concept definition + if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT): + with open(CONCEPTS_FILE, "a") as f: + f.write(text + "\n") return ret def execute(self, execution_context, return_values, execution_steps, logger=None): @@ -549,8 +564,19 @@ class Sheerka(Concept): def history(self, page=10, start=0): """Gets the history of all commands""" - return self.sdp.load_events(page, start) + return self.history_handler.history(page, start) + def restore(self): + """ + Restore the state with all previous valid concept definitions + :return: + """ + try: + with open(CONCEPTS_FILE, "r") as f: + for line in f.readlines(): + self.evaluate_user_input(line) + except IOError: + pass def test(self): return f"I have access to Sheerka !" diff --git a/src/core/sheerka_transform.py b/src/core/sheerka_transform.py deleted file mode 100644 index 45fd829..0000000 --- a/src/core/sheerka_transform.py +++ /dev/null @@ -1,161 +0,0 @@ -import dataclasses -from enum import Enum - -from core.concept import Concept, PROPERTIES_TO_SERIALIZE -from core.sheerka.Sheerka import ExecutionContext -from core.tokenizer import Token -from evaluators.BaseEvaluator import BaseEvaluator -from parsers.BaseParser import BaseParser, Node -from parsers.BnfParser import BnfParser -from parsers.ConceptLexerParser import UnrecognizedTokensNode, ParsingExpression -from parsers.PythonParser import PythonNode -from sdp.sheerkaDataProvider import Event - -OBJ_TYPE_KEY = "__type__" -OBJ_ID_KEY = "__id__" -OBJ_NAME_KEY = "__name__" - -default_concept = Concept() - - -class SheerkaTransformType(Enum): - Concept = 1 - Reference = 2 - ExecutionContext = 3 - Event = 4 - Node = 5 - Exception = 6 - - def __repr__(self): - return self.__class__.__name__ + "." + self.name - - -class SheerkaTransform: - - def __init__(self, sheerka): - self.ids = {} - self.sheerka = sheerka - self.id_count = -1 - - def to_dict(self, obj): - - if isinstance(obj, (Concept, ExecutionContext, Event)): - exists, _id = self.exist(obj) - if exists: - return { - OBJ_TYPE_KEY: SheerkaTransformType.Reference, - OBJ_ID_KEY: _id - } - else: - self.id_count = self.id_count + 1 - self.ids[obj] = self.id_count - - if isinstance(obj, Concept): - return self.concept_to_dict(obj) - - elif isinstance(obj, ExecutionContext): - return self.execution_context_to_dict(obj) - - elif isinstance(obj, Event): - return { - OBJ_TYPE_KEY: SheerkaTransformType.Event, - OBJ_ID_KEY: self.id_count, - 'digest': obj.get_digest()} - - elif isinstance(obj, (BaseParser, BaseEvaluator, BnfParser)): - return obj.name - - elif isinstance(obj, Token): - return obj.__dict__ - - elif isinstance(obj, PythonNode): - return { - OBJ_TYPE_KEY: SheerkaTransformType.Node, - OBJ_NAME_KEY: "PythonNode", - 'source': obj.source, - 'ast_': obj.get_dump(obj.ast_) - } - - elif isinstance(obj, Node): - to_dict = { - OBJ_TYPE_KEY: SheerkaTransformType.Node, - OBJ_NAME_KEY: obj.__class__.__name__, - } - for k, v in obj.__dict__.items(): - to_dict[k] = self.to_dict(v) - - return to_dict - - elif isinstance(obj, Exception): - to_dict = { - OBJ_TYPE_KEY: SheerkaTransformType.Exception, - OBJ_NAME_KEY: obj.__class__.__name__, - } - for k, v in obj.__dict__.items(): - to_dict[k] = self.to_dict(v) - return to_dict - - elif isinstance(obj, ParsingExpression): - return obj.__repr__() - - elif isinstance(obj, dict): - return dict((str(k) if isinstance(k, Concept) else k, self.to_dict(v)) for k, v in obj.items()) - - elif hasattr(obj, "__iter__") and not isinstance(obj, str): - return list(self.to_dict(o) for o in obj) - - else: - return obj - - def concept_to_dict(self, obj: Concept): - to_dict = { - OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: self.id_count, - } - if obj.id: - ref = self.sheerka.get(obj.key, obj.id) - to_dict["id"] = obj.id - else: - ref = default_concept - - # transform metadata - for prop in PROPERTIES_TO_SERIALIZE: - value = getattr(obj.metadata, prop) - ref_value = getattr(ref.metadata, prop) - if value != ref_value: - to_dict["meta." + prop] = self.to_dict(value) - - # transform value - for metadata, value in obj.values.items(): - ref_value = ref.values[metadata] if metadata in ref.values else None - if value != ref_value: - to_dict[metadata.value] = self.to_dict(value) - - # transform properties - for prop in obj.props: - value = obj.props[prop].value - if prop not in ref.props or value != ref.props[prop].value: - if "props" not in to_dict: - to_dict["props"] = [] - to_dict["props"].append((prop, self.to_dict(value))) - - return to_dict - - def execution_context_to_dict(self, obj: ExecutionContext): - to_dict = { - OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext, - OBJ_ID_KEY: self.id_count - } - for property_name in obj.__dict__: - if property_name == "sheerka": - continue - to_dict[property_name] = self.to_dict(getattr(obj, property_name)) - - return to_dict - - def exist(self, obj): - for k, v in self.ids.items(): - if id(k) == id(obj) or k == obj: - return True, v - - return False, None diff --git a/src/core/utils.py b/src/core/utils.py index 4006d71..6a6cf3c 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -228,6 +228,7 @@ def escape_char(text, to_escape): return res + def pp(items): if not hasattr(items, "__iter__"): return str(items) @@ -236,3 +237,66 @@ def pp(items): return str(items) return " \n" + " \n".join(str(item) for item in items) + + +def decode_concept(concept_repr): + """ + if concept_repr is like :c:key:id: + return the key and the id + :param concept_repr: + :return: + """ + if not (concept_repr and isinstance(concept_repr, str) and concept_repr.startswith(":c:")): + return None, None + + i = 3 + length = len(concept_repr) + key = "" + while i < length: + if concept_repr[i] == ":": + break + key += concept_repr[i] + i += 1 + else: + return None, None + + i += 1 + if i >= length: + return key, None + + id = "" + while i < length: + if concept_repr[i] == ":": + break + id += concept_repr[i] + i += 1 + else: + return None, None + + return key, id + + +def decode_enum(enum_repr: str): + """ + Tries to transform ClassName.Name into an enum + :param enum_repr: + :return: + """ + if not (enum_repr and isinstance(enum_repr, str)): + return None + + try: + idx = enum_repr.rindex(".") + if idx == len(enum_repr): + return None + + cls_name = enum_repr[:idx] + cls = get_class(cls_name) + name = enum_repr[idx + 1:] + return cls[name] + + except ValueError: + return None + + except TypeError: + return None diff --git a/src/sdp/sheerkaDataProvider.py b/src/sdp/sheerkaDataProvider.py index ebe61f0..1c47ee6 100644 --- a/src/sdp/sheerkaDataProvider.py +++ b/src/sdp/sheerkaDataProvider.py @@ -36,6 +36,9 @@ class Event(object): def __str__(self): return f"{self.date.strftime('%d/%m/%Y %H:%M:%S')} {self.message}" + def __repr__(self): + return f"{self.get_digest()[:12]} {self.message}" + def get_digest(self): """ Returns the digest of the event @@ -64,6 +67,7 @@ class Event(object): self.date = datetime.fromisoformat(as_dict["date"]) self.message = as_dict["message"] self.parents = as_dict["parents"] + self._digest = as_dict["_digest"] # freeze the digest class ObjToUpdate: @@ -667,9 +671,9 @@ class SheerkaDataProvider: :param event: :return: digest of the event """ - digest = event.get_digest() parent = self.get_snapshot(SheerkaDataProvider.LastEventFile) event.parents = [parent] if parent else None + digest = event.get_digest() # must be call after setting the parents target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) if self.io.exists(target_path): @@ -712,7 +716,7 @@ class SheerkaDataProvider: digest = event.parents[0] count = 0 - while count < page_size: + while count < page_size or page_size <= 0: event = self.load_event(digest) if event is None: return @@ -725,12 +729,13 @@ class SheerkaDataProvider: digest = event.parents[0] count += 1 - def save_result(self, execution_context): + def save_result(self, sheerka, execution_context): """ Save the execution context associated with an event To make a long story short, for every single user input, there is an event (which is the first thing that is created) and a result (the ExecutionContext created by sheerka.evaluate_user_input() + :param sheerka: :param execution_context: :return: """ @@ -740,14 +745,16 @@ class SheerkaDataProvider: if self.io.exists(target_path): return digest - self.io.write_binary(target_path, self.serializer.serialize(execution_context, None).read()) + context = SerializerContext(sheerka=sheerka) + self.io.write_binary(target_path, self.serializer.serialize(execution_context, context).read()) return digest - def load_result(self, digest): + def load_result(self, sheerka, digest): target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + "_result" with self.io.open(target_path, "rb") as f: - return self.serializer.deserialize(f, None) + context = SerializerContext(sheerka=sheerka) + return self.serializer.deserialize(f, context) def save_state(self, state: State): digest = state.get_digest() diff --git a/src/sdp/sheerkaSerializer.py b/src/sdp/sheerkaSerializer.py index c9df1dd..b2ce8e9 100644 --- a/src/sdp/sheerkaSerializer.py +++ b/src/sdp/sheerkaSerializer.py @@ -1,18 +1,17 @@ -import dataclasses import json -import pickle import datetime +import pickle import struct import io from dataclasses import dataclass + +import sheerkapickle from core.sheerka_logger import get_logger from enum import Enum import core.utils from core.concept import Concept -from core.tokenizer import Token -from parsers.BaseParser import Node def json_default_converter(o): @@ -40,6 +39,7 @@ def json_default_converter(o): class SerializerContext: user_name: str = None origin: str = None + sheerka: object = None class Serializer: @@ -85,6 +85,7 @@ class Serializer: raise TypeError(f"Don't know how to serialize {type(obj)}") serializer = serializers[0] + self.log.debug(f"Serializing '{obj}' using '{serializer.name}'") stream = io.BytesIO() header = struct.pack(Serializer.HEADER_FORMAT, bytes(serializer.name, "utf-8"), serializer.version) @@ -248,23 +249,23 @@ class DictionarySerializer(PickleSerializer): class ExecutionContextSerializer(BaseSerializer): + CLASS_NAME = "core.sheerka.ExecutionContext.ExecutionContext" + def __init__(self): BaseSerializer.__init__(self, "R", 1) def matches(self, obj): - return core.utils.get_full_qualified_name(obj) == "core.sheerka.ExecutionContext.ExecutionContext" + return core.utils.get_full_qualified_name(obj) == self.CLASS_NAME def dump(self, stream, obj, context): - as_json = obj.to_dict() - stream.write(json.dumps(as_json, default=json_default_converter).encode("utf-8")) + stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8")) stream.seek(0) return stream def load(self, stream, context): json_stream = stream.read().decode("utf-8") - json_message = json.loads(json_stream) - obj = core.utils.get_class("core.sheerka.ExecutionContext")() - obj.from_dict(json_message) + obj = sheerkapickle.decode(context.sheerka, json_stream) + #json_message = json.loads(json_stream) return obj # diff --git a/src/sheerkapickle/SheerkaPickler.py b/src/sheerkapickle/SheerkaPickler.py new file mode 100644 index 0000000..15bcdea --- /dev/null +++ b/src/sheerkapickle/SheerkaPickler.py @@ -0,0 +1,132 @@ +import json +from logging import Logger + +import core.utils +from core.concept import Concept + +from sheerkapickle import utils, tags, handlers + + +def encode(sheerka, obj): + pickler = SheerkaPickler(sheerka) + data = pickler.flatten(obj) + return json.dumps(data) + + +class ToReduce: + def __init__(self, predicate, get_value): + self.predicate = predicate + self.get_value = get_value + + +class SheerkaPickler: + """ + Json sheerkapickle + Inspired by jsonpickle (https://github.com/jsonpickle/jsonpickle) + which failed to work in my environment + """ + + def __init__(self, sheerka): + self.ids = {} + self.objs = [] + self.id_count = -1 + self.sheerka = sheerka + self.to_reduce = [] + + self.to_reduce.append(ToReduce(lambda o: isinstance(o, Logger), lambda o: None)) + from parsers.BaseParser import BaseParser + from evaluators.BaseEvaluator import BaseEvaluator + self.to_reduce.append(ToReduce(lambda o: isinstance(o, (BaseParser, BaseEvaluator)), lambda o: o.name)) + + def flatten(self, obj): + if utils.is_primitive(obj): + return obj + + if utils.is_tuple(obj): + return {tags.TUPLE: [self.flatten(v) for v in obj]} + + if utils.is_set(obj): + return {tags.SET: [self.flatten(v) for v in obj]} + + if utils.is_list(obj): + return [self.flatten(v) for v in obj] + + if utils.is_dictionary(obj): + return self._flatten_dict(obj) + + if utils.is_enum(obj): + return self._flatten_enum(obj) + + if utils.is_object(obj): + return self._flatten_obj_instance(obj) + + raise Exception(f"Cannot flatten '{obj}'") + + def _flatten_dict(self, obj): + data = {} + for k, v in obj.items(): + if k is None: + k_str = "null" + elif utils.is_enum(k): + k_str = core.utils.get_full_qualified_name(k) + "." + k.name + elif isinstance(k, Concept): + k_str = f":c:{k.key}:{k.id}:" + else: + k_str = k + + data[k_str] = self.flatten(v) + + return data + + def _flatten_enum(self, obj): + # check if the object was already seen + exists, _id = self.exist(obj) + if exists: + return {tags.ID: _id} + else: + self.id_count = self.id_count + 1 + self.ids[id(obj)] = self.id_count + self.objs.append(obj) + + data = {} + class_name = core.utils.get_full_qualified_name(obj) + data[tags.ENUM] = class_name + "." + obj.name + return data + + def _flatten_obj_instance(self, obj): + for reduce in self.to_reduce: + if reduce.predicate(obj): + return reduce.get_value(obj) + + # check if the object was already seen + exists, _id = self.exist(obj) + if exists: + return {tags.ID: _id} + else: + self.id_count = self.id_count + 1 + self.ids[id(obj)] = self.id_count + self.objs.append(obj) + + # flatten + data = {} + cls = obj.__class__ if hasattr(obj, '__class__') else type(obj) + class_name = utils.importable_name(cls) + data[tags.OBJECT] = class_name + + handler = handlers.get(class_name) + if handler is not None: + return handler(self.sheerka, self).flatten(obj, data) + + if hasattr(obj, "__dict__"): + for k, v in obj.__dict__.items(): + data[k] = self.flatten(v) + return data + + return None + + def exist(self, obj): + for k, v in self.ids.items(): + if k == id(obj): + return True, v + + return False, None diff --git a/src/sheerkapickle/SheerkaUnpickler.py b/src/sheerkapickle/SheerkaUnpickler.py new file mode 100644 index 0000000..ab7f9e4 --- /dev/null +++ b/src/sheerkapickle/SheerkaUnpickler.py @@ -0,0 +1,105 @@ +import json + +import core.utils +from sheerkapickle import tags, utils, handlers + + +def decode(sheerka, obj): + return SheerkaUnpickler(sheerka).restore(json.loads(obj)) + + +class SheerkaUnpickler: + def __init__(self, sheerka): + self.sheerka = sheerka + self.objs = [] + + def restore(self, obj): + if has_tag(obj, tags.ID): + return self._restore_id(obj) + + if has_tag(obj, tags.TUPLE): + return self._restore_tuple(obj) + + if has_tag(obj, tags.SET): + return self._restore_set(obj) + + if has_tag(obj, tags.ENUM): + return self._restore_enum(obj) + + if has_tag(obj, tags.OBJECT): + return self._restore_obj(obj) + + if utils.is_list(obj): + return self._restore_list(obj) + + if utils.is_dictionary(obj): + return self._restore_dict(obj) + + return obj + + def _restore_list(self, obj): + return [self.restore(v) for v in obj] + + def _restore_tuple(self, obj): + return tuple([self.restore(v) for v in obj[tags.TUPLE]]) + + def _restore_set(self, obj): + return set([self.restore(v) for v in obj[tags.SET]]) + + def _restore_enum(self, obj): + instance = core.utils.decode_enum(obj[tags.ENUM]) + self.objs.append(instance) + return instance + + def _restore_dict(self, obj): + data = {} + for k, v in obj.items(): + resolved_key = self._resolve_key(k) + data[resolved_key] = self.restore(v) + return data + + def _restore_id(self, obj): + try: + return self.objs[obj[tags.ID]] + except IndexError: + pass + + def _restore_obj(self, obj): + handler = handlers.get(obj[tags.OBJECT]) + + if handler: + handler = handler(self.sheerka, self) + instance = handler.new(obj) + self.objs.append(instance) + instance = handler.restore(obj, instance) + else: + cls = core.utils.get_class(obj[tags.OBJECT]) + instance = cls.__new__(cls) + self.objs.append(instance) + + for k, v in obj.items(): + if k == tags.OBJECT: + continue + value = self.restore(v) + setattr(instance, k, value) + + return instance + + def _resolve_key(self, key): + + if key == "null": + return None + + concept_key, concept_id = core.utils.decode_concept(key) + if concept_key is not None: + return self.sheerka.new((concept_key, concept_id)) if concept_id else self.sheerka.new(concept_key) + + as_enum = core.utils.decode_enum(key) + if as_enum is not None: + return as_enum + + return key + + +def has_tag(obj, tag): + return type(obj) is dict and tag in obj diff --git a/src/sheerkapickle/__init__.py b/src/sheerkapickle/__init__.py new file mode 100644 index 0000000..ada700a --- /dev/null +++ b/src/sheerkapickle/__init__.py @@ -0,0 +1,7 @@ +from .SheerkaPickler import encode +from .SheerkaUnpickler import decode + +__all__ = ('encode', 'decode') + +# register built-in handlers +__import__('sheerkapickle.handlers', level=0) diff --git a/src/sheerkapickle/handlers.py b/src/sheerkapickle/handlers.py new file mode 100644 index 0000000..39a57b9 --- /dev/null +++ b/src/sheerkapickle/handlers.py @@ -0,0 +1,233 @@ +import datetime +import re +import threading +import uuid + +from sheerkapickle import utils + + +class ToReduce: + def __init__(self, predicate, get_value): + self.predicate = predicate + self.get_value = get_value + + +class SheerkaRegistry(object): + + def __init__(self): + self._handlers = {} + self._base_handlers = {} + + def get(self, cls_or_name, default=None): + """ + :param cls_or_name: the type or its fully qualified name + :param default: default value, if a matching handler is not found + + Looks up a handler by type reference or its fully + qualified name. If a direct match + is not found, the search is performed over all + handlers registered with base=True. + """ + handler = self._handlers.get(cls_or_name) + # attempt to find a base class + if handler is None and utils.is_type(cls_or_name): + for cls, base_handler in self._base_handlers.items(): + if issubclass(cls_or_name, cls): + return base_handler + return default if handler is None else handler + + def register(self, cls, handler=None, base=False): + """Register the a custom handler for a class + + :param cls: The custom object class to handle + :param handler: The custom handler class (if + None, a decorator wrapper is returned) + :param base: Indicates whether the handler should + be registered for all subclasses + + This function can be also used as a decorator + by omitting the `handler` argument:: + + @jsonpickle.handlers.register(Foo, base=True) + class FooHandler(jsonpickle.handlers.BaseHandler): + pass + + """ + if handler is None: + def _register(handler_cls): + self.register(cls, handler=handler_cls, base=base) + return handler_cls + + return _register + if not utils.is_type(cls): + raise TypeError('{!r} is not a class/type'.format(cls)) + # store both the name and the actual type for the ugly cases like + # _sre.SRE_Pattern that cannot be loaded back directly + self._handlers[utils.importable_name(cls)] = \ + self._handlers[cls] = handler + if base: + # only store the actual type for subclass checking + self._base_handlers[cls] = handler + + def unregister(self, cls): + self._handlers.pop(cls, None) + self._handlers.pop(utils.importable_name(cls), None) + self._base_handlers.pop(cls, None) + + +registry = SheerkaRegistry() +register = registry.register +unregister = registry.unregister +get = registry.get + + +class BaseHandler(object): + + def __init__(self, sheerka, context): + """ + Initialize a new handler to handle a registered type. + + :Parameters: + - `context`: reference to pickler/unpickler + + """ + self.sheerka = sheerka + self.context = context + + def __call__(self, sheerka, context): + """This permits registering either Handler instances or classes + + :Parameters: + - `context`: reference to pickler/unpickler + """ + self.sheerka = sheerka + self.context = context + return self + + def flatten(self, obj, data): + """ + Flatten `obj` into a json-friendly form and write result to `data`. + + :param object obj: The object to be serialized. + :param dict data: A partially filled dictionary which will contain the + json-friendly representation of `obj` once this method has + finished. + """ + raise NotImplementedError('You must implement flatten() in %s' % + self.__class__) + + def new(self, data): + raise NotImplementedError('You must implement new() in %s' % + self.__class__) + + def restore(self, data, instance): + """ + Restore an object of the registered type from the json-friendly + representation `obj` and return it. + """ + raise NotImplementedError('You must implement restore() in %s' % + self.__class__) + + @classmethod + def handles(self, cls): + """ + Register this handler for the given class. Suitable as a decorator, + e.g.:: + + @MyCustomHandler.handles + class MyCustomClass: + def __reduce__(self): + ... + """ + registry.register(cls, self) + return cls + + +# class DatetimeHandler(BaseHandler): +# """Custom handler for datetime objects +# +# Datetime objects use __reduce__, and they generate binary strings encoding +# the payload. This handler encodes that payload to reconstruct the +# object. +# +# """ +# +# def flatten(self, obj, data): +# pickler = self.context +# if not pickler.unpicklable: +# return str(obj) +# cls, args = obj.__reduce__() +# flatten = pickler.flatten +# payload = utils.b64encode(args[0]) +# args = [payload] + [flatten(i, reset=False) for i in args[1:]] +# data['__reduce__'] = (flatten(cls, reset=False), args) +# return data +# +# def restore(self, data): +# cls, args = data['__reduce__'] +# unpickler = self.context +# restore = unpickler.restore +# cls = restore(cls, reset=False) +# value = utils.b64decode(args[0]) +# params = (value,) + tuple([restore(i, reset=False) for i in args[1:]]) +# return cls.__new__(cls, *params) +# +# +# DatetimeHandler.handles(datetime.datetime) +# DatetimeHandler.handles(datetime.date) +# DatetimeHandler.handles(datetime.time) + + +class RegexHandler(BaseHandler): + """Flatten _sre.SRE_Pattern (compiled regex) objects""" + + def flatten(self, obj, data): + data['pattern'] = obj.pattern + return data + + def new(self, data): + return re.compile(data['pattern']) + + def restore(self, data, instance): + return instance + + +RegexHandler.handles(type(re.compile(''))) + + +class UUIDHandler(BaseHandler): + """Serialize uuid.UUID objects""" + + def flatten(self, obj, data): + data['hex'] = obj.hex + return data + + def new(self, data): + return uuid.UUID(data['hex']) + + def restore(self, data, instance): + return instance + + +UUIDHandler.handles(uuid.UUID) + + +class LockHandler(BaseHandler): + """Serialize threading.Lock objects""" + + def flatten(self, obj, data): + data['locked'] = obj.locked() + return data + + def new(self, data): + lock = threading.Lock() + if data.get('locked', False): + lock.acquire() + return lock + + def restore(self, data, instance): + return instance + + +_lock = threading.Lock() +LockHandler.handles(_lock.__class__) diff --git a/src/sheerkapickle/sheerka_handlers.py b/src/sheerkapickle/sheerka_handlers.py new file mode 100644 index 0000000..9670fa6 --- /dev/null +++ b/src/sheerkapickle/sheerka_handlers.py @@ -0,0 +1,182 @@ +from core.builtin_concepts import UserInputConcept, ReturnValueConcept, BuiltinConcepts +from core.sheerka.Sheerka import Sheerka +from evaluators.BaseEvaluator import BaseEvaluator +from parsers.BaseParser import BaseParser +from sheerkapickle.handlers import BaseHandler, registry +from core.concept import Concept, PROPERTIES_TO_SERIALIZE as CONCEPT_PROPERTIES_TO_SERIALIZE, ConceptParts +from core.sheerka.ExecutionContext import ExecutionContext, PROPERTIES_TO_SERIALIZE as CONTEXT_PROPERTIES_TO_SERIALIZE + +default_concept = Concept() +CONCEPT_ID = "concept/id" + + +class ConceptHandler(BaseHandler): + + def flatten(self, obj: Concept, data): + pickler = self.context + sheerka = self.sheerka + + if obj.id: + ref = sheerka.get_by_id(obj.id) + data[CONCEPT_ID] = (obj.key, obj.id) + else: + ref = default_concept + + # transform metadata + for prop in CONCEPT_PROPERTIES_TO_SERIALIZE: + value = getattr(obj.metadata, prop) + ref_value = getattr(ref.metadata, prop) + if value != ref_value: + data["meta." + prop] = pickler.flatten(value) + + # transform value + for metadata, value in obj.values.items(): + ref_value = ref.values[metadata] if metadata in ref.values else None + if value != ref_value: + data[metadata.value] = pickler.flatten(value) + + # transform properties + for prop in obj.props: + value = obj.props[prop].value + if prop not in ref.props or value != ref.props[prop].value: + if "props" not in data: + data["props"] = [] + data["props"].append((prop, pickler.flatten(value))) + + return data + + def new(self, data): + sheerka = self.sheerka + return sheerka.new(tuple(data[CONCEPT_ID])) if CONCEPT_ID in data else Concept() + + def restore(self, data, instance): + pickler = self.context + + for key, value in data.items(): + if key.startswith("_sheerka/") or key == CONCEPT_ID: + continue + + resolved_value = pickler.restore(data[key]) + + if key.startswith("meta."): + # get metadata + resolved_prop = key[5:] + if resolved_prop == "props": + for prop_name, prop_value in resolved_value: + instance.def_prop(prop_name, prop_value) + else: + setattr(instance.metadata, resolved_prop, resolved_value) + elif key == "props": + # get properties + for prop_name, prop_value in resolved_value: + instance.set_prop(prop_name, prop_value) + else: + # get value + instance.set_metadata_value(ConceptParts(key), resolved_value) + + return instance + + +class UserInputHandler(BaseHandler): + + def flatten(self, obj: UserInputConcept, data): + data[CONCEPT_ID] = (obj.key, obj.id) + data["user_name"] = obj.user_name + data["text"] = BaseParser.get_text_from_tokens(obj.text) if isinstance(obj.text, list) else obj.text + return data + + def new(self, data): + sheerka = self.sheerka + + instance = sheerka.new(tuple(data[CONCEPT_ID]), body=data["text"], user_name=data["user_name"]) + return instance + + def restore(self, data, instance): + return instance + + +class ReturnValueHandler(BaseHandler): + + def flatten(self, obj: ReturnValueConcept, data): + pickler = self.context + + data["who"] = f"c:{obj.who.id}:" if isinstance(obj.who, Concept) else \ + obj.who.name if isinstance(obj.who, (BaseParser, BaseEvaluator)) else \ + obj.who + data["status"] = obj.status + data["value"] = pickler.flatten(obj.value) + if obj.parents: + data["parents"] = pickler.flatten(obj.parents) + return data + + def new(self, data): + sheerka = self.sheerka + instance = sheerka.ret(data["who"], data["status"], None) + return instance + + def restore(self, data, instance): + pickler = self.context + + instance.value = pickler.restore(data["value"]) + if "parents" in data: + instance.parents = pickler.restore(data["parents"]) + return instance + + +# class BuiltinConceptsHandler(BaseHandler): +# +# def flatten(self, obj: BuiltinConcepts, data): +# return data +# +# def restore(self, obj): +# pass + + +class SheerkaHandler(BaseHandler): + + def flatten(self, obj: BuiltinConcepts, data): + return data + + def new(self, data): + return self.sheerka + + def restore(self, data, instance): + return instance + + +class ExecutionContextHandler(BaseHandler): + + def flatten(self, obj, data): + pickler = self.context + + for prop in CONTEXT_PROPERTIES_TO_SERIALIZE: + if prop == "who": + value = str(getattr(obj, prop)) + else: + value = getattr(obj, prop) + + if value is not None: + data[prop] = pickler.flatten(value) + + return data + + def new(self, data): + return ExecutionContext(data["who"], None, None) + + def restore(self, data, instance): + pickler = self.context + + for prop in CONTEXT_PROPERTIES_TO_SERIALIZE: + if prop not in data or prop == "who": + continue + setattr(instance, prop, pickler.restore(data[prop])) + + return instance + + +def initialize_pickle_handlers(): + registry.register(Concept, ConceptHandler, True) + registry.register(UserInputConcept, UserInputHandler, True) + registry.register(ReturnValueConcept, ReturnValueHandler, True) + registry.register(Sheerka, SheerkaHandler, True) + registry.register(ExecutionContext, ExecutionContextHandler, True) diff --git a/src/sheerkapickle/tags.py b/src/sheerkapickle/tags.py new file mode 100644 index 0000000..59e0976 --- /dev/null +++ b/src/sheerkapickle/tags.py @@ -0,0 +1,5 @@ +ID = "_sheerka/id" +TUPLE = "_sheerka/tuple" +SET = "_sheerka/set" +OBJECT = "_sheerka/obj" +ENUM = "_sheerka/enum" diff --git a/src/sheerkapickle/utils.py b/src/sheerkapickle/utils.py new file mode 100644 index 0000000..995dcd6 --- /dev/null +++ b/src/sheerkapickle/utils.py @@ -0,0 +1,85 @@ +import base64 +import types +from enum import Enum + +class_types = (type,) +PRIMITIVES = (str, bool, type(None), int, float) + + +def is_type(obj): + """Returns True is obj is a reference to a type. + """ + # use "isinstance" and not "is" to allow for metaclasses + return isinstance(obj, class_types) + + +def is_enum(obj): + return isinstance(obj, Enum) + + +def is_object(obj): + """Returns True is obj is a reference to an object instance.""" + + return (isinstance(obj, object) and + not isinstance(obj, (type, types.FunctionType, + types.BuiltinFunctionType))) + + +def is_primitive(obj): + return type(obj) in PRIMITIVES + + +def is_dictionary(obj): + return type(obj) is dict + + +def is_list(obj): + return type(obj) is list + + +def is_set(obj): + return type(obj) is set + + +def is_bytes(obj): + return type(obj) is bytes + + +def is_tuple(obj): + return type(obj) is tuple + + +def b64encode(data): + """ + Encode binary data to ascii text in base64. Data must be bytes. + """ + return base64.b64encode(data).decode('ascii') + + +def translate_module_name(module): + """Rename builtin modules to a consistent module name. + + Prefer the more modern naming. + + This is used so that references to Python's `builtins` module can + be loaded in both Python 2 and 3. We remap to the "__builtin__" + name and unmap it when importing. + + Map the Python2 `exceptions` module to `builtins` because + `builtins` is a superset and contains everything that is + available in `exceptions`, which makes the translation simpler. + + See untranslate_module_name() for the reverse operation. + """ + lookup = dict(__builtin__='builtins', exceptions='builtins') + return lookup.get(module, module) + + +def importable_name(cls): + """ + Fully qualified name (prefixed by builtin when needed) + """ + # Use the fully-qualified name if available (Python >= 3.3) + name = getattr(cls, '__qualname__', cls.__name__) + module = translate_module_name(cls.__module__) + return '{}.{}'.format(module, name) diff --git a/tests/core/test_ExecutionContext.py b/tests/core/test_ExecutionContext.py index 4797697..c4d09ab 100644 --- a/tests/core/test_ExecutionContext.py +++ b/tests/core/test_ExecutionContext.py @@ -2,7 +2,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept -from core.sheerka.Sheerka import ExecutionContext +from core.sheerka.ExecutionContext import ExecutionContext from sdp.sheerkaDataProvider import Event diff --git a/tests/core/test_SheerkaHistoryManager.py b/tests/core/test_SheerkaHistoryManager.py new file mode 100644 index 0000000..021ed98 --- /dev/null +++ b/tests/core/test_SheerkaHistoryManager.py @@ -0,0 +1,52 @@ +from core.sheerka.Services.SheerkaHistoryManager import hist +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaHistoryManager(TestUsingMemoryBasedSheerka): + def test_i_can_retrieve_history(self): + sheerka = self.get_sheerka(skip_builtins_in_db=False) + + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("one") + sheerka.evaluate_user_input("xxx") + sheerka.evaluate_user_input("def concept two as 2") + sheerka.evaluate_user_input("two") + sheerka.evaluate_user_input("def concept three as 3") + sheerka.evaluate_user_input("three") + sheerka.evaluate_user_input("def concept four as 4") + sheerka.evaluate_user_input("four") + sheerka.evaluate_user_input("def concept five as 5") + sheerka.evaluate_user_input("five") + + h = list(sheerka.history(-1)) # all + assert h == [ + hist("five", True), + hist("def concept five as 5", True), + hist("four", True), + hist("def concept four as 4", True), + hist("three", True), + hist("def concept three as 3", True), + hist("two", True), + hist("def concept two as 2", True), + hist("xxx", False), + hist("one", True), + hist("def concept one as 1", True), + hist("Initializing Sheerka.", True)] + + h = list(sheerka.history(2)) + assert h == [ + hist("two", True), + hist("def concept two as 2", True) + ] + + h = list(sheerka.history(2, 2)) + assert h == [ + hist("xxx", False), + hist("one", True), + ] + + h = list(sheerka.history(-1)) + assert h == [ + hist("xxx", False), + hist("one", True), + ] diff --git a/tests/core/test_sheerka_transform.py b/tests/core/test_sheerka_transform.py deleted file mode 100644 index 307b18b..0000000 --- a/tests/core/test_sheerka_transform.py +++ /dev/null @@ -1,318 +0,0 @@ -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts -from core.sheerka.ExecutionContext import ExecutionContext -from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY -from sdp.sheerkaDataProvider import Event - -from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka - - -class TestSheerkaTransform(TestUsingMemoryBasedSheerka): - def test_i_can_transform_an_unknown_concept(self): - sheerka = self.get_sheerka() - - foo = Concept("foo", body="body") - concept_with_sub = Concept("concept_with_sub", body=foo) - - concept = Concept( - name="concept_name", - is_builtin=True, - is_unique=True, - key="concept_key", - body=concept_with_sub, - where=[foo, 1, "1", True, 1.0], - pre=foo, - post=None, # will not appear - definition="it is a definition", - definition_type="def type", - desc="this this the desc" - ).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub") - - # add values and props - concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init() - concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0] - concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init() - concept.values[ConceptParts.POST] = "a value for POST" - concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub) - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(concept) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 0, - 'meta.name': 'concept_name', - 'meta.key': 'concept_key', - 'meta.is_builtin': True, - 'meta.is_unique': True, - 'meta.definition': 'it is a definition', - 'meta.definition_type': 'def type', - 'meta.desc': 'this this the desc', - 'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 1, - 'meta.body': 'body', - 'meta.name': 'foo'}, 1, '1', True, 1.0], - 'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, - 'meta.body': { - '__type__': SheerkaTransformType.Concept, - '__id__': 2, - 'meta.name': 'concept_with_sub', - 'meta.body': { - '__type__': SheerkaTransformType.Reference, - '__id__': 1}}, - 'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']], - 'pre': {'__type__': SheerkaTransformType.Concept, - '__id__': 4, - 'meta.body': 'body', - 'meta.name': 'foo', - 'body': 'body'}, - 'post': "a value for POST", - 'body': {'__type__': SheerkaTransformType.Concept, - '__id__': 3, - 'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}, - 'meta.name': 'concept_with_sub', - 'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}}, - 'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0], - 'props': [('a', 10), - ('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}), - ('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})], - } - - def test_i_can_transform_unknown_concept_with_almost_same_value(self): - sheerka = self.get_sheerka() - concept = Concept("foo") - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(concept) - - assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'} - - def test_i_can_transform_known_concept_when_the_values_are_the_same(self): - """ - Values are the same means that we are serializing a concept which has kept all its default values - There is not diff between the concept to serialize and the one which was registered with create_new_concept() - We serialize only the id of the concept - :return: - """ - sheerka = self.get_sheerka() - - concept = Concept( - name="concept_name", - is_builtin=True, - is_unique=False, - key="concept_key", - body="body definition", - where="where definition", - pre="pre definition", - post="post definition", - definition="it is a definition", - definition_type="def type", - desc="this this the desc" - ).def_prop("a").def_prop("b") - sheerka.create_new_concept(self.get_context(sheerka), concept) - - new_concept = sheerka.new(concept.key) - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(new_concept) - - assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"} - - def test_i_can_transform_known_concept_when_the_values_are_different(self): - """ - Values are the different means the concept was modified. - It's different from the one which was registered with create_new_concept() - We serialize only the differences - :return: - """ - sheerka = self.get_sheerka() - - concept = Concept( - name="concept_name", - is_builtin=True, - is_unique=False, - key="concept_key", - body="body definition", - where="where definition", - pre="pre definition", - post="post definition", - definition="it is a definition", - definition_type="def type", - desc="this this the desc" - ).def_prop("a").def_prop("b") - sheerka.create_new_concept(self.get_context(sheerka), concept) - - new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre") - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(new_concept) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 0, - "id": "1001", - 'pre': 'another pre', - "body": "another", - 'props': [('a', 10)] - } - - def test_i_can_transform_concept_with_circular_reference(self): - sheerka = self.get_sheerka() - foo = Concept("foo") - bar = Concept("bar", body=foo) - foo.metadata.body = bar - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(foo) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 0, - 'meta.name': 'foo', - 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 1, - 'meta.name': 'bar', - 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, - OBJ_ID_KEY: 0}, - }, - } - - def test_i_can_transform_concept_with_circular_reference_2(self): - sheerka = self.get_sheerka() - foo = Concept("foo") - bar = Concept("foo", body=foo) - foo.metadata.body = bar - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(foo) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 0, - 'meta.name': 'foo', - 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 1, - 'meta.name': 'foo', - 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, - OBJ_ID_KEY: 0}, - }, - } - - def test_i_can_transform_the_unknown_concept(self): - sheerka = self.get_sheerka(False) - - unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(unknown) - - assert len(to_dict) == 3 - assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept - assert to_dict[OBJ_ID_KEY] == 0 - assert "id" in to_dict - - def test_i_can_transform_simple_execution_context(self): - sheerka = self.get_sheerka() - ExecutionContext.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(context) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext, - OBJ_ID_KEY: 0, - '_parent': None, - '_id': 0, - '_tab': '', - '_bag': {}, - '_start': 0, - '_stop': 0, - 'who': 'requester', - 'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'}, - 'desc': 'this is the desc', - 'children': [], - 'preprocess': None, - 'inputs': {}, - 'values': {}, - 'obj': None, - 'concepts': {} - } - - def test_i_can_transform_list(self): - sheerka = self.get_sheerka() - ExecutionContext.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict([context]) - - assert len(to_dict) == 1 - assert isinstance(to_dict, list) - assert to_dict[0]["who"] == "requester" - assert to_dict[0]["desc"] == "this is the desc" - - def test_i_can_transform_set(self): - sheerka = self.get_sheerka() - ExecutionContext.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict({context}) - - assert len(to_dict) == 1 - assert isinstance(to_dict, list) - assert to_dict[0]["who"] == "requester" - assert to_dict[0]["desc"] == "this is the desc" - - def test_i_can_transform_dict(self): - sheerka = self.get_sheerka() - ExecutionContext.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key() - sheerka.create_new_concept(self.get_context(sheerka), known_concept) - unknown_concept = Concept("bar") - known = sheerka.new("foo") - - bag = { - "context": context, - "known_concept": known_concept, - "unknown_concept": unknown_concept, - "True": True, - "Number": 1.1, - "String": "a string value", - "None": None, - unknown_concept: "hello", - known: "world" - } - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(bag) - - assert isinstance(to_dict, dict) - assert to_dict['Number'] == 1.1 - assert to_dict['String'] == 'a string value' - assert to_dict['True'] - assert to_dict['None'] is None - assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext - assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept - assert to_dict["known_concept"]["id"] == '1001' - assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept - assert to_dict["(None)bar"] == "hello" - assert to_dict["(1001)foo"] == "world" - - def test_i_can_transform_when_circular_references(self): - sheerka = self.get_sheerka() - ExecutionContext.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - context.push("another requester", "another desc") - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(context) - - assert isinstance(to_dict, dict) - assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext - assert len(to_dict["children"]) == 1 - assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext - assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference - assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0 - assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference - assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1 diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index 5742c43..c303000 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -1,5 +1,6 @@ import core.utils import pytest +from core.concept import ConceptParts from core.tokenizer import Token, TokenKind @@ -130,6 +131,37 @@ def test_i_can_escape(): assert actual == "hello \\'world\\' my friend" +@pytest.mark.parametrize("text, expected_key, expected_id", [ + (None, None, None), + (10, None, None), + ("", None, None), + ("xxx", None, None), + (":c:", None, None), + (":c:key", None, None), + (":c:key:", "key", None), + (":c:key:id", None, None), + (":c:key:id:", "key", "id"), +]) +def test_i_can_decode_concept_repr(text, expected_key, expected_id): + k, i = core.utils.decode_concept(text) + assert k == expected_key + assert i == expected_id + + +@pytest.mark.parametrize("text, expected", [ + (None, None), + (10, None), + ("", None), + ("xxx", None), + ("xxx.", None), + ("xxx.yyy", None), + ("core.concept.ConceptParts.BODY", ConceptParts.BODY), +]) +def test_i_can_decode_enum(text, expected): + actual = core.utils.decode_enum(text) + assert actual == expected + + def get_tokens(lst): res = [] for e in lst: diff --git a/tests/sdp/test_sheerkaDataProvider.py b/tests/sdp/test_sheerkaDataProvider.py index 2674673..97f72f0 100644 --- a/tests/sdp/test_sheerkaDataProvider.py +++ b/tests/sdp/test_sheerkaDataProvider.py @@ -257,24 +257,27 @@ def test_i_can_load_events(root): for i in range(15): sdp.save_event(Event(f"Hello {i}")) - events = list(sdp.load_events(10)) + events = list(sdp.load_events(10)) # first ten assert len(events) == 10 assert events[0].message == "Hello 14" assert events[9].message == "Hello 5" - events = list(sdp.load_events(10, 5)) + events = list(sdp.load_events(10, 5)) # skip first 5, then take 10 assert len(events) == 10 assert events[0].message == "Hello 9" assert events[9].message == "Hello 0" - events = list(sdp.load_events(20, 10)) + events = list(sdp.load_events(20, 10)) # skip first 10, take 20,(but only 5 remaining) assert len(events) == 5 assert events[0].message == "Hello 4" assert events[4].message == "Hello 0" - events = list(sdp.load_events(1, 20)) + events = list(sdp.load_events(1, 20)) # skip first 20, take one assert len(events) == 0 + events = list(sdp.load_events(0)) # all + assert len(events) == 15 + @pytest.mark.parametrize("root", [ ".sheerka", diff --git a/tests/sheerkapickle/__init__.py b/tests/sheerkapickle/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sheerkapickle/test_SheerkaPickler.py b/tests/sheerkapickle/test_SheerkaPickler.py new file mode 100644 index 0000000..09e9e09 --- /dev/null +++ b/tests/sheerkapickle/test_SheerkaPickler.py @@ -0,0 +1,172 @@ +import logging + +import pytest +from core.concept import Concept, ConceptParts +from sheerkapickle import tags +from sheerkapickle.SheerkaPickler import SheerkaPickler +from sheerkapickle.SheerkaUnpickler import SheerkaUnpickler + +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka + + +class Obj: + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, Obj): + return False + + return self.a == other.a and self.b == other.b and self.c == other.c + + def __hash__(self): + return hash((self.a, self.b, self.c)) + + +class TestSheerkaPickler(TestUsingFileBasedSheerka): + + @pytest.mark.parametrize("obj, expected", [ + (1, 1), + (3.14, 3.14), + ("a string", "a string"), + (True, True), + (None, None), + ([1, 3.14, "a string"], [1, 3.14, "a string"]), + ((1, 3.14, "a string"), {tags.TUPLE: [1, 3.14, "a string"]}), + ({1}, {tags.SET: [1]}), + ({"a": "a", "b": 3.14, "c": True}, {"a": "a", "b": 3.14, "c": True}), + ({1: "a", 2: 3.14, 3: True}, {1: "a", 2: 3.14, 3: True}), + ([1, [3.14, "a string"]], [1, [3.14, "a string"]]), + ([1, (3.14, "a string")], [1, {tags.TUPLE: [3.14, "a string"]}]), + ([], []), + (ConceptParts.BODY, {tags.ENUM: "core.concept.ConceptParts.BODY"}), + ]) + def test_i_can_flatten_and_restore_primitives(self, obj, expected): + sheerka = self.get_sheerka() + flatten = SheerkaPickler(sheerka).flatten(obj) + assert flatten == expected + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj + + def test_i_can_flatten_and_restore_instances(self): + sheerka = self.get_sheerka() + + obj1 = Obj(1, "b", True) + obj2 = Obj(3.14, ("a", "b"), obj1) + + flatten = SheerkaPickler(sheerka).flatten(obj2) + assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj', + 'a': 3.14, + 'b': {'_sheerka/tuple': ['a', 'b']}, + 'c': {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj', + 'a': 1, + 'b': 'b', + 'c': True}} + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj2 + + def test_i_can_manage_circular_reference(self): + sheerka = self.get_sheerka() + + obj1 = Obj(1, "b", True) + obj1.c = obj1 + + flatten = SheerkaPickler(sheerka).flatten(obj1) + assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj', + 'a': 1, + 'b': 'b', + 'c': {'_sheerka/id': 0}} + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded.a == obj1.a + assert decoded.b == obj1.b + assert decoded.c == decoded + + def test_i_can_flatten_obj_with_new_props(self): + sheerka = self.get_sheerka() + + obj = Obj(1, "b", True) + obj.z = "new prop" + + flatten = SheerkaPickler(sheerka).flatten(obj) + assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj', + 'a': 1, + 'b': 'b', + 'c': True, + 'z': "new prop"} + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj + + @pytest.mark.parametrize("obj, expected", [ + ({None: "a"}, {'null': "a"}), + ({ConceptParts.BODY: "a"}, {'core.concept.ConceptParts.BODY': 'a'}), + ({(1, 2): "a"}, {(1, 2): "a"}), + ]) + def test_i_can_manage_specific_keys_in_dictionaries(self, obj, expected): + sheerka = self.get_sheerka() + + flatten = SheerkaPickler(sheerka).flatten(obj) + assert flatten == expected + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj + + def test_i_can_use_concept_as_dictionary_key(self): + sheerka = self.get_sheerka() + + concept = Concept("foo").init_key() + sheerka.set_id_if_needed(concept, False) + sheerka.add_in_cache(concept) + obj = {concept: "a"} + flatten = SheerkaPickler(sheerka).flatten(obj) + assert flatten == {':c:foo:1001:': 'a'} + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj + + def test_i_can_manage_references(self): + sheerka = self.get_sheerka() + + foo = Obj("foo", "bar", "baz") + obj = [ConceptParts.BODY, foo, ConceptParts.WHERE, ConceptParts.BODY, foo] + flatten = SheerkaPickler(sheerka).flatten(obj) + + assert flatten == [{'_sheerka/enum': 'core.concept.ConceptParts.BODY'}, + {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj', + 'a': 'foo', + 'b': 'bar', + 'c': 'baz'}, + {'_sheerka/enum': 'core.concept.ConceptParts.WHERE'}, + {'_sheerka/id': 0}, + {'_sheerka/id': 1}] + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj + + def test_serialize_concept(self): + sheerka = self.get_sheerka() + + foo = Concept("doo") + flatten = SheerkaPickler(sheerka).flatten(foo) + restored = SheerkaUnpickler(sheerka).restore(flatten) + + assert restored == foo + + def test_i_do_not_encode_logger(self): + sheerka = self.get_sheerka() + + logger = logging.getLogger("log_name") + logger2 = logging.getLogger("log_name2") + obj = Obj("foo", logger, {"a": logger, "b": logger2}) + + flatten = SheerkaPickler(sheerka).flatten(obj) + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == Obj("foo", None, {"a": None, "b": None}) diff --git a/tests/sheerkapickle/test_sheerka_handlers.py b/tests/sheerkapickle/test_sheerka_handlers.py new file mode 100644 index 0000000..fbbefd6 --- /dev/null +++ b/tests/sheerkapickle/test_sheerka_handlers.py @@ -0,0 +1,285 @@ +import sheerkapickle +from core.builtin_concepts import BuiltinConcepts, UserInputConcept, ReturnValueConcept +from core.concept import Concept, ConceptParts +from core.sheerka.ExecutionContext import ExecutionContext +from core.tokenizer import Tokenizer +from evaluators.ConceptEvaluator import ConceptEvaluator +from parsers.DefaultParser import DefaultParser +from sdp.sheerkaDataProvider import Event + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): + + def test_i_can_encode_decode_unknown_concept_metadata(self): + sheerka = self.get_sheerka() + + concept = Concept(name="foo", key="my_key") + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.key": "my_key"}' + assert decoded == concept + + concept = Concept("foo", is_builtin=True, is_unique=True) + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.is_builtin": true, "meta.is_unique": true}' + + concept = Concept("foo", body="my_body") + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "my_body"}' + + concept = Concept("foo", pre="my_pre") + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.pre": "my_pre"}' + + concept = Concept("foo", post="my_post") + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.post": "my_post"}' + + concept = Concept("foo", where="my_where") + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.where": "my_where"}' + + concept = Concept("foo").def_prop("a", "value_a").def_prop("b", "value_b") + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + + def test_i_can_encode_decode_unknown_concept_values(self): + sheerka = self.get_sheerka() + + concept = Concept("foo") + concept.values[ConceptParts.PRE] = 10 # an int + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "pre": 10}' + + concept = Concept("foo") + concept.values[ConceptParts.POST] = 'a string' # an int + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "post": "a string"}' + + concept = Concept("foo") + concept.values[ConceptParts.WHERE] = ['a string', 3.14] # a list + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": ["a string", 3.14]}' + + concept = Concept("foo") + concept.values[ConceptParts.WHERE] = ('a string', 3.14) # a tuple + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": {"_sheerka/tuple": ["a string", 3.14]}}' + + concept = Concept("foo") + concept.values[ConceptParts.BODY] = Concept("foo", body="foo_body") + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "foo_body"}}' + + def test_i_can_encode_decode_unknown_concept_properties(self): + sheerka = self.get_sheerka() + + concept = Concept("foo") + concept.set_prop("a", "value_a") # string + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "value_a"]]}' + + concept = Concept("foo") + concept.set_prop("a", 10) # int + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", 10]]}' + + concept = Concept("foo") + concept.set_prop("a", Concept("bar")) # another concept + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}' + + concept = Concept("foo") + concept.set_prop("a", "a").set_prop("b", "b") # at least two props + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "a"], ["b", "b"]]}' + + def test_i_can_encode_decode_known_concepts(self): + sheerka = self.get_sheerka() + + ref_concept = Concept("my_name", True, True, "my_key", "my_body", "my_where", "my_pre", "my_post", "my_def") + ref_concept.def_prop("a", "value_a").def_prop("b", "value_b") + + sheerka.create_new_concept(self.get_context(sheerka), ref_concept) + + to_string = sheerkapickle.encode(sheerka, ref_concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == ref_concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"]}' + + concept = Concept().update_from(sheerka.get_by_id(ref_concept.id)) + concept.set_metadata_value(ConceptParts.BODY, Concept("bar")) + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"], "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}}' + + def test_i_can_manage_reference_of_the_same_object(self): + sheerka = self.get_sheerka() + + concept_ref = Concept("foo") + + concept = Concept("bar") + concept.set_metadata_value(ConceptParts.PRE, concept_ref) + concept.set_metadata_value(ConceptParts.BODY, concept_ref) + + to_string = sheerkapickle.encode(sheerka, concept) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == concept + assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "bar", "pre": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo"}, "body": {"_sheerka/id": 1}}' + + def test_i_can_encode_decode_user_input(self): + sheerka = self.get_sheerka() + + user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body="my_text", user_name="my_user_name") + + to_string = sheerkapickle.encode(sheerka, user_input) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == user_input + assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "my_text"}' + + def test_i_can_encode_decode_user_input_when_tokens(self): + sheerka = self.get_sheerka() + + text = "I have 'a complicated' 10 text" + tokens = list(Tokenizer(text)) + user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens, user_name="my_user_name") + + to_string = sheerkapickle.encode(sheerka, user_input) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == UserInputConcept(text, "my_user_name") + assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "{text}"' + '}' + + def test_i_can_encode_decode_return_value(self): + sheerka = self.get_sheerka() + + ret_val = sheerka.ret("who", True, 10) + + to_string = sheerkapickle.encode(sheerka, ret_val) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == ret_val + assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10}' + + def test_i_can_encode_decode_return_value_with_parent(self): + sheerka = self.get_sheerka() + + ret_val = sheerka.ret("who", True, 10) + ret_val_parent = sheerka.ret("parent_who", True, "10") + ret_val.parents = [ret_val_parent, ret_val_parent] + + to_string = sheerkapickle.encode(sheerka, ret_val) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == ret_val + assert decoded.parents == ret_val.parents + parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]' + assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}' + + def test_i_can_encode_decode_return_values_with_complex_body(self): + sheerka = self.get_sheerka() + + ret_val = sheerka.ret("who", True, Concept("foo", body="bar")) + + to_string = sheerkapickle.encode(sheerka, ret_val) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == ret_val + + def test_i_can_encode_decode_return_values_from_concepts_parsers_or_evaluators(self): + sheerka = self.get_sheerka() + + foo = Concept("foo") + sheerka.set_id_if_needed(foo, False) + ret_val = sheerka.ret(foo, True, 10) + to_string = sheerkapickle.encode(sheerka, ret_val) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == sheerka.ret("c:1001:", True, 10) + + ret_val = sheerka.ret(DefaultParser(), True, 10) + to_string = sheerkapickle.encode(sheerka, ret_val) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == sheerka.ret("parsers.Default", True, 10) + + ret_val = sheerka.ret(ConceptEvaluator(), True, 10) + to_string = sheerkapickle.encode(sheerka, ret_val) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == sheerka.ret("evaluators.Concept", True, 10) + + def test_i_can_encode_decode_execution_context(self): + sheerka = self.get_sheerka() + + context = ExecutionContext("who", Event("xxx"), sheerka, "my desc") + input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)] + context.inputs = {"a": input_list, "b": Concept("foo")} + context.values = {"c": input_list, "d": Concept("bar")} + context.obj = Concept("baz") + context.push("who3", "sub_child1") + context.push("who4", "sub_child2") + + to_string = sheerkapickle.encode(sheerka, context) + decoded = sheerkapickle.decode(sheerka, to_string) + assert decoded == context + + def test_complicated_execution_context(self): + sheerka = self.get_sheerka(skip_builtins_in_db=False) + + text = "def concept one as 1" + execution_context = ExecutionContext("s", Event(), sheerka, f"Evaluating '{text}'") + user_input = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="n")) + reduce_requested = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + + steps = [ + BuiltinConcepts.BEFORE_PARSING, + BuiltinConcepts.PARSING, + BuiltinConcepts.AFTER_PARSING, + BuiltinConcepts.BEFORE_EVALUATION, + BuiltinConcepts.EVALUATION, + BuiltinConcepts.AFTER_EVALUATION + ] + + ret = sheerka.execute(execution_context, [user_input, reduce_requested], steps) + execution_context.add_values(return_values=ret) + + to_string = sheerkapickle.encode(sheerka, execution_context) + decoded = sheerkapickle.decode(sheerka, to_string) + + return_value = decoded.values["return_values"][0].value + assert sheerka.isinstance(return_value, BuiltinConcepts.NEW_CONCEPT) + + def test_encode_simple_concept(self): + sheerka = self.get_sheerka() + + foo = Concept("foo") + to_string = sheerkapickle.encode(sheerka, foo) + decoded = sheerkapickle.decode(sheerka, to_string) + + assert decoded == foo