Refactored ExecutionContext serialization (added sheerkapickle) and added History management

_concepts.txt

@@ -0,0 +1,10 @@
+def concept one as 1
+def concept two as 2
+def concept three as 3
+def concept one as 1
+
+def concept two as 2
+
+def concept three as 3
+
+def concept four as 4

src/core/concept.py

@@ -343,6 +343,8 @@ class Concept:
         :param metadata:
         :return:
         """
+        if metadata not in self.values:
+            return None
         return self.values[metadata]
 
     def auto_init(self):

src/core/sheerka/ExecutionContext.py

@@ -4,9 +4,22 @@ import time
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from sdp.sheerkaDataProvider import Event
+from sheerkapickle.SheerkaPickler import SheerkaPickler
 
 DEBUG_TAB_SIZE = 4
 
+PROPERTIES_TO_SERIALIZE = ("_id",
+                           "_bag",
+                           "_start",
+                           "_stop",
+                           "who",
+                           "desc",
+                           "children",
+                           "inputs",
+                           "values",
+                           "obj",
+                           "concepts")
+
 
 class ExecutionContext:
     """
@@ -31,7 +44,7 @@ class ExecutionContext:
                  **kwargs):
 
         self._parent = None
-        self._id = ExecutionContext.get_id(event.get_digest())
+        self._id = ExecutionContext.get_id(event.get_digest()) if event else None
         self._tab = ""
         self._bag = {}  # other variables
         self._start = 0
@@ -90,6 +103,26 @@ class ExecutionContext:
         msg += ")"
         return msg
 
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, ExecutionContext):
+            return False
+
+        for prop in PROPERTIES_TO_SERIALIZE:
+            if prop == "who":
+                value = str(getattr(self, prop))
+                other_value = str(getattr(other, prop))
+            else:
+                value = getattr(self, prop)
+                other_value = getattr(other, prop)
+
+            if value != other_value:
+                return False
+
+        return True
+
     def add_preprocess(self, name, **kwargs):
         preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
         preprocess.set_prop("name", name)
@@ -189,10 +222,6 @@ class ExecutionContext:
             to_str = self.return_value_to_str(r)
             logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
-    def to_dict(self):
-        from core.sheerka_transform import SheerkaTransform
-        st = SheerkaTransform(self.sheerka)
-        return st.to_dict(self)
 
     @staticmethod
     def return_value_to_str(r):

src/core/sheerka/Services/SheerkaDump.py

@@ -39,7 +39,8 @@ class SheerkaDump:
                 self.sheerka.log.info(f"name   : {c.name}")
                 self.sheerka.log.info(f"bnf    : {c.metadata.definition}")
                 self.sheerka.log.info(f"key    : {c.key}")
-                self.sheerka.log.info(f"body   : {c.body}")
+                self.sheerka.log.info(f"body   : {c.metadata.body}")
+                self.sheerka.log.info(f"value  : {c.body}")
                 self.sheerka.log.info(f"digest : {c.get_digest()}")
                 first = False
 
@@ -57,7 +58,7 @@ class SheerkaDump:
 
             while True:
                 try:
-                    if h.user != self.sheerka.name:
+                    if h.event.user != self.sheerka.name:
                         self.sheerka.log.info(h)
                         count += 1
                     h = next(history)

src/core/sheerka/Services/SheerkaExecute.py

@@ -218,7 +218,7 @@ class SheerkaExecute:
 
         for step in execution_steps:
             copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values]
-            with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context:
+            with execution_context.push(step=step, iteration=0, desc=f"{step=}") as sub_context:
                 sub_context.log(logger or self.sheerka.log, f"{step=}, context='{sub_context}'")
 
                 if step == BuiltinConcepts.PARSING:

src/core/sheerka/Services/SheerkaHistoryManager.py

@@ -0,0 +1,74 @@
+from collections import namedtuple
+
+from sdp.sheerkaDataProvider import Event
+
+hist = namedtuple("History", "text status")  # tests purposes only
+
+
+class History:
+    def __init__(self, event: Event, result):
+        self.event = event
+        self.result = result
+        self._status = None
+
+    def __str__(self):
+        msg = f"{self.event.get_digest()} {self.event.date.strftime('%d/%m/%Y %H:%M:%S')} : {self.event.message}"
+        status = self.status
+        if status is not None:
+            msg += f" => {status}"
+        return msg
+
+    def __repr__(self):
+        return f"event={self.event!r}, status={self.status}, result={self.result}"
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if isinstance(other, hist):
+            return self.event.message == other.text and self.status == other.status
+
+        if not isinstance(other, History):
+            return False
+
+        return self.event == other.event and self.result == other.result
+
+    @property
+    def status(self):
+        if self._status:
+            return self._status
+
+        if not self.result or "return_values" not in self.result.values:
+            return
+
+        if hasattr(self.result.values["return_values"], "__iter__"):
+            if len(self.result.values["return_values"]) != 1:
+                self._status = False
+                return self._status
+            else:
+                self._status = self.result.values["return_values"][0].status
+                return self._status
+        else:
+            self._status = self.result.values["return_values"].status
+            return self._status
+
+
+class SheerkaHistoryManager:
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+
+    def history(self, depth_or_digest, start):
+        """
+        Load history
+        :param depth_or_digest: number of items or digest
+        :param start:
+        :return:
+        """
+
+        events = list(self.sheerka.sdp.load_events(depth_or_digest, start))
+        for event in events:
+            try:
+                result = self.sheerka.sdp.load_result(self.sheerka, event.get_digest())
+            except (IOError, KeyError):
+                result = None
+            yield History(event, result)

src/core/sheerka/Sheerka.py

@@ -2,11 +2,12 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConc
     UnknownConcept
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
 from core.sheerka.ExecutionContext import ExecutionContext
-from core.sheerka.SheerkaCreateNewConcept import SheerkaCreateNewConcept
+from core.sheerka.Services.SheerkaCreateNewConcept import SheerkaCreateNewConcept
-from core.sheerka.SheerkaDump import SheerkaDump
+from core.sheerka.Services.SheerkaDump import SheerkaDump
-from core.sheerka.SheerkaEvaluateConcept import SheerkaEvaluateConcept
+from core.sheerka.Services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
-from core.sheerka.SheerkaExecute import SheerkaExecute
+from core.sheerka.Services.SheerkaExecute import SheerkaExecute
-from core.sheerka.SheerkaSetsManager import SheerkaSetsManager
+from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager
+from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 import core.utils
 import core.builtin_helpers
@@ -21,7 +22,7 @@ import logging
 #     BuiltinConcepts.AFTER_EVALUATION]
 
 CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
-
+CONCEPTS_FILE = "_concepts.txt"
 
 class Sheerka(Concept):
     """
@@ -81,6 +82,7 @@ class Sheerka(Concept):
         self.dump_handler = SheerkaDump(self)
         self.sets_handler = SheerkaSetsManager(self)
         self.evaluate_concept_handler = SheerkaEvaluateConcept(self)
+        self.history_handler = SheerkaHistoryManager(self)
 
     def initialize(self, root_folder: str = None):
         """
@@ -92,6 +94,9 @@ class Sheerka(Concept):
         """
 
         try:
+            from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
+            initialize_pickle_handlers()
+
             self.sdp = SheerkaDataProvider(root_folder)
             if self.sdp.first_time:
                 self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
@@ -104,11 +109,16 @@ class Sheerka(Concept):
             self.initialize_builtin_parsers()
             self.initialize_builtin_evaluators()
             self.initialize_concepts_definitions(exec_context)
+            res = ReturnValueConcept(self, True, self)
+
+            exec_context.add_values(return_values=res)
+            if not self.skip_builtins_in_db:
+                self.sdp.save_result(self, exec_context)
 
         except IOError as e:
-            return ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
+            res = ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
 
-        return ReturnValueConcept(self, True, self)
+        return res
 
     def initialize_builtin_concepts(self):
         """
@@ -232,7 +242,12 @@ class Sheerka(Concept):
             execution_context.add_values(return_values=ret)
 
             if not self.skip_builtins_in_db:
-                self.sdp.save_result(execution_context)
+                self.sdp.save_result(self, execution_context)
+
+            #hack to save valid concept definition
+            if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
+                with open(CONCEPTS_FILE, "a") as f:
+                    f.write(text + "\n")
         return ret
 
     def execute(self, execution_context, return_values, execution_steps, logger=None):
@@ -549,8 +564,19 @@ class Sheerka(Concept):
 
     def history(self, page=10, start=0):
         """Gets the history of all commands"""
-        return self.sdp.load_events(page, start)
+        return self.history_handler.history(page, start)
 
+    def restore(self):
+        """
+        Restore the state with all previous valid concept definitions
+        :return:
+        """
+        try:
+            with open(CONCEPTS_FILE, "r") as f:
+                for line in f.readlines():
+                    self.evaluate_user_input(line)
+        except IOError:
+            pass
 
     def test(self):
         return f"I have access to Sheerka !"

src/core/sheerka_transform.py

@@ -1,161 +0,0 @@
-import dataclasses
-from enum import Enum
-
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE
-from core.sheerka.Sheerka import ExecutionContext
-from core.tokenizer import Token
-from evaluators.BaseEvaluator import BaseEvaluator
-from parsers.BaseParser import BaseParser, Node
-from parsers.BnfParser import BnfParser
-from parsers.ConceptLexerParser import UnrecognizedTokensNode, ParsingExpression
-from parsers.PythonParser import PythonNode
-from sdp.sheerkaDataProvider import Event
-
-OBJ_TYPE_KEY = "__type__"
-OBJ_ID_KEY = "__id__"
-OBJ_NAME_KEY = "__name__"
-
-default_concept = Concept()
-
-
-class SheerkaTransformType(Enum):
-    Concept = 1
-    Reference = 2
-    ExecutionContext = 3
-    Event = 4
-    Node = 5
-    Exception = 6
-
-    def __repr__(self):
-        return self.__class__.__name__ + "." + self.name
-
-
-class SheerkaTransform:
-
-    def __init__(self, sheerka):
-        self.ids = {}
-        self.sheerka = sheerka
-        self.id_count = -1
-
-    def to_dict(self, obj):
-
-        if isinstance(obj, (Concept, ExecutionContext, Event)):
-            exists, _id = self.exist(obj)
-            if exists:
-                return {
-                    OBJ_TYPE_KEY: SheerkaTransformType.Reference,
-                    OBJ_ID_KEY: _id
-                }
-            else:
-                self.id_count = self.id_count + 1
-                self.ids[obj] = self.id_count
-
-        if isinstance(obj, Concept):
-            return self.concept_to_dict(obj)
-
-        elif isinstance(obj, ExecutionContext):
-            return self.execution_context_to_dict(obj)
-
-        elif isinstance(obj, Event):
-            return {
-                OBJ_TYPE_KEY: SheerkaTransformType.Event,
-                OBJ_ID_KEY: self.id_count,
-                'digest': obj.get_digest()}
-
-        elif isinstance(obj, (BaseParser, BaseEvaluator, BnfParser)):
-            return obj.name
-
-        elif isinstance(obj, Token):
-            return obj.__dict__
-
-        elif isinstance(obj, PythonNode):
-            return {
-                OBJ_TYPE_KEY: SheerkaTransformType.Node,
-                OBJ_NAME_KEY: "PythonNode",
-                'source': obj.source,
-                'ast_': obj.get_dump(obj.ast_)
-            }
-
-        elif isinstance(obj, Node):
-            to_dict = {
-                OBJ_TYPE_KEY: SheerkaTransformType.Node,
-                OBJ_NAME_KEY: obj.__class__.__name__,
-            }
-            for k, v in obj.__dict__.items():
-                to_dict[k] = self.to_dict(v)
-
-            return to_dict
-
-        elif isinstance(obj, Exception):
-            to_dict = {
-                OBJ_TYPE_KEY: SheerkaTransformType.Exception,
-                OBJ_NAME_KEY: obj.__class__.__name__,
-            }
-            for k, v in obj.__dict__.items():
-                to_dict[k] = self.to_dict(v)
-            return to_dict
-
-        elif isinstance(obj, ParsingExpression):
-            return obj.__repr__()
-
-        elif isinstance(obj, dict):
-            return dict((str(k) if isinstance(k, Concept) else k, self.to_dict(v)) for k, v in obj.items())
-
-        elif hasattr(obj, "__iter__") and not isinstance(obj, str):
-            return list(self.to_dict(o) for o in obj)
-
-        else:
-            return obj
-
-    def concept_to_dict(self, obj: Concept):
-        to_dict = {
-            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-            OBJ_ID_KEY: self.id_count,
-        }
-        if obj.id:
-            ref = self.sheerka.get(obj.key, obj.id)
-            to_dict["id"] = obj.id
-        else:
-            ref = default_concept
-
-        # transform metadata
-        for prop in PROPERTIES_TO_SERIALIZE:
-            value = getattr(obj.metadata, prop)
-            ref_value = getattr(ref.metadata, prop)
-            if value != ref_value:
-                to_dict["meta." + prop] = self.to_dict(value)
-
-        # transform value
-        for metadata, value in obj.values.items():
-            ref_value = ref.values[metadata] if metadata in ref.values else None
-            if value != ref_value:
-                to_dict[metadata.value] = self.to_dict(value)
-
-        # transform properties
-        for prop in obj.props:
-            value = obj.props[prop].value
-            if prop not in ref.props or value != ref.props[prop].value:
-                if "props" not in to_dict:
-                    to_dict["props"] = []
-                to_dict["props"].append((prop, self.to_dict(value)))
-
-        return to_dict
-
-    def execution_context_to_dict(self, obj: ExecutionContext):
-        to_dict = {
-            OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
-            OBJ_ID_KEY: self.id_count
-        }
-        for property_name in obj.__dict__:
-            if property_name == "sheerka":
-                continue
-            to_dict[property_name] = self.to_dict(getattr(obj, property_name))
-
-        return to_dict
-
-    def exist(self, obj):
-        for k, v in self.ids.items():
-            if id(k) == id(obj) or k == obj:
-                return True, v
-
-        return False, None

src/core/utils.py

@@ -228,6 +228,7 @@ def escape_char(text, to_escape):
 
     return res
 
+
 def pp(items):
     if not hasattr(items, "__iter__"):
         return str(items)
@@ -236,3 +237,66 @@ def pp(items):
         return str(items)
 
     return "    \n" + "    \n".join(str(item) for item in items)
+
+
+def decode_concept(concept_repr):
+    """
+    if concept_repr is like :c:key:id:
+    return the key and the id
+    :param concept_repr:
+    :return:
+    """
+    if not (concept_repr and isinstance(concept_repr, str) and concept_repr.startswith(":c:")):
+        return None, None
+
+    i = 3
+    length = len(concept_repr)
+    key = ""
+    while i < length:
+        if concept_repr[i] == ":":
+            break
+        key += concept_repr[i]
+        i += 1
+    else:
+        return None, None
+
+    i += 1
+    if i >= length:
+        return key, None
+
+    id = ""
+    while i < length:
+        if concept_repr[i] == ":":
+            break
+        id += concept_repr[i]
+        i += 1
+    else:
+        return None, None
+
+    return key, id
+
+
+def decode_enum(enum_repr: str):
+    """
+    Tries to transform ClassName.Name into an enum
+    :param enum_repr:
+    :return:
+    """
+    if not (enum_repr and isinstance(enum_repr, str)):
+        return None
+
+    try:
+        idx = enum_repr.rindex(".")
+        if idx == len(enum_repr):
+            return None
+
+        cls_name = enum_repr[:idx]
+        cls = get_class(cls_name)
+        name = enum_repr[idx + 1:]
+        return cls[name]
+
+    except ValueError:
+        return None
+
+    except TypeError:
+        return None

src/sdp/sheerkaDataProvider.py

@@ -36,6 +36,9 @@ class Event(object):
     def __str__(self):
         return f"{self.date.strftime('%d/%m/%Y %H:%M:%S')} {self.message}"
 
+    def __repr__(self):
+        return f"{self.get_digest()[:12]} {self.message}"
+
     def get_digest(self):
         """
         Returns the digest of the event
@@ -64,6 +67,7 @@ class Event(object):
         self.date = datetime.fromisoformat(as_dict["date"])
         self.message = as_dict["message"]
         self.parents = as_dict["parents"]
+        self._digest = as_dict["_digest"]  # freeze the digest
 
 
 class ObjToUpdate:
@@ -667,9 +671,9 @@ class SheerkaDataProvider:
         :param event:
         :return: digest of the event
         """
-        digest = event.get_digest()
         parent = self.get_snapshot(SheerkaDataProvider.LastEventFile)
         event.parents = [parent] if parent else None
+        digest = event.get_digest()  # must be call after setting the parents
 
         target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest)
         if self.io.exists(target_path):
@@ -712,7 +716,7 @@ class SheerkaDataProvider:
                 digest = event.parents[0]
 
         count = 0
-        while count < page_size:
+        while count < page_size or page_size <= 0:
             event = self.load_event(digest)
             if event is None:
                 return
@@ -725,12 +729,13 @@ class SheerkaDataProvider:
             digest = event.parents[0]
             count += 1
 
-    def save_result(self, execution_context):
+    def save_result(self, sheerka, execution_context):
         """
         Save the execution context associated with an event
         To make a long story short,
         for every single user input, there is an event (which is the first thing that is created)
         and a result (the ExecutionContext created by sheerka.evaluate_user_input()
+        :param sheerka:
         :param execution_context:
         :return:
         """
@@ -740,14 +745,16 @@ class SheerkaDataProvider:
         if self.io.exists(target_path):
             return digest
 
-        self.io.write_binary(target_path, self.serializer.serialize(execution_context, None).read())
+        context = SerializerContext(sheerka=sheerka)
+        self.io.write_binary(target_path, self.serializer.serialize(execution_context, context).read())
         return digest
 
-    def load_result(self, digest):
+    def load_result(self, sheerka, digest):
         target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + "_result"
 
         with self.io.open(target_path, "rb") as f:
-            return self.serializer.deserialize(f, None)
+            context = SerializerContext(sheerka=sheerka)
+            return self.serializer.deserialize(f, context)
 
     def save_state(self, state: State):
         digest = state.get_digest()

src/sdp/sheerkaSerializer.py

@@ -1,18 +1,17 @@
-import dataclasses
 import json
-import pickle
 import datetime
+import pickle
 import struct
 import io
 from dataclasses import dataclass
+
+import sheerkapickle
 from core.sheerka_logger import get_logger
 from enum import Enum
 
 import core.utils
 
 from core.concept import Concept
-from core.tokenizer import Token
-from parsers.BaseParser import Node
 
 
 def json_default_converter(o):
@@ -40,6 +39,7 @@ def json_default_converter(o):
 class SerializerContext:
     user_name: str = None
     origin: str = None
+    sheerka: object = None
 
 
 class Serializer:
@@ -85,6 +85,7 @@ class Serializer:
             raise TypeError(f"Don't know how to serialize {type(obj)}")
 
         serializer = serializers[0]
+        self.log.debug(f"Serializing '{obj}' using '{serializer.name}'")
 
         stream = io.BytesIO()
         header = struct.pack(Serializer.HEADER_FORMAT, bytes(serializer.name, "utf-8"), serializer.version)
@@ -248,23 +249,23 @@ class DictionarySerializer(PickleSerializer):
 
 
 class ExecutionContextSerializer(BaseSerializer):
+    CLASS_NAME = "core.sheerka.ExecutionContext.ExecutionContext"
+
     def __init__(self):
         BaseSerializer.__init__(self, "R", 1)
 
     def matches(self, obj):
-        return core.utils.get_full_qualified_name(obj) == "core.sheerka.ExecutionContext.ExecutionContext"
+        return core.utils.get_full_qualified_name(obj) == self.CLASS_NAME
 
     def dump(self, stream, obj, context):
-        as_json = obj.to_dict()
+        stream.write(sheerkapickle.encode(context.sheerka, obj).encode("utf-8"))
-        stream.write(json.dumps(as_json, default=json_default_converter).encode("utf-8"))
         stream.seek(0)
         return stream
 
     def load(self, stream, context):
         json_stream = stream.read().decode("utf-8")
-        json_message = json.loads(json_stream)
+        obj = sheerkapickle.decode(context.sheerka, json_stream)
-        obj = core.utils.get_class("core.sheerka.ExecutionContext")()
+        #json_message = json.loads(json_stream)
-        obj.from_dict(json_message)
         return obj
 
 #

src/sheerkapickle/SheerkaPickler.py

@@ -0,0 +1,132 @@
+import json
+from logging import Logger
+
+import core.utils
+from core.concept import Concept
+
+from sheerkapickle import utils, tags, handlers
+
+
+def encode(sheerka, obj):
+    pickler = SheerkaPickler(sheerka)
+    data = pickler.flatten(obj)
+    return json.dumps(data)
+
+
+class ToReduce:
+    def __init__(self, predicate, get_value):
+        self.predicate = predicate
+        self.get_value = get_value
+
+
+class SheerkaPickler:
+    """
+    Json sheerkapickle
+    Inspired by jsonpickle (https://github.com/jsonpickle/jsonpickle)
+    which failed to work in my environment
+    """
+
+    def __init__(self, sheerka):
+        self.ids = {}
+        self.objs = []
+        self.id_count = -1
+        self.sheerka = sheerka
+        self.to_reduce = []
+
+        self.to_reduce.append(ToReduce(lambda o: isinstance(o, Logger), lambda o: None))
+        from parsers.BaseParser import BaseParser
+        from evaluators.BaseEvaluator import BaseEvaluator
+        self.to_reduce.append(ToReduce(lambda o: isinstance(o, (BaseParser, BaseEvaluator)), lambda o: o.name))
+
+    def flatten(self, obj):
+        if utils.is_primitive(obj):
+            return obj
+
+        if utils.is_tuple(obj):
+            return {tags.TUPLE: [self.flatten(v) for v in obj]}
+
+        if utils.is_set(obj):
+            return {tags.SET: [self.flatten(v) for v in obj]}
+
+        if utils.is_list(obj):
+            return [self.flatten(v) for v in obj]
+
+        if utils.is_dictionary(obj):
+            return self._flatten_dict(obj)
+
+        if utils.is_enum(obj):
+            return self._flatten_enum(obj)
+
+        if utils.is_object(obj):
+            return self._flatten_obj_instance(obj)
+
+        raise Exception(f"Cannot flatten '{obj}'")
+
+    def _flatten_dict(self, obj):
+        data = {}
+        for k, v in obj.items():
+            if k is None:
+                k_str = "null"
+            elif utils.is_enum(k):
+                k_str = core.utils.get_full_qualified_name(k) + "." + k.name
+            elif isinstance(k, Concept):
+                k_str = f":c:{k.key}:{k.id}:"
+            else:
+                k_str = k
+
+            data[k_str] = self.flatten(v)
+
+        return data
+
+    def _flatten_enum(self, obj):
+        # check if the object was already seen
+        exists, _id = self.exist(obj)
+        if exists:
+            return {tags.ID: _id}
+        else:
+            self.id_count = self.id_count + 1
+            self.ids[id(obj)] = self.id_count
+            self.objs.append(obj)
+
+        data = {}
+        class_name = core.utils.get_full_qualified_name(obj)
+        data[tags.ENUM] = class_name + "." + obj.name
+        return data
+
+    def _flatten_obj_instance(self, obj):
+        for reduce in self.to_reduce:
+            if reduce.predicate(obj):
+                return reduce.get_value(obj)
+
+        # check if the object was already seen
+        exists, _id = self.exist(obj)
+        if exists:
+            return {tags.ID: _id}
+        else:
+            self.id_count = self.id_count + 1
+            self.ids[id(obj)] = self.id_count
+            self.objs.append(obj)
+
+        # flatten
+        data = {}
+        cls = obj.__class__ if hasattr(obj, '__class__') else type(obj)
+        class_name = utils.importable_name(cls)
+        data[tags.OBJECT] = class_name
+
+        handler = handlers.get(class_name)
+        if handler is not None:
+            return handler(self.sheerka, self).flatten(obj, data)
+
+        if hasattr(obj, "__dict__"):
+            for k, v in obj.__dict__.items():
+                data[k] = self.flatten(v)
+            return data
+
+        return None
+
+    def exist(self, obj):
+        for k, v in self.ids.items():
+            if k == id(obj):
+                return True, v
+
+        return False, None

src/sheerkapickle/SheerkaUnpickler.py

@@ -0,0 +1,105 @@
+import json
+
+import core.utils
+from sheerkapickle import tags, utils, handlers
+
+
+def decode(sheerka, obj):
+    return SheerkaUnpickler(sheerka).restore(json.loads(obj))
+
+
+class SheerkaUnpickler:
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+        self.objs = []
+
+    def restore(self, obj):
+        if has_tag(obj, tags.ID):
+            return self._restore_id(obj)
+
+        if has_tag(obj, tags.TUPLE):
+            return self._restore_tuple(obj)
+
+        if has_tag(obj, tags.SET):
+            return self._restore_set(obj)
+
+        if has_tag(obj, tags.ENUM):
+            return self._restore_enum(obj)
+
+        if has_tag(obj, tags.OBJECT):
+            return self._restore_obj(obj)
+
+        if utils.is_list(obj):
+            return self._restore_list(obj)
+
+        if utils.is_dictionary(obj):
+            return self._restore_dict(obj)
+
+        return obj
+
+    def _restore_list(self, obj):
+        return [self.restore(v) for v in obj]
+
+    def _restore_tuple(self, obj):
+        return tuple([self.restore(v) for v in obj[tags.TUPLE]])
+
+    def _restore_set(self, obj):
+        return set([self.restore(v) for v in obj[tags.SET]])
+
+    def _restore_enum(self, obj):
+        instance = core.utils.decode_enum(obj[tags.ENUM])
+        self.objs.append(instance)
+        return instance
+
+    def _restore_dict(self, obj):
+        data = {}
+        for k, v in obj.items():
+            resolved_key = self._resolve_key(k)
+            data[resolved_key] = self.restore(v)
+        return data
+
+    def _restore_id(self, obj):
+        try:
+            return self.objs[obj[tags.ID]]
+        except IndexError:
+            pass
+
+    def _restore_obj(self, obj):
+        handler = handlers.get(obj[tags.OBJECT])
+
+        if handler:
+            handler = handler(self.sheerka, self)
+            instance = handler.new(obj)
+            self.objs.append(instance)
+            instance = handler.restore(obj, instance)
+        else:
+            cls = core.utils.get_class(obj[tags.OBJECT])
+            instance = cls.__new__(cls)
+            self.objs.append(instance)
+
+            for k, v in obj.items():
+                if k == tags.OBJECT:
+                    continue
+                value = self.restore(v)
+                setattr(instance, k, value)
+
+        return instance
+
+    def _resolve_key(self, key):
+
+        if key == "null":
+            return None
+
+        concept_key, concept_id = core.utils.decode_concept(key)
+        if concept_key is not None:
+            return self.sheerka.new((concept_key, concept_id)) if concept_id else self.sheerka.new(concept_key)
+
+        as_enum = core.utils.decode_enum(key)
+        if as_enum is not None:
+            return as_enum
+
+        return key
+
+
+def has_tag(obj, tag):
+    return type(obj) is dict and tag in obj

src/sheerkapickle/__init__.py

@@ -0,0 +1,7 @@
+from .SheerkaPickler import encode
+from .SheerkaUnpickler import decode
+
+__all__ = ('encode', 'decode')
+
+# register built-in handlers
+__import__('sheerkapickle.handlers', level=0)

src/sheerkapickle/handlers.py

@@ -0,0 +1,233 @@
+import datetime
+import re
+import threading
+import uuid
+
+from sheerkapickle import utils
+
+
+class ToReduce:
+    def __init__(self, predicate, get_value):
+        self.predicate = predicate
+        self.get_value = get_value
+
+
+class SheerkaRegistry(object):
+
+    def __init__(self):
+        self._handlers = {}
+        self._base_handlers = {}
+
+    def get(self, cls_or_name, default=None):
+        """
+        :param cls_or_name: the type or its fully qualified name
+        :param default: default value, if a matching handler is not found
+
+        Looks up a handler by type reference or its fully
+        qualified name. If a direct match
+        is not found, the search is performed over all
+        handlers registered with base=True.
+        """
+        handler = self._handlers.get(cls_or_name)
+        # attempt to find a base class
+        if handler is None and utils.is_type(cls_or_name):
+            for cls, base_handler in self._base_handlers.items():
+                if issubclass(cls_or_name, cls):
+                    return base_handler
+        return default if handler is None else handler
+
+    def register(self, cls, handler=None, base=False):
+        """Register the a custom handler for a class
+
+        :param cls: The custom object class to handle
+        :param handler: The custom handler class (if
+            None, a decorator wrapper is returned)
+        :param base: Indicates whether the handler should
+            be registered for all subclasses
+
+        This function can be also used as a decorator
+        by omitting the `handler` argument::
+
+            @jsonpickle.handlers.register(Foo, base=True)
+            class FooHandler(jsonpickle.handlers.BaseHandler):
+                pass
+
+        """
+        if handler is None:
+            def _register(handler_cls):
+                self.register(cls, handler=handler_cls, base=base)
+                return handler_cls
+
+            return _register
+        if not utils.is_type(cls):
+            raise TypeError('{!r} is not a class/type'.format(cls))
+        # store both the name and the actual type for the ugly cases like
+        # _sre.SRE_Pattern that cannot be loaded back directly
+        self._handlers[utils.importable_name(cls)] = \
+            self._handlers[cls] = handler
+        if base:
+            # only store the actual type for subclass checking
+            self._base_handlers[cls] = handler
+
+    def unregister(self, cls):
+        self._handlers.pop(cls, None)
+        self._handlers.pop(utils.importable_name(cls), None)
+        self._base_handlers.pop(cls, None)
+
+
+registry = SheerkaRegistry()
+register = registry.register
+unregister = registry.unregister
+get = registry.get
+
+
+class BaseHandler(object):
+
+    def __init__(self, sheerka, context):
+        """
+        Initialize a new handler to handle a registered type.
+
+        :Parameters:
+          - `context`: reference to pickler/unpickler
+
+        """
+        self.sheerka = sheerka
+        self.context = context
+
+    def __call__(self, sheerka, context):
+        """This permits registering either Handler instances or classes
+
+        :Parameters:
+          - `context`: reference to pickler/unpickler
+        """
+        self.sheerka = sheerka
+        self.context = context
+        return self
+
+    def flatten(self, obj, data):
+        """
+        Flatten `obj` into a json-friendly form and write result to `data`.
+
+        :param object obj: The object to be serialized.
+        :param dict data: A partially filled dictionary which will contain the
+            json-friendly representation of `obj` once this method has
+            finished.
+        """
+        raise NotImplementedError('You must implement flatten() in %s' %
+                                  self.__class__)
+
+    def new(self, data):
+        raise NotImplementedError('You must implement new() in %s' %
+                                  self.__class__)
+
+    def restore(self, data, instance):
+        """
+        Restore an object of the registered type from the json-friendly
+        representation `obj` and return it.
+        """
+        raise NotImplementedError('You must implement restore() in %s' %
+                                  self.__class__)
+
+    @classmethod
+    def handles(self, cls):
+        """
+        Register this handler for the given class. Suitable as a decorator,
+        e.g.::
+
+            @MyCustomHandler.handles
+            class MyCustomClass:
+                def __reduce__(self):
+                    ...
+        """
+        registry.register(cls, self)
+        return cls
+
+
+# class DatetimeHandler(BaseHandler):
+#     """Custom handler for datetime objects
+#
+#     Datetime objects use __reduce__, and they generate binary strings encoding
+#     the payload. This handler encodes that payload to reconstruct the
+#     object.
+#
+#     """
+#
+#     def flatten(self, obj, data):
+#         pickler = self.context
+#         if not pickler.unpicklable:
+#             return str(obj)
+#         cls, args = obj.__reduce__()
+#         flatten = pickler.flatten
+#         payload = utils.b64encode(args[0])
+#         args = [payload] + [flatten(i, reset=False) for i in args[1:]]
+#         data['__reduce__'] = (flatten(cls, reset=False), args)
+#         return data
+#
+#     def restore(self, data):
+#         cls, args = data['__reduce__']
+#         unpickler = self.context
+#         restore = unpickler.restore
+#         cls = restore(cls, reset=False)
+#         value = utils.b64decode(args[0])
+#         params = (value,) + tuple([restore(i, reset=False) for i in args[1:]])
+#         return cls.__new__(cls, *params)
+#
+#
+# DatetimeHandler.handles(datetime.datetime)
+# DatetimeHandler.handles(datetime.date)
+# DatetimeHandler.handles(datetime.time)
+
+
+class RegexHandler(BaseHandler):
+    """Flatten _sre.SRE_Pattern (compiled regex) objects"""
+
+    def flatten(self, obj, data):
+        data['pattern'] = obj.pattern
+        return data
+
+    def new(self, data):
+        return re.compile(data['pattern'])
+
+    def restore(self, data, instance):
+        return instance
+
+
+RegexHandler.handles(type(re.compile('')))
+
+
+class UUIDHandler(BaseHandler):
+    """Serialize uuid.UUID objects"""
+
+    def flatten(self, obj, data):
+        data['hex'] = obj.hex
+        return data
+
+    def new(self, data):
+        return uuid.UUID(data['hex'])
+
+    def restore(self, data, instance):
+        return instance
+
+
+UUIDHandler.handles(uuid.UUID)
+
+
+class LockHandler(BaseHandler):
+    """Serialize threading.Lock objects"""
+
+    def flatten(self, obj, data):
+        data['locked'] = obj.locked()
+        return data
+
+    def new(self, data):
+        lock = threading.Lock()
+        if data.get('locked', False):
+            lock.acquire()
+        return lock
+
+    def restore(self, data, instance):
+        return instance
+
+
+_lock = threading.Lock()
+LockHandler.handles(_lock.__class__)

src/sheerkapickle/sheerka_handlers.py

@@ -0,0 +1,182 @@
+from core.builtin_concepts import UserInputConcept, ReturnValueConcept, BuiltinConcepts
+from core.sheerka.Sheerka import Sheerka
+from evaluators.BaseEvaluator import BaseEvaluator
+from parsers.BaseParser import BaseParser
+from sheerkapickle.handlers import BaseHandler, registry
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE as CONCEPT_PROPERTIES_TO_SERIALIZE, ConceptParts
+from core.sheerka.ExecutionContext import ExecutionContext, PROPERTIES_TO_SERIALIZE as CONTEXT_PROPERTIES_TO_SERIALIZE
+
+default_concept = Concept()
+CONCEPT_ID = "concept/id"
+
+
+class ConceptHandler(BaseHandler):
+
+    def flatten(self, obj: Concept, data):
+        pickler = self.context
+        sheerka = self.sheerka
+
+        if obj.id:
+            ref = sheerka.get_by_id(obj.id)
+            data[CONCEPT_ID] = (obj.key, obj.id)
+        else:
+            ref = default_concept
+
+        # transform metadata
+        for prop in CONCEPT_PROPERTIES_TO_SERIALIZE:
+            value = getattr(obj.metadata, prop)
+            ref_value = getattr(ref.metadata, prop)
+            if value != ref_value:
+                data["meta." + prop] = pickler.flatten(value)
+
+        # transform value
+        for metadata, value in obj.values.items():
+            ref_value = ref.values[metadata] if metadata in ref.values else None
+            if value != ref_value:
+                data[metadata.value] = pickler.flatten(value)
+
+        # transform properties
+        for prop in obj.props:
+            value = obj.props[prop].value
+            if prop not in ref.props or value != ref.props[prop].value:
+                if "props" not in data:
+                    data["props"] = []
+                data["props"].append((prop, pickler.flatten(value)))
+
+        return data
+
+    def new(self, data):
+        sheerka = self.sheerka
+        return sheerka.new(tuple(data[CONCEPT_ID])) if CONCEPT_ID in data else Concept()
+
+    def restore(self, data, instance):
+        pickler = self.context
+
+        for key, value in data.items():
+            if key.startswith("_sheerka/") or key == CONCEPT_ID:
+                continue
+
+            resolved_value = pickler.restore(data[key])
+
+            if key.startswith("meta."):
+                # get metadata
+                resolved_prop = key[5:]
+                if resolved_prop == "props":
+                    for prop_name, prop_value in resolved_value:
+                        instance.def_prop(prop_name, prop_value)
+                else:
+                    setattr(instance.metadata, resolved_prop, resolved_value)
+            elif key == "props":
+                # get properties
+                for prop_name, prop_value in resolved_value:
+                    instance.set_prop(prop_name, prop_value)
+            else:
+                # get value
+                instance.set_metadata_value(ConceptParts(key), resolved_value)
+
+        return instance
+
+
+class UserInputHandler(BaseHandler):
+
+    def flatten(self, obj: UserInputConcept, data):
+        data[CONCEPT_ID] = (obj.key, obj.id)
+        data["user_name"] = obj.user_name
+        data["text"] = BaseParser.get_text_from_tokens(obj.text) if isinstance(obj.text, list) else obj.text
+        return data
+
+    def new(self, data):
+        sheerka = self.sheerka
+
+        instance = sheerka.new(tuple(data[CONCEPT_ID]), body=data["text"], user_name=data["user_name"])
+        return instance
+
+    def restore(self, data, instance):
+        return instance
+
+
+class ReturnValueHandler(BaseHandler):
+
+    def flatten(self, obj: ReturnValueConcept, data):
+        pickler = self.context
+
+        data["who"] = f"c:{obj.who.id}:" if isinstance(obj.who, Concept) else \
+            obj.who.name if isinstance(obj.who, (BaseParser, BaseEvaluator)) else \
+                obj.who
+        data["status"] = obj.status
+        data["value"] = pickler.flatten(obj.value)
+        if obj.parents:
+            data["parents"] = pickler.flatten(obj.parents)
+        return data
+
+    def new(self, data):
+        sheerka = self.sheerka
+        instance = sheerka.ret(data["who"], data["status"], None)
+        return instance
+
+    def restore(self, data, instance):
+        pickler = self.context
+
+        instance.value = pickler.restore(data["value"])
+        if "parents" in data:
+            instance.parents = pickler.restore(data["parents"])
+        return instance
+
+
+# class BuiltinConceptsHandler(BaseHandler):
+#
+#     def flatten(self, obj: BuiltinConcepts, data):
+#         return data
+#
+#     def restore(self, obj):
+#         pass
+
+
+class SheerkaHandler(BaseHandler):
+
+    def flatten(self, obj: BuiltinConcepts, data):
+        return data
+
+    def new(self, data):
+        return self.sheerka
+
+    def restore(self, data, instance):
+        return instance
+
+
+class ExecutionContextHandler(BaseHandler):
+
+    def flatten(self, obj, data):
+        pickler = self.context
+
+        for prop in CONTEXT_PROPERTIES_TO_SERIALIZE:
+            if prop == "who":
+                value = str(getattr(obj, prop))
+            else:
+                value = getattr(obj, prop)
+
+            if value is not None:
+                data[prop] = pickler.flatten(value)
+
+        return data
+
+    def new(self, data):
+        return ExecutionContext(data["who"], None, None)
+
+    def restore(self, data, instance):
+        pickler = self.context
+
+        for prop in CONTEXT_PROPERTIES_TO_SERIALIZE:
+            if prop not in data or prop == "who":
+                continue
+            setattr(instance, prop, pickler.restore(data[prop]))
+
+        return instance
+
+
+def initialize_pickle_handlers():
+    registry.register(Concept, ConceptHandler, True)
+    registry.register(UserInputConcept, UserInputHandler, True)
+    registry.register(ReturnValueConcept, ReturnValueHandler, True)
+    registry.register(Sheerka, SheerkaHandler, True)
+    registry.register(ExecutionContext, ExecutionContextHandler, True)

src/sheerkapickle/tags.py

@@ -0,0 +1,5 @@
+ID = "_sheerka/id"
+TUPLE = "_sheerka/tuple"
+SET = "_sheerka/set"
+OBJECT = "_sheerka/obj"
+ENUM = "_sheerka/enum"

src/sheerkapickle/utils.py

@@ -0,0 +1,85 @@
+import base64
+import types
+from enum import Enum
+
+class_types = (type,)
+PRIMITIVES = (str, bool, type(None), int, float)
+
+
+def is_type(obj):
+    """Returns True is obj is a reference to a type.
+    """
+    # use "isinstance" and not "is" to allow for metaclasses
+    return isinstance(obj, class_types)
+
+
+def is_enum(obj):
+    return isinstance(obj, Enum)
+
+
+def is_object(obj):
+    """Returns True is obj is a reference to an object instance."""
+
+    return (isinstance(obj, object) and
+            not isinstance(obj, (type, types.FunctionType,
+                                 types.BuiltinFunctionType)))
+
+
+def is_primitive(obj):
+    return type(obj) in PRIMITIVES
+
+
+def is_dictionary(obj):
+    return type(obj) is dict
+
+
+def is_list(obj):
+    return type(obj) is list
+
+
+def is_set(obj):
+    return type(obj) is set
+
+
+def is_bytes(obj):
+    return type(obj) is bytes
+
+
+def is_tuple(obj):
+    return type(obj) is tuple
+
+
+def b64encode(data):
+    """
+    Encode binary data to ascii text in base64. Data must be bytes.
+    """
+    return base64.b64encode(data).decode('ascii')
+
+
+def translate_module_name(module):
+    """Rename builtin modules to a consistent module name.
+
+    Prefer the more modern naming.
+
+    This is used so that references to Python's `builtins` module can
+    be loaded in both Python 2 and 3.  We remap to the "__builtin__"
+    name and unmap it when importing.
+
+    Map the Python2 `exceptions` module to `builtins` because
+    `builtins` is a superset and contains everything that is
+    available in `exceptions`, which makes the translation simpler.
+
+    See untranslate_module_name() for the reverse operation.
+    """
+    lookup = dict(__builtin__='builtins', exceptions='builtins')
+    return lookup.get(module, module)
+
+
+def importable_name(cls):
+    """
+    Fully qualified name (prefixed by builtin when needed)
+    """
+    # Use the fully-qualified name if available (Python >= 3.3)
+    name = getattr(cls, '__qualname__', cls.__name__)
+    module = translate_module_name(cls.__module__)
+    return '{}.{}'.format(module, name)

tests/core/test_ExecutionContext.py

@@ -2,7 +2,7 @@ import pytest
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.sheerka.Sheerka import ExecutionContext
+from core.sheerka.ExecutionContext import ExecutionContext
 from sdp.sheerkaDataProvider import Event
 
 

tests/core/test_SheerkaHistoryManager.py

@@ -0,0 +1,52 @@
+from core.sheerka.Services.SheerkaHistoryManager import hist
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaHistoryManager(TestUsingMemoryBasedSheerka):
+    def test_i_can_retrieve_history(self):
+        sheerka = self.get_sheerka(skip_builtins_in_db=False)
+
+        sheerka.evaluate_user_input("def concept one as 1")
+        sheerka.evaluate_user_input("one")
+        sheerka.evaluate_user_input("xxx")
+        sheerka.evaluate_user_input("def concept two as 2")
+        sheerka.evaluate_user_input("two")
+        sheerka.evaluate_user_input("def concept three as 3")
+        sheerka.evaluate_user_input("three")
+        sheerka.evaluate_user_input("def concept four as 4")
+        sheerka.evaluate_user_input("four")
+        sheerka.evaluate_user_input("def concept five as 5")
+        sheerka.evaluate_user_input("five")
+
+        h = list(sheerka.history(-1)) # all
+        assert h == [
+            hist("five", True),
+            hist("def concept five as 5", True),
+            hist("four", True),
+            hist("def concept four as 4", True),
+            hist("three", True),
+            hist("def concept three as 3", True),
+            hist("two", True),
+            hist("def concept two as 2", True),
+            hist("xxx", False),
+            hist("one", True),
+            hist("def concept one as 1", True),
+            hist("Initializing Sheerka.", True)]
+
+        h = list(sheerka.history(2))
+        assert h == [
+            hist("two", True),
+            hist("def concept two as 2", True)
+        ]
+
+        h = list(sheerka.history(2, 2))
+        assert h == [
+            hist("xxx", False),
+            hist("one", True),
+        ]
+
+        h = list(sheerka.history(-1))
+        assert h == [
+            hist("xxx", False),
+            hist("one", True),
+        ]

tests/core/test_sheerka_transform.py

@@ -1,318 +0,0 @@
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts
-from core.sheerka.ExecutionContext import ExecutionContext
-from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY
-from sdp.sheerkaDataProvider import Event
-
-from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-
-
-class TestSheerkaTransform(TestUsingMemoryBasedSheerka):
-    def test_i_can_transform_an_unknown_concept(self):
-        sheerka = self.get_sheerka()
-
-        foo = Concept("foo", body="body")
-        concept_with_sub = Concept("concept_with_sub", body=foo)
-
-        concept = Concept(
-            name="concept_name",
-            is_builtin=True,
-            is_unique=True,
-            key="concept_key",
-            body=concept_with_sub,
-            where=[foo, 1, "1", True, 1.0],
-            pre=foo,
-            post=None,  # will not appear
-            definition="it is a definition",
-            definition_type="def type",
-            desc="this this the desc"
-        ).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub")
-
-        # add values and props
-        concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init()
-        concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0]
-        concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init()
-        concept.values[ConceptParts.POST] = "a value for POST"
-        concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub)
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(concept)
-
-        assert to_dict == {
-            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-            OBJ_ID_KEY: 0,
-            'meta.name': 'concept_name',
-            'meta.key': 'concept_key',
-            'meta.is_builtin': True,
-            'meta.is_unique': True,
-            'meta.definition': 'it is a definition',
-            'meta.definition_type': 'def type',
-            'meta.desc': 'this this the desc',
-            'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-                            OBJ_ID_KEY: 1,
-                            'meta.body': 'body',
-                            'meta.name': 'foo'}, 1, '1', True, 1.0],
-            'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1},
-            'meta.body': {
-                '__type__': SheerkaTransformType.Concept,
-                '__id__': 2,
-                'meta.name': 'concept_with_sub',
-                'meta.body': {
-                    '__type__': SheerkaTransformType.Reference,
-                    '__id__': 1}},
-            'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']],
-            'pre': {'__type__': SheerkaTransformType.Concept,
-                    '__id__': 4,
-                    'meta.body': 'body',
-                    'meta.name': 'foo',
-                    'body': 'body'},
-            'post': "a value for POST",
-            'body': {'__type__': SheerkaTransformType.Concept,
-                     '__id__': 3,
-                     'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference},
-                     'meta.name': 'concept_with_sub',
-                     'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}},
-            'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0],
-            'props': [('a', 10),
-                      ('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}),
-                      ('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})],
-        }
-
-    def test_i_can_transform_unknown_concept_with_almost_same_value(self):
-        sheerka = self.get_sheerka()
-        concept = Concept("foo")
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(concept)
-
-        assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'}
-
-    def test_i_can_transform_known_concept_when_the_values_are_the_same(self):
-        """
-        Values are the same means that we are serializing a concept which has kept all its default values
-        There is not diff between the concept to serialize and the one which was registered with create_new_concept()
-        We serialize only the id of the concept
-        :return:
-        """
-        sheerka = self.get_sheerka()
-
-        concept = Concept(
-            name="concept_name",
-            is_builtin=True,
-            is_unique=False,
-            key="concept_key",
-            body="body definition",
-            where="where definition",
-            pre="pre definition",
-            post="post definition",
-            definition="it is a definition",
-            definition_type="def type",
-            desc="this this the desc"
-        ).def_prop("a").def_prop("b")
-        sheerka.create_new_concept(self.get_context(sheerka), concept)
-
-        new_concept = sheerka.new(concept.key)
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(new_concept)
-
-        assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"}
-
-    def test_i_can_transform_known_concept_when_the_values_are_different(self):
-        """
-        Values are the different means the concept was modified.
-        It's different from the one which was registered with create_new_concept()
-        We serialize only the differences
-        :return:
-        """
-        sheerka = self.get_sheerka()
-
-        concept = Concept(
-            name="concept_name",
-            is_builtin=True,
-            is_unique=False,
-            key="concept_key",
-            body="body definition",
-            where="where definition",
-            pre="pre definition",
-            post="post definition",
-            definition="it is a definition",
-            definition_type="def type",
-            desc="this this the desc"
-        ).def_prop("a").def_prop("b")
-        sheerka.create_new_concept(self.get_context(sheerka), concept)
-
-        new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre")
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(new_concept)
-
-        assert to_dict == {
-            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-            OBJ_ID_KEY: 0,
-            "id": "1001",
-            'pre': 'another pre',
-            "body": "another",
-            'props': [('a', 10)]
-        }
-
-    def test_i_can_transform_concept_with_circular_reference(self):
-        sheerka = self.get_sheerka()
-        foo = Concept("foo")
-        bar = Concept("bar", body=foo)
-        foo.metadata.body = bar
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(foo)
-
-        assert to_dict == {
-            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-            OBJ_ID_KEY: 0,
-            'meta.name': 'foo',
-            'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-                          OBJ_ID_KEY: 1,
-                          'meta.name': 'bar',
-                          'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
-                                        OBJ_ID_KEY: 0},
-                          },
-        }
-
-    def test_i_can_transform_concept_with_circular_reference_2(self):
-        sheerka = self.get_sheerka()
-        foo = Concept("foo")
-        bar = Concept("foo", body=foo)
-        foo.metadata.body = bar
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(foo)
-
-        assert to_dict == {
-            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-            OBJ_ID_KEY: 0,
-            'meta.name': 'foo',
-            'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
-                          OBJ_ID_KEY: 1,
-                          'meta.name': 'foo',
-                          'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
-                                        OBJ_ID_KEY: 0},
-                          },
-        }
-
-    def test_i_can_transform_the_unknown_concept(self):
-        sheerka = self.get_sheerka(False)
-
-        unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(unknown)
-
-        assert len(to_dict) == 3
-        assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept
-        assert to_dict[OBJ_ID_KEY] == 0
-        assert "id" in to_dict
-
-    def test_i_can_transform_simple_execution_context(self):
-        sheerka = self.get_sheerka()
-        ExecutionContext.ids = {}
-        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(context)
-
-        assert to_dict == {
-            OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
-            OBJ_ID_KEY: 0,
-            '_parent': None,
-            '_id': 0,
-            '_tab': '',
-            '_bag': {},
-            '_start': 0,
-            '_stop': 0,
-            'who': 'requester',
-            'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'},
-            'desc': 'this is the desc',
-            'children': [],
-            'preprocess': None,
-            'inputs': {},
-            'values': {},
-            'obj': None,
-            'concepts': {}
-        }
-
-    def test_i_can_transform_list(self):
-        sheerka = self.get_sheerka()
-        ExecutionContext.ids = {}
-        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict([context])
-
-        assert len(to_dict) == 1
-        assert isinstance(to_dict, list)
-        assert to_dict[0]["who"] == "requester"
-        assert to_dict[0]["desc"] == "this is the desc"
-
-    def test_i_can_transform_set(self):
-        sheerka = self.get_sheerka()
-        ExecutionContext.ids = {}
-        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict({context})
-
-        assert len(to_dict) == 1
-        assert isinstance(to_dict, list)
-        assert to_dict[0]["who"] == "requester"
-        assert to_dict[0]["desc"] == "this is the desc"
-
-    def test_i_can_transform_dict(self):
-        sheerka = self.get_sheerka()
-        ExecutionContext.ids = {}
-        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-        known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key()
-        sheerka.create_new_concept(self.get_context(sheerka), known_concept)
-        unknown_concept = Concept("bar")
-        known = sheerka.new("foo")
-
-        bag = {
-            "context": context,
-            "known_concept": known_concept,
-            "unknown_concept": unknown_concept,
-            "True": True,
-            "Number": 1.1,
-            "String": "a string value",
-            "None": None,
-            unknown_concept: "hello",
-            known: "world"
-        }
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(bag)
-
-        assert isinstance(to_dict, dict)
-        assert to_dict['Number'] == 1.1
-        assert to_dict['String'] == 'a string value'
-        assert to_dict['True']
-        assert to_dict['None'] is None
-        assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
-        assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
-        assert to_dict["known_concept"]["id"] == '1001'
-        assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
-        assert to_dict["(None)bar"] == "hello"
-        assert to_dict["(1001)foo"] == "world"
-
-    def test_i_can_transform_when_circular_references(self):
-        sheerka = self.get_sheerka()
-        ExecutionContext.ids = {}
-        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
-        context.push("another requester", "another desc")
-
-        st = SheerkaTransform(sheerka)
-        to_dict = st.to_dict(context)
-
-        assert isinstance(to_dict, dict)
-        assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
-        assert len(to_dict["children"]) == 1
-        assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
-        assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
-        assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0
-        assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
-        assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1

tests/core/test_utils.py

@@ -1,5 +1,6 @@
 import core.utils
 import pytest
+from core.concept import ConceptParts
 
 from core.tokenizer import Token, TokenKind
 
@@ -130,6 +131,37 @@ def test_i_can_escape():
     assert actual == "hello \\'world\\' my friend"
 
 
+@pytest.mark.parametrize("text, expected_key, expected_id", [
+    (None, None, None),
+    (10, None, None),
+    ("", None, None),
+    ("xxx", None, None),
+    (":c:", None, None),
+    (":c:key", None, None),
+    (":c:key:", "key", None),
+    (":c:key:id", None, None),
+    (":c:key:id:", "key", "id"),
+])
+def test_i_can_decode_concept_repr(text, expected_key, expected_id):
+    k, i = core.utils.decode_concept(text)
+    assert k == expected_key
+    assert i == expected_id
+
+
+@pytest.mark.parametrize("text, expected", [
+    (None, None),
+    (10, None),
+    ("", None),
+    ("xxx", None),
+    ("xxx.", None),
+    ("xxx.yyy", None),
+    ("core.concept.ConceptParts.BODY", ConceptParts.BODY),
+])
+def test_i_can_decode_enum(text, expected):
+    actual = core.utils.decode_enum(text)
+    assert actual == expected
+
+
 def get_tokens(lst):
     res = []
     for e in lst:

tests/sdp/test_sheerkaDataProvider.py

@@ -257,24 +257,27 @@ def test_i_can_load_events(root):
     for i in range(15):
         sdp.save_event(Event(f"Hello {i}"))
 
-    events = list(sdp.load_events(10))
+    events = list(sdp.load_events(10))  # first ten
     assert len(events) == 10
     assert events[0].message == "Hello 14"
     assert events[9].message == "Hello 5"
 
-    events = list(sdp.load_events(10, 5))
+    events = list(sdp.load_events(10, 5))  # skip first 5, then take 10
     assert len(events) == 10
     assert events[0].message == "Hello 9"
     assert events[9].message == "Hello 0"
 
-    events = list(sdp.load_events(20, 10))
+    events = list(sdp.load_events(20, 10))  # skip first 10, take 20,(but only 5 remaining)
     assert len(events) == 5
     assert events[0].message == "Hello 4"
     assert events[4].message == "Hello 0"
 
-    events = list(sdp.load_events(1, 20))
+    events = list(sdp.load_events(1, 20))  # skip first 20, take one
     assert len(events) == 0
 
+    events = list(sdp.load_events(0))  # all
+    assert len(events) == 15
+
 
 @pytest.mark.parametrize("root", [
     ".sheerka",

tests/sheerkapickle/test_SheerkaPickler.py

@@ -0,0 +1,172 @@
+import logging
+
+import pytest
+from core.concept import Concept, ConceptParts
+from sheerkapickle import tags
+from sheerkapickle.SheerkaPickler import SheerkaPickler
+from sheerkapickle.SheerkaUnpickler import SheerkaUnpickler
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class Obj:
+    def __init__(self, a, b, c):
+        self.a = a
+        self.b = b
+        self.c = c
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, Obj):
+            return False
+
+        return self.a == other.a and self.b == other.b and self.c == other.c
+
+    def __hash__(self):
+        return hash((self.a, self.b, self.c))
+
+
+class TestSheerkaPickler(TestUsingFileBasedSheerka):
+
+    @pytest.mark.parametrize("obj, expected", [
+        (1, 1),
+        (3.14, 3.14),
+        ("a string", "a string"),
+        (True, True),
+        (None, None),
+        ([1, 3.14, "a string"], [1, 3.14, "a string"]),
+        ((1, 3.14, "a string"), {tags.TUPLE: [1, 3.14, "a string"]}),
+        ({1}, {tags.SET: [1]}),
+        ({"a": "a", "b": 3.14, "c": True}, {"a": "a", "b": 3.14, "c": True}),
+        ({1: "a", 2: 3.14, 3: True}, {1: "a", 2: 3.14, 3: True}),
+        ([1, [3.14, "a string"]], [1, [3.14, "a string"]]),
+        ([1, (3.14, "a string")], [1, {tags.TUPLE: [3.14, "a string"]}]),
+        ([], []),
+        (ConceptParts.BODY, {tags.ENUM: "core.concept.ConceptParts.BODY"}),
+    ])
+    def test_i_can_flatten_and_restore_primitives(self, obj, expected):
+        sheerka = self.get_sheerka()
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == expected
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_can_flatten_and_restore_instances(self):
+        sheerka = self.get_sheerka()
+
+        obj1 = Obj(1, "b", True)
+        obj2 = Obj(3.14, ("a", "b"), obj1)
+
+        flatten = SheerkaPickler(sheerka).flatten(obj2)
+        assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                           'a': 3.14,
+                           'b': {'_sheerka/tuple': ['a', 'b']},
+                           'c': {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                                 'a': 1,
+                                 'b': 'b',
+                                 'c': True}}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj2
+
+    def test_i_can_manage_circular_reference(self):
+        sheerka = self.get_sheerka()
+
+        obj1 = Obj(1, "b", True)
+        obj1.c = obj1
+
+        flatten = SheerkaPickler(sheerka).flatten(obj1)
+        assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                           'a': 1,
+                           'b': 'b',
+                           'c': {'_sheerka/id': 0}}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded.a == obj1.a
+        assert decoded.b == obj1.b
+        assert decoded.c == decoded
+
+    def test_i_can_flatten_obj_with_new_props(self):
+        sheerka = self.get_sheerka()
+
+        obj = Obj(1, "b", True)
+        obj.z = "new prop"
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                           'a': 1,
+                           'b': 'b',
+                           'c': True,
+                           'z': "new prop"}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    @pytest.mark.parametrize("obj, expected", [
+        ({None: "a"}, {'null': "a"}),
+        ({ConceptParts.BODY: "a"}, {'core.concept.ConceptParts.BODY': 'a'}),
+        ({(1, 2): "a"}, {(1, 2): "a"}),
+    ])
+    def test_i_can_manage_specific_keys_in_dictionaries(self, obj, expected):
+        sheerka = self.get_sheerka()
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == expected
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_can_use_concept_as_dictionary_key(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo").init_key()
+        sheerka.set_id_if_needed(concept, False)
+        sheerka.add_in_cache(concept)
+        obj = {concept: "a"}
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == {':c:foo:1001:': 'a'}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_can_manage_references(self):
+        sheerka = self.get_sheerka()
+
+        foo = Obj("foo", "bar", "baz")
+        obj = [ConceptParts.BODY, foo, ConceptParts.WHERE, ConceptParts.BODY, foo]
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+
+        assert flatten == [{'_sheerka/enum': 'core.concept.ConceptParts.BODY'},
+                           {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                            'a': 'foo',
+                            'b': 'bar',
+                            'c': 'baz'},
+                           {'_sheerka/enum': 'core.concept.ConceptParts.WHERE'},
+                           {'_sheerka/id': 0},
+                           {'_sheerka/id': 1}]
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_serialize_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("doo")
+        flatten = SheerkaPickler(sheerka).flatten(foo)
+        restored = SheerkaUnpickler(sheerka).restore(flatten)
+
+        assert restored == foo
+
+    def test_i_do_not_encode_logger(self):
+        sheerka = self.get_sheerka()
+
+        logger = logging.getLogger("log_name")
+        logger2 = logging.getLogger("log_name2")
+        obj = Obj("foo", logger, {"a": logger, "b": logger2})
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == Obj("foo", None, {"a": None, "b": None})

tests/sheerkapickle/test_sheerka_handlers.py

@@ -0,0 +1,285 @@
+import sheerkapickle
+from core.builtin_concepts import BuiltinConcepts, UserInputConcept, ReturnValueConcept
+from core.concept import Concept, ConceptParts
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.tokenizer import Tokenizer
+from evaluators.ConceptEvaluator import ConceptEvaluator
+from parsers.DefaultParser import DefaultParser
+from sdp.sheerkaDataProvider import Event
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_encode_decode_unknown_concept_metadata(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept(name="foo", key="my_key")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.key": "my_key"}'
+        assert decoded == concept
+
+        concept = Concept("foo", is_builtin=True, is_unique=True)
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.is_builtin": true, "meta.is_unique": true}'
+
+        concept = Concept("foo", body="my_body")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "my_body"}'
+
+        concept = Concept("foo", pre="my_pre")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.pre": "my_pre"}'
+
+        concept = Concept("foo", post="my_post")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.post": "my_post"}'
+
+        concept = Concept("foo", where="my_where")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.where": "my_where"}'
+
+        concept = Concept("foo").def_prop("a", "value_a").def_prop("b", "value_b")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+
+    def test_i_can_encode_decode_unknown_concept_values(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.PRE] = 10  # an int
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "pre": 10}'
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.POST] = 'a string'  # an int
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "post": "a string"}'
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.WHERE] = ['a string', 3.14]  # a list
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": ["a string", 3.14]}'
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.WHERE] = ('a string', 3.14)  # a tuple
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "where": {"_sheerka/tuple": ["a string", 3.14]}}'
+
+        concept = Concept("foo")
+        concept.values[ConceptParts.BODY] = Concept("foo", body="foo_body")
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "foo_body"}}'
+
+    def test_i_can_encode_decode_unknown_concept_properties(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo")
+        concept.set_prop("a", "value_a")  # string
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "value_a"]]}'
+
+        concept = Concept("foo")
+        concept.set_prop("a", 10)  # int
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", 10]]}'
+
+        concept = Concept("foo")
+        concept.set_prop("a", Concept("bar"))  # another concept
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}'
+
+        concept = Concept("foo")
+        concept.set_prop("a", "a").set_prop("b", "b")  # at least two props
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "props": [["a", "a"], ["b", "b"]]}'
+
+    def test_i_can_encode_decode_known_concepts(self):
+        sheerka = self.get_sheerka()
+
+        ref_concept = Concept("my_name", True, True, "my_key", "my_body", "my_where", "my_pre", "my_post", "my_def")
+        ref_concept.def_prop("a", "value_a").def_prop("b", "value_b")
+
+        sheerka.create_new_concept(self.get_context(sheerka), ref_concept)
+
+        to_string = sheerkapickle.encode(sheerka, ref_concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ref_concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"]}'
+
+        concept = Concept().update_from(sheerka.get_by_id(ref_concept.id))
+        concept.set_metadata_value(ConceptParts.BODY, Concept("bar"))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": ["my_key", "1001"], "body": {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}}'
+
+    def test_i_can_manage_reference_of_the_same_object(self):
+        sheerka = self.get_sheerka()
+
+        concept_ref = Concept("foo")
+
+        concept = Concept("bar")
+        concept.set_metadata_value(ConceptParts.PRE, concept_ref)
+        concept.set_metadata_value(ConceptParts.BODY, concept_ref)
+
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "bar", "pre": {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo"}, "body": {"_sheerka/id": 1}}'
+
+    def test_i_can_encode_decode_user_input(self):
+        sheerka = self.get_sheerka()
+
+        user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body="my_text", user_name="my_user_name")
+
+        to_string = sheerkapickle.encode(sheerka, user_input)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == user_input
+        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "my_text"}'
+
+    def test_i_can_encode_decode_user_input_when_tokens(self):
+        sheerka = self.get_sheerka()
+
+        text = "I have 'a complicated' 10 text"
+        tokens = list(Tokenizer(text))
+        user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens, user_name="my_user_name")
+
+        to_string = sheerkapickle.encode(sheerka, user_input)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == UserInputConcept(text, "my_user_name")
+        assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", null], "user_name": "my_user_name", "text": "{text}"' + '}'
+
+    def test_i_can_encode_decode_return_value(self):
+        sheerka = self.get_sheerka()
+
+        ret_val = sheerka.ret("who", True, 10)
+
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ret_val
+        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10}'
+
+    def test_i_can_encode_decode_return_value_with_parent(self):
+        sheerka = self.get_sheerka()
+
+        ret_val = sheerka.ret("who", True, 10)
+        ret_val_parent = sheerka.ret("parent_who", True, "10")
+        ret_val.parents = [ret_val_parent, ret_val_parent]
+
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ret_val
+        assert decoded.parents == ret_val.parents
+        parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]'
+        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}'
+
+    def test_i_can_encode_decode_return_values_with_complex_body(self):
+        sheerka = self.get_sheerka()
+
+        ret_val = sheerka.ret("who", True, Concept("foo", body="bar"))
+
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ret_val
+
+    def test_i_can_encode_decode_return_values_from_concepts_parsers_or_evaluators(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+        ret_val = sheerka.ret(foo, True, 10)
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.ret("c:1001:", True, 10)
+
+        ret_val = sheerka.ret(DefaultParser(), True, 10)
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.ret("parsers.Default", True, 10)
+
+        ret_val = sheerka.ret(ConceptEvaluator(), True, 10)
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.ret("evaluators.Concept", True, 10)
+
+    def test_i_can_encode_decode_execution_context(self):
+        sheerka = self.get_sheerka()
+
+        context = ExecutionContext("who", Event("xxx"), sheerka, "my desc")
+        input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)]
+        context.inputs = {"a": input_list, "b": Concept("foo")}
+        context.values = {"c": input_list, "d": Concept("bar")}
+        context.obj = Concept("baz")
+        context.push("who3", "sub_child1")
+        context.push("who4", "sub_child2")
+
+        to_string = sheerkapickle.encode(sheerka, context)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == context
+
+    def test_complicated_execution_context(self):
+        sheerka = self.get_sheerka(skip_builtins_in_db=False)
+
+        text = "def concept one as 1"
+        execution_context = ExecutionContext("s", Event(), sheerka, f"Evaluating '{text}'")
+        user_input = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="n"))
+        reduce_requested = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+
+        steps = [
+            BuiltinConcepts.BEFORE_PARSING,
+            BuiltinConcepts.PARSING,
+            BuiltinConcepts.AFTER_PARSING,
+            BuiltinConcepts.BEFORE_EVALUATION,
+            BuiltinConcepts.EVALUATION,
+            BuiltinConcepts.AFTER_EVALUATION
+        ]
+
+        ret = sheerka.execute(execution_context, [user_input, reduce_requested], steps)
+        execution_context.add_values(return_values=ret)
+
+        to_string = sheerkapickle.encode(sheerka, execution_context)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+
+        return_value = decoded.values["return_values"][0].value
+        assert sheerka.isinstance(return_value, BuiltinConcepts.NEW_CONCEPT)
+
+    def test_encode_simple_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo")
+        to_string = sheerkapickle.encode(sheerka, foo)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+
+        assert decoded == foo