Enhanced ExecutionContext to keep track of the execution flow

core/builtin_concepts.py

@@ -186,7 +186,12 @@ class ReturnValueConcept(Concept):
                self.message == other.message
 
     def __hash__(self):
-        return hash((self.who, self.status, self.value))
+        if hasattr(self.value, "__iter__") and not isinstance(self.value, str):
+            value_hash = hash(tuple(self.value))
+        else:
+            value_hash = hash(self.value)
+
+        return hash((self.who, self.status, value_hash))
 
 
 class UnknownPropertyConcept(Concept):
@@ -233,6 +238,9 @@ class ParserResultConcept(Concept):
                self.body == other.body and \
                self.try_parsed == other.try_parsed
 
+    def __hash__(self):
+        return hash(self.metadata.name)
+
     @property
     def value(self):
         return self.body

core/builtin_helpers.py

@@ -8,6 +8,7 @@ from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
 
 
+
 def is_same_success(sheerka, return_values):
     """
     Returns True if all returns values are successful and have the same value
@@ -209,3 +210,4 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
 
     return predicates
 
+

core/concept.py

@@ -98,8 +98,25 @@ class Concept:
 
         # check the attributes
         for prop in PROPERTIES_TO_SERIALIZE:
-            if getattr(self.metadata, prop) != getattr(other.metadata, prop):
             # print(prop) # use full to know which id does not match
+            my_value = getattr(self.metadata, prop)
+            other_value = getattr(other.metadata, prop)
+            if isinstance(my_value, Concept) and isinstance(other_value, Concept):
+                # need to check if circular references
+                if id(self) == id(other):
+                    continue
+
+                sub_value = getattr(other_value.metadata, prop)
+                while isinstance(sub_value, Concept):
+                    if id(self) == id(sub_value):
+                        return False  # circular reference
+                    sub_value = getattr(sub_value.metadata, prop)
+
+                if my_value != other_value:
+                    return False
+
+            else:
+                if my_value != other_value:
                     return False
 
         # check the props (Concept variables)

core/sheerka.py

@@ -1,5 +1,3 @@
-from dataclasses import dataclass, field
-
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
 from parsers.BaseParser import BaseParser
@@ -10,6 +8,7 @@ import core.builtin_helpers
 from core.sheerka_logger import console_handler
 
 import logging
+import time
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,
@@ -189,8 +188,8 @@ class Sheerka(Concept):
         event = Event(text, user_name)
         evt_digest = self.sdp.save_event(event)
         self.log.debug(f"{evt_digest=}")
-        execution_context = ExecutionContext(self.key, event, self)
 
+        with ExecutionContext(self.key, event, self, f"Evaluating '{text}'") as execution_context:
             user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
             reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
 
@@ -203,7 +202,12 @@ class Sheerka(Concept):
                 BuiltinConcepts.AFTER_EVALUATION
             ]
 
-        return self.execute(execution_context, [user_input, reduce_requested], steps)
+            ret = self.execute(execution_context, [user_input, reduce_requested], steps)
+            execution_context.add_values(return_values=ret)
+
+            if not self.skip_builtins_in_db:
+                self.sdp.save_result(execution_context)
+        return ret
 
     def _call_parsers(self, execution_context, return_values, logger=None):
 
@@ -229,7 +233,9 @@ class Sheerka(Concept):
                 p = parser(sheerka=self)
                 if logger:
                     p.log = logger
-                res = p.parse(execution_context, to_parse)
+
+                with execution_context.push(desc=f"Parsing using {p.name}") as sub_context:
+                    res = p.parse(sub_context, to_parse)
 
                     if hasattr(res, "__iter__"):
                         for r in res:
@@ -239,6 +245,8 @@ class Sheerka(Concept):
                         res.parents = [return_value]
                         result.append(res)
 
+                    sub_context.add_values(return_values=res)
+
         return result
 
     def _call_evaluators(self, execution_context, return_values, process_step, evaluation_context=None, logger=None):
@@ -368,10 +376,9 @@ class Sheerka(Concept):
         """
 
         for step in execution_steps:
-            sub_context = execution_context.push(step=step)
-            sub_context.log(logger or self.log, f"{step=}, context='{sub_context}'")
-
             copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values]
+            with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context:
+                sub_context.log(logger or self.log, f"{step=}, context='{sub_context}'")
 
                 if step == BuiltinConcepts.PARSING:
                     return_values = self._call_parsers(sub_context, return_values, logger)
@@ -381,6 +388,8 @@ class Sheerka(Concept):
                 if copy != return_values:
                     sub_context.log_result(logger or self.log, return_values)
 
+                sub_context.add_values(return_values=return_values)
+
         return return_values
 
     def set_id_if_needed(self, obj: Concept, is_builtin: bool):
@@ -430,14 +439,15 @@ class Sheerka(Concept):
 
             # check if it's a valid BNF or whether it breaks the known rules
             concept_lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]()
-            sub_context = context.push(self.name, desc=f"Initializing concept definition for {concept}")
+            with context.push(self.name, desc=f"Initializing concept definition for {concept}") as sub_context:
                 sub_context.concepts[concept.key] = concept  # the concept is not in the real cache yet
                 sub_context.log_new(logger)
                 init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions)
+                sub_context.add_values(return_values=init_ret_value)
                 if not init_ret_value.status:
                     return self.ret(self.create_new_concept.__name__, False, ErrorConcept(init_ret_value.value))
 
-        # save the new context in sdp
+        # save the new concept in sdp
         try:
             self.sdp.add(context.event.get_digest(), self.CONCEPTS_ENTRY, concept, use_ref=True)
             if concepts_definitions is not None:
@@ -507,10 +517,12 @@ class Sheerka(Concept):
                 # I refuse empty strings for performance matters, I don't want to handle useless NOPConcepts
                 continue
             else:
-                sub_context = context.push(desc=f"Initializing AST for {part_key}")
+                with context.push(desc=f"Initializing AST for {part_key}") as sub_context:
                     sub_context.log_new(logger)
                     to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=source))
-                concept.cached_asts[part_key] = self.execute(sub_context, to_parse, steps, logger)
+                    res = self.execute(sub_context, to_parse, steps, logger)
+                    concept.cached_asts[part_key] = res
+                    sub_context.add_values(return_values=res)
 
         for prop in concept.props:
             value = concept.props[prop].value
@@ -522,9 +534,11 @@ class Sheerka(Concept):
                         context.who,
                         True,
                         self.new(BuiltinConcepts.USER_INPUT, body=value))
-                    sub_context = context.push(desc=f"Initializing AST for property {prop}")
+                    with context.push(desc=f"Initializing AST for property {prop}") as sub_context:
                         sub_context.log_new(logger)
-                    concept.cached_asts[prop] = self.execute(context, to_parse, steps)
+                        res = self.execute(context, to_parse, steps)
+                        concept.cached_asts[prop] = res
+                        sub_context.add_values(return_values=res)
 
         # Updates the cache of concepts when possible
         if concept.key in self.concepts_cache:
@@ -552,10 +566,12 @@ class Sheerka(Concept):
 
         def _resolve(return_value, desc, obj):
             context.log(logger, desc, self.evaluate_concept.__name__)
-            sub_context = context.push(desc=desc, obj=obj)
+            with context.push(desc=desc, obj=obj) as sub_context:
                 sub_context.log_new(logger)
                 r = self.execute(sub_context, return_value, CONCEPT_EVALUATION_STEPS, logger)
-            return core.builtin_helpers.expect_one(context, r)
+                one_r = core.builtin_helpers.expect_one(context, r)
+                sub_context.add_values(return_values=one_r)
+                return one_r
 
         # WHERE condition should already be validated by the parser.
         # It's a mandatory condition for the concept before it can be recognized
@@ -579,9 +595,10 @@ class Sheerka(Concept):
                     if isinstance(concept.cached_asts[prop_name], Concept):
                         context.log(
                             logger, f"Evaluation prop={prop_name}, value={prop_ast}", self.evaluate_concept.__name__)
-                        sub_context = context.push(f"Evaluation property '{prop_name}', value='{prop_ast}'")
+                        with context.push(f"Evaluation property '{prop_name}', value='{prop_ast}'") as sub_context:
                             sub_context.log_new(logger)
                             evaluated = self.evaluate_concept(sub_context, prop_ast)
+                            sub_context.add_values(return_values=evaluated)
                             concept.set_prop(prop_name, evaluated)
                     else:
                         res = _resolve(prop_ast, f"Evaluating property '{prop_name}'", None)
@@ -631,26 +648,36 @@ class Sheerka(Concept):
         self.concepts_cache[concept.key] = concept
         return concept
 
-    def get(self, concept_key):
+    def get(self, concept_key, concept_id=None):
         """
         Tries to find a concept
         What is return must be used a template for another concept.
         You must not modify the returned concept
-        :param concept_key:
+        :param concept_key: key of the concept
+        :param concept_id: when multiple concepts with the same key, use the id
         :return:
         """
 
+        if concept_key is None:
+            return ErrorConcept("Concept key is undefined.")
+
         if isinstance(concept_key, BuiltinConcepts):
             concept_key = str(concept_key)
 
         # first search in cache
-        if concept_key in self.concepts_cache:
+        result = self.concepts_cache[concept_key] if concept_key in self.concepts_cache else \
-            return self.concepts_cache[concept_key]
+            self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key)
 
-        # else look in sdp
+        if result and (concept_id is None or not isinstance(result, list)):
-        from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key)
+            return result
-        if from_db is not None:
+
-            return from_db
+        if isinstance(result, list):
+            if concept_id:
+                for c in result:
+                    if c.id == concept_id:
+                        return c
+            else:
+                return result
 
         # else return new Unknown concept
         # Note that I don't call the new() method to prevent cyclic call
@@ -875,6 +902,7 @@ class Sheerka(Concept):
             self.log.info(f"bnf    : {c.metadata.definition}")
             self.log.info(f"key    : {c.key}")
             self.log.info(f"body   : {c.body}")
+            self.log.info(f"digest : {c.get_digest()}")
             first = False
 
     @staticmethod
@@ -900,7 +928,6 @@ class Sheerka(Concept):
         logging.basicConfig(format=log_format, level=log_level, handlers=[console_handler])
 
 
-@dataclass
 class ExecutionContext:
     """
     To keep track of the execution of a request
@@ -910,28 +937,66 @@ class ExecutionContext:
                  who,
                  event: Event,
                  sheerka: Sheerka,
-                 /,
                  desc: str = None,
-                 obj: Concept = None,
+                 **kwargs):
-                 step: BuiltinConcepts = None,
+
-                 iteration: int = 0,
+        self._parent = None
-                 concepts: dict = None):
+        self._id = ExecutionContextIdManager.get_id(event.get_digest())
+        self._tab = ""
+        self._bag = {}  # other variables
+        self._start = 0
+        self._stop = 0
 
         self.who = who  # who is asking
         self.event = event  # what was the (original) trigger
         self.sheerka = sheerka  # sheerka
-
-        self.step = step
-        self.iteration = iteration
-        self.preprocess = None
-
         self.desc = desc  # human description of what is going on
-        self.obj = obj  # what is the subject of the execution context (if known)
+        self.children = []
+        self.preprocess = None
+        self.values = {}  # what was produced by the execution context
 
-        self.concepts = concepts or {}  # cache for concepts that are specific to this execution
+        self.obj = kwargs.pop("obj", None)
+        self.concepts = kwargs.pop("concepts", {})
+        # update the other elements
+        for k, v in kwargs.items():
+            self._bag[k] = v
 
-        self._id = ExecutionContextIdManager.get_id(event.get_digest())
+    @property
-        self._tab = ""
+    def elapsed(self):
+        if self._start == 0:
+            return 0
+
+        return (self._stop if self._stop > 0 else time.time_ns()) - self._start
+
+    @property
+    def elapsed_str(self):
+        nano_sec = self.elapsed
+        dt = nano_sec / 1e6
+        return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
+
+    @property
+    def id(self):
+        return self._id
+
+    def __getattr__(self, item):
+        if item in self._bag:
+            return self._bag[item]
+
+        raise AttributeError(f"'ExecutionContext' object has no attribute '{item}'")
+
+    def __enter__(self):
+        self._start = time.time_ns()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._stop = time.time_ns()
+
+    def __repr__(self):
+        msg = f"ExecutionContext(who={self.who}, id={self._id}"
+        if self.desc:
+            msg += f", desc='{self.desc}'"
+        msg += ")"
+        return msg
 
     def add_preprocess(self, name, **kwargs):
         preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
@@ -944,6 +1009,11 @@ class ExecutionContext:
         self.preprocess.add(preprocess)
         return self
 
+    def add_values(self, **kwargs):
+        for k, v in kwargs.items():
+            self.values[k] = v
+        return self
+
     def new_concept(self, key, **kwargs):
         # search in obj
         if self.obj:
@@ -964,29 +1034,23 @@ class ExecutionContext:
 
         return self.sheerka.new(key, **kwargs)
 
-    @property
+    def push(self, who=None, desc=None, **kwargs):
-    def id(self):
-        return self._id
-
-    def push(self, who=None, /, **kwargs):
         who = who or self.who
-        desc = kwargs.get("desc", "")
+        _kwargs = {"obj": self.obj, "concepts": self.concepts}
-        obj = kwargs.get("obj", self.obj)
+        _kwargs.update(self._bag)
-        concepts = kwargs.get("concepts", self.concepts)
+        _kwargs.update(kwargs)
-        step = kwargs.get("step", self.step)
-        iteration = kwargs.get("iteration", self.iteration)
         new = ExecutionContext(
             who,
             self.event,
             self.sheerka,
-            desc=desc,
+            desc,
-            obj=obj,
+            **_kwargs,
-            concepts=concepts,
-            step=step,
-            iteration=iteration,
         )
+        new._parent = self
         new._tab = self._tab + " " * DEBUG_TAB_SIZE
         new.preprocess = self.preprocess
+
+        self.children.append(new)
         return new
 
     def log_new(self, logger):
@@ -1009,6 +1073,11 @@ class ExecutionContext:
             to_str = self.return_value_to_str(r)
             logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
+    def to_dict(self):
+        from core.sheerka_transform import SheerkaTransform
+        st = SheerkaTransform(self.sheerka)
+        return st.to_dict(self)
+
     @staticmethod
     def return_value_to_str(r):
         value = str(r.value)
@@ -1017,13 +1086,6 @@ class ExecutionContext:
         to_str = f"ReturnValue(who={r.who}, status={r.status}, value={value})"
         return to_str
 
-    def __repr__(self):
-        msg = f"ExecutionContext(who={self.who}, id={self._id}"
-        if self.desc:
-            msg += f", desc='{self.desc}'"
-        msg += ")"
-        return msg
-
 
 class ExecutionContextIdManager:
     ids = {}

core/sheerka_transform.py

@@ -0,0 +1,152 @@
+import dataclasses
+from enum import Enum
+
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE
+from core.sheerka import ExecutionContext
+from core.tokenizer import Token
+from evaluators.BaseEvaluator import BaseEvaluator
+from parsers.BaseParser import BaseParser, Node
+from parsers.BnfParser import BnfParser
+from parsers.ConceptLexerParser import UnrecognizedTokensNode, ParsingExpression
+from parsers.PythonParser import PythonNode
+from sdp.sheerkaDataProvider import Event
+
+OBJ_TYPE_KEY = "__type__"
+OBJ_ID_KEY = "__id__"
+OBJ_NAME_KEY = "__name__"
+
+default_concept = Concept()
+
+
+class SheerkaTransformType(Enum):
+    Concept = 1
+    Reference = 2
+    ExecutionContext = 3
+    Event = 4
+    Node = 5
+    Exception = 6
+
+
+class SheerkaTransform:
+
+    def __init__(self, sheerka):
+        self.ids = {}
+        self.sheerka = sheerka
+        self.id_count = -1
+
+    def to_dict(self, obj):
+
+        if isinstance(obj, (Concept, ExecutionContext, Event)):
+            exists, _id = self.exist(obj)
+            if exists:
+                return {
+                    OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                    OBJ_ID_KEY: _id
+                }
+            else:
+                self.id_count = self.id_count + 1
+                self.ids[obj] = self.id_count
+
+        if isinstance(obj, Concept):
+            return self.context_to_dict(obj)
+
+        elif isinstance(obj, ExecutionContext):
+            return self.execution_context_to_dict(obj)
+
+        elif isinstance(obj, Event):
+            return {
+                OBJ_TYPE_KEY: SheerkaTransformType.Event,
+                OBJ_ID_KEY: self.id_count,
+                'digest': obj.get_digest()}
+
+        elif isinstance(obj, (BaseParser, BaseEvaluator, BnfParser)):
+            return obj.name
+
+        elif isinstance(obj, Token):
+            return obj.__dict__
+
+        elif isinstance(obj, PythonNode):
+            return {
+                OBJ_TYPE_KEY: SheerkaTransformType.Node,
+                OBJ_NAME_KEY: "PythonNode",
+                'source': obj.source,
+                'ast_': obj.get_dump(obj.ast_)
+            }
+
+        elif isinstance(obj, Node):
+            to_dict = {
+                OBJ_TYPE_KEY: SheerkaTransformType.Node,
+                OBJ_NAME_KEY: obj.__class__.__name__,
+            }
+            for k, v in obj.__dict__.items():
+                to_dict[k] = self.to_dict(v)
+
+            return to_dict
+
+        elif isinstance(obj, Exception):
+            to_dict = {
+                OBJ_TYPE_KEY: SheerkaTransformType.Exception,
+                OBJ_NAME_KEY: obj.__class__.__name__,
+            }
+            for k, v in obj.__dict__.items():
+                to_dict[k] = self.to_dict(v)
+            return to_dict
+
+        elif isinstance(obj, ParsingExpression):
+            return obj.__repr__()
+
+        elif isinstance(obj, dict):
+            return dict((str(k) if isinstance(k, Concept) else k, self.to_dict(v)) for k, v in obj.items())
+
+        elif hasattr(obj, "__iter__") and not isinstance(obj, str):
+            return list(self.to_dict(o) for o in obj)
+
+        else:
+            return obj
+
+    def context_to_dict(self, obj: Concept):
+        to_dict = {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: self.id_count,
+        }
+        if obj.id:
+            ref = self.sheerka.get(obj.key, obj.id)
+            to_dict["id"] = obj.id
+        else:
+            ref = default_concept
+
+        # transform metadata
+        for prop in PROPERTIES_TO_SERIALIZE:
+            value = self.to_dict(getattr(obj.metadata, prop))
+            ref_value = getattr(ref.metadata, prop)
+            if value != ref_value:
+                to_dict[prop] = value
+
+        # transform properties
+        for prop in obj.props:
+            value = self.to_dict(obj.props[prop].value)
+            if prop not in ref.props or value != ref.props[prop].value:
+                if "props" not in to_dict:
+                    to_dict["props"] = []
+                to_dict["props"].append((prop, value))
+
+        return to_dict
+
+    def execution_context_to_dict(self, obj: ExecutionContext):
+        to_dict = {
+            OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
+            OBJ_ID_KEY: self.id_count
+        }
+        for property_name in obj.__dict__:
+            if property_name == "sheerka":
+                continue
+            to_dict[property_name] = self.to_dict(getattr(obj, property_name))
+
+        return to_dict
+
+    def exist(self, obj):
+        for k, v in self.ids.items():
+            if id(k) == id(obj) or k == obj:
+                return True, v
+
+        return False, None

evaluators/AddConceptInSetEvaluator.py

@@ -32,9 +32,12 @@ class AddConceptInSetEvaluator(OneReturnValueEvaluator):
                 self.name,
                 True,
                 sheerka.new(BuiltinConcepts.USER_INPUT, body=name_node.tokens, user_name="N/A"))
-            sub_context = context.push(desc=f"Recognizing '{name_node}'")
+
+            with context.push(desc=f"Recognizing '{name_node}'") as sub_context:
                 r = sheerka.execute(sub_context, ret_val, ALL_STEPS, self.verbose_log)
-            return core.builtin_helpers.expect_one(context, r)
+                one_r = core.builtin_helpers.expect_one(context, r)
+                sub_context.add_values(return_values=one_r)
+                return one_r
 
         isa_node = return_value.value.value
         sheerka = context.sheerka

evaluators/ConceptComposerEvaluator.py

@@ -66,17 +66,20 @@ class ConceptComposerEvaluator(AllReturnValuesEvaluator):
                         if sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
                             has_error = True
                         else:
-                            sub_context = context.push(self.name, desc=f"Evaluating '{concept}'")
+                            with context.push(self.name, desc=f"Evaluating '{concept}'") as sub_context:
                                 sub_context.log_new(self.verbose_log)
                                 concept = sheerka.evaluate_concept(sub_context, concept, self.verbose_log)
+                                sub_context.add_values(return_values=concept)
                                 temp_res.append(concept)
+
                     else:
                         temp_res.append(core.utils.strip_quotes(token.value))
                         concepts_only &= token.type == TokenKind.WHITESPACE or token.type == TokenKind.NEWLINE
             else:
-                sub_context = context.push(self.name, desc=f"Evaluating '{node.concept}'")
+                with context.push(self.name, desc=f"Evaluating '{node.concept}'") as sub_context:
                     sub_context.log_new(self.verbose_log)
                     concept = sheerka.evaluate_concept(sub_context, node.concept, self.verbose_log)
+                    sub_context.add_values(return_values=concept)
                     temp_res.append(concept)
 
         if has_error:
@@ -104,7 +107,3 @@ class ConceptComposerEvaluator(AllReturnValuesEvaluator):
             True,
             res,
             parents=[self.eaten])
-
-
-
-

evaluators/PythonEvaluator.py

@@ -94,9 +94,10 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 continue
 
             context.log(self.verbose_log, f"'{name_resolved}' is a concept. Evaluating.", self.name)
-            sub_context = context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept)
+            with context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept) as sub_context:
                 sub_context.log_new(self.verbose_log)
                 evaluated = context.sheerka.evaluate_concept(sub_context, concept, self.verbose_log)
+                sub_context.add_values(return_values=evaluated)
 
                 if evaluated.key == concept.key:
                     my_locals[name] = evaluated if return_concept else \

parsers/DefaultParser.py

@@ -372,8 +372,10 @@ class DefaultParser(BaseParser):
             return NotInitializedNode()
 
         regex_parser = BnfParser()
-        new_context = self.context.push(self.name)
+        with self.context.push(self.name) as sub_context:
-        parsing_result = regex_parser.parse(new_context, tokens)
+            parsing_result = regex_parser.parse(sub_context, tokens)
+            sub_context.add_values(return_values=parsing_result)
+
             if not parsing_result.status:
                 self.add_error(parsing_result.value)
                 return NotInitializedNode()
@@ -406,15 +408,17 @@ class DefaultParser(BaseParser):
                 continue
 
             # ask the other parsers if they recognize the tokens
-            new_context = self.context.push(self.name, desc=f"Parsing {keyword}")
+            with self.context.push(self.name, desc=f"Parsing {keyword}") as sub_context:
-            new_context.log_new(self.verbose_log)
+                sub_context.log_new(self.verbose_log)
                 to_parse = self.sheerka.ret(
-                new_context.who,
+                    sub_context.who,
                     True,
                     self.sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens))
                 steps = [BuiltinConcepts.PARSING]
-            parsed = self.sheerka.execute(new_context, to_parse, steps, self.verbose_log)
+                parsed = self.sheerka.execute(sub_context, to_parse, steps, self.verbose_log)
-            parsing_result = core.builtin_helpers.expect_one(new_context, parsed, self.verbose_log)
+                parsing_result = core.builtin_helpers.expect_one(sub_context, parsed, self.verbose_log)
+                sub_context.add_values(return_values=parsing_result)
+
                 if not parsing_result.status:
                     self.add_error(parsing_result.value)
                     continue

sdp/readme.md

@@ -7,11 +7,12 @@
 
 ### Current supported types
 - E : events
-- O : object (with history management)
+- J : Json object (with history management)
-- P : pickle 
+- P : pickle (no history)
-- S : state 
+- S : state (history, but not managed by the serializer )
-- C : concept
+- C : concept (with history management)
-- D : concept definitions 
+- D : concept definitions (no history management)
+- R : executionContext ('R' stands for Result or ReturnValue, no history management)
 
 ## How concepts are serialized ?
 - get the id of the concept

sdp/sheerkaDataProvider.py

@@ -656,6 +656,30 @@ class SheerkaDataProvider:
         with self.io.open(target_path, "rb") as f:
             return self.serializer.deserialize(f, None)
 
+    def save_result(self, execution_context):
+        """
+        Save the execution context associated with an event
+        To make a long story short,
+        for every single user input, there is an event (which is the first thing that is created)
+        and a result (the ExecutionContext created by sheerka.evaluate_user_input()
+        :param execution_context:
+        :return:
+        """
+        digest = execution_context.event.get_digest()
+        self.log.debug(f"Saving execution context. digest={digest}")
+        target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + "_result"
+        if self.io.exists(target_path):
+            return digest
+
+        self.io.write_binary(target_path, self.serializer.serialize(execution_context, None).read())
+        return digest
+
+    def load_result(self, digest):
+        target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + "_result"
+
+        with self.io.open(target_path, "rb") as f:
+            return self.serializer.deserialize(f, None)
+
     def save_state(self, state: State):
         digest = state.get_digest()
         self.log.debug(f"Saving new state. digest={digest}")

sdp/sheerkaSerializer.py

@@ -1,3 +1,4 @@
+import dataclasses
 import json
 import pickle
 import datetime
@@ -10,6 +11,9 @@ from enum import Enum
 import core.utils
 
 from core.concept import Concept
+from core.tokenizer import Token
+from parsers.BaseParser import Node
+
 
 def json_default_converter(o):
     """
@@ -23,7 +27,13 @@ def json_default_converter(o):
         return o.isoformat()
 
     if isinstance(o, Enum):
-        return o.key
+        return o.name
+
+    raise Exception("Cannot serialize " + o.__class__.__name__)
+    # with open("json_encoding_error.txt", "a") as f:
+    #     f.write(o.__class__.__name__ + "\n")
+
+
 
 
 @dataclass()
@@ -51,6 +61,7 @@ class Serializer:
         self.register(StateSerializer())
         self.register(ConceptSerializer())
         self.register(DictionarySerializer())
+        self.register(ExecutionContextSerializer())
 
     def register(self, serializer):
         """
@@ -161,9 +172,9 @@ class EventSerializer(BaseSerializer):
         return event
 
 
-class ObjectSerializer(BaseSerializer):
+class JsonSerializer(BaseSerializer):
 
-    def __init__(self, fully_qualified_name, name="O", version=1):
+    def __init__(self, fully_qualified_name, name="J", version=1):
         BaseSerializer.__init__(self, name, version)
         self.fully_qualified_name = fully_qualified_name
 
@@ -219,9 +230,9 @@ class StateSerializer(PickleSerializer):
             1)
 
 
-class ConceptSerializer(ObjectSerializer):
+class ConceptSerializer(JsonSerializer):
     def __init__(self):
-        ObjectSerializer.__init__(self, "core.concept.Concept", "C", 1)
+        JsonSerializer.__init__(self, "core.concept.Concept", "C", 1)
 
     def matches(self, obj):
         return isinstance(obj, Concept)
@@ -235,6 +246,27 @@ class DictionarySerializer(PickleSerializer):
             "D",
             1)
 
+
+class ExecutionContextSerializer(BaseSerializer):
+    def __init__(self):
+        BaseSerializer.__init__(self, "R", 1)
+
+    def matches(self, obj):
+        return core.utils.get_full_qualified_name(obj) == "core.sheerka.ExecutionContext"
+
+    def dump(self, stream, obj, context):
+        as_json = obj.to_dict()
+        stream.write(json.dumps(as_json, default=json_default_converter).encode("utf-8"))
+        stream.seek(0)
+        return stream
+
+    def load(self, stream, context):
+        json_stream = stream.read().decode("utf-8")
+        json_message = json.loads(json_stream)
+        obj = core.utils.get_class("core.sheerka.ExecutionContext")()
+        obj.from_dict(json_message)
+        return obj
+
 #
 # class SheerkaSerializer(ObjectSerializer):
 #     def __init__(self):

tests/test_ExecutionContext.py

@@ -1,3 +1,5 @@
+import pytest
+
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka import ExecutionContext
@@ -18,9 +20,14 @@ def test_id_is_incremented_by_event_digest():
     assert e.id == 1
 
 
-def test_some_properties_are_given_to_the_child():
+def test_i_can_use_with_statement():
-    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka",
+    with ExecutionContext("who_", Event("event"), "fake_sheerka") as e:
-                         desc="some description",
+        pass
+    assert e.elapsed > 0
+
+
+def test_i_can_push():
+    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", "some description",
                          obj=Concept("foo"),
                          step=BuiltinConcepts.EVALUATION,
                          iteration=15,
@@ -30,10 +37,11 @@ def test_some_properties_are_given_to_the_child():
 
     b = a.push()
 
+    assert b._parent == a
     assert b.who == a.who
     assert b.event == a.event
     assert b.sheerka == a.sheerka
-    assert b.desc == ""
+    assert b.desc is None
     assert b.obj == a.obj
     assert b.step == a.step
     assert b.iteration == a.iteration
@@ -41,3 +49,24 @@ def test_some_properties_are_given_to_the_child():
     assert b.id == a.id + 1
     assert b._tab == a._tab + "    "
     assert b.preprocess == a.preprocess
+
+
+def test_children_i_created_when_i_push():
+    e = ExecutionContext("who_", Event("event"), "fake_sheerka")
+    e.push("a", desc="I do something")
+    e.push("b", desc="oups! I did a again")
+    e.push("c", desc="I do something else")
+
+    assert len(e.children) == 3
+    assert e.children[0].who, e.children[0].who == ("a", "I do something")
+    assert e.children[1].who, e.children[1].who == ("b", "oups! I did a again")
+    assert e.children[2].who, e.children[2].who == ("c", "I do something else")
+
+
+def test_i_can_add_variable_when_i_push():
+    e = ExecutionContext("who_", Event("event"), "fake_sheerka")
+    sub_e = e.push("a", my_new_var="new var value")
+
+    assert sub_e.my_new_var == "new var value"
+    with pytest.raises(AttributeError):
+        assert e.my_new_var == ""  # my_new_var does not exist in parent

tests/test_concept.py

@@ -32,8 +32,37 @@ def test_i_can_serialize():
     Test concept.to_dict()
     :return:
     """
-    # TODO
+    concept = Concept(
-    pass
+        name="concept_name",
+        is_builtin=True,
+        is_unique=True,
+        key="concept_key",
+        body="definition of the body",
+        where="definition of the where",
+        pre="definition of the pre",
+        post="definition of the post",
+        definition="bnf definition",
+        definition_type="def type",
+        desc="this this the desc",
+        id="123456"
+    ).set_prop("a", 10).set_prop("b", None)
+
+    to_dict = concept.to_dict()
+    assert to_dict == {
+        'body': 'definition of the body',
+        'definition': 'bnf definition',
+        'definition_type': 'def type',
+        'desc': 'this this the desc',
+        'id': '123456',
+        'is_builtin': True,
+        'is_unique': True,
+        'key': 'concept_key',
+        'name': 'concept_name',
+        'post': 'definition of the post',
+        'pre': 'definition of the pre',
+        'props': [('a', 10), ('b', None)],
+        'where': 'definition of the where'
+    }
 
 
 def test_i_can_deserialize():
@@ -41,5 +70,61 @@ def test_i_can_deserialize():
     Test concept.from_dict()
     :return:
     """
-    # TODO
+
-    pass
+    from_dict = {
+        'body': 'definition of the body',
+        'definition': 'bnf definition',
+        'definition_type': 'def type',
+        'desc': 'this this the desc',
+        'id': '123456',
+        'is_builtin': True,
+        'is_unique': True,
+        'key': 'concept_key',
+        'name': 'concept_name',
+        'post': 'definition of the post',
+        'pre': 'definition of the pre',
+        'props': [('a', 10), ('b', None)],
+        'where': 'definition of the where'
+    }
+
+    concept = Concept().from_dict(from_dict)
+
+    assert concept == Concept(
+        name="concept_name",
+        is_builtin=True,
+        is_unique=True,
+        key="concept_key",
+        body="definition of the body",
+        where="definition of the where",
+        pre="definition of the pre",
+        post="definition of the post",
+        definition="bnf definition",
+        definition_type="def type",
+        desc="this this the desc",
+        id="123456"
+    ).set_prop("a", 10).set_prop("b", None)
+
+
+def test_i_can_compare_concept_with_circular_reference():
+    foo = Concept("foo")
+    foo.metadata.body = foo
+
+    assert foo == foo
+
+
+def test_i_can_compare_concept_with_sophisticated_circular_reference():
+    foo = Concept("foo")
+    bar = Concept("foo", body=foo)
+    baz = Concept("foo", body=bar)
+    foo.metadata.body = baz
+
+    assert foo != bar
+
+
+def test_i_can_compare_concept_with_sophisticated_circular_reference_in_other_metadata():
+    foo = Concept("foo")
+    bar = Concept("foo", pre=foo)
+    baz = Concept("foo", pre=bar)
+    foo.metadata.pre = baz
+
+    assert foo != bar

tests/test_sheerka.py

@@ -201,10 +201,10 @@ def test_i_can_get_list_of_concept_when_same_key_when_no_cache():
 
     sheerka.concepts_cache = {}  # reset the cache
 
-    from_cache = sheerka.get(concept1.key)
+    result = sheerka.get(concept1.key)
-    assert len(from_cache) == 2
+    assert len(result) == 2
-    assert from_cache[0] == concept1
+    assert result[0] == concept1
-    assert from_cache[1] == concept2
+    assert result[1] == concept2
 
 
 def test_i_can_get_list_of_concept_when_same_key_when_cache():
@@ -220,10 +220,65 @@ def test_i_can_get_list_of_concept_when_same_key_when_cache():
 
     # sheerka.concepts_cache = {}  # Do not reset the cache
 
-    from_cache = sheerka.get(concept1.key)
+    result = sheerka.get(concept1.key)
-    assert len(from_cache) == 2
+    assert len(result) == 2
-    assert from_cache[0] == concept1
+    assert result[0] == concept1
-    assert from_cache[1] == concept2
+    assert result[1] == concept2
+
+
+def test_i_can_get_the_correct_concept_using_the_id_when_same_key_when_no_cache():
+    sheerka = get_sheerka()
+    concept1 = get_default_concept()
+    concept2 = get_default_concept()
+    concept2.metadata.body = "a+b"
+
+    res1 = sheerka.create_new_concept(get_context(sheerka), concept1)
+    res2 = sheerka.create_new_concept(get_context(sheerka), concept2)
+
+    assert res1.value.body.key == res2.value.body.key  # same key
+
+    result = sheerka.get(concept1.key, res2.body.body.id)
+    assert result.name == "a + b"
+    assert result.body == "a+b"
+
+
+def test_i_can_get_the_correct_concept_using_the_id__when_same_key_when_cache():
+    sheerka = get_sheerka()
+    concept1 = get_default_concept()
+    concept2 = get_default_concept()
+    concept2.metadata.body = "a+b"
+
+    res1 = sheerka.create_new_concept(get_context(sheerka), concept1)
+    res2 = sheerka.create_new_concept(get_context(sheerka), concept2)
+
+    assert res1.value.body.key == res2.value.body.key  # same key
+
+    result = sheerka.get(concept1.key, res2.body.body.id)
+    assert result.name == "a + b"
+    assert result.body == "a+b"
+
+
+def test_i_cannot_get_the_correct_concept_id_the_id_is_wrong():
+    sheerka = get_sheerka()
+    concept1 = get_default_concept()
+    concept2 = get_default_concept()
+    concept2.metadata.body = "a+b"
+
+    res1 = sheerka.create_new_concept(get_context(sheerka), concept1)
+    res2 = sheerka.create_new_concept(get_context(sheerka), concept2)
+
+    assert res1.value.body.key == res2.value.body.key  # same key
+
+    result = sheerka.get(concept1.key, "wrong id")
+    assert sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT)
+
+
+def test_i_cannot_get_when_key_is_none():
+    sheerka = get_sheerka()
+
+    res = sheerka.get(None)
+    assert sheerka.isinstance(res, BuiltinConcepts.ERROR)
+    assert res.body == "Concept key is undefined."
 
 
 def test_unknown_concept_is_return_when_the_concept_is_not_found():

tests/test_sheerkaDataProvider.py

@@ -9,7 +9,7 @@ from datetime import date, datetime
 import shutil
 import json
 
-from sdp.sheerkaSerializer import ObjectSerializer, Serializer, PickleSerializer
+from sdp.sheerkaSerializer import JsonSerializer, Serializer, PickleSerializer
 import core.utils
 
 tests_root = path.abspath("../build/tests")
@@ -789,7 +789,7 @@ def test_i_can_set_using_reference(root):
 def test_i_can_add_an_object_with_a_key_as_a_reference(root):
     sdp = SheerkaDataProvider(root)
     obj = ObjDumpJson("my_key", "value1")
-    obj_serializer = ObjectSerializer(core.utils.get_full_qualified_name(obj))
+    obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
     sdp.serializer.register(obj_serializer)
 
     entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
@@ -813,7 +813,7 @@ def test_i_can_add_a_dictionary_as_a_reference(root):
     sdp = SheerkaDataProvider(root)
     obj = {"my_key": "value1"}
 
-    obj_serializer = ObjectSerializer(core.utils.get_full_qualified_name(obj))
+    obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
     sdp.serializer.register(obj_serializer)
 
     entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
@@ -1499,7 +1499,7 @@ def test_i_can_get_an_entry_by_key(root):
 def test_i_can_get_object_saved_by_reference(root):
     sdp = SheerkaDataProvider(root)
     obj = ObjDumpJson("my_key", "value1")
-    sdp.serializer.register(ObjectSerializer(core.utils.get_full_qualified_name(obj)))
+    sdp.serializer.register(JsonSerializer(core.utils.get_full_qualified_name(obj)))
 
     entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
     loaded = sdp.get(entry, key)
@@ -1714,7 +1714,7 @@ def test_i_can_test_than_the_object_exists_when_using_references(root):
 def test_i_can_save_and_load_object_ref_with_history(root):
     sdp = SheerkaDataProvider(root)
     obj = ObjDumpJson("my_key", "value1")
-    sdp.serializer.register(ObjectSerializer(core.utils.get_full_qualified_name(obj)))
+    sdp.serializer.register(JsonSerializer(core.utils.get_full_qualified_name(obj)))
 
     entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
     loaded = sdp.get(entry, key)
@@ -1770,7 +1770,7 @@ def test_i_can_add_obj_with_same_key_and_get_them_back(root):
     sdp = SheerkaDataProvider(root)
     obj1 = ObjDumpJson("key", "value1")
     obj2 = ObjDumpJson("key", "value2")
-    sdp.serializer.register(ObjectSerializer(core.utils.get_full_qualified_name(obj1)))
+    sdp.serializer.register(JsonSerializer(core.utils.get_full_qualified_name(obj1)))
 
     entry1, key1 = sdp.add(evt_digest, "entry", obj1, use_ref=True)
     entry2, key2 = sdp.add(evt_digest, "entry", obj2, use_ref=True)
@@ -1790,7 +1790,7 @@ def test_i_get_safe_dictionary_without_origin(root):
     sdp = SheerkaDataProvider(root)
     obj = {"my_key": "value1"}
 
-    obj_serializer = ObjectSerializer(core.utils.get_full_qualified_name(obj))
+    obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
     sdp.serializer.register(obj_serializer)
 
     entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
@@ -1814,7 +1814,7 @@ def test_i_get_dictionary_without_origin(root):
     sdp = SheerkaDataProvider(root)
     obj = {"my_key": "value1"}
 
-    obj_serializer = ObjectSerializer(core.utils.get_full_qualified_name(obj))
+    obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
     sdp.serializer.register(obj_serializer)
 
     entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
@@ -1838,7 +1838,7 @@ def test_i_get_safe_object_without_origin(root):
     sdp = SheerkaDataProvider(root)
     obj = ObjDumpJson("my_key", "value1")
 
-    obj_serializer = ObjectSerializer(core.utils.get_full_qualified_name(obj))
+    obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
     sdp.serializer.register(obj_serializer)
 
     entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)

tests/test_sheerkaSerializer.py

@@ -2,7 +2,7 @@ import pytest
 from dataclasses import dataclass
 
 from sdp.sheerkaDataProvider import Event
-from sdp.sheerkaSerializer import Serializer, ObjectSerializer, SerializerContext
+from sdp.sheerkaSerializer import Serializer, JsonSerializer, SerializerContext
 from datetime import datetime
 import core.utils
 
@@ -37,7 +37,7 @@ def test_i_can_serialize_an_event():
 def test_i_can_serialize_an_object():
     obj = Obj("10", "value")
     serializer = Serializer()
-    serializer.register(ObjectSerializer("tests.test_sheerkaSerializer.Obj"))
+    serializer.register(JsonSerializer("tests.test_sheerkaSerializer.Obj"))
     context = SerializerContext("kodjo", "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b")
 
     stream = serializer.serialize(obj, context)

tests/test_sheerka_non_reg.py

@@ -1,15 +1,14 @@
-import pytest
 import os
-from os import path
 import shutil
+from os import path
 
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, PROPERTIES_TO_SERIALIZE, Property
 from core.sheerka import Sheerka, ExecutionContext
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
-from parsers.BaseParser import BaseParser
+from parsers.ConceptLexerParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptMatch
-from parsers.ConceptLexerParser import Sequence, ZeroOrMore, StrMatch, OrderedChoice, Optional, ConceptMatch, \
-    ConceptLexerParser
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 
 tests_root = path.abspath("../build/tests")

tests/test_sheerka_transform.py

@@ -0,0 +1,303 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka import Sheerka, ExecutionContext, ExecutionContextIdManager
+from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY
+from sdp.sheerkaDataProvider import Event
+
+
+def get_sheerka():
+    sheerka = Sheerka()
+    sheerka.initialize("mem://")
+    return sheerka
+
+
+def get_context(sheerka):
+    return ExecutionContext("test", Event(), sheerka)
+
+
+def test_i_can_transform_an_unknown_concept():
+    sheerka = get_sheerka()
+
+    foo = Concept("foo", body="body")
+    concept_with_sub = Concept("concept_with_sub", body=foo)
+
+    concept = Concept(
+        name="concept_name",
+        is_builtin=True,
+        is_unique=True,
+        key="concept_key",
+        body=concept_with_sub,
+        where=[foo, 1, "1", True, 1.0],
+        pre=foo,
+        post=None,  # will not appear
+        definition="it is a definition",
+        definition_type="def type",
+        desc="this this the desc"
+    ).set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub)
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(concept)
+
+    assert to_dict == {
+        OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+        OBJ_ID_KEY: 0,
+        'name': 'concept_name',
+        'key': 'concept_key',
+        'is_builtin': True,
+        'is_unique': True,
+        'definition': 'it is a definition',
+        'definition_type': 'def type',
+        'desc': 'this this the desc',
+        'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                   OBJ_ID_KEY: 1,
+                   'body': 'body',
+                   'name': 'foo'}, 1, '1', True, 1.0],
+        'pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1},
+        'body': {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 2,
+            'name': 'concept_with_sub',
+            'body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}},
+        'props': [
+            ('a', 10),
+            ('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}),
+            ('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})
+        ]
+    }
+
+
+def test_i_can_transform_unknown_concept_with_almost_same_value():
+    sheerka = get_sheerka()
+    concept = Concept("foo")
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(concept)
+
+    assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'name': 'foo'}
+
+
+def test_i_can_transform_known_concept_when_the_values_are_the_same():
+    sheerka = get_sheerka()
+
+    concept = Concept(
+        name="concept_name",
+        is_builtin=True,
+        is_unique=False,
+        key="concept_key",
+        body="body definition",
+        where="where definition",
+        pre="pre definition",
+        post="post definition",
+        definition="it is a definition",
+        definition_type="def type",
+        desc="this this the desc"
+    ).set_prop("a").set_prop("b")
+    sheerka.create_new_concept(get_context(sheerka), concept)
+
+    new_concept = sheerka.new(concept.key)
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(new_concept)
+
+    assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"}
+
+
+def test_i_can_transform_known_concept_when_the_values_are_different():
+    sheerka = get_sheerka()
+
+    concept = Concept(
+        name="concept_name",
+        is_builtin=True,
+        is_unique=False,
+        key="concept_key",
+        body="body definition",
+        where="where definition",
+        pre="pre definition",
+        post="post definition",
+        definition="it is a definition",
+        definition_type="def type",
+        desc="this this the desc"
+    ).set_prop("a").set_prop("b")
+    sheerka.create_new_concept(get_context(sheerka), concept)
+
+    new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre")
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(new_concept)
+
+    assert to_dict == {
+        OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+        OBJ_ID_KEY: 0,
+        "id": "1001",
+        'pre': 'another pre',
+        "body": "another",
+        'props': [('a', 10)]
+    }
+
+
+def test_i_can_transform_concept_with_circular_reference():
+    sheerka = get_sheerka()
+    foo = Concept("foo", )
+    bar = Concept("bar", body=foo)
+    foo.metadata.body = bar
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(foo)
+
+    assert to_dict == {
+        OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+        OBJ_ID_KEY: 0,
+        'name': 'foo',
+        'body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                 OBJ_ID_KEY: 1,
+                 'name': 'bar',
+                 'body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                          OBJ_ID_KEY: 0},
+                 },
+    }
+
+
+def test_i_can_transform_concept_with_circular_reference_2():
+    sheerka = get_sheerka()
+    foo = Concept("foo", )
+    bar = Concept("foo", body=foo)
+    foo.metadata.body = bar
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(foo)
+
+    assert to_dict == {
+        OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+        OBJ_ID_KEY: 0,
+        'name': 'foo',
+        'body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                 OBJ_ID_KEY: 1,
+                 'name': 'foo',
+                 'body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                          OBJ_ID_KEY: 0},
+                 },
+    }
+
+
+def test_i_can_transform_the_unknown_concept():
+    sheerka = get_sheerka()
+
+    unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(unknown)
+
+    assert len(to_dict) == 3
+    assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+    assert to_dict[OBJ_ID_KEY] == 0
+    assert "id" in to_dict
+
+
+def test_i_can_transform_simple_execution_context():
+    sheerka = get_sheerka()
+    ExecutionContextIdManager.ids = {}
+    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(context)
+
+    assert to_dict == {
+        OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
+        OBJ_ID_KEY: 0,
+        '_parent': None,
+        '_id': 0,
+        '_tab': '',
+        '_bag': {},
+        '_start': 0,
+        '_stop': 0,
+        'who': 'requester',
+        'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'},
+        'desc': 'this is the desc',
+        'children': [],
+        'preprocess': None,
+        'values': {},
+        'obj': None,
+        'concepts': {}
+    }
+
+
+def test_i_can_transform_list():
+    sheerka = get_sheerka()
+    ExecutionContextIdManager.ids = {}
+    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict([context])
+
+    assert len(to_dict) == 1
+    assert isinstance(to_dict, list)
+    assert to_dict[0]["who"] == "requester"
+    assert to_dict[0]["desc"] == "this is the desc"
+
+
+def test_i_can_transform_set():
+    sheerka = get_sheerka()
+    ExecutionContextIdManager.ids = {}
+    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict({context})
+
+    assert len(to_dict) == 1
+    assert isinstance(to_dict, list)
+    assert to_dict[0]["who"] == "requester"
+    assert to_dict[0]["desc"] == "this is the desc"
+
+
+def test_i_can_transform_dict():
+    sheerka = get_sheerka()
+    ExecutionContextIdManager.ids = {}
+    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+    known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key()
+    sheerka.create_new_concept(get_context(sheerka), known_concept)
+    unknown_concept = Concept("bar")
+    known = sheerka.new("foo")
+
+    bag = {
+        "context": context,
+        "known_concept": known_concept,
+        "unknown_concept": unknown_concept,
+        "True": True,
+        "Number": 1.1,
+        "String": "a string value",
+        "None": None,
+        unknown_concept: "hello",
+        known: "world"
+    }
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(bag)
+
+    assert isinstance(to_dict, dict)
+    assert to_dict['Number'] == 1.1
+    assert to_dict['String'] == 'a string value'
+    assert to_dict['True']
+    assert to_dict['None'] is None
+    assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+    assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+    assert to_dict["known_concept"]["id"] == '1001'
+    assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+    assert to_dict["(None)bar"] == "hello"
+    assert to_dict["(1001)foo"] == "world"
+
+
+def test_i_can_transform_when_circular_references():
+    sheerka = get_sheerka()
+    ExecutionContextIdManager.ids = {}
+    context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+    context.push("another requester", "another desc")
+
+    st = SheerkaTransform(sheerka)
+    to_dict = st.to_dict(context)
+
+    assert isinstance(to_dict, dict)
+    assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+    assert len(to_dict["children"]) == 1
+    assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+    assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
+    assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0
+    assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
+    assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1