Enhanced ExecutionContext to keep track of the execution flow

core/builtin_concepts.py

@@ -186,7 +186,12 @@ class ReturnValueConcept(Concept):
                self.message == other.message
 
     def __hash__(self):
-        return hash((self.who, self.status, self.value))
+        if hasattr(self.value, "__iter__") and not isinstance(self.value, str):
+            value_hash = hash(tuple(self.value))
+        else:
+            value_hash = hash(self.value)
+
+        return hash((self.who, self.status, value_hash))
 
 
 class UnknownPropertyConcept(Concept):
@@ -233,6 +238,9 @@ class ParserResultConcept(Concept):
                self.body == other.body and \
                self.try_parsed == other.try_parsed
 
+    def __hash__(self):
+        return hash(self.metadata.name)
+
     @property
     def value(self):
         return self.body

core/builtin_helpers.py

@@ -8,6 +8,7 @@ from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
 
 
+
 def is_same_success(sheerka, return_values):
     """
     Returns True if all returns values are successful and have the same value
@@ -209,3 +210,4 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
 
     return predicates
 
+

core/concept.py

@@ -98,9 +98,26 @@ class Concept:
 
         # check the attributes
         for prop in PROPERTIES_TO_SERIALIZE:
-            if getattr(self.metadata, prop) != getattr(other.metadata, prop):
-                # print(prop) # use full to know which id does not match
-                return False
+            # print(prop) # use full to know which id does not match
+            my_value = getattr(self.metadata, prop)
+            other_value = getattr(other.metadata, prop)
+            if isinstance(my_value, Concept) and isinstance(other_value, Concept):
+                # need to check if circular references
+                if id(self) == id(other):
+                    continue
+
+                sub_value = getattr(other_value.metadata, prop)
+                while isinstance(sub_value, Concept):
+                    if id(self) == id(sub_value):
+                        return False  # circular reference
+                    sub_value = getattr(sub_value.metadata, prop)
+
+                if my_value != other_value:
+                    return False
+
+            else:
+                if my_value != other_value:
+                    return False
 
         # check the props (Concept variables)
         for var_name, p in self.props.items():

core/sheerka.py

@@ -1,5 +1,3 @@
-from dataclasses import dataclass, field
-
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
 from parsers.BaseParser import BaseParser
@@ -10,6 +8,7 @@ import core.builtin_helpers
 from core.sheerka_logger import console_handler
 
 import logging
+import time
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,
@@ -189,21 +188,26 @@ class Sheerka(Concept):
         event = Event(text, user_name)
         evt_digest = self.sdp.save_event(event)
         self.log.debug(f"{evt_digest=}")
-        execution_context = ExecutionContext(self.key, event, self)
 
-        user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
-        reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
+        with ExecutionContext(self.key, event, self, f"Evaluating '{text}'") as execution_context:
+            user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
+            reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
 
-        steps = [
-            BuiltinConcepts.BEFORE_PARSING,
-            BuiltinConcepts.PARSING,
-            BuiltinConcepts.AFTER_PARSING,
-            BuiltinConcepts.BEFORE_EVALUATION,
-            BuiltinConcepts.EVALUATION,
-            BuiltinConcepts.AFTER_EVALUATION
-        ]
+            steps = [
+                BuiltinConcepts.BEFORE_PARSING,
+                BuiltinConcepts.PARSING,
+                BuiltinConcepts.AFTER_PARSING,
+                BuiltinConcepts.BEFORE_EVALUATION,
+                BuiltinConcepts.EVALUATION,
+                BuiltinConcepts.AFTER_EVALUATION
+            ]
 
-        return self.execute(execution_context, [user_input, reduce_requested], steps)
+            ret = self.execute(execution_context, [user_input, reduce_requested], steps)
+            execution_context.add_values(return_values=ret)
+
+            if not self.skip_builtins_in_db:
+                self.sdp.save_result(execution_context)
+        return ret
 
     def _call_parsers(self, execution_context, return_values, logger=None):
 
@@ -229,15 +233,19 @@ class Sheerka(Concept):
                 p = parser(sheerka=self)
                 if logger:
                     p.log = logger
-                res = p.parse(execution_context, to_parse)
 
-                if hasattr(res, "__iter__"):
-                    for r in res:
-                        r.parents = [return_value]
-                        result.append(r)
-                else:
-                    res.parents = [return_value]
-                    result.append(res)
+                with execution_context.push(desc=f"Parsing using {p.name}") as sub_context:
+                    res = p.parse(sub_context, to_parse)
+
+                    if hasattr(res, "__iter__"):
+                        for r in res:
+                            r.parents = [return_value]
+                            result.append(r)
+                    else:
+                        res.parents = [return_value]
+                        result.append(res)
+
+                    sub_context.add_values(return_values=res)
 
         return result
 
@@ -368,18 +376,19 @@ class Sheerka(Concept):
         """
 
         for step in execution_steps:
-            sub_context = execution_context.push(step=step)
-            sub_context.log(logger or self.log, f"{step=}, context='{sub_context}'")
-
             copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values]
+            with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context:
+                sub_context.log(logger or self.log, f"{step=}, context='{sub_context}'")
 
-            if step == BuiltinConcepts.PARSING:
-                return_values = self._call_parsers(sub_context, return_values, logger)
-            else:
-                return_values = self._call_evaluators(sub_context, return_values, step, None, logger)
+                if step == BuiltinConcepts.PARSING:
+                    return_values = self._call_parsers(sub_context, return_values, logger)
+                else:
+                    return_values = self._call_evaluators(sub_context, return_values, step, None, logger)
 
-            if copy != return_values:
-                sub_context.log_result(logger or self.log, return_values)
+                if copy != return_values:
+                    sub_context.log_result(logger or self.log, return_values)
+
+                sub_context.add_values(return_values=return_values)
 
         return return_values
 
@@ -430,14 +439,15 @@ class Sheerka(Concept):
 
             # check if it's a valid BNF or whether it breaks the known rules
             concept_lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]()
-            sub_context = context.push(self.name, desc=f"Initializing concept definition for {concept}")
-            sub_context.concepts[concept.key] = concept  # the concept is not in the real cache yet
-            sub_context.log_new(logger)
-            init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions)
-            if not init_ret_value.status:
-                return self.ret(self.create_new_concept.__name__, False, ErrorConcept(init_ret_value.value))
+            with context.push(self.name, desc=f"Initializing concept definition for {concept}") as sub_context:
+                sub_context.concepts[concept.key] = concept  # the concept is not in the real cache yet
+                sub_context.log_new(logger)
+                init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions)
+                sub_context.add_values(return_values=init_ret_value)
+                if not init_ret_value.status:
+                    return self.ret(self.create_new_concept.__name__, False, ErrorConcept(init_ret_value.value))
 
-        # save the new context in sdp
+        # save the new concept in sdp
         try:
             self.sdp.add(context.event.get_digest(), self.CONCEPTS_ENTRY, concept, use_ref=True)
             if concepts_definitions is not None:
@@ -507,10 +517,12 @@ class Sheerka(Concept):
                 # I refuse empty strings for performance matters, I don't want to handle useless NOPConcepts
                 continue
             else:
-                sub_context = context.push(desc=f"Initializing AST for {part_key}")
-                sub_context.log_new(logger)
-                to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=source))
-                concept.cached_asts[part_key] = self.execute(sub_context, to_parse, steps, logger)
+                with context.push(desc=f"Initializing AST for {part_key}") as sub_context:
+                    sub_context.log_new(logger)
+                    to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=source))
+                    res = self.execute(sub_context, to_parse, steps, logger)
+                    concept.cached_asts[part_key] = res
+                    sub_context.add_values(return_values=res)
 
         for prop in concept.props:
             value = concept.props[prop].value
@@ -522,9 +534,11 @@ class Sheerka(Concept):
                         context.who,
                         True,
                         self.new(BuiltinConcepts.USER_INPUT, body=value))
-                    sub_context = context.push(desc=f"Initializing AST for property {prop}")
-                    sub_context.log_new(logger)
-                    concept.cached_asts[prop] = self.execute(context, to_parse, steps)
+                    with context.push(desc=f"Initializing AST for property {prop}") as sub_context:
+                        sub_context.log_new(logger)
+                        res = self.execute(context, to_parse, steps)
+                        concept.cached_asts[prop] = res
+                        sub_context.add_values(return_values=res)
 
         # Updates the cache of concepts when possible
         if concept.key in self.concepts_cache:
@@ -552,10 +566,12 @@ class Sheerka(Concept):
 
         def _resolve(return_value, desc, obj):
             context.log(logger, desc, self.evaluate_concept.__name__)
-            sub_context = context.push(desc=desc, obj=obj)
-            sub_context.log_new(logger)
-            r = self.execute(sub_context, return_value, CONCEPT_EVALUATION_STEPS, logger)
-            return core.builtin_helpers.expect_one(context, r)
+            with context.push(desc=desc, obj=obj) as sub_context:
+                sub_context.log_new(logger)
+                r = self.execute(sub_context, return_value, CONCEPT_EVALUATION_STEPS, logger)
+                one_r = core.builtin_helpers.expect_one(context, r)
+                sub_context.add_values(return_values=one_r)
+                return one_r
 
         # WHERE condition should already be validated by the parser.
         # It's a mandatory condition for the concept before it can be recognized
@@ -579,10 +595,11 @@ class Sheerka(Concept):
                     if isinstance(concept.cached_asts[prop_name], Concept):
                         context.log(
                             logger, f"Evaluation prop={prop_name}, value={prop_ast}", self.evaluate_concept.__name__)
-                        sub_context = context.push(f"Evaluation property '{prop_name}', value='{prop_ast}'")
-                        sub_context.log_new(logger)
-                        evaluated = self.evaluate_concept(sub_context, prop_ast)
-                        concept.set_prop(prop_name, evaluated)
+                        with context.push(f"Evaluation property '{prop_name}', value='{prop_ast}'") as sub_context:
+                            sub_context.log_new(logger)
+                            evaluated = self.evaluate_concept(sub_context, prop_ast)
+                            sub_context.add_values(return_values=evaluated)
+                            concept.set_prop(prop_name, evaluated)
                     else:
                         res = _resolve(prop_ast, f"Evaluating property '{prop_name}'", None)
                         if res.status:
@@ -631,26 +648,36 @@ class Sheerka(Concept):
         self.concepts_cache[concept.key] = concept
         return concept
 
-    def get(self, concept_key):
+    def get(self, concept_key, concept_id=None):
         """
         Tries to find a concept
         What is return must be used a template for another concept.
         You must not modify the returned concept
-        :param concept_key:
+        :param concept_key: key of the concept
+        :param concept_id: when multiple concepts with the same key, use the id
         :return:
         """
 
+        if concept_key is None:
+            return ErrorConcept("Concept key is undefined.")
+
         if isinstance(concept_key, BuiltinConcepts):
             concept_key = str(concept_key)
 
         # first search in cache
-        if concept_key in self.concepts_cache:
-            return self.concepts_cache[concept_key]
+        result = self.concepts_cache[concept_key] if concept_key in self.concepts_cache else \
+            self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key)
 
-        # else look in sdp
-        from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key)
-        if from_db is not None:
-            return from_db
+        if result and (concept_id is None or not isinstance(result, list)):
+            return result
+
+        if isinstance(result, list):
+            if concept_id:
+                for c in result:
+                    if c.id == concept_id:
+                        return c
+            else:
+                return result
 
         # else return new Unknown concept
         # Note that I don't call the new() method to prevent cyclic call
@@ -871,10 +898,11 @@ class Sheerka(Concept):
         for c in concepts:
             if not first:
                 self.log.info("")
-            self.log.info(f"name : {c.name}")
-            self.log.info(f"bnf  : {c.metadata.definition}")
-            self.log.info(f"key  : {c.key}")
-            self.log.info(f"body : {c.body}")
+            self.log.info(f"name   : {c.name}")
+            self.log.info(f"bnf    : {c.metadata.definition}")
+            self.log.info(f"key    : {c.key}")
+            self.log.info(f"body   : {c.body}")
+            self.log.info(f"digest : {c.get_digest()}")
             first = False
 
     @staticmethod
@@ -900,7 +928,6 @@ class Sheerka(Concept):
         logging.basicConfig(format=log_format, level=log_level, handlers=[console_handler])
 
 
-@dataclass
 class ExecutionContext:
     """
     To keep track of the execution of a request
@@ -910,28 +937,66 @@ class ExecutionContext:
                  who,
                  event: Event,
                  sheerka: Sheerka,
-                 /,
                  desc: str = None,
-                 obj: Concept = None,
-                 step: BuiltinConcepts = None,
-                 iteration: int = 0,
-                 concepts: dict = None):
+                 **kwargs):
+
+        self._parent = None
+        self._id = ExecutionContextIdManager.get_id(event.get_digest())
+        self._tab = ""
+        self._bag = {}  # other variables
+        self._start = 0
+        self._stop = 0
 
         self.who = who  # who is asking
         self.event = event  # what was the (original) trigger
         self.sheerka = sheerka  # sheerka
-
-        self.step = step
-        self.iteration = iteration
-        self.preprocess = None
-
         self.desc = desc  # human description of what is going on
-        self.obj = obj  # what is the subject of the execution context (if known)
+        self.children = []
+        self.preprocess = None
+        self.values = {}  # what was produced by the execution context
 
-        self.concepts = concepts or {}  # cache for concepts that are specific to this execution
+        self.obj = kwargs.pop("obj", None)
+        self.concepts = kwargs.pop("concepts", {})
+        # update the other elements
+        for k, v in kwargs.items():
+            self._bag[k] = v
 
-        self._id = ExecutionContextIdManager.get_id(event.get_digest())
-        self._tab = ""
+    @property
+    def elapsed(self):
+        if self._start == 0:
+            return 0
+
+        return (self._stop if self._stop > 0 else time.time_ns()) - self._start
+
+    @property
+    def elapsed_str(self):
+        nano_sec = self.elapsed
+        dt = nano_sec / 1e6
+        return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
+
+    @property
+    def id(self):
+        return self._id
+
+    def __getattr__(self, item):
+        if item in self._bag:
+            return self._bag[item]
+
+        raise AttributeError(f"'ExecutionContext' object has no attribute '{item}'")
+
+    def __enter__(self):
+        self._start = time.time_ns()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._stop = time.time_ns()
+
+    def __repr__(self):
+        msg = f"ExecutionContext(who={self.who}, id={self._id}"
+        if self.desc:
+            msg += f", desc='{self.desc}'"
+        msg += ")"
+        return msg
 
     def add_preprocess(self, name, **kwargs):
         preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
@@ -944,6 +1009,11 @@ class ExecutionContext:
         self.preprocess.add(preprocess)
         return self
 
+    def add_values(self, **kwargs):
+        for k, v in kwargs.items():
+            self.values[k] = v
+        return self
+
     def new_concept(self, key, **kwargs):
         # search in obj
         if self.obj:
@@ -964,29 +1034,23 @@ class ExecutionContext:
 
         return self.sheerka.new(key, **kwargs)
 
-    @property
-    def id(self):
-        return self._id
-
-    def push(self, who=None, /, **kwargs):
+    def push(self, who=None, desc=None, **kwargs):
         who = who or self.who
-        desc = kwargs.get("desc", "")
-        obj = kwargs.get("obj", self.obj)
-        concepts = kwargs.get("concepts", self.concepts)
-        step = kwargs.get("step", self.step)
-        iteration = kwargs.get("iteration", self.iteration)
+        _kwargs = {"obj": self.obj, "concepts": self.concepts}
+        _kwargs.update(self._bag)
+        _kwargs.update(kwargs)
         new = ExecutionContext(
             who,
             self.event,
             self.sheerka,
-            desc=desc,
-            obj=obj,
-            concepts=concepts,
-            step=step,
-            iteration=iteration,
+            desc,
+            **_kwargs,
         )
+        new._parent = self
         new._tab = self._tab + " " * DEBUG_TAB_SIZE
         new.preprocess = self.preprocess
+
+        self.children.append(new)
         return new
 
     def log_new(self, logger):
@@ -1009,6 +1073,11 @@ class ExecutionContext:
             to_str = self.return_value_to_str(r)
             logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
+    def to_dict(self):
+        from core.sheerka_transform import SheerkaTransform
+        st = SheerkaTransform(self.sheerka)
+        return st.to_dict(self)
+
     @staticmethod
     def return_value_to_str(r):
         value = str(r.value)
@@ -1017,13 +1086,6 @@ class ExecutionContext:
         to_str = f"ReturnValue(who={r.who}, status={r.status}, value={value})"
         return to_str
 
-    def __repr__(self):
-        msg = f"ExecutionContext(who={self.who}, id={self._id}"
-        if self.desc:
-            msg += f", desc='{self.desc}'"
-        msg += ")"
-        return msg
-
 
 class ExecutionContextIdManager:
     ids = {}

core/sheerka_transform.py

@@ -0,0 +1,152 @@
+import dataclasses
+from enum import Enum
+
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE
+from core.sheerka import ExecutionContext
+from core.tokenizer import Token
+from evaluators.BaseEvaluator import BaseEvaluator
+from parsers.BaseParser import BaseParser, Node
+from parsers.BnfParser import BnfParser
+from parsers.ConceptLexerParser import UnrecognizedTokensNode, ParsingExpression
+from parsers.PythonParser import PythonNode
+from sdp.sheerkaDataProvider import Event
+
+OBJ_TYPE_KEY = "__type__"
+OBJ_ID_KEY = "__id__"
+OBJ_NAME_KEY = "__name__"
+
+default_concept = Concept()
+
+
+class SheerkaTransformType(Enum):
+    Concept = 1
+    Reference = 2
+    ExecutionContext = 3
+    Event = 4
+    Node = 5
+    Exception = 6
+
+
+class SheerkaTransform:
+
+    def __init__(self, sheerka):
+        self.ids = {}
+        self.sheerka = sheerka
+        self.id_count = -1
+
+    def to_dict(self, obj):
+
+        if isinstance(obj, (Concept, ExecutionContext, Event)):
+            exists, _id = self.exist(obj)
+            if exists:
+                return {
+                    OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                    OBJ_ID_KEY: _id
+                }
+            else:
+                self.id_count = self.id_count + 1
+                self.ids[obj] = self.id_count
+
+        if isinstance(obj, Concept):
+            return self.context_to_dict(obj)
+
+        elif isinstance(obj, ExecutionContext):
+            return self.execution_context_to_dict(obj)
+
+        elif isinstance(obj, Event):
+            return {
+                OBJ_TYPE_KEY: SheerkaTransformType.Event,
+                OBJ_ID_KEY: self.id_count,
+                'digest': obj.get_digest()}
+
+        elif isinstance(obj, (BaseParser, BaseEvaluator, BnfParser)):
+            return obj.name
+
+        elif isinstance(obj, Token):
+            return obj.__dict__
+
+        elif isinstance(obj, PythonNode):
+            return {
+                OBJ_TYPE_KEY: SheerkaTransformType.Node,
+                OBJ_NAME_KEY: "PythonNode",
+                'source': obj.source,
+                'ast_': obj.get_dump(obj.ast_)
+            }
+
+        elif isinstance(obj, Node):
+            to_dict = {
+                OBJ_TYPE_KEY: SheerkaTransformType.Node,
+                OBJ_NAME_KEY: obj.__class__.__name__,
+            }
+            for k, v in obj.__dict__.items():
+                to_dict[k] = self.to_dict(v)
+
+            return to_dict
+
+        elif isinstance(obj, Exception):
+            to_dict = {
+                OBJ_TYPE_KEY: SheerkaTransformType.Exception,
+                OBJ_NAME_KEY: obj.__class__.__name__,
+            }
+            for k, v in obj.__dict__.items():
+                to_dict[k] = self.to_dict(v)
+            return to_dict
+
+        elif isinstance(obj, ParsingExpression):
+            return obj.__repr__()
+
+        elif isinstance(obj, dict):
+            return dict((str(k) if isinstance(k, Concept) else k, self.to_dict(v)) for k, v in obj.items())
+
+        elif hasattr(obj, "__iter__") and not isinstance(obj, str):
+            return list(self.to_dict(o) for o in obj)
+
+        else:
+            return obj
+
+    def context_to_dict(self, obj: Concept):
+        to_dict = {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: self.id_count,
+        }
+        if obj.id:
+            ref = self.sheerka.get(obj.key, obj.id)
+            to_dict["id"] = obj.id
+        else:
+            ref = default_concept
+
+        # transform metadata
+        for prop in PROPERTIES_TO_SERIALIZE:
+            value = self.to_dict(getattr(obj.metadata, prop))
+            ref_value = getattr(ref.metadata, prop)
+            if value != ref_value:
+                to_dict[prop] = value
+
+        # transform properties
+        for prop in obj.props:
+            value = self.to_dict(obj.props[prop].value)
+            if prop not in ref.props or value != ref.props[prop].value:
+                if "props" not in to_dict:
+                    to_dict["props"] = []
+                to_dict["props"].append((prop, value))
+
+        return to_dict
+
+    def execution_context_to_dict(self, obj: ExecutionContext):
+        to_dict = {
+            OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
+            OBJ_ID_KEY: self.id_count
+        }
+        for property_name in obj.__dict__:
+            if property_name == "sheerka":
+                continue
+            to_dict[property_name] = self.to_dict(getattr(obj, property_name))
+
+        return to_dict
+
+    def exist(self, obj):
+        for k, v in self.ids.items():
+            if id(k) == id(obj) or k == obj:
+                return True, v
+
+        return False, None