kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhanced ExecutionContext to keep track of the execution flow

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-01-07 15:47:43 +01:00
parent ffd98d7407
commit b4346b5af0
19 changed files with 966 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/sheerka.py
+164 -102
View File
@@ -1,5 +1,3 @@
from dataclasses import dataclass, field
from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
from parsers.BaseParser import BaseParser
@@ -10,6 +8,7 @@ import core.builtin_helpers
from core.sheerka_logger import console_handler
import logging
import time
CONCEPT_EVALUATION_STEPS = [
BuiltinConcepts.BEFORE_EVALUATION,
@@ -189,21 +188,26 @@ class Sheerka(Concept):
event = Event(text, user_name)
evt_digest = self.sdp.save_event(event)
self.log.debug(f"{evt_digest=}")
execution_context = ExecutionContext(self.key, event, self)
user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
with ExecutionContext(self.key, event, self, f"Evaluating '{text}'") as execution_context:
user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
steps = [
BuiltinConcepts.BEFORE_PARSING,
BuiltinConcepts.PARSING,
BuiltinConcepts.AFTER_PARSING,
BuiltinConcepts.BEFORE_EVALUATION,
BuiltinConcepts.EVALUATION,
BuiltinConcepts.AFTER_EVALUATION
]
steps = [
BuiltinConcepts.BEFORE_PARSING,
BuiltinConcepts.PARSING,
BuiltinConcepts.AFTER_PARSING,
BuiltinConcepts.BEFORE_EVALUATION,
BuiltinConcepts.EVALUATION,
BuiltinConcepts.AFTER_EVALUATION
]
return self.execute(execution_context, [user_input, reduce_requested], steps)
ret = self.execute(execution_context, [user_input, reduce_requested], steps)
execution_context.add_values(return_values=ret)
if not self.skip_builtins_in_db:
self.sdp.save_result(execution_context)
return ret
def _call_parsers(self, execution_context, return_values, logger=None):
@@ -229,15 +233,19 @@ class Sheerka(Concept):
p = parser(sheerka=self)
if logger:
p.log = logger
res = p.parse(execution_context, to_parse)
if hasattr(res, "__iter__"):
for r in res:
r.parents = [return_value]
result.append(r)
else:
res.parents = [return_value]
result.append(res)
with execution_context.push(desc=f"Parsing using {p.name}") as sub_context:
res = p.parse(sub_context, to_parse)
if hasattr(res, "__iter__"):
for r in res:
r.parents = [return_value]
result.append(r)
else:
res.parents = [return_value]
result.append(res)
sub_context.add_values(return_values=res)
return result
@@ -368,18 +376,19 @@ class Sheerka(Concept):
"""
for step in execution_steps:
sub_context = execution_context.push(step=step)
sub_context.log(logger or self.log, f"{step=}, context='{sub_context}'")
copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values]
with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context:
sub_context.log(logger or self.log, f"{step=}, context='{sub_context}'")
if step == BuiltinConcepts.PARSING:
return_values = self._call_parsers(sub_context, return_values, logger)
else:
return_values = self._call_evaluators(sub_context, return_values, step, None, logger)
if step == BuiltinConcepts.PARSING:
return_values = self._call_parsers(sub_context, return_values, logger)
else:
return_values = self._call_evaluators(sub_context, return_values, step, None, logger)
if copy != return_values:
sub_context.log_result(logger or self.log, return_values)
if copy != return_values:
sub_context.log_result(logger or self.log, return_values)
sub_context.add_values(return_values=return_values)
return return_values
@@ -430,14 +439,15 @@ class Sheerka(Concept):
# check if it's a valid BNF or whether it breaks the known rules
concept_lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]()
sub_context = context.push(self.name, desc=f"Initializing concept definition for {concept}")
sub_context.concepts[concept.key] = concept # the concept is not in the real cache yet
sub_context.log_new(logger)
init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions)
if not init_ret_value.status:
return self.ret(self.create_new_concept.__name__, False, ErrorConcept(init_ret_value.value))
with context.push(self.name, desc=f"Initializing concept definition for {concept}") as sub_context:
sub_context.concepts[concept.key] = concept # the concept is not in the real cache yet
sub_context.log_new(logger)
init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions)
sub_context.add_values(return_values=init_ret_value)
if not init_ret_value.status:
return self.ret(self.create_new_concept.__name__, False, ErrorConcept(init_ret_value.value))
# save the new context in sdp
# save the new concept in sdp
try:
self.sdp.add(context.event.get_digest(), self.CONCEPTS_ENTRY, concept, use_ref=True)
if concepts_definitions is not None:
@@ -507,10 +517,12 @@ class Sheerka(Concept):
# I refuse empty strings for performance matters, I don't want to handle useless NOPConcepts
continue
else:
sub_context = context.push(desc=f"Initializing AST for {part_key}")
sub_context.log_new(logger)
to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=source))
concept.cached_asts[part_key] = self.execute(sub_context, to_parse, steps, logger)
with context.push(desc=f"Initializing AST for {part_key}") as sub_context:
sub_context.log_new(logger)
to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=source))
res = self.execute(sub_context, to_parse, steps, logger)
concept.cached_asts[part_key] = res
sub_context.add_values(return_values=res)
for prop in concept.props:
value = concept.props[prop].value
@@ -522,9 +534,11 @@ class Sheerka(Concept):
context.who,
True,
self.new(BuiltinConcepts.USER_INPUT, body=value))
sub_context = context.push(desc=f"Initializing AST for property {prop}")
sub_context.log_new(logger)
concept.cached_asts[prop] = self.execute(context, to_parse, steps)
with context.push(desc=f"Initializing AST for property {prop}") as sub_context:
sub_context.log_new(logger)
res = self.execute(context, to_parse, steps)
concept.cached_asts[prop] = res
sub_context.add_values(return_values=res)
# Updates the cache of concepts when possible
if concept.key in self.concepts_cache:
@@ -552,10 +566,12 @@ class Sheerka(Concept):
def _resolve(return_value, desc, obj):
context.log(logger, desc, self.evaluate_concept.__name__)
sub_context = context.push(desc=desc, obj=obj)
sub_context.log_new(logger)
r = self.execute(sub_context, return_value, CONCEPT_EVALUATION_STEPS, logger)
return core.builtin_helpers.expect_one(context, r)
with context.push(desc=desc, obj=obj) as sub_context:
sub_context.log_new(logger)
r = self.execute(sub_context, return_value, CONCEPT_EVALUATION_STEPS, logger)
one_r = core.builtin_helpers.expect_one(context, r)
sub_context.add_values(return_values=one_r)
return one_r
# WHERE condition should already be validated by the parser.
# It's a mandatory condition for the concept before it can be recognized
@@ -579,10 +595,11 @@ class Sheerka(Concept):
if isinstance(concept.cached_asts[prop_name], Concept):
context.log(
logger, f"Evaluation prop={prop_name}, value={prop_ast}", self.evaluate_concept.__name__)
sub_context = context.push(f"Evaluation property '{prop_name}', value='{prop_ast}'")
sub_context.log_new(logger)
evaluated = self.evaluate_concept(sub_context, prop_ast)
concept.set_prop(prop_name, evaluated)
with context.push(f"Evaluation property '{prop_name}', value='{prop_ast}'") as sub_context:
sub_context.log_new(logger)
evaluated = self.evaluate_concept(sub_context, prop_ast)
sub_context.add_values(return_values=evaluated)
concept.set_prop(prop_name, evaluated)
else:
res = _resolve(prop_ast, f"Evaluating property '{prop_name}'", None)
if res.status:
@@ -631,26 +648,36 @@ class Sheerka(Concept):
self.concepts_cache[concept.key] = concept
return concept
def get(self, concept_key):
def get(self, concept_key, concept_id=None):
"""
Tries to find a concept
What is return must be used a template for another concept.
You must not modify the returned concept
:param concept_key:
:param concept_key: key of the concept
:param concept_id: when multiple concepts with the same key, use the id
:return:
"""
if concept_key is None:
return ErrorConcept("Concept key is undefined.")
if isinstance(concept_key, BuiltinConcepts):
concept_key = str(concept_key)
# first search in cache
if concept_key in self.concepts_cache:
return self.concepts_cache[concept_key]
result = self.concepts_cache[concept_key] if concept_key in self.concepts_cache else \
self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key)
# else look in sdp
from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key)
if from_db is not None:
return from_db
if result and (concept_id is None or not isinstance(result, list)):
return result
if isinstance(result, list):
if concept_id:
for c in result:
if c.id == concept_id:
return c
else:
return result
# else return new Unknown concept
# Note that I don't call the new() method to prevent cyclic call
@@ -871,10 +898,11 @@ class Sheerka(Concept):
for c in concepts:
if not first:
self.log.info("")
self.log.info(f"name : {c.name}")
self.log.info(f"bnf : {c.metadata.definition}")
self.log.info(f"key : {c.key}")
self.log.info(f"body : {c.body}")
self.log.info(f"name : {c.name}")
self.log.info(f"bnf : {c.metadata.definition}")
self.log.info(f"key : {c.key}")
self.log.info(f"body : {c.body}")
self.log.info(f"digest : {c.get_digest()}")
first = False
@staticmethod
@@ -900,7 +928,6 @@ class Sheerka(Concept):
logging.basicConfig(format=log_format, level=log_level, handlers=[console_handler])
@dataclass
class ExecutionContext:
"""
To keep track of the execution of a request
@@ -910,28 +937,66 @@ class ExecutionContext:
who,
event: Event,
sheerka: Sheerka,
/,
desc: str = None,
obj: Concept = None,
step: BuiltinConcepts = None,
iteration: int = 0,
concepts: dict = None):
**kwargs):
self._parent = None
self._id = ExecutionContextIdManager.get_id(event.get_digest())
self._tab = ""
self._bag = {} # other variables
self._start = 0
self._stop = 0
self.who = who # who is asking
self.event = event # what was the (original) trigger
self.sheerka = sheerka # sheerka
self.step = step
self.iteration = iteration
self.preprocess = None
self.desc = desc # human description of what is going on
self.obj = obj # what is the subject of the execution context (if known)
self.children = []
self.preprocess = None
self.values = {} # what was produced by the execution context
self.concepts = concepts or {} # cache for concepts that are specific to this execution
self.obj = kwargs.pop("obj", None)
self.concepts = kwargs.pop("concepts", {})
# update the other elements
for k, v in kwargs.items():
self._bag[k] = v
self._id = ExecutionContextIdManager.get_id(event.get_digest())
self._tab = ""
@property
def elapsed(self):
if self._start == 0:
return 0
return (self._stop if self._stop > 0 else time.time_ns()) - self._start
@property
def elapsed_str(self):
nano_sec = self.elapsed
dt = nano_sec / 1e6
return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
@property
def id(self):
return self._id
def __getattr__(self, item):
if item in self._bag:
return self._bag[item]
raise AttributeError(f"'ExecutionContext' object has no attribute '{item}'")
def __enter__(self):
self._start = time.time_ns()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._stop = time.time_ns()
def __repr__(self):
msg = f"ExecutionContext(who={self.who}, id={self._id}"
if self.desc:
msg += f", desc='{self.desc}'"
msg += ")"
return msg
def add_preprocess(self, name, **kwargs):
preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
@@ -944,6 +1009,11 @@ class ExecutionContext:
self.preprocess.add(preprocess)
return self
def add_values(self, **kwargs):
for k, v in kwargs.items():
self.values[k] = v
return self
def new_concept(self, key, **kwargs):
# search in obj
if self.obj:
@@ -964,29 +1034,23 @@ class ExecutionContext:
return self.sheerka.new(key, **kwargs)
@property
def id(self):
return self._id
def push(self, who=None, /, **kwargs):
def push(self, who=None, desc=None, **kwargs):
who = who or self.who
desc = kwargs.get("desc", "")
obj = kwargs.get("obj", self.obj)
concepts = kwargs.get("concepts", self.concepts)
step = kwargs.get("step", self.step)
iteration = kwargs.get("iteration", self.iteration)
_kwargs = {"obj": self.obj, "concepts": self.concepts}
_kwargs.update(self._bag)
_kwargs.update(kwargs)
new = ExecutionContext(
who,
self.event,
self.sheerka,
desc=desc,
obj=obj,
concepts=concepts,
step=step,
iteration=iteration,
desc,
**_kwargs,
)
new._parent = self
new._tab = self._tab + " " * DEBUG_TAB_SIZE
new.preprocess = self.preprocess
self.children.append(new)
return new
def log_new(self, logger):
@@ -1009,6 +1073,11 @@ class ExecutionContext:
to_str = self.return_value_to_str(r)
logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
def to_dict(self):
from core.sheerka_transform import SheerkaTransform
st = SheerkaTransform(self.sheerka)
return st.to_dict(self)
@staticmethod
def return_value_to_str(r):
value = str(r.value)
@@ -1017,13 +1086,6 @@ class ExecutionContext:
to_str = f"ReturnValue(who={r.who}, status={r.status}, value={value})"
return to_str
def __repr__(self):
msg = f"ExecutionContext(who={self.who}, id={self._id}"
if self.desc:
msg += f", desc='{self.desc}'"
msg += ")"
return msg
class ExecutionContextIdManager:
ids = {}