From 912455c343ef93e0f256e1717d715c5e63660dd8 Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Fri, 12 Jun 2020 17:47:29 +0200 Subject: [PATCH] EvalEvaluator is called only if in root context. Added action and action_context to ExecutionContext --- src/core/builtin_concepts.py | 13 +++ src/core/builtin_helpers.py | 8 +- src/core/concept.py | 32 ++++++- src/core/sheerka/ExecutionContext.py | 87 ++++++++++++------- src/core/sheerka/Sheerka.py | 37 +++----- src/core/sheerka/services/SheerkaAdmin.py | 23 +++++ src/core/sheerka/services/SheerkaDump.py | 2 +- .../services/SheerkaEvaluateConcept.py | 18 +++- src/core/sheerka/services/SheerkaExecute.py | 18 +++- .../sheerka/services/SheerkaSetsManager.py | 4 +- src/evaluators/AddConceptInSetEvaluator.py | 2 +- src/evaluators/ConceptEvaluator.py | 7 +- src/evaluators/EvalEvaluator.py | 4 +- src/evaluators/PythonEvaluator.py | 6 +- src/parsers/BaseNodeParser.py | 6 +- src/parsers/BnfNodeParser.py | 2 +- src/parsers/DefaultParser.py | 8 +- src/parsers/PythonParser.py | 4 +- src/parsers/PythonWithConceptsParser.py | 4 +- src/sheerkapickle/sheerka_handlers.py | 4 +- tests/BaseTest.py | 2 +- tests/core/test_ExecutionContext.py | 41 +++++---- tests/core/test_SheerkaFilter.py | 2 +- tests/core/test_sheerka_printer.py | 5 +- tests/evaluators/test_EvalEvaluator.py | 25 +++++- tests/non_reg/test_sheerka_non_reg.py | 29 ++++++- tests/sheerkapickle/test_sheerka_handlers.py | 16 ++-- 27 files changed, 292 insertions(+), 117 deletions(-) diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 39b2c3d..20d7c93 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -15,6 +15,10 @@ class BuiltinConcepts(Enum): """ SHEERKA = "sheerka" + # Execution context actions + INIT_SHEERKA = "init sheerka" # + PROCESS_INPUT = "process input" # Processing user input or other input + PROCESSING = "processing input" # Processing user input or other input BEFORE_PARSING = "before parsing" # activated before evaluation by the parsers PARSING = "parsing" # activated during the parsing. It contains the text to parse AFTER_PARSING = "after parsing" # after parsing @@ -24,6 +28,15 @@ class BuiltinConcepts(Enum): BEFORE_RENDERING = "before rendering" # activate before the output is rendered RENDERING = "rendering" # rendering the response from sheerka AFTER_RENDERING = "after rendering" # rendering the response from sheerka + EVALUATE_CONCEPT = "evaluate concept" # a concept will be evaluated + EVALUATING_CONCEPT = "evaluating concept" # a concept will be evaluated + VALIDATE_CONCEPT = "validate concept" + VALIDATING_CONCEPT = "validating concept" + INIT_COMPILED = "initializing concept compiled" + INIT_BNF = "ensure bnf" + MANAGE_INFINITE_RECURSION = "manage infinite recursion" + PARSE_CODE = "execute source code" + EXEC_CODE = "execute source code" USER_INPUT = "user input" # represent an input from an user SUCCESS = "success" diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index 7539ce2..32d18d2 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -25,7 +25,9 @@ def is_same_success(context, return_values): if isinstance(ret_val.body, Concept): if not ret_val.body.metadata.is_evaluated: - with context.push(desc=f"Evaluating concept '{ret_val.body}'") as sub_context: + with context.push(BuiltinConcepts.EVALUATE_CONCEPT, + ret_val.body, + desc=f"Evaluating concept '{ret_val.body}'") as sub_context: sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) evaluated = context.sheerka.evaluate_concept(sub_context, ret_val.body) if evaluated.key != ret_val.body.key: @@ -237,7 +239,7 @@ def parse_unrecognized(context, source, parsers): steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING] sheerka = context.sheerka - with context.push(desc=f"Parsing unrecognized '{source}'") as sub_context: + with context.push(BuiltinConcepts.PARSING, source, desc=f"Parsing unrecognized '{source}'") as sub_context: # disable all parsers but the following ones sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False) for parser in parsers: @@ -322,7 +324,7 @@ def ensure_evaluated(context, concept): if concept.metadata.is_evaluated: return concept - with context.push(desc=f"Evaluating concept {concept}") as sub_context: + with context.push(BuiltinConcepts.EVALUATE_CONCEPT, concept, desc=f"Evaluating concept {concept}") as sub_context: sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) evaluated = context.sheerka.evaluate_concept(sub_context, concept) sub_context.add_values(return_values=evaluated) diff --git a/src/core/concept.py b/src/core/concept.py index fa24993..5272911 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -115,7 +115,7 @@ class Concept: if isinstance(other, simplec): return self.name == other.name and self.body == other.body - if isinstance(other, (CC, CB, CMV)): + if isinstance(other, (CC, CB, CV, CMV)): return other == self if not isinstance(other, Concept): @@ -589,7 +589,7 @@ class CC: class CB: """ Concept with body only - Test class that test only the body of the concept + Test class that tests only the body of the concept """ concept: Union[str, Concept] body: object @@ -611,6 +611,34 @@ class CB: return f"CB({self.body})" +class CV: + """ + Concept with all values + Test class that tests all the values (not the metadata, so not the properties) of a concept + """ + + def __init__(self, concept, body, **kwargs): + self.concept_key = concept.key if isinstance(concept, Concept) else concept + self.concept = concept if isinstance(concept, Concept) else None + self.values = kwargs + self.values[ConceptParts.BODY] = body + + def __eq__(self, other): + if isinstance(other, Concept): + return self.concept_key == other.key and self.values == other.values + + if not isinstance(other, CV): + return False + + return self.concept_key == other.concept_key and self.values == other.values + + def __hash__(self): + return hash((self.concept_key, self.values)) + + def __repr__(self): + return f"CV(key={self.concept_key}, values={self.values})" + + class CMV: """ Concept with metadata variables diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index f5ac46e..69257ef 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -41,6 +41,8 @@ class ExecutionContext: who, event: Event, sheerka, + action: BuiltinConcepts, + action_context, desc: str = None, logger=None, global_hints=None, @@ -53,11 +55,13 @@ class ExecutionContext: self._bag = {} # context variables self._start = 0 # when the execution starts (to measure elapsed time) self._stop = 0 # when the execution stops (to measure elapses time) - self._format_instructions = None # how to print the execution context + self._format_instructions = None # how to print the execution context self.who = who # who is asking self.event = event # what was the (original) trigger self.sheerka = sheerka # sheerka + self.action = action + self.action_context = action_context self.desc = desc # human description of what is going on self.children = [] self.preprocess = None @@ -66,7 +70,6 @@ class ExecutionContext: self.global_hints = set() if global_hints is None else global_hints self.global_errors = [] if global_errors is None else global_errors - self.inputs = {} # what was the parameters of the execution context self.values = {} # what was produced by the execution context @@ -114,16 +117,16 @@ class ExecutionContext: self.stat_log.debug(f"[{self._id:2}]" + self._tab + "Execution time: " + self.elapsed_str) def __repr__(self): - msg = f"ExecutionContext(who={self.who}, id={self._id}" + msg = f"ExecutionContext(who={self.who}, id={self._id}, action={self.action}, context={self.action_context}" if self.desc: msg += f", desc='{self.desc}'" msg += ")" return msg - def __str__(self): - msg = self.desc or "New Context" - msg += f", who={self.who}, id={self.id}" - return msg + # def __str__(self): + # msg = self.desc or "New Context" + # msg += f", who={self.who}, id={self.id}" + # return msg def __eq__(self, other): if id(self) == id(other): @@ -145,6 +148,31 @@ class ExecutionContext: return True + def push(self, action: BuiltinConcepts, action_context, who=None, desc=None, logger=None, **kwargs): + who = who or self.who + logger = logger or self.logger + _kwargs = {"obj": self.obj, "concepts": self.concepts} + _kwargs.update(self._bag) + _kwargs.update(kwargs) + new = ExecutionContext( + who, + self.event, + self.sheerka, + action, + action_context, + desc, + logger, + self.global_hints, + self.global_errors, + **_kwargs) + new._parent = self + new._tab = self._tab + " " * DEBUG_TAB_SIZE + new.preprocess = self.preprocess + new.local_hints.update(self.local_hints) + + self.children.append(new) + return new + def add_preprocess(self, name, **kwargs): preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS) preprocess.set_value("name", name) @@ -205,29 +233,6 @@ class ExecutionContext: return self.sheerka.new(key, **kwargs) - def push(self, who=None, desc=None, logger=None, **kwargs): - who = who or self.who - logger = logger or self.logger - _kwargs = {"obj": self.obj, "concepts": self.concepts} - _kwargs.update(self._bag) - _kwargs.update(kwargs) - new = ExecutionContext( - who, - self.event, - self.sheerka, - desc, - logger, - self.global_hints, - self.global_errors, - **_kwargs) - new._parent = self - new._tab = self._tab + " " * DEBUG_TAB_SIZE - new.preprocess = self.preprocess - new.local_hints.update(self.local_hints) - - self.children.append(new) - return new - def log_new(self): if self.logger and not self.logger.disabled: self.logger.debug(f"[{self._id:2}]" + self._tab + str(self)) @@ -265,6 +270,9 @@ class ExecutionContext: return False + def in_current_context(self, concept_key): + return concept_key in self.local_hints + @staticmethod def _is_return_value(obj): return isinstance(obj, Concept) and obj.key == str(BuiltinConcepts.RETURN_VALUE) @@ -347,3 +355,22 @@ class ExecutionContext: def set_format_instructions(self, instructions): self._format_instructions = instructions + + def get_parents(self, predicate=None): + """ + Gets all the parents that match the given predicate + :param predicate: + :return: + """ + res = [] + current = self + while True: + parent = current._parent + if parent: + if predicate is None or predicate(parent): + res.append(parent) + current = parent + else: + break + + return res diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index e9214ca..9de89c1 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -18,7 +18,6 @@ from printer.SheerkaPrinter import SheerkaPrinter from sdp.sheerkaDataProvider import SheerkaDataProvider, Event BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser" -CONCEPTS_FILE = "_concepts.txt" EXIT_COMMANDS = ("quit", "exit", "bye") @@ -156,7 +155,13 @@ class Sheerka(Concept): event = Event("Initializing Sheerka.", user_id=self.name) self.sdp.save_event(event) - with ExecutionContext(self.key, event, self, "Initializing Sheerka.", self.init_log) as exec_context: + with ExecutionContext(self.key, + event, + self, + BuiltinConcepts.INIT_SHEERKA, + None, + desc="Initializing Sheerka.", + logger=self.init_log) as exec_context: if self.sdp.first_time: self.first_time_initialisation(exec_context) @@ -355,7 +360,13 @@ class Sheerka(Concept): evt_digest = self.sdp.save_event(event) self.log.debug(f"{evt_digest=}") - with ExecutionContext(self.key, event, self, f"Evaluating '{text}'", self.log) as execution_context: + with ExecutionContext(self.key, + event, + self, + BuiltinConcepts.PROCESS_INPUT, + text, + desc=f"Evaluating '{text}'", + logger=self.log) as execution_context: user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name)) reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED)) @@ -757,26 +768,6 @@ class Sheerka(Concept): return sorted(res, key=lambda i: int(i.id)) - def restore(self): - """ - Restore the state with all previous valid concept definitions - :return: - """ - try: - self.during_restore = True - with open(CONCEPTS_FILE, "r") as f: - for line in f.readlines(): - line = line.strip() - if line == "" or line.startswith("#"): - continue - self.log.info(line) - res = self.evaluate_user_input(line) - if len(res) > 1 or not res[0].status: - self.log.error("Error detected !") - self.during_restore = False - except IOError: - pass - def get_last_execution(self): return self._last_execution diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index 663c759..3de1a87 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -1,6 +1,8 @@ from core.builtin_concepts import BuiltinConcepts from core.sheerka.services.sheerka_service import BaseService +CONCEPTS_FILE = "_concepts.txt" + class SheerkaAdmin(BaseService): NAME = "Admin" @@ -11,6 +13,7 @@ class SheerkaAdmin(BaseService): def initialize(self): self.sheerka.bind_service_method(self.caches_names) self.sheerka.bind_service_method(self.cache) + self.sheerka.bind_service_method(self.restore) def caches_names(self): """ @@ -29,3 +32,23 @@ class SheerkaAdmin(BaseService): return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"cache": name}) return self.sheerka.cache_manager.caches[name].cache.copy() + + def restore(self): + """ + Restore the state with all previous valid concept definitions + :return: + """ + try: + self.sheerka.during_restore = True + with open(CONCEPTS_FILE, "r") as f: + for line in f.readlines(): + line = line.strip() + if line == "" or line.startswith("#"): + continue + self.sheerka.log.info(line) + res = self.sheerka.evaluate_user_input(line) + if len(res) > 1 or not res[0].status: + self.sheerka.log.error("Error detected !") + self.sheerka.during_restore = False + except IOError: + pass diff --git a/src/core/sheerka/services/SheerkaDump.py b/src/core/sheerka/services/SheerkaDump.py index c785729..b1917a3 100644 --- a/src/core/sheerka/services/SheerkaDump.py +++ b/src/core/sheerka/services/SheerkaDump.py @@ -38,7 +38,7 @@ class SheerkaDump(BaseService): def dump_desc(self, *concept_names, eval=False): first = True event = Event(f"Dumping description", "") - context = ExecutionContext("dump_desc", event, self.sheerka) + context = ExecutionContext("dump_desc", event, self.sheerka, BuiltinConcepts.RENDERING, concept_names) for concept_name in concept_names: if isinstance(concept_name, Concept): concepts = concept_name diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index e13c8d7..95fb2bb 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -104,7 +104,9 @@ class SheerkaEvaluateConcept(BaseService): context.log(f"Recognized concept '{concept_found}'", self.NAME) concept.compiled[part_key] = concept_found else: - with context.push(desc=f"Initializing *compiled* for {part_key}") as sub_context: + with context.push(BuiltinConcepts.INIT_COMPILED, + {"part": part_key, "source": source}, + desc=f"Initializing *compiled* for {part_key}") as sub_context: sub_context.add_inputs(source=source) to_parse = self.sheerka.ret(context.who, True, self.sheerka.new(BuiltinConcepts.USER_INPUT, body=source)) @@ -128,7 +130,9 @@ class SheerkaEvaluateConcept(BaseService): context.log(f"Recognized concept '{concept_found}'", self.NAME) concept.compiled[var_name] = concept_found else: - with context.push(desc=f"Initializing *compiled* for property {var_name}") as sub_context: + with context.push(BuiltinConcepts.INIT_COMPILED, + {"property": var_name, "source": default_value}, + desc=f"Initializing *compiled* for property {var_name}") as sub_context: sub_context.add_inputs(source=default_value) to_parse = self.sheerka.ret(context.who, True, self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value)) @@ -148,7 +152,10 @@ class SheerkaEvaluateConcept(BaseService): # manage infinite loop if self.infinite_recursion_detected(context, current_concept): - with context.push(desc="Infinite recursion detected", obj=current_concept) as sub_context: + with context.push(BuiltinConcepts.MANAGE_INFINITE_RECURSION, + current_concept, + desc="Infinite recursion detected", + obj=current_concept) as sub_context: # I create a sub context in order to log what happened ret_val = self.manage_infinite_recursion(context) sub_context.add_values(return_values=ret_val) @@ -156,7 +163,10 @@ class SheerkaEvaluateConcept(BaseService): desc = f"Evaluating {current_prop} (concept={current_concept})" context.log(desc, self.NAME) - with context.push(desc=desc, obj=current_concept) as sub_context: + with context.push(BuiltinConcepts.EVALUATING_CONCEPT, + current_prop, + desc=desc, + obj=current_concept) as sub_context: if force_evaluation: sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) diff --git a/src/core/sheerka/services/SheerkaExecute.py b/src/core/sheerka/services/SheerkaExecute.py index d56c5e3..e56fb2b 100644 --- a/src/core/sheerka/services/SheerkaExecute.py +++ b/src/core/sheerka/services/SheerkaExecute.py @@ -219,7 +219,9 @@ class SheerkaExecute(BaseService): # else "'" + BaseParser.get_text_from_tokens(to_parse) + "' as tokens" # context.log(f"Parsing {debug_text}") - with context.push(desc=f"Parsing using {parser.name}", + with context.push(BuiltinConcepts.PARSING, + {"parser": parser.name}, + desc=f"Parsing using {parser.name}", logger=parser.verbose_log) as sub_context: sub_context.add_inputs(to_parse=to_parse) res = parser.parse(sub_context, to_parse) @@ -277,7 +279,10 @@ class SheerkaExecute(BaseService): # process iteration = 0 while True: - with context.push(desc=f"iteration #{iteration}", iteration=iteration) as iteration_context: + with context.push(process_step, + {"iteration": iteration}, + desc=f"iteration #{iteration}", + iteration=iteration) as iteration_context: simple_digest = return_values[:] iteration_context.add_inputs(return_values=simple_digest) @@ -290,7 +295,10 @@ class SheerkaExecute(BaseService): evaluator = self.preprocess(context, evaluator.__class__()) # fresh copy sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})" - with iteration_context.push(desc=sub_context_desc, logger=evaluator.verbose_log) as sub_context: + with iteration_context.push(process_step, + {"iteration": iteration, "evaluator": evaluator.name}, + desc=sub_context_desc, + logger=evaluator.verbose_log) as sub_context: sub_context.add_inputs(return_values=original_items) # process evaluators that work on one simple return value at the time @@ -375,7 +383,9 @@ class SheerkaExecute(BaseService): for step in execution_steps: copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values] - with context.push(step=step, iteration=0, desc=f"{step=}") as sub_context: + with context.push(BuiltinConcepts.PROCESSING, + {"step": step}, + step=step, iteration=0, desc=f"{step=}") as sub_context: if step == BuiltinConcepts.PARSING: return_values = self.call_parsers(sub_context, return_values) diff --git a/src/core/sheerka/services/SheerkaSetsManager.py b/src/core/sheerka/services/SheerkaSetsManager.py index d92786d..165a3f4 100644 --- a/src/core/sheerka/services/SheerkaSetsManager.py +++ b/src/core/sheerka/services/SheerkaSetsManager.py @@ -235,7 +235,9 @@ for x in xx__concepts__xx: return [self.sheerka.get_by_id(element_id) for element_id in ids] result = [] - with context.push(desc=f"Evaluating concepts of a set") as sub_context: + with context.push(BuiltinConcepts.EVALUATE_CONCEPT, + {"ids": ids}, + desc=f"Evaluating concepts of a set") as sub_context: sub_context.add_inputs(ids=ids) sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) for element_id in ids: diff --git a/src/evaluators/AddConceptInSetEvaluator.py b/src/evaluators/AddConceptInSetEvaluator.py index 85a368e..008837a 100644 --- a/src/evaluators/AddConceptInSetEvaluator.py +++ b/src/evaluators/AddConceptInSetEvaluator.py @@ -35,7 +35,7 @@ class AddConceptInSetEvaluator(OneReturnValueEvaluator): True, sheerka.new(BuiltinConcepts.USER_INPUT, body=parser_input, user_name="N/A")) - with context.push(desc=f"Recognizing '{name_node}'") as sub_context: + with context.push(BuiltinConcepts.PROCESS_INPUT, name_node, desc=f"Recognizing '{name_node}'") as sub_context: r = sheerka.execute(sub_context, ret_val, ALL_STEPS) one_r = core.builtin_helpers.expect_one(context, r) sub_context.add_values(return_values=one_r) diff --git a/src/evaluators/ConceptEvaluator.py b/src/evaluators/ConceptEvaluator.py index 6c271b6..e25c34f 100644 --- a/src/evaluators/ConceptEvaluator.py +++ b/src/evaluators/ConceptEvaluator.py @@ -62,7 +62,8 @@ class ConceptEvaluator(OneReturnValueEvaluator): evaluated, parents=[return_value]) - if not self.return_body or ConceptParts.BODY not in evaluated.compiled: - return sheerka.ret(self.name, True, evaluated, parents=[return_value]) - else: + if self.return_body and ConceptParts.BODY in evaluated.compiled: return sheerka.ret(self.name, True, evaluated.body, parents=[return_value]) + else: + return sheerka.ret(self.name, True, evaluated, parents=[return_value]) + diff --git a/src/evaluators/EvalEvaluator.py b/src/evaluators/EvalEvaluator.py index c046b33..efee9c4 100644 --- a/src/evaluators/EvalEvaluator.py +++ b/src/evaluators/EvalEvaluator.py @@ -14,7 +14,9 @@ class EvalEvaluator(AllReturnValuesEvaluator): super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 80) def matches(self, context, return_values): - return context.in_context(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) + evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING) + is_root = len(evaluation_parents) <= 1 + return context.in_context(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) and is_root def eval(self, context, return_values): sheerka = context.sheerka diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index c37d386..ac13891 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -200,7 +200,11 @@ class PythonEvaluator(OneReturnValueEvaluator): context.log(f"Concept {name} is already evaluated.", self.name) else: context.log(f"Evaluating '{concept}'", self.name) - with context.push(self.name, desc=f"Evaluating '{concept}'", obj=concept) as sub_context: + with context.push(BuiltinConcepts.EVALUATE_CONCEPT, + concept, + who=self.name, + desc=f"Evaluating '{concept}'", + obj=concept) as sub_context: sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) evaluated = context.sheerka.evaluate_concept(sub_context, concept) sub_context.add_values(return_values=evaluated) diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index 3ee27f8..90e220a 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -798,7 +798,11 @@ class BaseNodeParser(BaseParser): from parsers.BnfParser import BnfParser regex_parser = BnfParser() desc = f"Resolving BNF {concept.metadata.definition}" - with context.push(parser_name, obj=concept, desc=desc) as sub_context: + with context.push(BuiltinConcepts.INIT_BNF, + concept, + who=parser_name, + obj=concept, + desc=desc) as sub_context: sub_context.add_inputs(parser_input=concept.metadata.definition) bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.metadata.definition) sub_context.add_values(return_values=bnf_parsing_ret_val) diff --git a/src/parsers/BnfNodeParser.py b/src/parsers/BnfNodeParser.py index 925ae8c..5fa9e54 100644 --- a/src/parsers/BnfNodeParser.py +++ b/src/parsers/BnfNodeParser.py @@ -845,7 +845,7 @@ class BnfNodeParser(BaseNodeParser): expression = concept.bnf desc = f"Resolving parsing expression {expression}" - with self.context.push(self.name, obj=concept, desc=desc) as sub_context: + with self.context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context: sub_context.add_inputs(expression=expression) resolved = self.resolve_parsing_expression(expression, already_seen or set()) sub_context.add_values(return_values=resolved) diff --git a/src/parsers/DefaultParser.py b/src/parsers/DefaultParser.py index 6093c48..5c4deff 100644 --- a/src/parsers/DefaultParser.py +++ b/src/parsers/DefaultParser.py @@ -357,7 +357,11 @@ class DefaultParser(BaseParser): regex_parser = BnfParser() desc = f"Resolving BNF {current_concept_def.definition}" - with self.context.push(self.name, obj=current_concept_def, desc=desc) as sub_context: + with self.context.push(BuiltinConcepts.INIT_BNF, + current_concept_def, + who=self.name, + obj=current_concept_def, + desc=desc) as sub_context: parsing_result = regex_parser.parse(sub_context, tokens) sub_context.add_values(return_values=parsing_result) @@ -402,7 +406,7 @@ class DefaultParser(BaseParser): continue # ask the other parsers if they recognize the tokens - with self.context.push(self.name, desc=f"Parsing {keyword}") as sub_context: + with self.context.push(BuiltinConcepts.PARSING, keyword, who=self.name, desc=f"Parsing {keyword}") as sub_context: parser_input = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens) to_parse = self.sheerka.ret( sub_context.who, diff --git a/src/parsers/PythonParser.py b/src/parsers/PythonParser.py index 1f652a5..1e0b2b6 100644 --- a/src/parsers/PythonParser.py +++ b/src/parsers/PythonParser.py @@ -133,7 +133,9 @@ class LexerNodeParserHelperForPython: source += node.source to_parse += node.source - with context.push(self, desc="Trying Python for '" + to_parse + "'") as sub_context: + with context.push(BuiltinConcepts.PARSE_CODE, + {"language": "Python", "source": to_parse}, + desc="Trying Python for '" + to_parse + "'") as sub_context: sub_context.add_inputs(to_parse=to_parse) python_parser = PythonParser() parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(to_parse) diff --git a/src/parsers/PythonWithConceptsParser.py b/src/parsers/PythonWithConceptsParser.py index 92dc4f9..b8ca142 100644 --- a/src/parsers/PythonWithConceptsParser.py +++ b/src/parsers/PythonWithConceptsParser.py @@ -77,7 +77,9 @@ class PythonWithConceptsParser(BaseParser): source += node.source to_parse += node.source - with context.push(self, "Trying Python for '" + to_parse + "'") as sub_context: + with context.push(BuiltinConcepts.PARSE_CODE, + {"language": "Python", "source": to_parse}, + "Trying Python for '" + to_parse + "'") as sub_context: parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(to_parse) python_parser = PythonParser() result = python_parser.parse(sub_context, parser_input) diff --git a/src/sheerkapickle/sheerka_handlers.py b/src/sheerkapickle/sheerka_handlers.py index a3f41c6..c02a022 100644 --- a/src/sheerkapickle/sheerka_handlers.py +++ b/src/sheerkapickle/sheerka_handlers.py @@ -145,7 +145,7 @@ class ExecutionContextHandler(BaseHandler): pickler = self.context for prop in CONTEXT_PROPERTIES_TO_SERIALIZE: - if prop == "who": + if prop in ("who", "action", "action_context"): value = str(getattr(obj, prop)) else: value = getattr(obj, prop) @@ -156,7 +156,7 @@ class ExecutionContextHandler(BaseHandler): return data def new(self, data): - return ExecutionContext(data["who"], None, None) + return ExecutionContext(data["who"], None, None, BuiltinConcepts.NOP, None) def restore(self, data, instance): pickler = self.context diff --git a/tests/BaseTest.py b/tests/BaseTest.py index 647654a..36b27b6 100644 --- a/tests/BaseTest.py +++ b/tests/BaseTest.py @@ -13,7 +13,7 @@ class BaseTest: pass def get_context(self, sheerka=None, eval_body=False, eval_where=False): - context = ExecutionContext("test", Event(), sheerka or self.get_sheerka()) + context = ExecutionContext("test", Event(), sheerka or self.get_sheerka(), BuiltinConcepts.NOP, None) if eval_body: context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) if eval_where: diff --git a/tests/core/test_ExecutionContext.py b/tests/core/test_ExecutionContext.py index 6b69353..a70bd06 100644 --- a/tests/core/test_ExecutionContext.py +++ b/tests/core/test_ExecutionContext.py @@ -7,11 +7,11 @@ from sdp.sheerkaDataProvider import Event def test_id_is_incremented_by_event_digest(): - a = ExecutionContext("foo", Event("event_1"), None) - b = ExecutionContext("foo", Event("event_1"), None) - c = ExecutionContext("foo", Event("event_2"), None) - d = b.push() - e = c.push() + a = ExecutionContext("foo", Event("event_1"), None, BuiltinConcepts.NOP, None) + b = ExecutionContext("foo", Event("event_1"), None, BuiltinConcepts.NOP, None) + c = ExecutionContext("foo", Event("event_2"), None, BuiltinConcepts.NOP, None) + d = b.push(BuiltinConcepts.NOP, None) + e = c.push(BuiltinConcepts.NOP, None) assert a.id == 0 assert b.id == 1 @@ -21,13 +21,14 @@ def test_id_is_incremented_by_event_digest(): def test_i_can_use_with_statement(): - with ExecutionContext("who_", Event("event"), "fake_sheerka") as e: + with ExecutionContext("who_", Event("event"), "fake_sheerka", BuiltinConcepts.NOP, None) as e: pass assert e.elapsed > 0 def test_i_can_push(): - a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", "some description", + a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None, + desc="some description", obj=Concept("foo"), step=BuiltinConcepts.EVALUATION, iteration=15, @@ -37,13 +38,15 @@ def test_i_can_push(): a.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) a.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - b = a.push() + b = a.push(BuiltinConcepts.EVALUATION, "sub action context", desc="sub description") assert b._parent == a assert b.who == a.who assert b.event == a.event assert b.sheerka == a.sheerka - assert b.desc is None + assert b.action == BuiltinConcepts.EVALUATION + assert b.action_context == "sub action context" + assert b.desc == "sub description" assert b.obj == a.obj assert b.step == a.step assert b.iteration == a.iteration @@ -56,10 +59,10 @@ def test_i_can_push(): def test_children_i_created_when_i_push(): - e = ExecutionContext("who_", Event("event"), "fake_sheerka") - e.push("a", desc="I do something") - e.push("b", desc="oups! I did a again") - e.push("c", desc="I do something else") + e = ExecutionContext("who_", Event("event"), "fake_sheerka", BuiltinConcepts.NOP, None) + e.push(BuiltinConcepts.NOP, None, who="a", desc="I do something") + e.push(BuiltinConcepts.NOP, None, who="b", desc="oups! I did a again") + e.push(BuiltinConcepts.NOP, None, who="c", desc="I do something else") assert len(e.children) == 3 assert e.children[0].who, e.children[0].who == ("a", "I do something") @@ -68,8 +71,8 @@ def test_children_i_created_when_i_push(): def test_i_can_add_variable_when_i_push(): - e = ExecutionContext("who_", Event("event"), "fake_sheerka") - sub_e = e.push("a", my_new_var="new var value") + e = ExecutionContext("who_", Event("event"), "fake_sheerka", BuiltinConcepts.NOP, None) + sub_e = e.push(BuiltinConcepts.NOP, None, who="a", my_new_var="new var value") assert sub_e.my_new_var == "new var value" with pytest.raises(AttributeError): @@ -77,11 +80,11 @@ def test_i_can_add_variable_when_i_push(): def test_local_hints_are_local_and_global_hints_are_global(): - a = ExecutionContext("foo", Event("event_1"), "fake_sheerka") + a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None) a.local_hints.add("local hint 1") a.global_hints.add("global hint 1") - b = a.push() + b = a.push(BuiltinConcepts.NOP, None) b.local_hints.add("local hint 2") b.global_hints.add("global hint 2") @@ -93,9 +96,9 @@ def test_local_hints_are_local_and_global_hints_are_global(): def test_global_hits_are_global_even_when_empty(): - a = ExecutionContext("foo", Event("event_1"), "fake_sheerka") + a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.NOP, None) - b = a.push() + b = a.push(BuiltinConcepts.NOP, None) b.global_hints.add("global hint 2") assert a.global_hints == {"global hint 2"} diff --git a/tests/core/test_SheerkaFilter.py b/tests/core/test_SheerkaFilter.py index 4130b8d..7e4b6dd 100644 --- a/tests/core/test_SheerkaFilter.py +++ b/tests/core/test_SheerkaFilter.py @@ -68,7 +68,7 @@ class TestSheerkaFilter(TestUsingMemoryBasedSheerka): def test_i_can_pipe_explanation_concept(self): sheerka, context = self.init_concepts() - execution_contexts = [context.push(desc=f"desc_{i}") for i in range(4)] + execution_contexts = [context.push(BuiltinConcepts.NOP, None, desc=f"desc_{i}") for i in range(4)] explanation_node = sheerka.new(BuiltinConcepts.EXPLANATION, body=execution_contexts) @Pipe diff --git a/tests/core/test_sheerka_printer.py b/tests/core/test_sheerka_printer.py index 85770c5..1900b0a 100644 --- a/tests/core/test_sheerka_printer.py +++ b/tests/core/test_sheerka_printer.py @@ -219,7 +219,10 @@ class TestSheerkaPrinter(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() context = self.get_context(sheerka) - execution_context = context.push("test_sheerka_printer", "Testing Execution Context Printing") + execution_context = context.push(BuiltinConcepts.NOP, + None, + who="test_sheerka_printer", + desc="Testing Execution Context Printing") ret_val = sheerka.ret("test_sheerka_printer", True, sheerka.new(BuiltinConcepts.SUCCESS)) execution_context.add_values(return_value=ret_val) ec = execution_context.as_bag() diff --git a/tests/evaluators/test_EvalEvaluator.py b/tests/evaluators/test_EvalEvaluator.py index 363bff6..75cd9a0 100644 --- a/tests/evaluators/test_EvalEvaluator.py +++ b/tests/evaluators/test_EvalEvaluator.py @@ -1,9 +1,9 @@ import pytest - from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts from core.concept import Concept from core.sheerka.services.SheerkaSetsManager import SheerkaSetsManager from evaluators.EvalEvaluator import EvalEvaluator + from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -22,8 +22,8 @@ class TestEvalEvaluator(TestUsingMemoryBasedSheerka): return_values = [ ReturnValueConcept("some_name", True, "not to eval"), - ReturnValueConcept("some_name", True, Concept(name="not to eval")), - ReturnValueConcept("some_name", False, Concept(name="1", body="'not to eval'")), + ReturnValueConcept("some_name", True, Concept(name="not to eval")), # no body + ReturnValueConcept("some_name", False, Concept(name="1", body="'not to eval'")), # status is false to_eval1, to_eval2, ] @@ -52,6 +52,25 @@ class TestEvalEvaluator(TestUsingMemoryBasedSheerka): context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) assert EvalEvaluator().matches(context, [return_value]) + def test_i_can_match_depending_on_builtin_concept_processing(self): + context = self.get_context() + context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) + return_values = [ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init())] + evaluator = EvalEvaluator() + + # i match when no BuiltinConcepts.PROCESSING is found + assert evaluator.matches(context, return_values) + + # i match when one BuiltinConcepts.PROCESSING is found + root_processing = context.push(BuiltinConcepts.PROCESSING, {"step": BuiltinConcepts.EVALUATION}). \ + push(BuiltinConcepts.NOP, None) + assert evaluator.matches(root_processing, return_values) + + # otherwise, i cannot match + sub_root = root_processing.push(BuiltinConcepts.PROCESSING, {"step": BuiltinConcepts.EVALUATION}). \ + push(BuiltinConcepts.NOP, None) + assert not evaluator.matches(sub_root, return_values) + def test_concept_eval_requested_is_reduced_when_nothing_to_reduce(self): context = self.get_context() diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index eab0f56..fece58a 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -1,7 +1,8 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV +from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, CB, CC, CV from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator +from evaluators.PythonEvaluator import PythonEvalError from parsers.BaseNodeParser import SyaAssociativity from parsers.BnfNodeParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression @@ -916,10 +917,34 @@ as: assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2} + # it now also works using the concepts names expression = "eval one < two" res = sheerka.evaluate_user_input(expression) - assert not res[0].status + assert res[0].status + assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) + assert sheerka.get_concepts_weights("some_prop") == {'1001': 1, '1002': 2} + def test_i_can_detect_multiple_errors_when_evaluating_a_concept(self): + sheerka, context, foo, plus_one = self.init_concepts( + Concept("foo", body="'string'"), + Concept("a plus one", body="a + 1").def_var("a") + ) + + res = sheerka.evaluate_user_input("eval foo plus one") + + assert not res[0].status + assert context.sheerka.isinstance(res[0].body, BuiltinConcepts.CONCEPT_EVAL_ERROR) + assert context.sheerka.isinstance(res[0].body.body, BuiltinConcepts.TOO_MANY_ERRORS) + + error0 = res[0].body.body.body[0] + assert isinstance(error0, PythonEvalError) + assert isinstance(error0.error, TypeError) + assert error0.error.args[0] == 'can only concatenate str (not "int") to str' + + error1 = res[0].body.body.body[1] + assert isinstance(error1, PythonEvalError) + assert isinstance(error1.error, TypeError) + assert error1.error.args[0] == "unsupported operand type(s) for +: 'Concept' and 'int'" class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): diff --git a/tests/sheerkapickle/test_sheerka_handlers.py b/tests/sheerkapickle/test_sheerka_handlers.py index 7f6871c..091ea79 100644 --- a/tests/sheerkapickle/test_sheerka_handlers.py +++ b/tests/sheerkapickle/test_sheerka_handlers.py @@ -201,7 +201,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): to_string = sheerkapickle.encode(sheerka, user_input) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == user_input - assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "11"], "user_name": "my_user_name", "text": "my_text"}' + assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "22"], "user_name": "my_user_name", "text": "my_text"}' def test_i_can_encode_decode_user_input_when_tokens(self): sheerka = self.get_sheerka() @@ -213,7 +213,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): to_string = sheerkapickle.encode(sheerka, user_input) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="my_user_name") - assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "11"], "user_name": "my_user_name", "text": "{text}"' + '}' + assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "22"], "user_name": "my_user_name", "text": "{text}"' + '}' def test_i_can_encode_decode_return_value(self): sheerka = self.get_sheerka() @@ -223,7 +223,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): to_string = sheerkapickle.encode(sheerka, ret_val) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ret_val - assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "16"], "who": "who", "status": true, "value": 10}' + assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "27"], "who": "who", "status": true, "value": 10}' def test_i_can_encode_decode_return_value_with_parent(self): sheerka = self.get_sheerka() @@ -236,7 +236,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ret_val assert decoded.parents == ret_val.parents - id_str = ', "concept/id": ["__RETURN_VALUE", "16"]' + id_str = ', "concept/id": ["__RETURN_VALUE", "27"]' parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]' assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}' @@ -272,13 +272,13 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): def test_i_can_encode_decode_execution_context(self): sheerka = self.get_sheerka() - context = ExecutionContext("who", Event("xxx"), sheerka, "my desc") + context = ExecutionContext("who", Event("xxx"), sheerka, BuiltinConcepts.NOP, None, "my desc") input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)] context.inputs = {"a": input_list, "b": set_full_serialization(Concept("foo"))} context.values = {"c": input_list, "d": set_full_serialization(Concept("bar"))} context.obj = set_full_serialization(Concept("baz")) - context.push("who3", "sub_child1") - context.push("who4", "sub_child2") + context.push(BuiltinConcepts.NOP, None, who="who3", desc="sub_child1") + context.push(BuiltinConcepts.NOP, None, who="who4", desc="sub_child2") to_string = sheerkapickle.encode(sheerka, context) decoded = sheerkapickle.decode(sheerka, to_string) @@ -288,7 +288,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka(skip_builtins_in_db=False) text = "def concept one as 1" - execution_context = ExecutionContext("s", Event(), sheerka, f"Evaluating '{text}'") + execution_context = ExecutionContext("s", Event(), sheerka, BuiltinConcepts.NOP, None, f"Evaluating '{text}'") user_input = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="n")) reduce_requested = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))