From 87cab44fb88a7f8dca241f91bb892a56f000929d Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Thu, 28 Oct 2021 14:04:41 +0200 Subject: [PATCH] Fixed #125: SheerkaErrorManager Fixed #135: Change services service priorities Fixed #136: ErrorManager: Implement recognize_error Fixed #137: BNFNodeParser : Error when parsing regex with sub parsers Fixed #138: get_last_errors(): real errors sources are lost Fixed #139: OneError return value removes the origin of the error Fixed #140: Concept variables are not correctly handled when parsing sub expression Fixed #143: Implement has_unknown_concepts() --- sheerka_backup/admin.sb | 8 +- sheerka_backup/python.sb | 3 +- src/cache/FastCache.py | 9 + src/core/builtin_concepts.py | 20 +- src/core/builtin_helpers.py | 4 +- src/core/global_symbols.py | 26 + src/core/sheerka/Sheerka.py | 24 +- .../sheerka/services/SheerkaConceptManager.py | 3 +- .../sheerka/services/SheerkaDebugManager.py | 12 +- .../sheerka/services/SheerkaErrorManager.py | 365 ++++++++++++-- .../services/SheerkaEvaluateConcept.py | 19 +- src/core/sheerka/services/SheerkaExecute.py | 44 +- .../sheerka/services/SheerkaHistoryManager.py | 4 +- src/core/sheerka/services/SheerkaMemory.py | 28 +- src/core/sheerka/services/SheerkaOut.py | 2 +- .../sheerka/services/SheerkaQueryManager.py | 16 +- .../sheerka/services/SheerkaResultManager.py | 20 +- .../sheerka/services/SheerkaRuleManager.py | 67 ++- src/core/utils.py | 21 - src/evaluators/OneErrorEvaluator.py | 2 +- src/parsers/BaseExpressionParser.py | 74 +-- src/parsers/BaseNodeParser.py | 4 + src/parsers/BnfNodeParser.py | 106 +++- src/parsers/ExpressionParser.py | 23 +- src/parsers/FunctionParserOld.py | 343 ------------- src/parsers/RelationalOperatorParser.py | 30 +- src/parsers/SyaNodeParser.py | 3 + src/sdp/sheerkaDataProvider.py | 2 +- src/sheerkapython/BaseExprTransform.py | 12 +- src/sheerkapython/ExprToConditions.py | 3 +- src/sheerkapython/ExprToPython.py | 18 +- src/sheerkapython/python_wrapper.py | 28 +- tests/BaseTest.py | 21 +- tests/TestUsingFileBasedSheerka.py | 19 +- tests/core/test_SheerkaErrorManager.py | 463 ++++++++++++++++++ tests/core/test_SheerkaHistoryManager.py | 10 +- tests/core/test_SheerkaMemory.py | 32 +- tests/core/test_SheerkaQueryManager.py | 23 + tests/core/test_SheerkaRuleManager.py | 20 +- ...test_SheerkaRuleManagerRulesCompilation.py | 4 +- tests/core/test_sheerka.py | 171 +------ tests/core/test_sheerka_ontology.py | 12 +- tests/evaluators/test_OneErrorEvaluator.py | 12 +- tests/evaluators/test_PythonEvaluator.py | 10 +- .../test_ValidateConceptEvaluator.py | 13 + tests/non_reg/test_sheerka_non_reg.py | 12 +- tests/non_reg/test_sheerka_non_reg2.py | 2 +- tests/non_reg/test_sheerka_non_reg_out.py | 39 ++ tests/parsers/parsers_utils.py | 95 ---- tests/parsers/test_BnfNodeParser.py | 43 +- tests/parsers/test_DefConceptParser.py | 4 +- tests/parsers/test_FunctionParserOld.py | 249 ---------- tests/parsers/test_SyaNodeParser.py | 30 +- tests/parsers/test_parsers_utils.py | 31 +- .../test_ExprToConditionsVisitor.py | 17 +- tests/sheerkapython/test_PythonExprVisitor.py | 2 +- 56 files changed, 1391 insertions(+), 1286 deletions(-) delete mode 100644 src/parsers/FunctionParserOld.py create mode 100644 tests/core/test_SheerkaErrorManager.py delete mode 100644 tests/parsers/test_FunctionParserOld.py diff --git a/sheerka_backup/admin.sb b/sheerka_backup/admin.sb index 60ae33a..b11db4e 100644 --- a/sheerka_backup/admin.sb +++ b/sheerka_backup/admin.sb @@ -2,7 +2,11 @@ push_ontology("admin") def concept explain as get_results(id=0, depth=2) auto_eval True def concept explain last as get_last_results(id=0, depth=2) auto_eval True +def concept explain last results as get_last_results(id=0, depth=2) auto_eval True def concept explain x as get_results(id=x, depth=3) auto_eval True +def concept explain last errors as get_last_errors(level=0) auto_eval True +def concept explain errors as get_errors(level=0) auto_eval True +def concept explain error x as get_errors(id=x, depth=99) where isinstance(x, int) auto_eval True def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b, 'Sya') auto_eval True @@ -16,13 +20,15 @@ def concept debug off as set_debug(False) auto_eval True def concept activate debug on x as set_debug_var(x) auto_eval True def concept activate debug on x id=y as set_debug_var(x, y) auto_eval True def concept debug x as set_debug_var(x) auto_eval True +def concept activate concept debug on x as set_debug_concept(x) auto_eval True def concept debug var x as set_debug_var(x) auto_eval True def concept debug variable x as set_debug_var(variable=x) auto_eval True def concept debug method x as set_debug_var(method=x) auto_eval True def concept debug service x as set_debug_var(service=x) auto_eval True -def concept deactivate debug on x as debug_var(x, enabled=False) where x auto_eval True +def concept deactivate debug on x as set_debug_var(x, enabled=False) where x auto_eval True +def concept deactivate concept debug on x as set_debug_concept(x, enabled=False) auto_eval True def concept activate return values processing as set_var("sheerka.enable_process_return_values", True) auto_eval True def concept deactivate return values processing as set_var("sheerka.enable_process_return_values", False) auto_eval True diff --git a/sheerka_backup/python.sb b/sheerka_backup/python.sb index 16bb077..4683098 100644 --- a/sheerka_backup/python.sb +++ b/sheerka_backup/python.sb @@ -1,6 +1,7 @@ push_ontology("english") def concept x is a string pre is_question() as isinstance(x, str) -def concept x is a integer pre is_question() as isinstance(x, int) +def concept x is an integer pre is_question() as isinstance(x, int) +def concept x is an int pre is_question() as isinstance(x, int) def concept x starts with y pre is_question() where x is a string as x.startswith(y) def concept sha256 from bnf r'[a-f0-9]{64}' def concept sha512 from bnf r'[a-f0-9]{128}' diff --git a/src/cache/FastCache.py b/src/cache/FastCache.py index 0d853af..602193f 100644 --- a/src/cache/FastCache.py +++ b/src/cache/FastCache.py @@ -16,6 +16,15 @@ class FastCache: def __contains__(self, item): return self.has(item) + def __iter__(self): + yield from self.cache + + def __next__(self): + return next(iter(self.cache)) + + def __len__(self): + return len(self.cache) + def put(self, key, value): if len(self.cache) == self.max_size: del self.cache[self.lru.pop(0)] diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 352517c..7da12e8 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -165,6 +165,9 @@ class ParserResultConcept(Concept): from parsers.BaseParser import BaseParser return parser.name if isinstance(parser, BaseParser) else str(parser) + def get_error(self): + return None + class RuleEvaluationResultConcept(Concept): """ @@ -273,6 +276,9 @@ class FilteredConcept(Concept): self.set_value("reason", reason) self._hints.is_evaluated = True + def get_error(self): + return self.get_value('reason') + class ConceptAlreadyInSet(Concept, ErrorObj): ALL_ATTRIBUTES = ["concept", "concept_set"] @@ -312,9 +318,9 @@ class PropertyAlreadyDefined(Concept, ErrorObj): class ConditionFailed(Concept, ErrorObj): - ALL_ATTRIBUTES = ["condition", "concept", "prop", "reason"] + ALL_ATTRIBUTES = ["condition", "concept", "prop", "reason", "args"] - def __init__(self, condition=None, concept=None, prop=None, reason=None): + def __init__(self, condition=None, concept=None, prop=None, reason=None, args=None): Concept.__init__(self, BuiltinConcepts.CONDITION_FAILED, True, @@ -325,10 +331,15 @@ class ConditionFailed(Concept, ErrorObj): self.set_value("concept", concept) self.set_value("prop", prop) self.set_value("reason", reason) + self.set_value("args", args) self._hints.is_evaluated = True def __repr__(self): - return f"ConditionFailed(condition='{self.body}', concept='{self.concept}', prop='{self.prop}')" + text = f"ConditionFailed(condition='{self.body}', concept='{self.concept}', prop='{self.prop}'" + if self.args: + for k, v in self.args.items(): + text += f", {k}={v}" + return text + ")" class NotForMeConcept(Concept): # Not considered as an error ? @@ -348,6 +359,9 @@ class NotForMeConcept(Concept): # Not considered as an error ? def __repr__(self): return f"NotForMeConcept(source={self.body}, reason={self.get_value('reason')})" + def get_error(self): + return self.get_value('reason') + class ExplanationConcept(Concept): ALL_ATTRIBUTES = ["digest", "command", "title", "instructions", "execution_result"] diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index be38d1f..c8cacd3 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -930,11 +930,11 @@ def get_inner_body(context, concept): return concept.body -def get_possible_variables_from_concept(context, concept): +def get_possible_parameters_from_concept(context, concept): """ Given concept definition, gives the variables of the concept that can be considered as a parameter in another function - >>> gpvfc = get_possible_variables_from_concept + >>> gpvfc = get_possible_parameters_from_concept >>> assert gpvfc(Concept("a plus b").def_var("a").def_var("b")) == {"a", "b"} >>> assert gpvfc(Concept("twenties", definition="twenty (one|two)=n").def_var("n")) == set() :param context: diff --git a/src/core/global_symbols.py b/src/core/global_symbols.py index 2c75a08..add7349 100644 --- a/src/core/global_symbols.py +++ b/src/core/global_symbols.py @@ -70,6 +70,12 @@ class ErrorObj: pass +class SheerkaException(Exception, ErrorObj): + def __init__(self, value): + super().__init__(value) + self.value = value + + CURRENT_OBJ = "__obj" @@ -100,3 +106,23 @@ INIT_AST_QUESTION_PARSERS = ["Expression"] DEFAULT_EVALUATORS = ["Python", "Concept", "LexerNode", "Expression", "ValidateConcept", "ResolveAmbiguity"] OBJECTS_COUNTER_KEY = "__#OBJECTS_COUNTER_KEY#__" + + +class ErrorItem: + """" + Description of an error + """ + + def __init__(self, identifier: int, level: int, root, error: object, children=None): + self.id = identifier + self.level = level + self.root = root + self.error = error + self.children = children + + def __repr__(self): + return f"ErrorItem(id={self.id}, level={self.level}, error={self.error}, source={self.source})" + + @property + def source(self): + return self.root.who if hasattr(self.root, "who") else str(self.root) diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index f8b7306..e02ef6e 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -102,7 +102,8 @@ class Sheerka(Concept): self.services = {} # sheerka plugins - self.builtin_cache = {} # cache for builtin concepts + self.builtin_cache = {} # cache for builtin concepts + self.builtin_cache_by_class_name = {} # self.parsers = {} # cache for builtin parsers self.evaluators = [] # cache for builtin evaluators @@ -203,7 +204,7 @@ class Sheerka(Concept): self.bind_service_method(self.name, self.objvalue, False) self.om = SheerkaOntologyManager(self, root_folder, self.cache_only) - self.builtin_cache = self.get_builtins_classes_as_dict() + self.builtin_cache, self.builtin_cache_by_class_name = self.get_builtins_classes_as_dict() self.initialize_caching() self.get_builtin_parsers() @@ -849,7 +850,7 @@ class Sheerka(Concept): if isinstance(b, tuple): for item in b: item_key = item.key if isinstance(item, Concept) else str(item) - if a.key == item_key.key: + if a.key == item_key: return True return False @@ -903,12 +904,14 @@ class Sheerka(Concept): @staticmethod def get_builtins_classes_as_dict(): - res = {} + by_concept_name = {} + by_class_name = {} for c in core.utils.get_classes("core.builtin_concepts"): if issubclass(c, Concept) and c != Concept: - res[c()._metadata.key] = c - - return res + concept = c() + by_concept_name[concept._metadata.key] = c + by_class_name[c.__name__] = concept._metadata.name + return by_concept_name, by_class_name @staticmethod def init_logging(debug, loggers): @@ -993,13 +996,6 @@ class Sheerka(Concept): return concept - @staticmethod - def deepdiff(a, b): - from deepdiff import DeepDiff - ddiff = DeepDiff(a, b, ignore_order=True) - print(ddiff) - return ddiff - def to_profile(): sheerka = Sheerka() diff --git a/src/core/sheerka/services/SheerkaConceptManager.py b/src/core/sheerka/services/SheerkaConceptManager.py index 7171bb0..fc35fd3 100644 --- a/src/core/sheerka/services/SheerkaConceptManager.py +++ b/src/core/sheerka/services/SheerkaConceptManager.py @@ -1184,14 +1184,13 @@ class SheerkaConceptManager(BaseService): """ Go thru all the declared regular expressions and try to see if there is a match :param expr: - :param pos: + :param pos: starting pos (try to match at this starting position) :return: """ result = [] for compiled_regex, concept_ids in self.compiled_concepts_by_regex: if compiled_regex.match(expr, pos): result.extend([self.sheerka.get_by_id(concept_id) for concept_id in concept_ids]) - return result def get_concepts_bnf_definitions(self): diff --git a/src/core/sheerka/services/SheerkaDebugManager.py b/src/core/sheerka/services/SheerkaDebugManager.py index 1ef5769..34e3998 100644 --- a/src/core/sheerka/services/SheerkaDebugManager.py +++ b/src/core/sheerka/services/SheerkaDebugManager.py @@ -417,8 +417,16 @@ class ListDebugLogger(BaseDebugLogger): if not self.should_i_log_concept(concept.id): return - concept_id = f"{concept.id}{text}" if text else f"{concept.id}" - self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_CONCEPT, concept_id, False, kwargs)) + raw = kwargs.pop('raw', None) + str_vars = raw if raw else pp.pformat(kwargs) if kwargs else "" + text = " - " + text if text is not None else "" + colon = ": " if str_vars else "" + str_text = f"..concept#{concept.id}{text}{colon}" + + self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_CONCEPT, + str_text + str_vars, + False, + kwargs)) class TeeDebugLogger(BaseDebugLogger): diff --git a/src/core/sheerka/services/SheerkaErrorManager.py b/src/core/sheerka/services/SheerkaErrorManager.py index 33005cd..0db75ae 100644 --- a/src/core/sheerka/services/SheerkaErrorManager.py +++ b/src/core/sheerka/services/SheerkaErrorManager.py @@ -1,78 +1,121 @@ import core.builtin_helpers import core.utils +from cache.FastCache import FastCache from core.builtin_concepts_ids import BuiltinConcepts, BuiltinErrors from core.concept import Concept -from core.global_symbols import ErrorObj, NotInit +from core.global_symbols import EVENT_USER_INPUT_EVALUATED, ErrorItem, ErrorObj, NotInit, SheerkaException from core.sheerka.services.sheerka_service import BaseService +from core.tokenizer import TokenKind, Tokenizer +MAX_EXECUTION_HISTORY = 100 +CURRENT_ERRORS_EVENT_ID = "current_errors_event_id" +CURRENT_ERRORS_EVENT_MESSAGE = "current_errors_event_message" +LAST_DETECTED_ERRORS_EVENT_ID = "last_detected_errors_event_id" +LAST_DETECTED_ERRORS_EVENT_MESSAGE = "last_detected_errors_event_message" -class ErrorItem: - def __init__(self, level: int, parent, error: object): - self.level = level - self.parent = parent - self.error = error +LAST_UNKNOWN_CONCEPTS = "__last_unknown_concepts" class SheerkaErrorManager(BaseService): NAME = "Error" def __init__(self, sheerka): - super().__init__(sheerka, order=1) + super().__init__(sheerka, order=6) + self.last_detected_errors_event_id = None # updated every time an user input fails + self.last_detected_errors_event_message = None # updated every time an user input fails + + self.current_errors_event_id = None # updated only when get_last_errors() is called + self.current_errors_event_message = None # updated only when get_last_errors() is called + + self.last_return_values = None + + self.state_vars = [CURRENT_ERRORS_EVENT_ID, + CURRENT_ERRORS_EVENT_MESSAGE, + LAST_DETECTED_ERRORS_EVENT_ID, + LAST_DETECTED_ERRORS_EVENT_MESSAGE] + + self.executions_contexts_cache = FastCache(MAX_EXECUTION_HISTORY) + self.get_unknown_names_chain = [self._get_unknown_from_unknown_concept, + self._get_unknown_from_name_error, + self._get_unknown_from_undefined_symbol] def initialize(self): - self.sheerka.bind_service_method(self.NAME, self.get_errors, False) + self.sheerka.bind_service_method(self.NAME, self.get_obj_errors, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.has_error, False) self.sheerka.bind_service_method(self.NAME, self.get_error_cause, False) + self.sheerka.bind_service_method(self.NAME, self.get_errors, False) + self.sheerka.bind_service_method(self.NAME, self.get_last_errors, False) + self.sheerka.bind_service_method(self.NAME, self.recognize_error, False) + self.sheerka.bind_service_method(self.NAME, self.has_unknown_concepts, False) - def get_errors(self, context, obj, **kwargs): + self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.on_user_input_evaluated) + + def initialize_deferred(self, context, is_first_time): + self.restore_values(*self.state_vars) + + def reset_state(self): + self.last_detected_errors_event_id = None + self.last_detected_errors_event_message = None + self.current_errors_event_id = None + self.current_errors_event_message = None + self.last_return_values = None + + def is_error(self, obj): + if isinstance(obj, (ErrorObj, Exception)): + return True + + if isinstance(obj, Concept) and obj.get_metadata().is_builtin and obj.key in BuiltinErrors: + return True + + if self.sheerka.isinstance(obj, BuiltinConcepts.NOT_FOR_ME): + return True # In this situation, we want NOT_FOR_ME to appear in the list of errors + + return False + + def get_obj_errors(self, context, obj, predicate=None, **kwargs): """ - Browse obj, looking for error + Quickly retrieve all errors from obj (which can be a list) + All errors are flatten. No hierarchy :param context: :param obj: + :param predicate: if defined, filtering function :param kwargs: if defined, specialize the error (example __type="PythonEvalError") :return: """ - def is_error(_obj): - if isinstance(_obj, (ErrorObj, Exception)): - return True - - if isinstance(_obj, Concept) and _obj.get_metadata().is_builtin and _obj.key in BuiltinErrors: - return True - - return False - - def inner_get_errors(_obj, level, parent): + def inner_get_errors(_obj): if self.sheerka.isinstance(_obj, BuiltinConcepts.RETURN_VALUE) and _obj.status: return [] if isinstance(_obj, (list, set, tuple)): - return core.utils.flatten([inner_get_errors(o, level, parent) for o in _obj]) + return core.utils.flatten([inner_get_errors(o) for o in _obj]) + + if self.is_error(_obj): + if hasattr(_obj, "get_error"): + return [_obj] + inner_get_errors(_obj.get_error()) + + elif isinstance(_obj, Concept) and _obj.body not in (NotInit, None): + return [_obj] + inner_get_errors(_obj.body) - if is_error(_obj): - #error_item = ErrorItem(level, parent, _obj) - error_item = _obj - if isinstance(_obj, Concept) and _obj.body not in (NotInit, None): - return [error_item] + inner_get_errors(_obj.body, level + 1, error_item) - if isinstance(_obj, ErrorObj) and hasattr(_obj, "get_error"): - return [error_item] + inner_get_errors(_obj.get_error(), level + 1, error_item) else: - return [error_item] + return [_obj] if self.sheerka.isinstance(_obj, BuiltinConcepts.FILTERED): - return inner_get_errors(_obj.reason, level, parent) + return inner_get_errors(_obj.reason) if isinstance(_obj, Concept) and _obj.body != NotInit: - return inner_get_errors(_obj.body, level, parent) + return inner_get_errors(_obj.body) return [] - errors = inner_get_errors(obj, 0, None) - #return self.sheerka.filter_objects(context, errors, mapping=lambda o: o.error, **kwargs) - return self.sheerka.filter_objects(context, errors, **kwargs) + errors = inner_get_errors(obj) - def has_error(self, context, obj, **kwargs): - errors = self.get_errors(context, obj, **kwargs) + predicate = self.enhance_error_predicate(predicate) + + return self.sheerka.filter_objects(context, errors, predicate=predicate, **kwargs) + + def has_error(self, context, obj, predicate=None, **kwargs): + errors = self.get_obj_errors(context, obj, predicate, **kwargs) return len(errors) > 0 def get_error_cause(self, obj): @@ -87,3 +130,253 @@ class SheerkaErrorManager(BaseService): return res[0] else: return res + + def get_errors_items(self, returns_values): + """ + Returns a list or ErrorItems + They are numbered and have hierarchy + :param returns_values: + :return: + """ + identifier = 0 + + def inner_get_errors_items(obj, level, root): + nonlocal identifier + + if self.is_error(obj) or hasattr(obj, "get_error"): + + error_item = ErrorItem(identifier, level, root, obj) + identifier += 1 + + if hasattr(obj, "get_error"): + error_item.children = list(inner_get_errors_items(obj.get_error(), level + 1, root)) + + elif isinstance(obj, Concept) and obj.body not in (NotInit, None): + error_item.children = list(inner_get_errors_items(obj.body, level + 1, root)) + + else: + error_item.children = [] + + yield error_item + + elif isinstance(obj, Concept) and obj.body != NotInit: + if self.sheerka.isinstance(obj, BuiltinConcepts.RETURN_VALUE) and not obj.status: + root_to_use = obj + else: + root_to_use = root + yield from inner_get_errors_items(obj.body, level, root_to_use) + + elif hasattr(obj, "__iter__") and not isinstance(obj, str): + for item in obj: + yield from inner_get_errors_items(item, level, root) + + for ret_val in [r for r in returns_values if not r.status]: + yield from inner_get_errors_items(ret_val.body, 0, ret_val) + + def get_all_error_items(self, return_values): + def _as_list(error_items): + for error_item in error_items: + yield error_item + yield from _as_list(error_item.children) + + return _as_list(self.get_errors_items(return_values)) + + def get_return_values_in_error(self): + if self.current_errors_event_id is None: + error = self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"}) + raise SheerkaException(error) + + # search in history + if self.current_errors_event_id in self.executions_contexts_cache: + execution_result = self.executions_contexts_cache.get(self.current_errors_event_id) + else: + try: + execution_result = self.sheerka.om.current_sdp().load_result(self.current_errors_event_id) + self.executions_contexts_cache.put(self.current_errors_event_id, execution_result) + except FileNotFoundError as ex: + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": self.current_errors_event_id}) + + return [ret for ret in execution_result.values["return_values"] if not ret.status] + + def get_errors(self, context, predicate=None, **kwargs): + """ + Returns + :param context: + :param predicate: + :param kwargs: + :return: + """ + try: + in_error = self.get_return_values_in_error() + error_items = self.get_all_error_items(in_error) + depth = kwargs.pop("depth", None) + depth = kwargs.pop("recursion_depth", depth) + + filtered = self.filter_error_items(context, list(error_items), predicate, **kwargs) + + explanation = self.sheerka.new(BuiltinConcepts.EXPLANATION, + digest=self.current_errors_event_id, + command=self.current_errors_event_message, + body=filtered) + + # add format instructions if applicable + if depth is not None: + explanation.set_format_instr(recursion_depth=depth, recurse_on="children") + + return explanation + except SheerkaException as ex: + return ex.value + + def get_last_errors(self, context, predicate=None, **kwargs): + """ + + :param context: + :param predicate: + :param kwargs: + :return: + """ + if self.last_detected_errors_event_id is None: + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"}) + + self.current_errors_event_id = self.last_detected_errors_event_id + self.current_errors_event_message = self.last_detected_errors_event_message + self.store_values(context, CURRENT_ERRORS_EVENT_ID, CURRENT_ERRORS_EVENT_MESSAGE) + return self.get_errors(context, predicate, **kwargs) + + def recognize_error(self, context, predicate=None, **kwargs): + """ + Browse the last errors to see if the given error can be recognized + :param context: + :param predicate: + :param kwargs: + :return: + """ + if self.last_return_values is None: + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"}) + + error_items = self.get_all_error_items(self.last_return_values) + predicate = self.enhance_error_predicate(predicate) + filtered = self.filter_error_items(context, list(error_items), predicate, **kwargs) + + if len(filtered) == 0: + return False + + return True + + def has_unknown_concepts(self, context): + """ + Browse the last errors looking for unknown concepts + :param context: + :return: + """ + if self.last_return_values is None: + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"}) + + errors = self.get_obj_errors(context, self.last_return_values) + + # use all the known ways to detect unrecognized concepts + unknown_concepts = set() + for get_unknown_names in self.get_unknown_names_chain: + items = get_unknown_names(context, errors) + if items: + unknown_concepts.update(items) + + if len(unknown_concepts) > 0: + # use all the ways to reduce the list ? + errors_names = self._remove_recognized(unknown_concepts) + + if len(errors_names) == 1: + self.sheerka.set_in_memory(context, LAST_UNKNOWN_CONCEPTS, next(iter(errors_names))) + else: + self.sheerka.set_in_memory(context, LAST_UNKNOWN_CONCEPTS, errors_names) + return True + + def _get_unknown_from_unknown_concept(self, context, errors): + from_unknown = self.sheerka.filter_objects(context, errors, __type=BuiltinConcepts.UNKNOWN_CONCEPT) + return [e.body for e in from_unknown] if from_unknown else None + + def _get_unknown_from_name_error(self, context, errors): + from_name_error = self.sheerka.filter_objects(context, errors, __type="NameError") + if not from_name_error: + return None + + return [list(Tokenizer(e.args[0], yield_eof=False))[2].strip_quote for e in from_name_error] + + def _get_unknown_from_undefined_symbol(self, context, errors): + from_undefined_error = self.sheerka.filter_objects(context, errors, __type="UndefinedSymbolError") + return [e.symbol for e in from_undefined_error] if from_undefined_error else None + + def _remove_recognized(self, errors_names): + """ + parse error names a split it if some parts are known concept + ex 'dress' is a known concept + 'red dress' will not be recognized, but only 'red' should be in error + :param errors_names: + :return: + """ + # remove parts that are known + res = set() + unrecognized = [] + for error_name in [e.strip() for e in errors_names]: + unrecognized.clear() + + for token in Tokenizer(error_name, yield_eof=False): + if (token.type == TokenKind.WHITESPACE and + len(unrecognized) > 0 and + unrecognized[-1].type != TokenKind.WHITESPACE): + unrecognized.append(token) + continue + + elif self.sheerka.fast_resolve(token.value, return_new=False): + unrecognized_name = core.utils.get_text_from_tokens(unrecognized).strip() + if unrecognized_name: + res.add(unrecognized_name) + unrecognized.clear() + + else: + unrecognized.append(token) + + if unrecognized: + unrecognized_name = core.utils.get_text_from_tokens(unrecognized).strip() + if unrecognized_name: + res.add(unrecognized_name) + + return res + + def on_user_input_evaluated(self, execution_context): + """ + Track user input result in error + :param execution_context: + :return: + """ + self.last_return_values = [ret for ret in execution_context.values["return_values"] if not ret.status] + if self.last_return_values: + self.last_detected_errors_event_id = execution_context.event.get_digest() + self.last_detected_errors_event_message = execution_context.event.message + self.store_values(execution_context, LAST_DETECTED_ERRORS_EVENT_ID, LAST_DETECTED_ERRORS_EVENT_MESSAGE) + self.executions_contexts_cache.put(execution_context.event.get_digest(), execution_context) + + def filter_error_items(self, context, error_items, predicate=None, **kwargs): + error_items_args = {} + for key in ["id", "level", "source"]: + if key in kwargs: + error_items_args[key] = kwargs.pop(key) + + filtered = self.sheerka.filter_objects(context, list(error_items), **error_items_args) + return self.sheerka.filter_objects(context, list(filtered), lambda x: x.error, predicate, **kwargs) + + def enhance_error_predicate(self, predicate): + if isinstance(predicate, type): + if issubclass(predicate, Concept): + c = predicate() + predicate = f"get_type(self) == {core.utils.get_safe_str_value(c.name)}" + else: + predicate = f"get_type(self) == {core.utils.get_safe_str_value(predicate.__name__)}" + elif isinstance(predicate, str) and " " not in predicate.strip(): + if predicate in self.sheerka.builtin_cache_by_class_name: + concept_name = self.sheerka.builtin_cache_by_class_name[predicate] + predicate = f"get_type(self) == {core.utils.get_safe_str_value(concept_name)}" + else: + predicate = f"get_type(self) == {core.utils.get_safe_str_value(predicate)}" + + return predicate diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index 4d496e3..c62a955 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from core.builtin_concepts import BuiltinConcepts -from core.builtin_helpers import ensure_bnf, ensure_concept, expect_one, get_parsed_concept, is_only_successful, \ +from core.builtin_helpers import ensure_concept, expect_one, get_parsed_concept, is_only_successful, \ only_successful, variables_in_context from core.concept import AllConceptParts, Concept, ConceptParts, DoNotResolve, InfiniteRecursionResolved, \ concept_part_value @@ -15,6 +15,7 @@ from core.utils import unstr_concept from parsers.BaseExpressionParser import TrueifyVisitor from parsers.BaseNodeParser import ConceptNode from sheerkapython.ExprToConditions import ExprToConditionsVisitor +from sheerkapython.python_wrapper import get_possible_variables_from_concept CONCEPT_EVALUATION_STEPS = [ BuiltinConcepts.BEFORE_EVALUATION, @@ -174,7 +175,8 @@ class SheerkaEvaluateConcept(BaseService): parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(trueified_where) parsed = ExpressionParser(auto_compile=False).parse(context, parser_input) - expr_to_cond_python_visitor = ExprToConditionsVisitor(context) + possible_variables = get_possible_variables_from_concept(context, concept) + expr_to_cond_python_visitor = ExprToConditionsVisitor(context, known_variables=possible_variables) conditions = expr_to_cond_python_visitor.get_conditions(parsed.body.body) return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, conditions) except FailedToCompileError: @@ -392,7 +394,8 @@ class SheerkaEvaluateConcept(BaseService): body=where_clause_def.clause, concept=where_clause_def.concept, prop=where_clause_def.prop, - reason=reason) + reason=reason, + args=where_clause_def.concept.variables()) def manage_infinite_recursion(self, context): """ @@ -434,15 +437,7 @@ class SheerkaEvaluateConcept(BaseService): # def concept foo a as 'foo' a where a == 'baz' # In the second concept (foo) as is a still a concept, but also a variable in the where clause - ensure_bnf(context, concept) - if concept.get_bnf(): - from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor - visitor = BnfNodeConceptExpressionVisitor() - visitor.visit(concept.get_bnf()) - possible_variables = [c.name if isinstance(c, Concept) else c for c in visitor.references] - else: - possible_variables = None - + possible_variables = get_possible_variables_from_concept(context, concept) for part_key in AllConceptParts: if part_key in concept.get_compiled(): continue diff --git a/src/core/sheerka/services/SheerkaExecute.py b/src/core/sheerka/services/SheerkaExecute.py index ebb540a..be43c2d 100644 --- a/src/core/sheerka/services/SheerkaExecute.py +++ b/src/core/sheerka/services/SheerkaExecute.py @@ -44,11 +44,17 @@ class ParserInput: if len(tokens) == 0: self.tokens = [Token(TokenKind.EOF, "", 0, 1, 1)] elif (last_token := tokens[-1]).type != TokenKind.EOF: + if isinstance(last_token.value, str): + offset = len(last_token.value) + else: + # KSI 2021-10-23 + # Not really true. So it will be to really handled later + offset = 1 self.tokens = tokens + [Token(TokenKind.EOF, "", - last_token.index + 1, + last_token.index + offset, last_token.line, - last_token.column + 1)] + last_token.column + offset)] else: self.tokens = tokens @@ -259,7 +265,6 @@ class SheerkaExecute(BaseService): self.sheerka.bind_service_method(self.NAME, self.execute, True, visible=False) self.sheerka.bind_service_method(self.NAME, self.execute_rules, True, visible=False) self.sheerka.bind_service_method(self.NAME, self.parse_unrecognized, False, visible=False) - self.sheerka.bind_service_method(self.NAME, self.parse_function, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.parse_python, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.parse_expression, False, visible=False) self.sheerka.bind_service_method(self.NAME, self.clear_parser_cache, True) @@ -893,39 +898,6 @@ class SheerkaExecute(BaseService): sub_context.add_values(return_values=res) return res - def parse_function(self, context, source, tokens=None, start=0): - """ - Helper function that parses what is supposed to be a function - :param context: - :param source: - :param tokens: - :param start: start index for the source code node - :return: - """ - from parsers.BaseNodeParser import SourceCodeWithConceptNode - - sheerka = context.sheerka - from parsers.FunctionParserOld import FunctionParserOld - parser = FunctionParserOld() - desc = f"Parsing function '{source}'" - with context.push(BuiltinConcepts.PARSE_CODE, source, desc=desc) as sub_context: - sheerka_execution = sheerka.services[SheerkaExecute.NAME] - res = parser.parse(sub_context, sheerka_execution.get_parser_input(source, tokens)) - - if not isinstance(res, list): - res = [res] - - for r in [r for r in res if sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]: - r.body.body.start += start - r.body.body.end += start - - if isinstance(r.body.body, SourceCodeWithConceptNode): - for n in [r.body.body.first, r.body.body.last] + r.body.body.nodes: - n.start += start - n.end += start - - return res - def parse_python(self, context, source, desc=None): """ Helper function that parses what is known to be Python source code diff --git a/src/core/sheerka/services/SheerkaHistoryManager.py b/src/core/sheerka/services/SheerkaHistoryManager.py index 65bf34e..2bec06f 100644 --- a/src/core/sheerka/services/SheerkaHistoryManager.py +++ b/src/core/sheerka/services/SheerkaHistoryManager.py @@ -4,7 +4,7 @@ from core.global_symbols import NotInit from core.sheerka.services.sheerka_service import BaseService from sdp.sheerkaDataProvider import Event -hist = namedtuple("HistoryTest", "text status") # tests purposes only +hist = namedtuple("HistoryTest", "message status") # tests purposes only class History: @@ -37,7 +37,7 @@ class History: return True if isinstance(other, hist): - return self.event.message == other.text and self.status == other.status + return self.event.message == other.message and self.status == other.status if not isinstance(other, History): return False diff --git a/src/core/sheerka/services/SheerkaMemory.py b/src/core/sheerka/services/SheerkaMemory.py index edeaeb7..6e5aa09 100644 --- a/src/core/sheerka/services/SheerkaMemory.py +++ b/src/core/sheerka/services/SheerkaMemory.py @@ -44,13 +44,14 @@ class SheerkaMemory(BaseService): self.sheerka.bind_service_method(self.NAME, self.add_to_short_term_memory, True, visible=False) self.sheerka.bind_service_method(self.NAME, self.remove_context, True, as_name="clear_short_term_memory", visible=False) - self.sheerka.bind_service_method(self.NAME, self.add_to_memory, True) self.sheerka.bind_service_method(self.NAME, self.add_many_to_short_term_memory, True, visible=False) self.sheerka.bind_service_method(self.NAME, self.get_from_memory, False) self.sheerka.bind_service_method(self.NAME, self.get_last_from_memory, False) self.sheerka.bind_service_method(self.NAME, self.register_object, True, visible=False) self.sheerka.bind_service_method(self.NAME, self.unregister_object, True, visible=False) self.sheerka.bind_service_method(self.NAME, self.commit_registered_objects, True, visible=False) + self.sheerka.bind_service_method(self.NAME, self.add_to_memory, True) + self.sheerka.bind_service_method(self.NAME, self.set_in_memory, True) self.sheerka.bind_service_method(self.NAME, self.memory, False) self.sheerka.bind_service_method(self.NAME, self.mem, False) @@ -123,7 +124,8 @@ class SheerkaMemory(BaseService): def add_to_memory(self, context, key, concept): """ - Adds an object to memory + Adds an object to memory. + Create a list if the entry already exists :param context: :param key: :param concept: @@ -157,6 +159,28 @@ class SheerkaMemory(BaseService): context.event.date.timestamp(), concept)) + def set_in_memory(self, context, key, value): + """ + Add an element in memory + Replace the old one if the entry already exists + :param context: + :param key: + :param value: + :return: + """ + last = self.sheerka.om.get(SheerkaMemory.OBJECTS_ENTRY, key) + if last is NotFound: + self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), + context.event.date.timestamp(), + value)) + return + + # replace with the new one + self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last) + self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(), + context.event.date.timestamp(), + value)) + def get_from_memory(self, context, key): """" """ diff --git a/src/core/sheerka/services/SheerkaOut.py b/src/core/sheerka/services/SheerkaOut.py index 7c505db..bf2243d 100644 --- a/src/core/sheerka/services/SheerkaOut.py +++ b/src/core/sheerka/services/SheerkaOut.py @@ -33,7 +33,7 @@ class SheerkaOut(BaseService): if valid_rules: if len(valid_rules) > 1: # TODO manage when too many rules - print("TODO: TOO MANY RULES !!!!!") + print("TODO: TOO MANY RULES !!!!!", [r.id for r in valid_rules]) pass rule = valid_rules[0] diff --git a/src/core/sheerka/services/SheerkaQueryManager.py b/src/core/sheerka/services/SheerkaQueryManager.py index 62a0b04..2354139 100644 --- a/src/core/sheerka/services/SheerkaQueryManager.py +++ b/src/core/sheerka/services/SheerkaQueryManager.py @@ -20,7 +20,7 @@ class SheerkaQueryManager(BaseService): MAPPING_PREFIX = "__xxx__map__xx__" def __init__(self, sheerka): - super().__init__(sheerka, order=16) + super().__init__(sheerka, order=5) self.queries = FastCache() self.conditions = FastCache() self.lexer = Lexer() @@ -31,10 +31,9 @@ class SheerkaQueryManager(BaseService): self.sheerka.bind_service_method(self.NAME, self.select_objects, False) self.sheerka.bind_service_method(self.NAME, self.collect_attributes, False) - self.sheerka.bind_service_method(self.NAME, self.filter_objects, False) + self.sheerka.bind_service_method(self.NAME, self.where_on_objects, False, as_name="pipe_where") self.sheerka.bind_service_method(self.NAME, self.select_objects, False, as_name="pipe_select") self.sheerka.bind_service_method(self.NAME, self.collect_attributes, False, as_name="pipe_props") - self.sheerka.bind_service_method(self.NAME, self.where_on_objects, False, as_name="pipe_where") self.sheerka.register_debug_vars(SheerkaQueryManager.NAME, "filter_objects", "query") @@ -62,6 +61,15 @@ class SheerkaQueryManager(BaseService): local_namespace[current_variable_name] = v if k == "__type": + if isinstance(v, type): + if issubclass(v, Concept): + c = v() + local_namespace[current_variable_name] = c.name + else: + local_namespace[current_variable_name] = v.__name__ + elif v in self.sheerka.builtin_cache_by_class_name: + local_namespace[current_variable_name] = self.sheerka.builtin_cache_by_class_name[v] + conditions.append(f"get_type({self_ident}) == {current_variable_name}") elif k in ("__self", "_"): @@ -97,7 +105,7 @@ class SheerkaQueryManager(BaseService): :param mapping: mapping to execute on each object before applying the predicate (lambda obj:obj) :param predicate: :param kwargs: - :return: + :return: empty list when nothing is found """ debugger = context.get_debugger(SheerkaQueryManager.NAME, "filter_objects") diff --git a/src/core/sheerka/services/SheerkaResultManager.py b/src/core/sheerka/services/SheerkaResultManager.py index cdc269b..3a70b59 100644 --- a/src/core/sheerka/services/SheerkaResultManager.py +++ b/src/core/sheerka/services/SheerkaResultManager.py @@ -1,11 +1,11 @@ import ast import os -from cache.Cache import Cache +from cache.FastCache import FastCache from core.builtin_concepts import BuiltinConcepts -from core.global_symbols import EVENT_USER_INPUT_EVALUATED, EVENT_CONCEPT_CREATED, NotFound, EVENT_CONCEPT_MODIFIED, \ - EVENT_CONCEPT_DELETED, EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_CREATED, EVENT_RULE_DELETED, \ - EVENT_RULE_PRECEDENCE_MODIFIED, SHEERKA_BACKUP_FOLDER, SHEERKA_BACKUP_FILE +from core.global_symbols import EVENT_CONCEPT_CREATED, EVENT_CONCEPT_DELETED, EVENT_CONCEPT_MODIFIED, \ + EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_CREATED, EVENT_RULE_DELETED, EVENT_RULE_PRECEDENCE_MODIFIED, \ + EVENT_USER_INPUT_EVALUATED, NotFound, SHEERKA_BACKUP_FILE, SHEERKA_BACKUP_FOLDER from core.sheerka.services.sheerka_service import BaseService from core.utils import CONSOLE_COLORS_MAP as CCM from core.utils import as_bag @@ -22,7 +22,7 @@ class SheerkaResultManager(BaseService): def __init__(self, sheerka, page_size=30): super().__init__(sheerka) self.page_size = page_size - self.executions_contexts_cache = Cache(MAX_EXECUTION_HISTORY) + self.executions_contexts_cache = FastCache(MAX_EXECUTION_HISTORY) self.last_execution = None self.last_created_concept = None self.last_created_concept_id = None @@ -42,7 +42,7 @@ class SheerkaResultManager(BaseService): self.sheerka.bind_service_method(self.NAME, self.get_last_error, False, as_name="last_err") self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.user_input_evaluated) - self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.new_concept_created) + self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.on_new_concept_created) self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.on_global_state_is_modified) self.sheerka.subscribe(EVENT_CONCEPT_MODIFIED, self.on_global_state_is_modified) @@ -144,7 +144,7 @@ class SheerkaResultManager(BaseService): # add format instructions if applicable if (depth := kwargs.get("depth", None)) is not None or \ - (depth := kwargs.get("recursion_depth", None)) is not None: + (depth := kwargs.get("recursion_depth", None)) is not None: explanation.set_format_instr(recursion_depth=depth, recurse_on="_children") return explanation @@ -331,7 +331,7 @@ class SheerkaResultManager(BaseService): return self.last_errors - def new_concept_created(self, context, concept): + def on_new_concept_created(self, context, concept): """ Subscriber (callback) when a new concept is created :param context: @@ -380,8 +380,8 @@ class SheerkaResultManager(BaseService): def backup_command(self, execution_context): if (self.sheerka.during_restore or - not execution_context.is_state_modified() or - not self.sheerka.enable_commands_backup): + not execution_context.is_state_modified() or + not self.sheerka.enable_commands_backup): return folder = os.getenv(SHEERKA_BACKUP_FOLDER) diff --git a/src/core/sheerka/services/SheerkaRuleManager.py b/src/core/sheerka/services/SheerkaRuleManager.py index c96980e..c692d92 100644 --- a/src/core/sheerka/services/SheerkaRuleManager.py +++ b/src/core/sheerka/services/SheerkaRuleManager.py @@ -13,8 +13,8 @@ from core.rule import ACTION_TYPE_PRINT, Rule from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, UnknownVariableError from core.tokenizer import Token, TokenKind -from parsers.BaseExpressionParser import AndNode, ComparisonNode, ExpressionVisitor, \ - FunctionNodeOld, NameExprNode, NotNode, VariableNode +from parsers.BaseExpressionParser import AndNode, ComparisonNode, ExpressionVisitor, NameExprNode, \ + NotNode, VariableNode from parsers.BaseNodeParser import ConceptNode from parsers.FormatRuleActionParser import FormatRuleActionParser from parsers.LogicalOperatorParser import LogicalOperatorParser @@ -271,9 +271,12 @@ class SheerkaRuleManager(BaseService): """ Remove a rule """ + original_rule = rule rule = self.resolve_rule(context, rule) if rule is None: - return + sheerka = context.sheerka + err = sheerka.new(BuiltinConcepts.NOT_FOUND, body={"rule": original_rule}) + return sheerka.ret(self.NAME, False, err) # rule will be deleted. publish the event first, as the rule may not be available after self.sheerka.publish(context, EVENT_RULE_DELETED, rule) @@ -292,6 +295,26 @@ class SheerkaRuleManager(BaseService): def init_builtin_rules(self, context): # self.sheerka.init_log.debug("Initializing default rules") + rules_by_names = {} + + def is_less_than(r1, r2): + self.sheerka.set_is_less_than(context, + BuiltinConcepts.PRECEDENCE, + rules_by_names[r1], rules_by_names[r2], + RULE_COMPARISON_CONTEXT) + + def is_greater_than(r1, r2): + self.sheerka.set_is_greater_than(context, + BuiltinConcepts.PRECEDENCE, + rules_by_names[r1], rules_by_names[r2], + RULE_COMPARISON_CONTEXT) + + def is_greatest(r1): + self.sheerka.set_is_greatest(context, + BuiltinConcepts.PRECEDENCE, + rules_by_names[r1], + RULE_COMPARISON_CONTEXT) + rules = [ # index=[0] in code, id=1 Rule #2 in debug Rule("print", "Print return values", "__rets", "list(__rets)"), @@ -336,25 +359,24 @@ class SheerkaRuleManager(BaseService): "isinstance(__ret_container, BuiltinConcepts.MULTIPLE_SUCCESS)", "list(__ret_container.body)"), - # # index=[9] in code, id=10 in debug - # Rule("print", "Display simple result when only one success", - # "len(__rets)==1 and __rets[0].status == True", - # "{__rets[0].body}"), + # index=[9] in code, id=10 in debug + Rule("print", "Print ErrorItem", + "isinstance(__obj, ErrorItem)", + "[{id:3}] {__tab}green(source) red(error)"), ] for r in rules: self.create_new_rule(context, r) + rules_by_names[r.name] = r - self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[2], RULE_COMPARISON_CONTEXT) - self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[3], RULE_COMPARISON_CONTEXT) - self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[5], RULE_COMPARISON_CONTEXT) - self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[6], RULE_COMPARISON_CONTEXT) - self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[8], RULE_COMPARISON_CONTEXT) - self.sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, rules[7], rules[6], - RULE_COMPARISON_CONTEXT) - self.sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, rules[7], rules[5], - RULE_COMPARISON_CONTEXT) - self.sheerka.set_is_greatest(context, BuiltinConcepts.PRECEDENCE, rules[0], RULE_COMPARISON_CONTEXT) + is_less_than("Print ReturnValue", "Failed ReturnValue in red") + is_less_than("Print ReturnValue", "List explanations") + is_less_than("Print ReturnValue", "Display formatted list") + is_less_than("Print ReturnValue", "Display formatted dict") + is_less_than("Print ReturnValue", "Display multiple success") + is_greater_than("Display multiple outputs", "Display formatted dict") + is_greater_than("Display multiple outputs", "Display formatted list") + is_greatest("Print return values") def get_rule_by_id(self, rule_id): """ @@ -585,7 +607,7 @@ class GetConditionExprVisitor(ExpressionVisitor): if return_body: if not res.status: - python_eval_error = self.context.sheerka.get_errors(self.context, res, __type="PythonEvalError") + python_eval_error = self.context.sheerka.get_obj_errors(self.context, res, __type="PythonEvalError") if python_eval_error and isinstance(python_eval_error[0].error, NameError): raise UnknownVariableError(python_eval_error[0].source) raise FailedToCompileError(res.body) @@ -706,15 +728,6 @@ class ReteConditionExprVisitor(GetConditionExprVisitor): else: raise FailedToCompileError([expr_node]) - def visit_FunctionNodeOld(self, expr_node: FunctionNodeOld): - if expr_node.first.value == "recognize(": - if not isinstance(expr_node.parameters[0].value, VariableNode): - return FailedToCompileError([f"Cannot recognize '{expr_node.parameters[0].value}'"]) - - return self.recognize_concept(expr_node.parameters[0].value.unpack(), - expr_node.parameters[1].value, - {}) - def visit_NotNode(self, expr_node: NotNode): def get_sub_conditions(conditions_): diff --git a/src/core/utils.py b/src/core/utils.py index 0a76030..4b6ae91 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -876,27 +876,6 @@ def escape_str(x): return x -def new_array(size): - res = [] - for _ in range(size): - res.append(0) - - return res - - -class NextIdManager: - """ - solely return the next integer - """ - - def __init__(self): - self.id = -1 - - def get_next_id(self): - self.id += 1 - return self.id - - def compute_hash(obj): try: if isinstance(obj, (list, tuple)): diff --git a/src/evaluators/OneErrorEvaluator.py b/src/evaluators/OneErrorEvaluator.py index bf61d25..e8685f8 100644 --- a/src/evaluators/OneErrorEvaluator.py +++ b/src/evaluators/OneErrorEvaluator.py @@ -54,5 +54,5 @@ class OneErrorEvaluator(AllReturnValuesEvaluator): context.log(f"{self.return_value_in_error}", who=self) sheerka = context.sheerka - to_return = self.return_value_in_error.value if self.return_value_in_error else self.return_value_filtered.body + to_return = self.return_value_in_error if self.return_value_in_error else self.return_value_filtered return sheerka.ret(self.name, False, to_return, parents=self.eaten.copy()) diff --git a/src/parsers/BaseExpressionParser.py b/src/parsers/BaseExpressionParser.py index 0c1de45..1cbfd2d 100644 --- a/src/parsers/BaseExpressionParser.py +++ b/src/parsers/BaseExpressionParser.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from itertools import product -from typing import List, Union +from typing import List from core.builtin_concepts_ids import BuiltinConcepts from core.sheerka.services.SheerkaExecute import ParserInput @@ -572,72 +572,6 @@ class FunctionNode(ExprNode): self.parameters.clone()) -@dataclass() -class FunctionParameter: - """ - class the represent result of the parameter parsing - """ - value: NameExprNode # value parsed - separator: NameExprNode = None # holds the value and the position of the separator - - def add_sep(self, start, end, tokens): - self.separator = NameExprNode(start, end, tokens) - - def value_to_unrecognized(self): - return UnrecognizedTokensNode(self.value.start, self.value.end, self.value.tokens).fix_source() - - def separator_to_unrecognized(self): - if self.separator is None: - return None - return UnrecognizedTokensNode(self.separator.start, self.separator.end, self.separator.tokens).fix_source() - - def clone(self): - return FunctionParameter(self.value.clone(), self.separator.clone()) - - -class FunctionNodeOld(ExprNode): - - def __init__(self, start, end, tokens, - first: NameExprNode, last: NameExprNode, parameters: Union[None, List[FunctionParameter]]): - super().__init__(start, end, tokens) - self.first = first - self.last = last - self.parameters = parameters - - def __eq__(self, other): - if id(self) == id(other): - return True - - if not isinstance(other, FunctionNodeOld): - return False - - return (self.first == other.first and - self.last == other.last and - self.parameters == other.parameters) - - def __hash__(self): - return hash((self.first, self.last, self.parameters)) - - def __repr__(self): - return f"FunctionNodeOld(start={self.start}, end={self.end}, {self.first!r} {self.last} {self.parameters!r})" - - def __str__(self): - return f"{self.first} {self.parameters} {self.last}" - - def clone(self): - if self.parameters is not None: - parameters = [p.clone for p in self.parameters] - else: - parameters = None - - return FunctionNodeOld(self.start, - self.end, - self.tokens.copy(), - self.first.clone(), - self.last.clone(), - parameters) - - @dataclass() class Comprehension: target: ExprNode @@ -1013,12 +947,6 @@ class IsAQuestionVisitor(ExpressionVisitor): return True return None - def visit_FunctionNodeOld(self, expr_node: FunctionNodeOld): - if tokens_are_matching(expr_node.tokens, is_question_tokens) or \ - tokens_are_matching(expr_node.tokens, in_context_tokens): - return True - return None - def visit_FunctionNode(self, expr_node: FunctionNode): if tokens_are_matching(expr_node.tokens, is_question_tokens) or \ tokens_are_matching(expr_node.tokens, in_context_tokens): diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index 7ec6551..35937c5 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -1,8 +1,10 @@ from dataclasses import dataclass +from typing import Union import core.utils from cache.FastCache import FastCache from core import builtin_helpers +from core.concept import Concept from core.sheerka.ExecutionContext import ExecutionContext from core.tokenizer import Token, TokenKind from core.var_ref import VariableRef @@ -483,11 +485,13 @@ class VariableNode(LexerNode): @dataclass() class GrammarErrorNode(ParsingError): message: str + concept: Union[Concept, None] = None @dataclass() class NoMatchingTokenError(ParsingError): pos: int + concept: Union[Concept, None] = None class UnrecognizedTokensCache: diff --git a/src/parsers/BnfNodeParser.py b/src/parsers/BnfNodeParser.py index cd58c63..9ee0b61 100644 --- a/src/parsers/BnfNodeParser.py +++ b/src/parsers/BnfNodeParser.py @@ -10,7 +10,7 @@ import re from collections import defaultdict from dataclasses import dataclass, field from operator import attrgetter -from typing import List +from typing import List, Union import core.builtin_helpers import core.utils @@ -23,11 +23,21 @@ from core.tokenizer import Token, TokenKind, Tokenizer from core.utils import CONSOLE_COLORS_MAP as CCM from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, GrammarErrorNode, NoMatchingTokenError, RuleNode, \ SourceCodeNode, SourceCodeWithConceptNode, UnrecognizedTokensCache, UnrecognizedTokensNode +from parsers.BaseParser import ParsingError PARSERS = ["Sequence", "Sya", "Python"] VARIABLE_EXPR_PARSER = ["Sequence", "Sya", "Python", "Bnf"] +@dataclass +class UndefinedSymbolError(ParsingError): + """ + When parsing a concept, returns an error when a symbol is not recognized as a concept, a rule, or python code... + """ + symbol: str + concept: Union[Concept, None] = None + + @dataclass(eq=True) class RegExDef: to_match: str = None @@ -469,6 +479,9 @@ class VariableExpression(ParsingExpression): def __hash__(self): return hash(("VariableExpression", self.rule_name)) + def debug_prefix(self, parser_helper): + return super().debug_prefix("VariableExpression", parser_helper) + def init_parsing(self): """ Get the instance of the following VariableExpression if they exists, @@ -485,6 +498,9 @@ class VariableExpression(ParsingExpression): def get_nodes_sequences_when_variables_are_first(self, parser_helper): if len(parser_helper.sequence) < len(self.expected_variables): # variable(s) is/are expected. But nothing found + if parser_helper.debugger.is_enabled(): + debug_prefix = self.debug_prefix(parser_helper) + parser_helper.debug_concept(debug_prefix, color="red", raw="Failed! Too few elements") return None # only take the requested number of variables @@ -499,6 +515,11 @@ class VariableExpression(ParsingExpression): end = parser_helper.get_last_token_pos() nodes_sequences = self.get_nodes_sequences_from_tokens(parser_helper, start, end, tokens) if not nodes_sequences: + unrecognized = core.utils.get_text_from_tokens(tokens) + parser_helper.add_error(UndefinedSymbolError(unrecognized, parser_helper.get_current_concept())) + if parser_helper.debugger.is_enabled(): + debug_prefix = self.debug_prefix(parser_helper) + parser_helper.debug_concept(debug_prefix, color="red", raw=f"Failed to recognize '{unrecognized}'") return nodes_sequences # only take the requested number of variables @@ -527,7 +548,7 @@ class VariableExpression(ParsingExpression): def _parse(self, parser_helper): if parser_helper.debugger.is_enabled(): - debug_prefix = self.debug_prefix("VariableExpression", parser_helper) + debug_prefix = self.debug_prefix(parser_helper) debug_vars = {"pos": parser_helper.pos, "expected variables": self.expected_variables, "next to match": self.next_node_to_parse} @@ -541,8 +562,14 @@ class VariableExpression(ParsingExpression): else: nodes_sequences = self.get_nodes_sequences_when_variables_are_in_between(parser_helper) - if nodes_sequences is None or self.has_unrecognized(nodes_sequences): - # nothing is recognized or only part is recognized + if nodes_sequences is None: + # nothing is recognized + return None + + if self.has_unrecognized(nodes_sequences): + if parser_helper.debugger.is_enabled(): + debug_prefix = self.debug_prefix(parser_helper) + parser_helper.debug_concept(debug_prefix, color="red", raw="Some parts are not recognized") return None all_results = [] @@ -558,7 +585,7 @@ class VariableExpression(ParsingExpression): resolved = self.get_resolved(node) if resolved is None: - parser_helper.errors.append(f"Failed to recognize {node.source}") + parser_helper.add_error(UndefinedSymbolError(node.source)) break ptree_nodes.append(TerminalNode(variable_expr, node.start, node.end, node.source, resolved)) @@ -575,6 +602,11 @@ class VariableExpression(ParsingExpression): all_results.append(ptree_nodes) if len(all_results) == 0: + if parser_helper.debugger.is_enabled(): + debug_prefix = self.debug_prefix(parser_helper) + parser_helper.debug_concept(debug_prefix, color="red", + raw="No result matching the expected number of variables", + nodes_sequences=nodes_sequences) return None # every seems to be fine. We can pop the nodes from parser_helper used as variable @@ -618,8 +650,9 @@ class VariableExpression(ParsingExpression): return None utn = UnrecognizedTokensNode(start, end, tokens) - nodes_sequences = parser_helper.parser.cache2.get_lexer_nodes_from_unrecognized(parser_helper.parser.context, - utn) + nodes_sequences = parser_helper.parser.variable_expr_cache.get_lexer_nodes_from_unrecognized( + parser_helper.parser.context, + utn) return nodes_sequences @staticmethod @@ -1043,21 +1076,24 @@ class StrMatch(Match): if parser_helper.debugger.is_enabled(): debug_prefix = self.debug_prefix("StrMatch", parser_helper) - debug_text = f"pos={parser_helper.pos}, to_match={self.to_match}, token={token.str_value}" - parser_helper.debug_concept(debug_prefix, raw=f"{CCM['green']}{debug_text}{CCM['reset']}") + debug_vars = {"pos": parser_helper.pos, + "to_match": self.to_match, + "token": token.str_value} + debug_text = self.debug_to_raw(debug_vars) + parser_helper.debug_concept(debug_prefix, color="cyan", raw=debug_text) m = token.str_value.lower() == self.to_match.lower() if self.ignore_case \ else token.strip_quote == self.to_match if m: if parser_helper.debugger.is_enabled(): - parser_helper.debug_concept(debug_prefix, raw=f"{CCM['green']}{debug_text}{CCM['reset']}") + parser_helper.debug_concept(debug_prefix, color="green", raw=f"matched") node = TerminalNode(self, parser_helper.pos, parser_helper.pos, token.str_value, token.str_value) parser_helper.next_token(self.skip_white_space) return node if parser_helper.debugger.is_enabled(): - parser_helper.debug_concept(debug_prefix, raw=f"{CCM['red']}{debug_text}{CCM['reset']}") + parser_helper.debug_concept(debug_prefix, color="red", raw=f"not matched") return None @@ -1110,17 +1146,18 @@ class RegExMatch(Match): # debug_text = f"pos={parser_helper.pos}, to_match={self.to_match}, text={text_debug}" # parser_helper.debug_concept(debug_prefix, raw=f"{CCM['green']}{debug_text}{CCM['reset']}") - m = self.regex.match(text, parser_helper.token.index) + m = self.regex.match(text, parser_helper.token.index - parser_helper.token_offset) if m: matched = m.group() # TODO: Add debug info here if matched: - # the match is only valid if it fits with the actual tokens + # the match is only valid it ends at the end of a token next_pos = parser_helper.get_next_matching_pos(m.end()) if next_pos is NotFound: - parser_helper.errors.append(NoMatchingTokenError(m.end())) + current_concept = parser_helper.get_current_concept() + parser_helper.add_error(NoMatchingTokenError(m.end(), concept=current_concept)) return None node = TerminalNode(self, parser_helper.pos, next_pos - 1, matched, matched) @@ -1242,11 +1279,11 @@ class HasAChoiceExpressionVisitor(ParsingExpressionVisitor): class BnfConceptParserHelper: - def __init__(self, parser, debugger): + def __init__(self, parser, token_offset, debugger): self.parser = parser self.debugger = debugger self.debug = [] # keep track of the tokens - self.errors = [] # sink of errors + self.errors = [] # sink of errors, sorted by bnf concepts self.sequence = [] # output. List of lexer nodes correctly parsed self.concepts = [] # stack of concepts being processed (fed by ConceptExpression) self.concepts_ids = [] # ids if the concept to increase speed @@ -1260,6 +1297,7 @@ class BnfConceptParserHelper: self.forked = [] self.token = None + self.token_offset = token_offset self.pos = -1 def __repr__(self): @@ -1282,6 +1320,12 @@ class BnfConceptParserHelper: if len(self.concepts) <= 2: self.debugger.debug_concept(self.concepts[0], text, **kwargs) + def get_current_concept(self): + return self.concepts[-1] if len(self.concepts) > 0 else None + + def add_error(self, error): + self.errors.append(error) + def get_current_rule_name(self): for rule_name in reversed(self.rules_names): if rule_name: @@ -1328,15 +1372,26 @@ class BnfConceptParserHelper: def get_next_matching_pos(self, token_index): """ Given the token, tries to find a token (within the remaining tokens) that matches the index + Why do we do that ? + The regex matching is done against the text of the parser input, not its tokens + With this function, we check that the matching found correspond to an exact number of tokens. + Token is "abcdef" and we manage to match "abc" + -> 'abc' falls in the middle of the token, so it's not a valid match :param token_index: :return: """ current = self.pos + token_index += self.token_offset # add the offset when using sub parser input while current <= self.parser.parser_input.end: if self.parser.parser_input.tokens[current].index == token_index: return current current += 1 + # last chance when dealing with sub parser input (that do not end with token(EOF)) + if current < self.parser.parser_input.length: + if self.parser.parser_input.tokens[current].index == token_index: + return current + # No matching token return NotFound @@ -1384,7 +1439,7 @@ class BnfConceptParserHelper: error_msg = f"Failed to parse concept '{concept}'" if parsing_expression is not None: error_msg += f". Reason: '{parsing_expression}'" - self.errors.append(GrammarErrorNode(error_msg)) + self.add_error(GrammarErrorNode(error_msg, concept=concept)) return self.pos = self.parser.parser_input.pos @@ -1468,7 +1523,7 @@ class BnfConceptParserHelper: self.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, []) def clone(self): - clone = BnfConceptParserHelper(self.parser, self.debugger) + clone = BnfConceptParserHelper(self.parser, self.token_offset, self.debugger) clone.debug = self.debug[:] clone.errors = self.errors[:] clone.sequence = self.sequence[:] @@ -1636,8 +1691,11 @@ class BnfNodeParser(BaseNodeParser): else: self.concepts_grammars = Cache() + # this cache is for unrecognized tokens that cannot be BNF expression self.cache = UnrecognizedTokensCache(PARSERS) - self.cache2 = UnrecognizedTokensCache(VARIABLE_EXPR_PARSER) + + # This cache is for variable expression. It can be anything + self.variable_expr_cache = UnrecognizedTokensCache(VARIABLE_EXPR_PARSER) self.ignore_case = True @staticmethod @@ -1730,10 +1788,12 @@ class BnfNodeParser(BaseNodeParser): return list1 + list2 + sheerka = context.sheerka forked = [] debugger = context.get_debugger(self.NAME, "parse") debugger.debug_entering(source=self.parser_input.as_text()) - concept_parser_helpers = [BnfConceptParserHelper(self, debugger)] + token_offset = self.parser_input.tokens[self.parser_input.start].index + concept_parser_helpers = [BnfConceptParserHelper(self, token_offset, debugger)] while self.parser_input.next_token(False): @@ -1753,8 +1813,8 @@ class BnfNodeParser(BaseNodeParser): debugger.debug_log(debug_prefix + ", all parsers are locked. Nothing to do.") continue - by_token = context.sheerka.get_concepts_by_first_token(token, self._is_eligible, strip_quotes=False) - by_regex = context.sheerka.get_concepts_by_first_regex(self.parser_input.sub_text, token.index) + by_token = sheerka.get_concepts_by_first_token(token, self._is_eligible, strip_quotes=False) + by_regex = sheerka.get_concepts_by_first_regex(self.parser_input.sub_text, token.index - token_offset) concepts = _merge(by_token, by_regex) @@ -2120,7 +2180,7 @@ class BnfNodeParser(BaseNodeParser): debugger = context.get_debugger(self.NAME, "parse") if debugger.is_enabled: debugger.debug_var("stats", self.cache.to_dict()) - #debugger.debug_var("stats", self.cache2.to_dict()) + # debugger.debug_var("stats", self.variable_expr_cache.to_dict()) if valid_parser_helpers is None: return self.sheerka.ret( diff --git a/src/parsers/ExpressionParser.py b/src/parsers/ExpressionParser.py index 11d6d7e..4068fa5 100644 --- a/src/parsers/ExpressionParser.py +++ b/src/parsers/ExpressionParser.py @@ -3,11 +3,9 @@ from core.sheerka.services.SheerkaExecute import ParserInput from parsers.ArithmericOperatorParser import ArithmeticOperatorParser from parsers.BaseExpressionParser import BaseExpressionParser, ChoiceParser from parsers.FunctionParser import FunctionParser -from parsers.FunctionParserOld import FunctionParserOld from parsers.ListParser import ListParser from parsers.LogicalOperatorParser import LogicalOperatorParser from parsers.RelationalOperatorParser import RelationalOperatorParser -from parsers.VariableOrNamesParser import VariableOrNamesParser from sheerkapython.ExprToConditions import ExprToConditionsVisitor from sheerkapython.ExprToPython import PythonExprVisitor @@ -19,24 +17,19 @@ class ExpressionParser(BaseExpressionParser): NAME = "Expression" - def __init__(self, auto_compile=True, old_style=False, known_variables=None, **kwargs): + def __init__(self, auto_compile=True, known_variables=None, **kwargs): super().__init__(ExpressionParser.NAME, 0, True, yield_eof=False, hints={BuiltinConcepts.EVAL_QUESTION_REQUESTED: 60}) - if old_style: - self.variable_parser = VariableOrNamesParser() - self.function_parser = FunctionParserOld(expr_parser=self, tokens_parser=self.variable_parser) - self.relational_parser = RelationalOperatorParser(old_style=old_style, expr_parser=self.function_parser) - self.logical_parser = LogicalOperatorParser(expr_parser=self.relational_parser) - else: - function_parser = FunctionParser(expr_parser=self) - list_parser = ListParser(expr_parser=self) - choice_parser = ChoiceParser(list_parser, function_parser) - arithmetic_parser = ArithmeticOperatorParser(expr_parser=self, function_parser=choice_parser) - relational_parser = RelationalOperatorParser(expr_parser=arithmetic_parser) - self.logical_parser = LogicalOperatorParser(expr_parser=relational_parser) + + function_parser = FunctionParser(expr_parser=self) + list_parser = ListParser(expr_parser=self) + choice_parser = ChoiceParser(list_parser, function_parser) + arithmetic_parser = ArithmeticOperatorParser(expr_parser=self, function_parser=choice_parser) + relational_parser = RelationalOperatorParser(expr_parser=arithmetic_parser) + self.logical_parser = LogicalOperatorParser(expr_parser=relational_parser) self.auto_compile = auto_compile self.known_variables = known_variables diff --git a/src/parsers/FunctionParserOld.py b/src/parsers/FunctionParserOld.py deleted file mode 100644 index ac5812b..0000000 --- a/src/parsers/FunctionParserOld.py +++ /dev/null @@ -1,343 +0,0 @@ -from core.builtin_concepts import BuiltinConcepts -from core.builtin_helpers import get_lexer_nodes_from_unrecognized, update_compiled -from core.concept import Concept -from core.sheerka.services.SheerkaExecute import ParserInput -from core.tokenizer import TokenKind -from core.utils import get_n_clones -from parsers.BaseExpressionParser import NameExprNode, FunctionNodeOld, FunctionParameter, BaseExpressionParser -from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode -from parsers.BaseParser import UnexpectedTokenParsingError, UnexpectedEofParsingError, ErrorSink -from parsers.BnfNodeParser import BnfNodeParser -from parsers.PythonWithConceptsParser import PythonWithConceptsParser -from parsers.RuleParser import RuleParser -from parsers.SequenceNodeParser import SequenceNodeParser -from parsers.SyaNodeParser import SyaNodeParser - -PARSERS = [RuleParser.NAME, - SequenceNodeParser.NAME, - BnfNodeParser.NAME, - SyaNodeParser.NAME] - - -class FunctionParserOld(BaseExpressionParser): - """ - The parser will be used to parse func(x, y, z) - where x, y and z can be source code, concepts or other functions - It will return a SourceCodeNode or SourceCodeNodeWithConcept - """ - - NAME = "FunctionOld" - - def __init__(self, sep=",", longest_concepts_only=True, strict=True, **kwargs): - """ - - :param sep: - :param longest_concepts_only: When multiples concepts are found, only keep the longest one - so 'twenty one' will resolve to [[c:twenty one:]], not [[c:twenty one:], [c:twenty:, c:one:]] - :param strict: Only allow expression that start by a function definition so 'xxx f(x)' will be refused - :param kwargs: - """ - super().__init__(self.NAME, 55, enabled=False) - self.sep = sep - self.longest_concepts_only = longest_concepts_only - self.strict = strict - self.expr_parser = kwargs.get("expr_parser", None) - self.tokens_parser = kwargs.get("tokens_parser", None) - - def function_parser_get_return_value_body(self, context, source, source_code_node): - if source_code_node.error_when_parsing: - return context.sheerka.new(BuiltinConcepts.ERROR, - body=source_code_node.error_when_parsing) - - return context.sheerka.new(BuiltinConcepts.PARSER_RESULT, - parser=self, - source=source, - body=source_code_node, - try_parsed=source_code_node) - - def parse(self, context, parser_input: ParserInput): - ret = super().parse(context, parser_input) - - if ret is None: - return None - - if not ret.status: - return ret - - # FunctionParserOld returns LexerNodes, rather than an ExprNode - # I know that is is not very logical, but at the beginning, the FunctionParserOld was - # uses exclusively by the SyaNodeParser. - # It has been refactored to fit in ExpressionParser. So it has two main usages - node = ret.body.body - source_code_nodes = self.to_source_code_node(context, node) - res = [] - for source_code_node in source_code_nodes: - body = self.function_parser_get_return_value_body(context, parser_input.as_text(), source_code_node) - res.append(context.sheerka.ret(self.name, source_code_node.python_node is not None, body)) - - return res[0] if len(res) == 1 else res - - def parse_input(self, context, parser_input, error_sink): - # when FunctionParserOld is used by LexerNode or SheerkaExecute, it must fail if no function is found - # when it is used by ExpressionParser, it must default to VariableOrNamesParser - # KSI 20210910 - Not quite quite it's the best approach. It seems a little bit complicated - pos = parser_input.pos - res = self.parse_function(context, parser_input, error_sink) - if (not res or error_sink.has_error) and self.tokens_parser: - parser_input.seek(pos) - error_sink.clear() - return self.tokens_parser.parse_input(context, parser_input, error_sink) - - return res - - def parse_function(self, context, parser_input, error_sink): - unrecognized_tokens_start = unrecognized_tokens_end = parser_input.pos - if not self.strict: - # eat everything that is not part of the function - token = parser_input.token - while token.type != TokenKind.EOF: - if token.type == TokenKind.IDENTIFIER and parser_input.the_token_after().type == TokenKind.LPAR: - break - else: - unrecognized_tokens_end += 1 - parser_input.next_token(skip_whitespace=False) - token = parser_input.token - - start = parser_input.pos - token = parser_input.token - if token.type != TokenKind.IDENTIFIER: - error_sink.add_error(UnexpectedTokenParsingError(f"{token.repr_value} is not a identifier", - token, - [TokenKind.IDENTIFIER])) - return None - - if not parser_input.next_token(): - error_sink.add_error(UnexpectedEofParsingError(f"while parsing left parenthesis")) - return None - - token = parser_input.token - if token.type != TokenKind.LPAR: - error_sink.add_error(UnexpectedTokenParsingError(f"{token.repr_value} is not a left parenthesis", - token, - [TokenKind.LPAR])) - return None - - start_node = NameExprNode(start, start + 1, parser_input.tokens[start:start + 2]) - if not parser_input.next_token(): - error_sink.add_error(UnexpectedEofParsingError(f"after left parenthesis")) - return FunctionNodeOld(start, start + 1, [], start_node, None, None) - - params = self.parse_parameters(context, parser_input, error_sink) - if error_sink.has_error: - return FunctionNodeOld(start, parser_input.pos, [], start_node, None, params) - - token = parser_input.token - if not token or token.type != TokenKind.RPAR: - error_sink.add_error(UnexpectedTokenParsingError(f"Right parenthesis not found", - token, - [TokenKind.RPAR])) - return FunctionNodeOld(start, parser_input.pos, [], start_node, None, params) - - function_node = FunctionNodeOld(start, - parser_input.pos, - parser_input.tokens[start:parser_input.pos + 1], - start_node, - NameExprNode(parser_input.pos, parser_input.pos, [token]), - params) - - parser_input.next_token() # do not forget to eat the trailing parenthesis - - # if unrecognized_tokens_end != unrecognized_tokens_start: - # if self.expr_parser: - # sub_parser = parser_input.sub_part(unrecognized_tokens_start, - # unrecognized_tokens_end, - # yield_oef=False).reset() - # sub_parser.next_token() - # expr_node = self.expr_parser(context, parser_input, error_sink) - # else: - # expr_node = NameExprNode(unrecognized_tokens_start, - # unrecognized_tokens_end, - # parser_input.tokens[unrecognized_tokens_start: unrecognized_tokens_end + 1]) - # - # return SequenceNode(unrecognized_tokens_start, function_node.end, - # parser_input.tokens[unrecognized_tokens_start: function_node.end + 1], - # expr_node, function_node) - - return function_node - - def parse_parameters(self, context, parser_input, error_sink): - nodes = [] - while True: - param_value = self.parse_parameter_value(context, parser_input, error_sink) - if not param_value: - break - - function_parameter = FunctionParameter(param_value) - nodes.append(function_parameter) - - token = parser_input.token - if token.type == TokenKind.EOF: - error_sink.add_error(UnexpectedEofParsingError(f"while parsing parameters")) - return None - - if token.type == TokenKind.RPAR: - break - - if token.value == self.sep: - sep_pos = parser_input.pos - has_next = parser_input.next_token() # it's before add_sep() to capture trailing whitespace - function_parameter.add_sep(sep_pos, - parser_input.pos - 1, - parser_input.tokens[sep_pos: parser_input.pos]) - if not has_next: - break - - return nodes - - def parse_parameter_value(self, context, parser_input, error_sink): - # check if the parameter is a function - start_pos = parser_input.pos - new_error_sink = ErrorSink() - func = self.parse_function(context, parser_input, new_error_sink) - if func and not new_error_sink.has_error: - return func - - # otherwise, eat until LPAR or separator - parser_input.seek(start_pos) - return self.parse_tokens(context, - parser_input, - error_sink, - self.parse_tokens_stop_condition, - self.expr_parser, - self.expr_parser) - - def parse_tokens_stop_condition(self, token, parser_input): - return token.value == self.sep - - def to_source_code_node(self, context, function_node: FunctionNodeOld): - python_parser = PythonWithConceptsParser() - - def update_source_code_node(scn, nodes, sep): - if hasattr(nodes, "__iter__"): - for n in nodes: - scn.add_node(n) - else: - scn.add_node(nodes) - - if sep: - scn.add_node(sep.to_unrecognized()) - - def get_errors_from_python_parsing(parsing_res): - if parsing_res.status: - return None - - if context.sheerka.isinstance(parsing_res.body, BuiltinConcepts.NOT_FOR_ME): - return parsing_res.body.reason - else: - return parsing_res.body.body - - if len(function_node.parameters) == 0: - # validate the source - nodes_to_parse = [function_node.first.to_unrecognized(), function_node.last.to_unrecognized()] - python_parsing_res = python_parser.parse_nodes(context, nodes_to_parse) - python_node = python_parsing_res.body.body if python_parsing_res.status else None - - return [SourceCodeNode(start=function_node.first.start, - end=function_node.last.end, - tokens=function_node.first.tokens + function_node.last.tokens, - python_node=python_node, - return_value=python_parsing_res, - error_when_parsing=get_errors_from_python_parsing(python_parsing_res))] - - res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())] - - # try to recognize every parameter, one by one - for param in function_node.parameters: - if isinstance(param.value, NameExprNode): - # try to recognize concepts - unrecognized = param.value.to_unrecognized() - nodes_sequences = get_lexer_nodes_from_unrecognized(context, - unrecognized, - PARSERS) - else: - # the parameter is also a function - nodes_sequences = self.to_source_code_node(context, param.value) - - if self.longest_concepts_only: - nodes_sequences = self.get_longest_concepts(nodes_sequences) - - if nodes_sequences is None: - # no concept found - for source_code_node in res: - update_source_code_node(source_code_node, unrecognized, param.separator) - - elif len(nodes_sequences) == 1: - # only one result - # It is the same code than when there are multiple results - # But here, we save the creation of the tmp_res object (not sure it worth it) - for source_code_node in res: - update_source_code_node(source_code_node, nodes_sequences[0], param.separator) - else: - # multiple result, make the cartesian product - tmp_res = [] - for source_code_node in res: - instances = get_n_clones(source_code_node, len(nodes_sequences)) - tmp_res.extend(instances) - for instance, node_sequence in zip(instances, nodes_sequences): - update_source_code_node(instance, node_sequence, param.separator) - res = tmp_res - - # check if it is a valid source code - for source_code_node in res: - source_code_node.fix_all_pos() - source_code_node.pseudo_fix_source() - - python_parsing_res = python_parser.parse_nodes(context, source_code_node.get_all_nodes()) - if python_parsing_res.status: - source_code_node.python_node = python_parsing_res.body.body - source_code_node.return_value = python_parsing_res - - # make sure that concepts found can be evaluated - errors = [] - for c in [c for c in source_code_node.python_node.objects.values() if isinstance(c, Concept)]: - update_compiled(context, c, errors) - - if errors: - source_code_node.error_when_parsing = errors - - else: - source_code_node.error_when_parsing = get_errors_from_python_parsing(python_parsing_res) - - return res - - @staticmethod - def get_longest_concepts(nodes_sequences): - """ - The longest sequences are the ones that have the less number of concepts - For example - 'twenty one' resolves to - [c:twenty one:] - [c:twenty:, c:one:] - [c:twenty one:] has only one concept, so it's the longest one (two tokens against one token twice) - :param nodes_sequences: - :return: - """ - if nodes_sequences is None: - return None - - res = [] - min_len = -1 - for current_sequence in nodes_sequences: - # awful hack to remove when NodeSequence and ConceptSequence will be implemented - current_len = len(current_sequence) if hasattr(current_sequence, "__len__") else 1 - if len(res) == 0: - res.append(current_sequence) - min_len = current_len - elif current_len == min_len: - res.append(current_sequence) - elif current_len < min_len: - res.clear() - res.append(current_sequence) - min_len = current_len - - return res diff --git a/src/parsers/RelationalOperatorParser.py b/src/parsers/RelationalOperatorParser.py index fefe482..e42ffdf 100644 --- a/src/parsers/RelationalOperatorParser.py +++ b/src/parsers/RelationalOperatorParser.py @@ -1,6 +1,6 @@ from core.tokenizer import TokenKind -from parsers.BaseExpressionParser import ComparisonNode, ComparisonType, \ - ParenthesisNode, BaseExpressionParser, open_parenthesis_mapping +from parsers.BaseExpressionParser import BaseExpressionParser, ComparisonNode, ComparisonType, ParenthesisNode, \ + open_parenthesis_mapping from parsers.BaseParser import UnexpectedTokenParsingError from parsers.ListParser import ListParser @@ -13,10 +13,9 @@ class RelationalOperatorParser(BaseExpressionParser): NAME = "RelationalOperator" - def __init__(self, old_style=False, **kwargs): + def __init__(self, **kwargs): super().__init__(self.NAME, 60, False, yield_eof=True) self.expr_parser = kwargs.get("expr_parser", None) - self.old_style = old_style self.list_parser = ListParser() def parse_input(self, context, parser_input, error_sink): @@ -36,29 +35,18 @@ class RelationalOperatorParser(BaseExpressionParser): if (comp := self.eat_comparison(parser_input)) is None: return left - if self.old_style: + if comp in (ComparisonType.IN, ComparisonType.NOT_IN): + token = parser_input.token + if token.type not in open_parenthesis_mapping: + error_sink.add_error(UnexpectedTokenParsingError(f"Expected parenthesis", token, [TokenKind.LPAR])) + right = self.list_parser.parse_input(context, parser_input, error_sink) + else: right = self.parse_tokens(context, parser_input, error_sink, self.parse_tokens_stop_condition, self.expr_parser, self) - if comp == ComparisonType.IN and not isinstance(right, ParenthesisNode): - t = right.tokens[0] - error_sink.add_error(UnexpectedTokenParsingError(f"Expected parenthesis", t, [TokenKind.LPAR])) - else: - if comp in (ComparisonType.IN, ComparisonType.NOT_IN): - token = parser_input.token - if token.type not in open_parenthesis_mapping: - error_sink.add_error(UnexpectedTokenParsingError(f"Expected parenthesis", token, [TokenKind.LPAR])) - right = self.list_parser.parse_input(context, parser_input, error_sink) - else: - right = self.parse_tokens(context, - parser_input, - error_sink, - self.parse_tokens_stop_condition, - self.expr_parser, - self) end = right.end if right else parser_input.pos diff --git a/src/parsers/SyaNodeParser.py b/src/parsers/SyaNodeParser.py index 13cea5f..db49ffd 100644 --- a/src/parsers/SyaNodeParser.py +++ b/src/parsers/SyaNodeParser.py @@ -1007,8 +1007,11 @@ class SyaNodeParser(BaseNodeParser): result.append(tokens_parser) elif tokens_parser.has_sya_concept and tokens_parser.has_error(): in_error.append(tokens_parser) + for err in tokens_parser.errors: + tokens_parser.debugger.debug_log(f"Validation Error: {err}", True) else: tokens_parser.errors.append(NoSyaConceptFound()) + tokens_parser.debugger.debug_log(f"Validation Error: No Sya Found", True) not_for_me.append(tokens_parser) # recurse on the forks diff --git a/src/sdp/sheerkaDataProvider.py b/src/sdp/sheerkaDataProvider.py index c1a7cab..c04ae77 100644 --- a/src/sdp/sheerkaDataProvider.py +++ b/src/sdp/sheerkaDataProvider.py @@ -52,7 +52,7 @@ class Event(object): if self._digest: return self._digest - if self.message == "" and self.user_id == "": + if self.user_id == "" and (self.message == "" or self.message.startswith("xxxTESTxxx::")): self._digest = "xxx" # to speed unit tests return self._digest diff --git a/src/sheerkapython/BaseExprTransform.py b/src/sheerkapython/BaseExprTransform.py index 7177d7d..c3df4e0 100644 --- a/src/sheerkapython/BaseExprTransform.py +++ b/src/sheerkapython/BaseExprTransform.py @@ -57,14 +57,15 @@ check_existence = ExprTransformHints(check_variable_existence=True, class BaseExprTransform(ExpressionVisitorWithHint): - def __init__(self, context, obj_counter): + def __init__(self, context, obj_counter, known_variables=None): self.context = context self.obj_counter = obj_counter self.objects_by_id = {} self.objects_by_name = {} self.errors = {} + self.known_variables = known_variables if known_variables is not None else set() from parsers.ExpressionParser import ExpressionParser - self.expression_parser = ExpressionParser(auto_compile=False, old_style=False) + self.expression_parser = ExpressionParser(auto_compile=False) def visit_NotNode(self, expr_node: NotNode, hint: ExprTransformHints): """ @@ -482,7 +483,12 @@ class BaseExprTransform(ExpressionVisitorWithHint): identifier = self.get_object_identifier(concept) objects = {} - concept_variables = get_variables_from_concept_asts(self.context, concept, variables, parameters_only=True) + known_variables = self.known_variables.copy() + known_variables.update(variables) + concept_variables = get_variables_from_concept_asts(self.context, + concept, + known_variables, + parameters_only=True) parameters_to_compute = {} for var_name, default_value in [(k, v) for k, v in concept.get_metadata().variables if k in concept_variables]: if isinstance(concept.get_compiled()[var_name], Concept): diff --git a/src/sheerkapython/ExprToConditions.py b/src/sheerkapython/ExprToConditions.py index b8b5e62..c12f86e 100644 --- a/src/sheerkapython/ExprToConditions.py +++ b/src/sheerkapython/ExprToConditions.py @@ -8,9 +8,8 @@ from sheerkapython.python_wrapper import is_variable class ExprToConditionsVisitor(BaseExprTransform): def __init__(self, context, obj_counter=0, known_variables=None): - super().__init__(context, obj_counter) + super().__init__(context, obj_counter, known_variables) self.intermediate_variables = {} - self.known_variables = known_variables if known_variables is not None else set() def get_conditions(self, expr_node): # first transform expr_node into list of conjunctions diff --git a/src/sheerkapython/ExprToPython.py b/src/sheerkapython/ExprToPython.py index 49d0c0b..d82408c 100644 --- a/src/sheerkapython/ExprToPython.py +++ b/src/sheerkapython/ExprToPython.py @@ -3,7 +3,7 @@ from itertools import product from core.builtin_concepts_ids import BuiltinConcepts from core.sheerka.services.sheerka_service import FailedToCompileError from core.tokenizer import TokenKind -from parsers.BaseExpressionParser import AndNode, FunctionNodeOld, ListComprehensionNode, NameExprNode, VariableNode, \ +from parsers.BaseExpressionParser import AndNode, ListComprehensionNode, NameExprNode, VariableNode, \ end_parenthesis_mapping, open_parenthesis_mapping from sheerkapython.BaseExprTransform import BaseExprTransform, ExprTransformHints, do_not_eval_source_hint, \ is_a_question_hint, not_a_question_hint, wrap_concept_call_hint @@ -128,19 +128,3 @@ class PythonExprVisitor(BaseExprTransform): :return: """ return self.visit_or_or_and_node("or", expr_node, hint) - - def visit_FunctionNodeOld(self, expr_node: FunctionNodeOld, hint: ExprTransformHints): - visitor_objects = [] - source = expr_node.get_source() - - parameters_objects = [] - for parameter in expr_node.parameters: - parameters_objects.append(self.visit(parameter.value, hint)) - - for parameters in product(*parameters_objects): - visitor_objects.append(self.create_function_old(source, - expr_node.first.get_source(), - expr_node.last.get_source(), - parameters)) - - return visitor_objects diff --git a/src/sheerkapython/python_wrapper.py b/src/sheerkapython/python_wrapper.py index 3633aae..0ed205e 100644 --- a/src/sheerkapython/python_wrapper.py +++ b/src/sheerkapython/python_wrapper.py @@ -6,7 +6,7 @@ from core.ast_helpers import UnreferencedVariablesVisitor from core.builtin_concepts import ReturnValueConcept from core.builtin_concepts_ids import BuiltinConcepts from core.concept import AllConceptParts, Concept -from core.global_symbols import ErrorObj, NotFound, NotInit, SyaAssociativity +from core.global_symbols import ErrorItem, ErrorObj, NotFound, NotInit, SyaAssociativity from core.rule import Rule from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka.services.SheerkaAdmin import SheerkaAdmin @@ -110,8 +110,14 @@ sheerka_globals = { "get_type": get_type, "hasattr": sheerka_hasattr, "getattr": sheerka_getattr, + "ErrorItem": ErrorItem } +# Adds all concepts that have their own class definition +for c in core.utils.get_classes("core.builtin_concepts"): + if issubclass(c, Concept) and c != Concept: + sheerka_globals[c.__name__] = c + def inject_context(context): """ @@ -285,6 +291,7 @@ def get_variables_from_concept_asts(context, concept, known_variables, parameter """ From a given concept that is already compiled, browse the compiled to see if there is any symbol that is unknown, eg variable + It is used to detect all mandatory variables before concept evaluation :param context: :param concept: :param known_variables: @@ -348,6 +355,25 @@ def get_variables_from_concept_asts(context, concept, known_variables, parameter return variables +def get_possible_variables_from_concept(context, concept): + """ + Given a concept, get its symbols that may be considered as variables for other concepts + :param context: + :param concept: + :return: + """ + possible_variables = set(concept.get_metadata().parameters) + + core.builtin_helpers.ensure_bnf(context, concept) + if concept.get_bnf(): + from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor + visitor = BnfNodeConceptExpressionVisitor() + visitor.visit(concept.get_bnf()) + possible_variables.update([c.name if isinstance(c, Concept) else c for c in visitor.references]) + + return possible_variables + + def is_variable(context, name): """ tells whether or not the name can be a variable diff --git a/tests/BaseTest.py b/tests/BaseTest.py index 41689ec..314c775 100644 --- a/tests/BaseTest.py +++ b/tests/BaseTest.py @@ -48,7 +48,7 @@ class InitTestHelper: c.get_metadata().definition_type = DEFINITION_TYPE_BNF else: raise Exception(f"Error in bnf definition '{c.get_metadata().definition}'", - self.sheerka.get_errors(self.context, res)) + self.sheerka.get_obj_errors(self.context, res)) self._update_concept_parameters(c) if create_new: @@ -92,8 +92,7 @@ class InitTestHelper: if create_new: res = self.sheerka.create_new_rule(self.context, rule) if not res.status: - raise Exception(f"Error in rule definition '{res.body}'", - self.sheerka.get_errors(res)) + raise Exception(f"Error in rule definition '{res.body}'", self.sheerka.get_obj_errors(res)) self.items.append(res.body.body) else: self.items.append(rule) @@ -294,16 +293,26 @@ class BaseTest: return [ret_val for ret_val in return_values if ret_val.status] @staticmethod - def activate_debug(context, pattern="Sya.*.*"): + def activate_debug(context, pattern="Sya.*.*", debug_var=True, debug_concept=False, debug_rule=False): sheerka = context.sheerka sheerka.set_debug(context, True) sheerka.set_debug_logger_definition(ListDebugLogger) if isinstance(pattern, list): for p in pattern: - sheerka.set_debug_var(context, p) + if debug_var: + sheerka.set_debug_var(context, p) + if debug_concept: + sheerka.set_debug_concept(context, p) + if debug_rule: + sheerka.set_debug_rule(context, p) else: - sheerka.set_debug_var(context, pattern) + if debug_var: + sheerka.set_debug_var(context, pattern) + if debug_concept: + sheerka.set_debug_concept(context, pattern) + if debug_rule: + sheerka.set_debug_rule(context, pattern) # the see the logs, do not forget to add # logs = sheerka.get_debugger_logs() diff --git a/tests/TestUsingFileBasedSheerka.py b/tests/TestUsingFileBasedSheerka.py index 58261df..7c110e9 100644 --- a/tests/TestUsingFileBasedSheerka.py +++ b/tests/TestUsingFileBasedSheerka.py @@ -1,8 +1,10 @@ +import shutil +from os import path + from conftest import SHEERKA_TEST_FOLDER from core.global_symbols import EVENT_ONTOLOGY_CREATED from core.sheerka.Sheerka import Sheerka from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager - from tests.BaseTest import BaseTest @@ -45,3 +47,18 @@ class TestUsingFileBasedSheerka(BaseTest): self.sheerka.push_ontology(self.context, ontology_name, cache_only=cache_only) return TestUsingFileBasedSheerka.sheerka + + @staticmethod + def commit(context): + sheerka = context.sheerka + event = context.event + sheerka.om.save_event(event) + + if sheerka.om.is_dirty(): + sheerka.om.commit(context) + + @staticmethod + def reset_hard_test_env(): + if path.exists(SHEERKA_TEST_FOLDER): + shutil.rmtree(SHEERKA_TEST_FOLDER) + TestUsingFileBasedSheerka.sheerka = None diff --git a/tests/core/test_SheerkaErrorManager.py b/tests/core/test_SheerkaErrorManager.py new file mode 100644 index 0000000..18143fa --- /dev/null +++ b/tests/core/test_SheerkaErrorManager.py @@ -0,0 +1,463 @@ +import pytest + +from core.builtin_concepts import ReturnValueConcept, UnknownConcept +from core.builtin_concepts_ids import BuiltinConcepts +from core.concept import Concept +from core.sheerka.services.SheerkaConceptManager import ValueNotFound +from core.sheerka.services.SheerkaErrorManager import LAST_UNKNOWN_CONCEPTS, SheerkaErrorManager +from core.sheerka.services.SheerkaExecute import ParserInput +from core.sheerka.services.SheerkaRuleManager import NoConditionFound +from evaluators.PythonEvaluator import PythonEvaluator +from parsers.BaseParser import ParsingError, UnexpectedEofParsingError +from parsers.BnfNodeParser import BnfNodeParser +from parsers.ExactConceptParser import ExactConceptParser +from parsers.PythonParser import PythonErrorNode, PythonParser +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def inner_errors(lst): + return [e.error for e in lst] + + +class TestSheerkaErrorManger(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("obj, expected", [ + ("a string", []), + (True, []), + (False, []), + (Concept("foo"), []), + (Concept("foo", body=False).auto_init(), []), + (UnknownConcept(), [UnknownConcept()]), + (Concept("foo", body=UnknownConcept()).auto_init(), [UnknownConcept()]), + (PythonErrorNode("msg", None), [PythonErrorNode("msg", None)]) + ]) + def test_i_can_get_error_for_simple_objects(self, obj, expected): + sheerka, context = self.init_test().unpack() + + assert sheerka.get_obj_errors(context, obj) == expected + + def test_i_can_get_error_when_builtin_concept_in_error(self): + sheerka, context = self.init_test().unpack() + + obj = sheerka.new(BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED) + assert sheerka.get_obj_errors(context, obj) == [obj] + + def test_i_can_get_error_when_return_value(self): + sheerka, context = self.init_test().unpack() + + error = sheerka.err("an error") + ret_val = ReturnValueConcept("Test", False, sheerka.err("an error")) + assert sheerka.get_obj_errors(context, ret_val) == [error] + + def test_i_can_get_inner_error(self): + sheerka, context = self.init_test().unpack() + + error = sheerka.err("an error") + ret_val = ReturnValueConcept("Test", False, sheerka.err("an error")) + assert sheerka.get_obj_errors(context, ret_val) == [error] + + def test_i_can_get_error_when_embedded_errors(self): + sheerka, context = self.init_test().unpack() + + concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) + unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) + not_an_error = sheerka.new(BuiltinConcepts.AUTO_EVAL) + error = sheerka.err([concept_eval_error, unknown_concept, not_an_error]) + + ret_val = ReturnValueConcept("Test", False, error) + errors_found = sheerka.get_obj_errors(context, ret_val) + + assert errors_found == [error, concept_eval_error, unknown_concept] + + def test_i_can_get_embedded_errors_from_get_error_method(self): + sheerka, context = self.init_test().unpack() + + embedded_error = NotImplementedError() + error = sheerka.new(BuiltinConcepts.NOT_FOR_ME, body="some text", reason=embedded_error) + ret_val = ReturnValueConcept("Test", False, error) + errors_found = sheerka.get_obj_errors(context, ret_val) + + assert errors_found == [error, embedded_error] + + def test_i_can_get_error_from_list(self): + sheerka, context = self.init_test().unpack() + + concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) + unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) + not_an_error = sheerka.new(BuiltinConcepts.AUTO_EVAL) + error = sheerka.err([concept_eval_error, unknown_concept, not_an_error]) + ret_val_1 = ReturnValueConcept("Test", False, error) + + python_error = PythonErrorNode("msg", Exception()) + value_not_found = ValueNotFound("item", "value") + multiple_error = sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=[python_error, value_not_found]) + ret_val_2 = ReturnValueConcept("Test", False, multiple_error) + + errors_found = sheerka.get_obj_errors(context, [ret_val_1, ret_val_2]) + + assert errors_found == [error, concept_eval_error, unknown_concept, + multiple_error, python_error, value_not_found] + + def test_i_can_filter_error_by_concept_key(self): + sheerka, context = self.init_test().unpack() + + concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) + unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) + python_error = PythonErrorNode("msg", Exception()) + error = sheerka.err([concept_eval_error, unknown_concept, python_error]) + ret_val = ReturnValueConcept("Test", False, error) + + errors_found = sheerka.get_obj_errors(context, ret_val, __type=BuiltinConcepts.CONCEPT_EVAL_ERROR) + assert errors_found == [concept_eval_error] + + def test_i_can_filter_error_by_class_name(self): + sheerka, context = self.init_test().unpack() + + concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) + unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) + python_error = PythonErrorNode("msg", Exception()) + error = sheerka.err([concept_eval_error, unknown_concept, python_error]) + ret_val = ReturnValueConcept("Test", False, error) + + errors_found = sheerka.get_obj_errors(context, ret_val, __type="PythonErrorNode") + assert errors_found == [python_error] + + def test_i_can_filter_error_by_concept_attribute(self): + sheerka, context = self.init_test().unpack() + + concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) + unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref") + python_error = PythonErrorNode("msg", Exception()) + error = sheerka.err([concept_eval_error, unknown_concept, python_error]) + ret_val = ReturnValueConcept("Test", False, error) + + errors_found = sheerka.get_obj_errors(context, ret_val, concept_ref="a_concept_ref") + assert errors_found == [unknown_concept] + + def test_i_can_filter_error_by_class_attribute(self): + sheerka, context = self.init_test().unpack() + + concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) + unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref") + python_error = PythonErrorNode("error source", Exception()) + error = sheerka.err([concept_eval_error, unknown_concept, python_error]) + ret_val = ReturnValueConcept("Test", False, error) + + errors_found = sheerka.get_obj_errors(context, ret_val, source="error source") + assert errors_found == [python_error] + + def test_i_can_filter_error_by_exception_name_and_type(self): + sheerka, context = self.init_test().unpack() + + from evaluators.PythonEvaluator import PythonEvalError + name_error = NameError("foo") + python_eval_error = PythonEvalError(name_error, "foo", None, None) + error = sheerka.err([python_eval_error]) + ret_val = ReturnValueConcept("Test", False, error) + + errors_found = sheerka.get_obj_errors(context, ret_val, __type="NameError") + assert errors_found == [name_error] + + errors_found = sheerka.get_obj_errors(context, ret_val, __type=NameError) + assert errors_found == [name_error] + + def test_i_can_filter_error_on_multiple_criteria(self): + sheerka, context = self.init_test().unpack() + + concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) + unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref") + value_not_found = ValueNotFound("an_item", "a value") + error = sheerka.err([concept_eval_error, unknown_concept, value_not_found]) + ret_val = ReturnValueConcept("Test", False, error) + + errors_found = sheerka.get_obj_errors(context, ret_val, __type="ValueNotFound", item="an_item", value="a value") + assert errors_found == [value_not_found] + + def test_i_cannot_get_error_when_return_value_s_status_is_true(self): + sheerka, context = self.init_test().unpack() + + ret_val = ReturnValueConcept("Test", True, sheerka.err("an error")) + assert sheerka.get_obj_errors(context, ret_val) == [] + + def test_i_can_get_error_items(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaErrorManager.NAME] + + simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo")) + ret1 = sheerka.ret("Test1", False, simple_error) + + sub_error_is_an_exception = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=NotImplementedError()) + ret2 = sheerka.ret("Test2", False, sub_error_is_an_exception) + + inner_error = sheerka.err([ParsingError(), UnexpectedEofParsingError("EOF", 0)]) + sub_error_is_an_error = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=inner_error) + ret3 = sheerka.ret("Test3", False, sub_error_is_an_error) + + parsing_error = sheerka.new(BuiltinConcepts.PARSER_RESULT, + parser=PythonParser(), + source="not a valid source", + body="Not totally valid parsing") + ret4 = sheerka.ret("Test4", False, parsing_error) + + multiple_level_errors1 = sheerka.err("Err1") + multiple_level_errors2 = sheerka.err(multiple_level_errors1) + multiple_level_errors3 = sheerka.err(multiple_level_errors2) + ret5 = sheerka.ret("Test5", False, multiple_level_errors3) + + filtered = sheerka.new(BuiltinConcepts.FILTERED, reason="not really given") + ret6 = sheerka.ret("Test6", False, filtered) + + ret7 = sheerka.ret("Test5", True, None) + + error_items = list(service.get_errors_items([ret1, ret2, ret3, ret4, ret5, ret6, ret7])) + assert len(error_items) == 6 + + # ret1 : simple_error + assert len(error_items[0].children) == 0 + + # ret2 : sub_error_is_an_exception + assert len(error_items[1].children) == 1 + assert len(error_items[1].children[0].children) == 0 + + # ret3 : inner_error + assert len(error_items[2].children) == 1 + assert len(error_items[2].children[0].children) == 2 + assert len(error_items[2].children[0].children[0].children) == 0 + assert len(error_items[2].children[0].children[1].children) == 0 + + # ret4 : parsing_error + assert len(error_items[3].children) == 0 + + # ret5 : multiple_level_errors + assert len(error_items[4].children) == 1 + assert len(error_items[4].children[0].children) == 1 + assert len(error_items[4].children[0].children[0].children) == 0 + + # ret6 Filtered without a real reason + assert len(error_items[5].children) == 0 + + def test_i_can_get_all_error_items(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaErrorManager.NAME] + + simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo")) + ret1 = sheerka.ret("Test1", False, simple_error) + + inner_error = sheerka.err([ParsingError(), UnexpectedEofParsingError("EOF", 0)]) + sub_error_is_an_error = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=inner_error) + ret2 = sheerka.ret("Test3", False, sub_error_is_an_error) + + multiple_level_errors1 = sheerka.err("Err1") + multiple_level_errors2 = sheerka.err(multiple_level_errors1) + multiple_level_errors3 = sheerka.err(multiple_level_errors2) + ret3 = sheerka.ret("Test5", False, multiple_level_errors3) + + all_error_items = list(service.get_all_error_items([ret1, ret2, ret3])) + + assert len(all_error_items) == 8 + + def test_i_can_keep_the_correct_error_source(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaErrorManager.NAME] + + simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo")) + ret1 = sheerka.ret("Source1", False, simple_error) + + sub_error_is_an_exception = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=NotImplementedError()) + ret2 = sheerka.ret("Source2", False, sub_error_is_an_exception) + + multiple_errors = sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=[ret1, ret2]) + ret3 = sheerka.ret("Source3", False, multiple_errors) + + error_items = list(service.get_errors_items([ret3])) + assert len(error_items) == 1 + assert error_items[0].source == "Source3" + assert error_items[0].children[0].source == "Source1" + assert error_items[0].children[1].source == "Source2" + + def test_i_can_get_filtered_error_explanation(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaErrorManager.NAME] + + simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo")) + ret1 = sheerka.ret("Test1", False, simple_error) + + sub_error_is_an_exception = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=NotImplementedError()) + ret2 = sheerka.ret("Test2", False, sub_error_is_an_exception) + context.add_values(return_values=[ret1, ret2]) + service.on_user_input_evaluated(context) + + res = sheerka.get_last_errors(context) + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert len(res.body) == 3 + + res = sheerka.get_last_errors(context, __type=BuiltinConcepts.UNKNOWN_CONCEPT) + assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION) + assert len(res.body) == 1 + + def test_i_cannot_recognize_error_when_no_error(self): + sheerka, context = self.init_test().unpack() + res = sheerka.recognize_error(context, "Some error") + assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) + + def test_i_can_recognized_error_when_error_is_a_builtin_concept(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaErrorManager.NAME] + + simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo")) + ret1 = sheerka.ret("Test1", False, simple_error) + context.add_values(return_values=[ret1]) + service.on_user_input_evaluated(context) + + assert sheerka.recognize_error(context, __type=BuiltinConcepts.UNKNOWN_CONCEPT) + assert sheerka.recognize_error(context, __type=UnknownConcept) + assert sheerka.recognize_error(context, __type="UnknownConcept") + assert sheerka.recognize_error(context, BuiltinConcepts.UNKNOWN_CONCEPT) + assert sheerka.recognize_error(context, UnknownConcept) + assert sheerka.recognize_error(context, "UnknownConcept") + assert sheerka.recognize_error(context, f"self.name == '{BuiltinConcepts.UNKNOWN_CONCEPT}'") + assert sheerka.recognize_error(context, f"get_type(self) == '{BuiltinConcepts.UNKNOWN_CONCEPT}'") + + def test_i_can_recognized_error_when_error_is_a_error_ErrorObj(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaErrorManager.NAME] + + simple_error = sheerka.new(BuiltinConcepts.ERROR, body=NoConditionFound()) + ret1 = sheerka.ret("Test1", False, simple_error) + context.add_values(return_values=[ret1]) + service.on_user_input_evaluated(context) + + assert sheerka.recognize_error(context, __type=NoConditionFound) + assert sheerka.recognize_error(context, __type="NoConditionFound") + + def test_i_cannot_recognize_unknown_concept_when_no_error(self): + sheerka, context = self.init_test().unpack() + + res = sheerka.has_unknown_concepts(context) + assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) + + def test_i_can_manage_direct_unknown_concept(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaErrorManager.NAME] + exact_concept_parser = ExactConceptParser() + + # a simple unknown concept + ret = exact_concept_parser.parse(context, ParserInput("foo bar")) + context.add_values(return_values=[ret]) + service.on_user_input_evaluated(context) + + assert sheerka.has_unknown_concepts(context) + assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == "foo bar" + + def test_i_can_manage_multiple_direct_unknown_concept(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaErrorManager.NAME] + exact_concept_parser = ExactConceptParser() + + # a simple unknown concept + ret1 = exact_concept_parser.parse(context, ParserInput("foo")) + ret2 = exact_concept_parser.parse(context, ParserInput("bar")) + context.add_values(return_values=[ret1, ret2]) + service.on_user_input_evaluated(context) + + assert sheerka.has_unknown_concepts(context) + assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == {"foo", "bar"} + + @pytest.mark.parametrize("source, expected", [ + ("foo bar", "foo"), + ("bar foo", "foo"), + ("foo bar baz", {"foo", "baz"}), + ("my foo bar my baz", {"my foo", "my baz"}), + ]) + def test_i_can_manage_unknown_concept_when_parts_are_recognized(self, source, expected): + sheerka, context, foo = self.init_concepts("bar") + service = sheerka.services[SheerkaErrorManager.NAME] + exact_concept_parser = ExactConceptParser() + + ret1 = exact_concept_parser.parse(context, ParserInput(source)) + context.add_values(return_values=[ret1]) + service.on_user_input_evaluated(context) + + assert sheerka.has_unknown_concepts(context) + assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == expected + + def test_i_can_manage_multiple_unknown_concept_when_parts_are_recognized(self): + sheerka, context, foo = self.init_concepts("bar") + service = sheerka.services[SheerkaErrorManager.NAME] + exact_concept_parser = ExactConceptParser() + + ret1 = exact_concept_parser.parse(context, ParserInput("foo bar")) + ret2 = exact_concept_parser.parse(context, ParserInput("my foo bar my baz")) + context.add_values(return_values=[ret1, ret2]) + service.on_user_input_evaluated(context) + + assert sheerka.has_unknown_concepts(context) + assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == {"foo", "my foo", "my baz"} + + def test_i_can_manage_unknown_concept_from_name_error(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaErrorManager.NAME] + python_parser = PythonParser() + python_evaluator = PythonEvaluator() + + # + ret = python_parser.parse(context, ParserInput("foo")) + ret = python_evaluator.eval(context, ret) + context.add_values(return_values=[ret]) + service.on_user_input_evaluated(context) + + assert sheerka.has_unknown_concepts(context) + assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == "foo" + + def test_i_can_manage_unknown_concept_from_bnf_error(self): + sheerka, context, quantify_x = self.init_concepts( + Concept("quantify x", definition="('one' | 'two')=unit x").def_var("x"), + create_new=True + ) + service = sheerka.services[SheerkaErrorManager.NAME] + bnf_node_parser = BnfNodeParser() + + # a simple unknown concept + ret = bnf_node_parser.parse(context, ParserInput("one foo")) + + context.add_values(return_values=[ret]) + service.on_user_input_evaluated(context) + + assert sheerka.has_unknown_concepts(context) + assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == "foo" + + +class TestFileBasedSheerkaErrorManger(TestUsingFileBasedSheerka): + def test_i_can_remember_last_error(self): + sheerka = self.get_sheerka() + context = self.get_context(message="TestingErrorManagement::get_last_errors()") + service = sheerka.services[SheerkaErrorManager.NAME] + + simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo")) + ret1 = sheerka.ret("Test1", False, simple_error) + + sub_error_is_an_exception = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=NotImplementedError()) + ret2 = sheerka.ret("Test2", False, sub_error_is_an_exception) + + context.add_values(return_values=[ret1, ret2]) + service.on_user_input_evaluated(context) + new_context = self.get_context(sheerka) + explanations = sheerka.get_last_errors(new_context) + self.commit(context) + + assert sheerka.isinstance(explanations, BuiltinConcepts.EXPLANATION) + assert len(explanations.body) == 3 + assert service.last_detected_errors_event_id == context.event.get_digest() + assert service.last_detected_errors_event_message == context.event.message + assert service.current_errors_event_id == context.event.get_digest() + assert service.current_errors_event_message == context.event.message + + sheerka = self.new_sheerka_instance() + service = sheerka.services[SheerkaErrorManager.NAME] + assert service.last_detected_errors_event_id == context.event.get_digest() + assert service.last_detected_errors_event_message == context.event.message + assert service.current_errors_event_id == context.event.get_digest() + assert service.current_errors_event_message == context.event.message + + self.reset_hard_test_env() diff --git a/tests/core/test_SheerkaHistoryManager.py b/tests/core/test_SheerkaHistoryManager.py index f94ec53..32e42f2 100644 --- a/tests/core/test_SheerkaHistoryManager.py +++ b/tests/core/test_SheerkaHistoryManager.py @@ -21,7 +21,9 @@ class TestSheerkaHistoryManager(TestUsingFileBasedSheerka): sheerka.evaluate_user_input("five") h = list(service.history(-1)) # all - assert h == [ + transformed_h = [hist(item.event.message, item.status) for item in h] + + assert transformed_h == [ hist("five", True), hist("def concept five as 5", True), hist("four", True), @@ -37,13 +39,15 @@ class TestSheerkaHistoryManager(TestUsingFileBasedSheerka): ] h = list(service.history(2)) - assert h == [ + transformed_h = [hist(item.event.message, item.status) for item in h] + assert transformed_h == [ hist("five", True), hist("def concept five as 5", True) ] h = list(service.history(2, 2)) - assert h == [ + transformed_h = [hist(item.event.message, item.status) for item in h] + assert transformed_h == [ hist("four", True), hist("def concept four as 4", True), ] diff --git a/tests/core/test_SheerkaMemory.py b/tests/core/test_SheerkaMemory.py index 82a1d81..fca6b32 100644 --- a/tests/core/test_SheerkaMemory.py +++ b/tests/core/test_SheerkaMemory.py @@ -3,11 +3,10 @@ from core.builtin_helpers import ensure_evaluated from core.concept import Concept from core.global_symbols import NotFound from core.sheerka.ExecutionContext import ExecutionContext -from core.sheerka.services.SheerkaMemory import SheerkaMemory, MemoryObject - +from core.sheerka.services.SheerkaMemory import MemoryObject, SheerkaMemory from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import CV, compare_with_test_object, CB +from tests.parsers.parsers_utils import CB, CV, compare_with_test_object class TestSheerkaMemory(TestUsingMemoryBasedSheerka): @@ -92,6 +91,33 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka): foo)} assert id(sheerka.get_from_memory(context, "a").obj) == id(foo) + # another object is appended + bar = Concept("bar") + sheerka.add_to_memory(context, "a", bar) + assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == { + "a": [ + MemoryObject(context.event.get_digest(), context.event.date.timestamp(), foo), + MemoryObject(context.event.get_digest(), context.event.date.timestamp(), bar) + ]} + + def test_i_can_set_and_retrieve_from_memory(self): + sheerka, context = self.init_test().unpack() + + assert sheerka.get_from_memory(context, "a") is NotFound + + foo = Concept("foo") + sheerka.set_in_memory(context, "a", foo) + assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(), + context.event.date.timestamp(), + foo)} + assert id(sheerka.get_from_memory(context, "a").obj) == id(foo) + + # another object replace the first one + sheerka.set_in_memory(context, "a", "foo") + assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(), + context.event.date.timestamp(), + "foo")} + def test_i_can_use_memory_with_a_string(self): sheerka, context = self.init_test().unpack() diff --git a/tests/core/test_SheerkaQueryManager.py b/tests/core/test_SheerkaQueryManager.py index 6059db7..ed43ad3 100644 --- a/tests/core/test_SheerkaQueryManager.py +++ b/tests/core/test_SheerkaQueryManager.py @@ -2,6 +2,7 @@ from dataclasses import dataclass import pytest +from core.builtin_concepts import NotFoundConcept from core.builtin_concepts_ids import BuiltinConcepts from core.concept import Concept from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -143,6 +144,28 @@ class TestSheerkaQueryManager(TestUsingMemoryBasedSheerka): res = sheerka.filter_objects(context, lst, mapping=lambda o: o.prop1, predicate="self is a concept") assert res == [lst[1], lst[3]] + def test_i_can_filter_objects_by_type(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", "a12"), Concept("foo"), NotImplementedError(), NotFoundConcept()] + + assert sheerka.filter_objects(context, lst, __type="A") == [lst[0]] + assert sheerka.filter_objects(context, lst, __type="foo") == [lst[1]] + + assert sheerka.filter_objects(context, lst, __type="NotImplementedError") == [lst[2]] + assert sheerka.filter_objects(context, lst, __type=NotImplementedError) == [lst[2]] + + assert sheerka.filter_objects(context, lst, __type=BuiltinConcepts.NOT_FOUND) == [lst[3]] + assert sheerka.filter_objects(context, lst, __type=NotFoundConcept) == [lst[3]] + assert sheerka.filter_objects(context, lst, __type="NotFoundConcept") == [lst[3]] + + def test_empty_list_is_returned_when_nothing_is_found(self): + sheerka, context = self.init_test().unpack() + lst = [A("a11", "a12")] + + assert sheerka.filter_objects(context, lst, __type="foo") == [] + assert sheerka.filter_objects(context, [], __type="foo") == [] + assert sheerka.filter_objects(context, lst, predicate="self.name == 'bar'") == [] + def test_i_must_select_object_property_using_string(self): sheerka, context = self.init_test().unpack() diff --git a/tests/core/test_SheerkaRuleManager.py b/tests/core/test_SheerkaRuleManager.py index c9a3ea4..267ca61 100644 --- a/tests/core/test_SheerkaRuleManager.py +++ b/tests/core/test_SheerkaRuleManager.py @@ -345,23 +345,23 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): ("not __ret.status", "__ret", [NEGCOND("#__x_00__|__name__|'__ret.status'")]), ("__ret and not __ret.status", "__ret", ["#__x_00__|__name__|'__ret'", NEGCOND("#__x_01__|__name__|'__ret.status'")]), - ("not recognize(__ret.body, hello sheerka)", "__ret", ["#__x_00__|__name__|'__ret'", - NCCOND(["#__x_00__|body|#__x_01__", - "#__x_01__|__is_concept__|True", - "#__x_01__|key|'hello __var__0'", - "#__x_01__|a|'__sheerka__'"])]), ("__ret and not __error", "__ret", ["#__x_00__|__name__|'__ret'", NEGCOND("#__x_01__|__name__|'__error'")]), - ("not recognize(self, hello sheerka)", "__ret", ["#__x_00__|__name__|'self'", - NCCOND(["#__x_00__|__is_concept__|True", - "#__x_00__|key|'hello __var__0'", - "#__x_00__|a|'__sheerka__'"])]), + # ("not recognize(__ret.body, hello sheerka)", "__ret", ["#__x_00__|__name__|'__ret'", + # NCCOND(["#__x_00__|body|#__x_01__", + # "#__x_01__|__is_concept__|True", + # "#__x_01__|key|'hello __var__0'", + # "#__x_01__|a|'__sheerka__'"])]), + # ("not recognize(self, hello sheerka)", "__ret", ["#__x_00__|__name__|'self'", + # NCCOND(["#__x_00__|__is_concept__|True", + # "#__x_00__|key|'hello __var__0'", + # "#__x_00__|a|'__sheerka__'"])]), ]) def test_i_can_get_rete_using_not(self, expression, bag_key, expected): sheerka, context, greetings, foo = self.init_test().with_concepts( Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), Concept("foo"), ).unpack() - parser = ExpressionParser(old_style=True) + parser = ExpressionParser() expected_conditions = get_rete_conditions(*expected) error_sink = ErrorSink() diff --git a/tests/core/test_SheerkaRuleManagerRulesCompilation.py b/tests/core/test_SheerkaRuleManagerRulesCompilation.py index eab6979..c343dbe 100644 --- a/tests/core/test_SheerkaRuleManagerRulesCompilation.py +++ b/tests/core/test_SheerkaRuleManagerRulesCompilation.py @@ -25,7 +25,7 @@ class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka): @staticmethod def get_conditions(context, expression): - parser = ExpressionParser(old_style=True) + parser = ExpressionParser() error_sink = ErrorSink() parser_input = ParserInput(expression) parser.reset_parser_input(parser_input, error_sink) @@ -927,7 +927,7 @@ class TestSheerkaRuleManagerRulesCompilationRecognizeConcept(BaseTestSheerkaRule Concept("my best friend"), create_new=True ).unpack() - parser = ExpressionParser(old_style=True) + parser = ExpressionParser() expected = get_rete_conditions(*expected_as_list_of_str) error_sink = ErrorSink() diff --git a/tests/core/test_sheerka.py b/tests/core/test_sheerka.py index f7d60c7..4908b26 100644 --- a/tests/core/test_sheerka.py +++ b/tests/core/test_sheerka.py @@ -3,14 +3,13 @@ import os import pytest from conftest import SHEERKA_TEST_FOLDER -from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UnknownConcept, UserInputConcept +from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept from core.builtin_concepts_ids import AllBuiltinConcepts from core.concept import Concept, ConceptParts, PROPERTIES_TO_SERIALIZE, get_concept_attrs from core.global_symbols import NotInit from core.sheerka.Sheerka import BASE_NODE_PARSER_CLASS, Sheerka -from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, ValueNotFound +from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from core.tokenizer import Token, TokenKind -from parsers.PythonParser import PythonErrorNode from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -44,21 +43,25 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): def test_i_can_list_builtin_concepts(self): sheerka = self.get_sheerka() - builtins = list(sheerka.get_builtins_classes_as_dict()) + builtins, _ = sheerka.get_builtins_classes_as_dict() - assert str(BuiltinConcepts.ERROR) in builtins - assert str(BuiltinConcepts.RETURN_VALUE) in builtins + assert BuiltinConcepts.ERROR in builtins + assert BuiltinConcepts.RETURN_VALUE in builtins def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string(self): """ Checks that a concept can be found its name even when there are variables in the name (ex 'hello + a' or 'a + b' ) + Also returns the mapping between concept name and specific class name :return: """ sheerka = self.get_sheerka() - for key in sheerka.get_builtins_classes_as_dict(): + builtins_by_key, builtins_by_class_name = sheerka.get_builtins_classes_as_dict() + for key in builtins_by_key: assert sheerka.get_by_key(key) is not None - assert sheerka.get_by_key(str(key)) is not None + + for k, v in builtins_by_class_name.items(): + assert type(sheerka.get_by_key(v)).__name__ == k def test_i_cannot_get_when_key_is_none(self): sheerka = self.get_sheerka() @@ -98,7 +101,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): assert ret.value == "value" # check the others - for key, concept_class in sheerka.get_builtins_classes_as_dict().items(): + builtins_by_key, _ = sheerka.get_builtins_classes_as_dict() + for key, concept_class in builtins_by_key.items(): assert isinstance(sheerka.get_by_key(key), concept_class) def test_i_can_instantiate_a_builtin_concept_when_no_specific_class(self): @@ -426,151 +430,6 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED) - @pytest.mark.parametrize("obj, expected", [ - ("a string", []), - (True, []), - (False, []), - (Concept("foo"), []), - (Concept("foo", body=False).auto_init(), []), - (UnknownConcept(), [UnknownConcept()]), - (Concept("foo", body=UnknownConcept()).auto_init(), [UnknownConcept()]), - (PythonErrorNode("msg", None), [PythonErrorNode("msg", None)]) - ]) - def test_i_can_get_error_for_simple_objects(self, obj, expected): - sheerka, context = self.init_test().unpack() - - assert sheerka.get_errors(context, obj) == expected - - def test_i_can_get_error_when_builtin_concept_in_error(self): - sheerka, context = self.init_test().unpack() - - obj = sheerka.new(BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED) - assert sheerka.get_errors(context, obj) == [obj] - - def test_i_can_get_error_when_return_value(self): - sheerka, context = self.init_test().unpack() - - error = sheerka.err("an error") - ret_val = ReturnValueConcept("Test", False, sheerka.err("an error")) - assert sheerka.get_errors(context, ret_val) == [error] - - def test_i_can_get_inner_error(self): - sheerka, context = self.init_test().unpack() - - error = sheerka.err("an error") - ret_val = ReturnValueConcept("Test", False, sheerka.err("an error")) - assert sheerka.get_errors(context, ret_val) == [error] - - def test_i_can_get_error_when_embedded_errors(self): - sheerka, context = self.init_test().unpack() - - concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) - unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) - not_an_error = sheerka.new(BuiltinConcepts.AUTO_EVAL) - error = sheerka.err([concept_eval_error, unknown_concept, not_an_error]) - - ret_val = ReturnValueConcept("Test", False, error) - errors_found = sheerka.get_errors(context, ret_val) - - assert errors_found == [error, concept_eval_error, unknown_concept] - - def test_i_can_get_error_from_list(self): - sheerka, context = self.init_test().unpack() - - concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) - unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) - not_an_error = sheerka.new(BuiltinConcepts.AUTO_EVAL) - error = sheerka.err([concept_eval_error, unknown_concept, not_an_error]) - ret_val_1 = ReturnValueConcept("Test", False, error) - - python_error = PythonErrorNode("msg", Exception()) - value_not_found = ValueNotFound("item", "value") - multiple_error = sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=[python_error, value_not_found]) - ret_val_2 = ReturnValueConcept("Test", False, multiple_error) - - errors_found = sheerka.get_errors(context, [ret_val_1, ret_val_2]) - - assert errors_found == [error, concept_eval_error, unknown_concept, - multiple_error, python_error, value_not_found] - - def test_i_can_filter_error_by_concept_key(self): - sheerka, context = self.init_test().unpack() - - concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) - unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) - python_error = PythonErrorNode("msg", Exception()) - error = sheerka.err([concept_eval_error, unknown_concept, python_error]) - ret_val = ReturnValueConcept("Test", False, error) - - errors_found = sheerka.get_errors(context, ret_val, __type=BuiltinConcepts.CONCEPT_EVAL_ERROR) - assert errors_found == [concept_eval_error] - - def test_i_can_filter_error_by_class_name(self): - sheerka, context = self.init_test().unpack() - - concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) - unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) - python_error = PythonErrorNode("msg", Exception()) - error = sheerka.err([concept_eval_error, unknown_concept, python_error]) - ret_val = ReturnValueConcept("Test", False, error) - - errors_found = sheerka.get_errors(context, ret_val, __type="PythonErrorNode") - assert errors_found == [python_error] - - def test_i_can_filter_error_by_concept_attribute(self): - sheerka, context = self.init_test().unpack() - - concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) - unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref") - python_error = PythonErrorNode("msg", Exception()) - error = sheerka.err([concept_eval_error, unknown_concept, python_error]) - ret_val = ReturnValueConcept("Test", False, error) - - errors_found = sheerka.get_errors(context, ret_val, concept_ref="a_concept_ref") - assert errors_found == [unknown_concept] - - def test_i_can_filter_error_by_class_attribute(self): - sheerka, context = self.init_test().unpack() - - concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) - unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref") - python_error = PythonErrorNode("error source", Exception()) - error = sheerka.err([concept_eval_error, unknown_concept, python_error]) - ret_val = ReturnValueConcept("Test", False, error) - - errors_found = sheerka.get_errors(context, ret_val, source="error source") - assert errors_found == [python_error] - - def test_i_can_filter_error_by_exception_name(self): - sheerka, context = self.init_test().unpack() - - from evaluators.PythonEvaluator import PythonEvalError - name_error = NameError("foo") - python_eval_error = PythonEvalError(name_error, "foo", None, None) - error = sheerka.err([python_eval_error]) - ret_val = ReturnValueConcept("Test", False, error) - - errors_found = sheerka.get_errors(context, ret_val, __type="NameError") - assert errors_found == [name_error] - - def test_i_can_filter_error_on_multiple_criteria(self): - sheerka, context = self.init_test().unpack() - - concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR) - unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref") - value_not_found = ValueNotFound("an_item", "a value") - error = sheerka.err([concept_eval_error, unknown_concept, value_not_found]) - ret_val = ReturnValueConcept("Test", False, error) - - errors_found = sheerka.get_errors(context, ret_val, __type="ValueNotFound", item="an_item", value="a value") - assert errors_found == [value_not_found] - - def test_i_cannot_get_error_when_return_value_s_status_is_true(self): - sheerka, context = self.init_test().unpack() - - ret_val = ReturnValueConcept("Test", True, sheerka.err("an error")) - assert sheerka.get_errors(context, ret_val) == [] - def test_i_can_add_and_remove_global_context_hints(self): sheerka, context = self.init_test().unpack() @@ -592,7 +451,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): ).unpack() res = sheerka.evaluate_user_input("eval foo", "testing_user") - assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONDITION_FAILED) # sanity check + assert sheerka.isinstance(res[0].value.value, BuiltinConcepts.CONDITION_FAILED) # sanity check sheerka.add_to_context(BuiltinConcepts.EVAL_GLOBAL_TRUTH_REQUESTED) res = sheerka.evaluate_user_input("eval foo", "testing_user") @@ -600,7 +459,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): sheerka.remove_fom_context(BuiltinConcepts.EVAL_GLOBAL_TRUTH_REQUESTED) res = sheerka.evaluate_user_input("eval foo", "testing_user") - assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONDITION_FAILED) + assert sheerka.isinstance(res[0].value.value, BuiltinConcepts.CONDITION_FAILED) class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka): diff --git a/tests/core/test_sheerka_ontology.py b/tests/core/test_sheerka_ontology.py index 5fa9093..032f89c 100644 --- a/tests/core/test_sheerka_ontology.py +++ b/tests/core/test_sheerka_ontology.py @@ -1316,11 +1316,11 @@ class TestSheerkaOntology(TestUsingMemoryBasedSheerka): sheerka.create_new_rule(context, Rule(ACTION_TYPE_EXEC, "rule3", "id3.attr3 == 'value'", "True")) sheerka.pop_ontology(context) - assert rules_by_ontology_from_cache() == {'#unit_test#': {'10'}} - assert ontologies_from_cache() == {'10': '#unit_test#'} + assert rules_by_ontology_from_cache() == {'#unit_test#': {'11'}} + assert ontologies_from_cache() == {'11': '#unit_test#'} # check that the 'rule is deleted' events are raised - assert events_raised == {'11', '12'} + assert events_raised == {'12', '13'} # def test_i_can_list_by_key_when_dictionaries(self): # sheerka = self.get_sheerka(cache_only=False) @@ -1664,6 +1664,6 @@ class TestSheerkaOntologyWithFileBasedSheerka(TestUsingFileBasedSheerka): rules_from_db = sheerka.om.self_cache_manager.sdp.get(SheerkaOntologyManager.RULES_BY_ONTOLOGY_ENTRY) del rules_from_db["__default__"] assert rules_from_db == { - '#unit_test#': {'10'}, - 'another ontology': {'13'}, - 'new ontology': {'12'}} + '#unit_test#': {'11'}, + 'another ontology': {'14'}, + 'new ontology': {'13'}} diff --git a/tests/evaluators/test_OneErrorEvaluator.py b/tests/evaluators/test_OneErrorEvaluator.py index 519bb53..1b82588 100644 --- a/tests/evaluators/test_OneErrorEvaluator.py +++ b/tests/evaluators/test_OneErrorEvaluator.py @@ -49,7 +49,8 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka): res = evaluator.eval(context, return_values) assert not res.status - assert res.body == "evaluators.one error" + assert context.sheerka.isinstance(res.body, BuiltinConcepts.RETURN_VALUE) + assert res.body.body == "evaluators.one error" assert len(res.parents) == 4 def test_unwanted_return_values_are_not_eaten(self): @@ -71,7 +72,8 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka): res = evaluator.eval(context, return_values) assert not res.status - assert res.body == "evaluators.one error" + assert context.sheerka.isinstance(res.body, BuiltinConcepts.RETURN_VALUE) + assert res.body.body == "evaluators.one error" assert len(res.parents) == 4 assert a_successful_concept not in res.parents @@ -89,7 +91,7 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka): assert evaluator.matches(context, return_values) res = evaluator.eval(context, return_values) assert not res.status - assert res.body == false_1.body + assert res.body.body == false_1.body assert res.parents == return_values evaluator.reset() @@ -97,7 +99,7 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka): assert evaluator.matches(context, return_values) res = evaluator.eval(context, return_values) assert not res.status - assert res.body == filtered + assert res.body.body == filtered assert res.parents == return_values evaluator.reset() @@ -105,5 +107,5 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka): assert evaluator.matches(context, return_values) res = evaluator.eval(context, return_values) assert not res.status - assert res.body == false_1.body + assert res.body.body == false_1.body assert res.parents == return_values diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py index 933a9b4..a7069df 100644 --- a/tests/evaluators/test_PythonEvaluator.py +++ b/tests/evaluators/test_PythonEvaluator.py @@ -11,7 +11,7 @@ from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode -from parsers.FunctionParserOld import FunctionParserOld +from parsers.FunctionParser import FunctionParser from parsers.PythonParser import PythonNode, PythonParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import CB, compare_with_test_object @@ -318,7 +318,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): context = self.get_context() context.add_to_short_term_memory("get_obj_name", get_obj_name) - parsed = FunctionParserOld().parse(context, ParserInput("get_obj_name(r:|1:)")) + parsed = FunctionParser().parse(context, ParserInput("get_obj_name(r:|1:)")) python_evaluator = PythonEvaluator() evaluated = python_evaluator.eval(context, parsed) @@ -344,7 +344,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): context = self.get_context() context.add_to_short_term_memory("return_return_value", return_return_value) - parsed = FunctionParserOld().parse(context, ParserInput(method)) + parsed = FunctionParser().parse(context, ParserInput(method)) python_evaluator = PythonEvaluator() evaluated = python_evaluator.eval(context, parsed) ret_val = return_return_value(expected_status) @@ -371,9 +371,9 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("parser, value", [ (PythonParser(), "3"), - (FunctionParserOld(), "3"), + (FunctionParser(), "3"), (PythonParser(), 3), - (FunctionParserOld(), 3), + (FunctionParser(), 3), ]) def test_i_can_eval_unresolved_rules(self, parser, value): context = self.get_context() diff --git a/tests/evaluators/test_ValidateConceptEvaluator.py b/tests/evaluators/test_ValidateConceptEvaluator.py index 9daaae6..342c27e 100644 --- a/tests/evaluators/test_ValidateConceptEvaluator.py +++ b/tests/evaluators/test_ValidateConceptEvaluator.py @@ -9,6 +9,7 @@ from evaluators.ValidateConceptEvaluator import ValidateConceptEvaluator from parsers.BaseNodeParser import ConceptNode from parsers.BaseParser import BaseParser from parsers.BnfNodeParser import BnfNodeParser +from parsers.ExactConceptParser import ExactConceptParser from parsers.SyaNodeParser import SyaNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.evaluators.EvaluatorTestsUtils import p_ret_val, pr_ret_val, ret_val @@ -185,3 +186,15 @@ class TestValidateConceptEvaluator(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED) assert res.body.body == ret_val.body.body + + def test_i_can_eval_exact_parser_concepts(self): + sheerka, context, explain_x = self.init_concepts( + Concept("explain x", where="isinstance(x, int)").def_var("x") + ) + evaluator = ValidateConceptEvaluator() + + ret_val = ExactConceptParser().parse(context, ParserInput("explain 1"))[0] + + assert evaluator.matches(context, ret_val) + res = evaluator.eval(context, ret_val) + assert sheerka.isinstance(res.body.body, explain_x) diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index f9ed549..726a04e 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -166,7 +166,7 @@ as: assert len(res) == 1 assert not res[0].status - assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED) + assert sheerka.isinstance(res[0].value.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED) @pytest.mark.parametrize("text", [ "", @@ -740,7 +740,7 @@ as: res = sheerka.evaluate_user_input("eval foobar") # where clause is forced assert len(res) == 1 # error assert not res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED) + assert sheerka.isinstance(res[0].body.value, BuiltinConcepts.CONDITION_FAILED) @pytest.mark.skip("Not ready for that") def test_i_can_manage_missing_variables_from_bnf_parsing(self): @@ -921,15 +921,15 @@ as: res = sheerka.evaluate_user_input("eval foo plus one") assert not res[0].status - assert context.sheerka.isinstance(res[0].body, BuiltinConcepts.CONCEPT_EVAL_ERROR) - assert context.sheerka.isinstance(res[0].body.body, BuiltinConcepts.ERROR) + assert context.sheerka.isinstance(res[0].body.body, BuiltinConcepts.CONCEPT_EVAL_ERROR) + assert context.sheerka.isinstance(res[0].body.body.body, BuiltinConcepts.ERROR) - error0 = res[0].body.body.body[0] + error0 = res[0].body.body.body.body[0] assert isinstance(error0, PythonEvalError) assert isinstance(error0.error, TypeError) assert error0.error.args[0] == "unsupported operand type(s) for +: 'Concept' and 'int'" - error1 = res[0].body.body.body[1] + error1 = res[0].body.body.body.body[1] assert isinstance(error1, PythonEvalError) assert isinstance(error1.error, TypeError) assert error1.error.args[0] == 'can only concatenate str (not "int") to str' diff --git a/tests/non_reg/test_sheerka_non_reg2.py b/tests/non_reg/test_sheerka_non_reg2.py index 26e3fa7..1dc6f26 100644 --- a/tests/non_reg/test_sheerka_non_reg2.py +++ b/tests/non_reg/test_sheerka_non_reg2.py @@ -123,7 +123,7 @@ class TestSheerkaNonRegMemory2(TestUsingMemoryBasedSheerka): assert len(res) == 1 assert not res[0].status - assert isinstance(res[0].body, ConceptEvalError) + assert isinstance(res[0].body.body, ConceptEvalError) def test_74_keyword_parameters_are_no_longer_recognized_when_a_concept_that_redefines_equality_is_created(self): init = [ diff --git a/tests/non_reg/test_sheerka_non_reg_out.py b/tests/non_reg/test_sheerka_non_reg_out.py index 52954e0..aeb4d17 100644 --- a/tests/non_reg/test_sheerka_non_reg_out.py +++ b/tests/non_reg/test_sheerka_non_reg_out.py @@ -1,3 +1,7 @@ +from core.builtin_concepts_ids import BuiltinConcepts +from core.concept import Concept +from core.sheerka.services.SheerkaErrorManager import SheerkaErrorManager +from core.utils import no_color_str from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -200,4 +204,39 @@ two: (1002)two assert captured.out == """DebugItem(type=vars, setting=Sya.parsers.*, context_id=45, debug_id=None, context_children=False, debug_children=False (enabled=True)) DebugItem(type=concepts, setting=c:|1015.*.*, context_id=13, debug_id=None, context_children=True, debug_children=False (enabled=True)) DebugItem(type=rules, setting=Out.*.*, context_id=None, debug_id=None, context_children=False, debug_children=False (enabled=True)) +""" + + def test_i_can_display_errors(self, capsys): + sheerka = self.get_sheerka() + context = self.get_context(message="xxxTESTxxx::get_last_errors()") + service = sheerka.services[SheerkaErrorManager.NAME] + + simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo")) + ret1 = sheerka.ret("Test1", False, simple_error) + + multiple_level_errors1 = sheerka.err("Err1") + multiple_level_errors2 = sheerka.err(multiple_level_errors1) + multiple_level_errors3 = sheerka.err(multiple_level_errors2) + ret2 = sheerka.ret("Test5", False, multiple_level_errors3) + + context.add_values(return_values=[ret1, ret2]) + service.on_user_input_evaluated(context) + + sheerka.enable_process_return_values = True + sheerka.evaluate_user_input("get_last_errors(level=0)") + + captured = capsys.readouterr() + colorless_captured = no_color_str(captured.out) + assert colorless_captured == """xxx : xxxTESTxxx::get_last_errors() +[ 0] Test1 (47)__UNKNOWN_CONCEPT: (None)foo +[ 1] Test5 (46)__ERROR: (46)__ERROR: (46)__ERROR: Err1 +""" + + sheerka.evaluate_user_input("get_last_errors(id=1, depth=3)") + captured = capsys.readouterr() + colorless_captured = no_color_str(captured.out) + assert colorless_captured == """xxx : xxxTESTxxx::get_last_errors() +[ 1] Test5 (46)__ERROR: (46)__ERROR: (46)__ERROR: Err1 +[ 2] Test5 (46)__ERROR: (46)__ERROR: Err1 +[ 3] Test5 (46)__ERROR: Err1 """ diff --git a/tests/parsers/parsers_utils.py b/tests/parsers/parsers_utils.py index df0f1e8..7797e5d 100644 --- a/tests/parsers/parsers_utils.py +++ b/tests/parsers/parsers_utils.py @@ -10,12 +10,10 @@ from core.tokenizer import Token, TokenKind, Tokenizer from core.utils import get_text_from_tokens, str_concept, tokens_index from parsers.BaseExpressionParser import AndNode, BinaryNode, ComparisonNode, ComparisonType, Comprehension, \ FunctionNode, \ - FunctionParameter, \ ListComprehensionNode, ListNode, NameExprNode, \ NotNode, OrNode, SequenceNode, VariableNode, t_comma from parsers.BaseNodeParser import ConceptNode, RuleNode, SourceCodeNode, SourceCodeWithConceptNode, \ UnrecognizedTokensNode -from parsers.FunctionParserOld import FunctionNodeOld from parsers.PythonParser import PythonNode from sheerkapython.python_wrapper import sheerka_globals from sheerkarete.common import V @@ -387,99 +385,6 @@ class LC(ExprTestObj): # for List Comprehension node return ListComprehensionNode(start, end, full_text_as_tokens[start: end + 1], element, comprehensions) -class FNOld(ExprTestObj): - """ - Test class only - It matches with FunctionNodeOld but with less constraints - - Thereby, - FNOld("first", "last", ["param1," ...]) can be compared to - FunctionNodeOld(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")]) - - Note that FunctionParameter can easily be defined with a single string - * "param" -> FunctionParameter(NameExprNode("param"), None) - * "param, " -> FunctionParameter(NameExprNode("param"), NameExprNode(", ")) - For more complicated situations, you can use a tuple (value, sep) to define the value part and the separator part - """ - - def __init__(self, first, last, parameters): - self.first = first - self.last = last - self.parameters = [] - for param in parameters: - if isinstance(param, tuple): - self.parameters.append(param) - elif isinstance(param, str) and (pos := param.find(",")) != -1: - self.parameters.append((param[:pos], param[pos:])) - else: - self.parameters.append((param, None)) - - def __repr__(self): - res = self.first - for param in self.parameters: - if param[1]: - res += f"{param[0]}{param[1]} " - else: - res += f"{param[0]}" - return res + self.last - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, FNOld): - return self.first == other.first and self.last == other.last and self.parameters == other.parameters - - return False - - def __hash__(self): - return hash((self.first, self.last, self.parameters)) - - def transform_real_obj(self, other, get_test_obj_delegate): - if isinstance(other, FNOld): - return other - - if isinstance(other, FunctionNodeOld): - params = [] - for self_parameter, other_parameter in zip(self.parameters, other.parameters): - if isinstance(self_parameter[0], str): - value = other_parameter.value.value - else: - value = get_test_obj_delegate(other_parameter.value, self_parameter[0]) - sep = other_parameter.separator.value if other_parameter.separator else None - params.append((value, sep)) - - return FNOld(other.first.value, other.last.value, params) - - raise Exception(f"Expecting FunctionNodeOld but received {other=}") - - def get_expr_node(self, full_text_as_tokens, default_expr_obj): - start, end = self.get_pos_from_source(self.first, full_text_as_tokens) - first = NameExprNode(start, end, full_text_as_tokens[start: end + 1]) - start, end = self.get_pos_from_source(self.last, full_text_as_tokens) - last = NameExprNode(start, end, full_text_as_tokens[start: end + 1]) - parameters = [] - for param_value, sep in self.parameters: - if isinstance(param_value, str): - start, end = self.get_pos_from_source(param_value, full_text_as_tokens) - param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1]) - else: - param_as_expr_node = param_value.get_expr_node(full_text_as_tokens, default_expr_obj) - - if sep: - sep_tokens = Tokenizer(sep, yield_eof=False) - start = param_as_expr_node.end + 1 - end = start + len(list(sep_tokens)) - 1 - sep_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1]) - else: - sep_as_expr_node = None - - parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node)) - - start, end = first.start, last.end - return FunctionNodeOld(start, end, full_text_as_tokens[start: end + 1], first, last, parameters) - - class HelperWithPos: def __init__(self, start=None, end=None): self.start = start diff --git a/tests/parsers/test_BnfNodeParser.py b/tests/parsers/test_BnfNodeParser.py index 85116de..1e727e3 100644 --- a/tests/parsers/test_BnfNodeParser.py +++ b/tests/parsers/test_BnfNodeParser.py @@ -9,6 +9,7 @@ from core.global_symbols import NotInit from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.SheerkaIsAManager import SheerkaIsAManager +from core.tokenizer import Tokenizer from parsers.BaseNodeParser import NoMatchingTokenError from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor, BnfNodeParser, ConceptExpression, Match, NonTerminalNode, \ @@ -1075,6 +1076,44 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expected = [CNC("foo", "one onyx", x=CC("one"))] self.validate_get_concepts_sequences(my_map, text, expected) + def test_i_can_match_regex_when_sub_parser_input(self): + my_map = { + "foo": self.bnf_concept("foo", RegExMatch("[a-f0-9]{4}")), + } + text = "begin 0af0 end" + parser_input = ParserInput(text).reset() + sub_parser = parser_input.sub_part(2, 3) + + sheerka, context, parser = self.init_parser(my_map) + + res = parser.parse(context, sub_parser) + + assert res.status + concept_nodes = res.body.body + + expected = [CN("foo", "0af0")] + actual, expected = tests.parsers.parsers_utils.prepare_nodes_comparison(my_map, text, concept_nodes, expected) + assert actual == expected + + def test_i_can_match_regex_when_from_tokens(self): + my_map = { + "foo": self.bnf_concept("foo", RegExMatch("[a-f0-9]{4}")), + } + text = "begin 0af0 end" + tokens = list(Tokenizer(text)) + parser_input = ParserInput(None, tokens[2:4]).reset() + + sheerka, context, parser = self.init_parser(my_map) + + res = parser.parse(context, parser_input) + + assert res.status + concept_nodes = res.body.body + + expected = [CN("foo", "0af0")] + actual, expected = tests.parsers.parsers_utils.prepare_nodes_comparison(my_map, "0af0", concept_nodes, expected) + assert actual == expected + def test_i_can_reuse_the_same_variable(self): # in this test, the variable appears several times, but only once in concept.compiled my_map = { @@ -1831,7 +1870,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) - assert res.body.reason == [NoMatchingTokenError(4)] + assert res.body.reason == [NoMatchingTokenError(4, concept=foo)] @pytest.mark.parametrize("text", [ "one", @@ -1852,7 +1891,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): (Sequence(StrMatch("foo"), VariableExpression("x")), "foo one"), (Sequence(StrMatch("foo"), VariableExpression("x"), StrMatch("bar")), "foo one bar"), ]) - def test_i_cannot_parse_variable_when_unrecognized_nodes(self, bnf, text): + def test_i_cannot_parse_variable_when_unrecognized_nodes(self, bnf, text): sheerka, context, foo = self.init_test().with_concepts( self.bnf_concept("foo", Sequence(VariableExpression("x"), StrMatch("shoe"))) ).unpack() diff --git a/tests/parsers/test_DefConceptParser.py b/tests/parsers/test_DefConceptParser.py index c6b51bd..66141d1 100644 --- a/tests/parsers/test_DefConceptParser.py +++ b/tests/parsers/test_DefConceptParser.py @@ -16,7 +16,7 @@ from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Se from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode, CannotHandleParsingError from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode from parsers.ExpressionParser import ExpressionParser -from parsers.FunctionParserOld import FunctionParserOld +from parsers.FunctionParser import FunctionParser from parsers.PythonParser import PythonParser, PythonNode from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import compute_expected_array, SCWC, compare_with_test_object, CIO @@ -77,7 +77,7 @@ def get_concept_part(part, use_expression=False): status=True, value=ParserResultConcept( source=part.source, - parser=FunctionParserOld(), + parser=FunctionParser(), value=nodes[0], try_parsed=nodes[0])) diff --git a/tests/parsers/test_FunctionParserOld.py b/tests/parsers/test_FunctionParserOld.py deleted file mode 100644 index 496aaa2..0000000 --- a/tests/parsers/test_FunctionParserOld.py +++ /dev/null @@ -1,249 +0,0 @@ -import pytest - -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept -from core.sheerka.services.SheerkaExecute import ParserInput -from parsers.BaseNodeParser import SourceCodeWithConceptNode -from parsers.BaseParser import ErrorSink -from parsers.FunctionParserOld import FunctionParserOld -from parsers.PythonParser import PythonErrorNode -from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import compute_expected_array, SCN, SCWC, CN, UTN, CNC, RN, FNOld, get_test_obj, \ - get_expr_node_from_test_node - -cmap = { - "one": Concept("one"), - "two": Concept("two"), - "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"), - "plus": Concept("a plus b").def_var("a").def_var("b"), -} - - -class TestFunctionParserOld(TestUsingMemoryBasedSheerka): - shared_ontology = None - - @classmethod - def setup_class(cls): - init_test_helper = cls().init_test(cache_only=False, ontology="#TestFunctionParserOld#") - sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack() - for i, concept_name in enumerate(cmap): - cmap[concept_name] = updated[i] - - cls.shared_ontology = sheerka.get_ontology(context) - sheerka.pop_ontology(context) - - def init_parser(self, my_concepts_map=None, **kwargs): - if my_concepts_map is None: - sheerka, context = self.init_test().unpack() - sheerka.add_ontology(context, self.shared_ontology) - else: - sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack() - for i, pair in enumerate(my_concepts_map): - my_concepts_map[pair] = updated[i] - - parser = FunctionParserOld() - return sheerka, context, parser - - def init_parser_with_source(self, source): - sheerka, context, parser = self.init_parser() - error_sink = ErrorSink() - parser_input = ParserInput(source) - parser.reset_parser_input(parser_input, error_sink) - return sheerka, context, parser, parser_input, error_sink - - def test_i_can_detect_empty_expression(self): - sheerka, context, parser = self.init_parser() - res = parser.parse(context, ParserInput("")) - - assert not res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY) - - def test_input_must_be_a_parser_input(self): - sheerka, context, parser = self.init_parser() - parser.parse(context, "not a parser input") is None - - def test_i_cannot_parse_when_not_a_function(self): - sheerka, context, parser = self.init_parser() - res = parser.parse(context, ParserInput("not a function")) - - assert not res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) - - @pytest.mark.parametrize("expression, expected", [ - ("func()", FNOld("func(", ")", [])), - ("concept(one)", FNOld("concept(", ")", ["one"])), - ("func(one)", FNOld("func(", ")", ["one"])), - ("func(a long two, 'three', ;:$*)", FNOld("func(", ")", ["a long two, ", "'three', ", ";:$*"])), - ("func(func1(one), two, func2(func3(), func4(three)))", FNOld("func(", (")", 4), [ - (FNOld("func1(", ")", ["one"]), ", "), - "two, ", - (FNOld("func2(", (")", 3), [ - (FNOld("func3(", (")", 1), []), ", "), - (FNOld("func4(", (")", 2), ["three"]), None), - ]), None) - ])), - ("func(r:|1:)", FNOld("func(", ")", ["r:|1:"])) - ]) - def test_i_can_parse_function(self, expression, expected): - sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression) - expected = get_expr_node_from_test_node(expression, expected) - - parsed = parser.parse_input(context, parser_input, error_sink) - - assert not error_sink.has_error - assert parsed == expected - - @pytest.mark.parametrize("text, expected", [ - ("func()", SCN("func()")), - (" func()", SCN("func()")), - ("func(one)", SCWC("func(", ")", CN("one"))), - ("func(one, unknown, two)", SCWC("func(", ")", CN("one"), ", ", UTN("unknown"), (", ", 1), CN("two"))), - ("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))), - ("func(one plus two, three)", SCWC("func(", ")", CNC("plus", a="one", b="two"), ", ", UTN("three"))), - ("func(func1(one), two)", SCWC("func(", (")", 1), SCWC("func1(", ")", "one"), ", ", "two")) - ]) - def test_i_can_parse(self, text, expected): - sheerka, context, parser = self.init_parser() - resolved_expected = compute_expected_array(cmap, text, [expected])[0] - res = parser.parse(context, ParserInput(text)) - parser_result = res.body - expression = res.body.body - - assert res.status - assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - transformed_expression = get_test_obj(expression, resolved_expected) - assert transformed_expression == resolved_expected - assert expression.python_node is not None - assert expression.return_value is not None - - def test_i_can_parse_when_multiple_results_when_requested(self): - # the previous output was - # [ - # SCWC("func(", ")", "one", ", ", "twenty ", "two"), - # SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two")) - # ] - # But the first one is now filtered out, as it's not a valid python function call - sheerka, context, parser = self.init_parser() - parser.longest_concepts_only = False - text = "func(one, twenty two)" - expected = [SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))] - resolved_expected = compute_expected_array(cmap, text, expected) - - results = parser.parse(context, ParserInput(text)) - - assert len(results) == 2 - - res = results[0] - assert not res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR) - assert len(res.body.body) == 1 - assert (res.body.body[0], PythonErrorNode) - - res = results[1] - parser_result = res.body - expressions = res.body.body - assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - transformed_expressions = get_test_obj(expressions, resolved_expected[0]) - assert transformed_expressions == resolved_expected[0] - - def test_i_can_parse_when_the_parameter_is_not_a_concept(self): - """ - It's not a concept, but it can be a valid short term memory object - :return: - """ - sheerka, context, parser = self.init_parser() - text = "func(unknown_concept)" - - res = parser.parse(context, ParserInput(text)) - expected = [SCWC("func(", ")", "unknown_concept")] - resolved_expected = compute_expected_array(cmap, text, expected) - - assert res.status - parsed = res.body.body - transformed_parsed = get_test_obj([parsed], resolved_expected) - assert transformed_parsed == resolved_expected - - def test_i_can_parse_when_the_concept_is_not_found(self): - """ - We do not check yet if it's a valid concept - If you find a cheap way to do so, simply remove this test - :return: - """ - sheerka, context, parser = self.init_parser() - text = "func(c:|xxx:)" - - res = parser.parse(context, ParserInput(text)) - - assert res.status - - def test_i_can_parse_when_rules(self): - sheerka, context, parser = self.init_parser() - text = "func(r:|1:)" - expected = SCWC("func(", ")", RN("1")) - resolved_expected = compute_expected_array(cmap, text, [expected])[0] - - res = parser.parse(context, ParserInput(text)) - parser_result = res.body - expression = res.body.body - transformed_expression = get_test_obj(expression, resolved_expected) - - assert res.status - assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) - assert transformed_expression == resolved_expected - assert expression.python_node is not None - assert expression.return_value is not None - - def test_i_can_parse_when_the_parameter_is_a_dynamic_concept(self): - sheerka, context, parser = self.init_parser() - - text = "func(ones)" - res = parser.parse(context, ParserInput(text)) - - assert res.status - assert isinstance(res.body.body, SourceCodeWithConceptNode) - assert res.body.body.python_node.source == 'func(__C__ones__1001___PLURAL__C__)' - assert "__C__ones__1001___PLURAL__C__" in res.body.body.python_node.objects - - @pytest.mark.parametrize("text, expected_error_type", [ - ("one", BuiltinConcepts.NOT_FOR_ME), # no function found - ("$*!", BuiltinConcepts.NOT_FOR_ME), # no function found - ("func(", BuiltinConcepts.ERROR), # function found, but incomplete - ("func(one", BuiltinConcepts.ERROR), # function found, but incomplete - ("func(one, two, ", BuiltinConcepts.ERROR), # function found, but incomplete - ("func(one) and func(two)", BuiltinConcepts.ERROR), # to many function - ("one func(one)", BuiltinConcepts.NOT_FOR_ME), # function not found ! (as it is not the first) - ("func(a=b, c)", BuiltinConcepts.ERROR), # function found, but cannot be parsed - ("func(one two)", BuiltinConcepts.ERROR), # function found, but cannot be parsed - ]) - def test_i_cannot_parse(self, text, expected_error_type): - sheerka, context, parser = self.init_parser() - - res = parser.parse(context, ParserInput(text)) - - assert not res.status - assert sheerka.isinstance(res.body, expected_error_type) - - @pytest.mark.parametrize("sequence, expected", [ - (None, None), - ([["a"]], [["a"]]), - ([["a"], ["b", "c"]], [["a"]]), - ([["b", "c"], ["a"]], [["a"]]), - ([["b", "c"], ["a"], ["d", "e"], ["f"]], [["a"], ["f"]]), - ]) - def test_i_can_get_the_longest_concept_sequence(self, sequence, expected): - assert FunctionParserOld.get_longest_concepts(sequence) == expected - - def test_concepts_found_are_fully_initialized(self): - sheerka, context, parser = self.init_parser() - - res = parser.parse(context, ParserInput("func(one plus three)")) - concept = res.body.body.nodes[0].concept - - assert res.status - assert isinstance(concept.get_compiled()["a"], Concept) - - # three is not recognized, - # so it will be transformed into list of ReturnValueConcept that indicate how to recognized it - assert isinstance(concept.get_compiled()["b"], list) - for item in concept.get_compiled()["b"]: - assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE) diff --git a/tests/parsers/test_SyaNodeParser.py b/tests/parsers/test_SyaNodeParser.py index a00bc7a..7cf4959 100644 --- a/tests/parsers/test_SyaNodeParser.py +++ b/tests/parsers/test_SyaNodeParser.py @@ -9,8 +9,7 @@ from core.sheerka.Sheerka import RECOGNIZED_BY_KEY from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Token, TokenKind, Tokenizer, comparable_tokens from core.utils import get_text_from_tokens -from parsers.BaseExpressionParser import BinaryNode, FunctionNode, FunctionNodeOld, FunctionParameter, ListNode, \ - NameExprNode, VariableNode +from parsers.BaseExpressionParser import BinaryNode, FunctionNode, ListNode, NameExprNode, VariableNode from parsers.BaseNodeParser import ConceptNode, SourceCodeNode, UnrecognizedTokensNode from parsers.PythonParser import PythonNode from parsers.SyaNodeParser import FunctionDetected, NoSyaConceptFound, NotEnoughParameters, SyaConceptParser, \ @@ -1564,33 +1563,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): _stack, _expected = prepare_nodes_comparison(concepts_map, text, lexer_nodes, expected) assert _stack == _expected - def test_i_can_parse_when_function_old_style_expr_tokens(self): - sheerka, context, parser = self.init_parser() - - text = "one plus func(twenty two)" - tokens = list(Tokenizer(text, yield_eof=False)) - fun_token = tokens[4] - expr = FunctionNodeOld(4, 9, tokens[4:10], - NameExprNode(4, 5, tokens[4:6]), - NameExprNode(9, 9, tokens[9:10]), - [FunctionParameter(NameExprNode(6, 8, tokens[6:9]), None)]) - tokens[4:] = [Token(TokenKind.EXPR, expr, fun_token.index, fun_token.line, fun_token.column)] - res = parser.parse(context, ParserInput(None, tokens=tokens)) - wrapper = res.body - lexer_nodes = res.body.body - - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - - expected_ret_val = RETVAL("func(twenty two)", objects={"__o_00__": CIO("twenties", source="twenty two")}) - expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)] - _stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected) - assert _stack == _expected - - # check the metadata - expected_concept = lexer_nodes[0].concept - assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")] - def test_i_can_parse_when_function_style_expr_tokens(self): sheerka, context, parser = self.init_parser() diff --git a/tests/parsers/test_parsers_utils.py b/tests/parsers/test_parsers_utils.py index 79a4bcd..38d01fd 100644 --- a/tests/parsers/test_parsers_utils.py +++ b/tests/parsers/test_parsers_utils.py @@ -1,13 +1,13 @@ from core.concept import Concept, ConceptParts from core.global_symbols import NotInit -from core.rule import Rule, ACTION_TYPE_EXEC +from core.rule import ACTION_TYPE_EXEC, Rule from core.sheerka.services.SheerkaExecute import ParserInput from parsers.BaseNodeParser import RuleNode from parsers.BnfNodeParser import BnfNodeParser -from parsers.FunctionParserOld import FunctionParserOld +from parsers.FunctionParser import FunctionParser from parsers.SyaNodeParser import SyaNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import get_test_obj, CNC, CC, CN, SCN, SCWC, UTN, RN, CB +from tests.parsers.parsers_utils import CB, CC, CN, CNC, RN, SCN, get_test_obj class TestParsersUtils(TestUsingMemoryBasedSheerka): @@ -133,7 +133,7 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka): def test_i_can_get_test_obj_when_SCN(self): sheerka, context = self.init_test().unpack() - parser = FunctionParserOld() + parser = FunctionParser() scn = parser.parse(context, ParserInput("test()")).body.body scn_res = get_test_obj(scn, SCN("", start=0, end=1)) @@ -145,29 +145,6 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka): assert isinstance(scn_res, SCN) assert scn_res == SCN("test()", start=None, end=None) - def test_i_can_get_test_obj_when_SCWC(self): - sheerka, context = self.init_test().unpack() - - parser = FunctionParserOld() - scwc = parser.parse(context, ParserInput("test(param1, test2())")).body.body - - scwc_res = get_test_obj(scwc, SCWC(UTN(""), UTN(""), UTN(""), UTN(""), SCN("", None, 0, 0))) - assert isinstance(scwc_res, SCWC) - expected = SCWC(UTN("test(", 0, 1), - UTN(")", 8, 8), - UTN("param1", 2, 2), - UTN(", ", 3, 4), - SCN("test2()", None, 5, 7)) - expected.start = 0 - expected.end = 8 - assert scwc_res == expected - - assert isinstance(scwc_res.first, UTN) - assert isinstance(scwc_res.last, UTN) - assert isinstance(scwc_res.content[0], UTN) - assert isinstance(scwc_res.content[1], UTN) - assert isinstance(scwc_res.content[2], SCN) - def test_i_can_get_test_obj_when_RN(self): rule = Rule(ACTION_TYPE_EXEC, "test_rule", "True", "True") rn = RuleNode(rule, 1, 1, source="r:|xxx:") diff --git a/tests/sheerkapython/test_ExprToConditionsVisitor.py b/tests/sheerkapython/test_ExprToConditionsVisitor.py index a718a0c..c910f04 100644 --- a/tests/sheerkapython/test_ExprToConditionsVisitor.py +++ b/tests/sheerkapython/test_ExprToConditionsVisitor.py @@ -87,7 +87,7 @@ class TestExprToConditionsVisitor(TestUsingMemoryBasedSheerka): @staticmethod def get_conditions_from_expression(context, expression, parser=None): - parser = parser or ExpressionParser(old_style=False) + parser = parser or ExpressionParser() error_sink = ErrorSink() parser_input = ParserInput(expression) parser.reset_parser_input(parser_input, error_sink) @@ -293,6 +293,21 @@ class TestExprToConditionsVisitor(TestUsingMemoryBasedSheerka): resolved_expected_objects = {k: cmap[v].id for k, v in {"__o_00__": "two"}.items()} assert resolved_objects == resolved_expected_objects + def test_i_can_manage_possible_variable_indication(self): + my_map = { + "isa": Concept("x is an int").def_var("x") # no pre condition + } + sheerka, context = self.initialize_test(my_map) + parser = ExpressionParser(known_variables={"x"}) + conditions = self.get_conditions_from_expression(context, "x is an int", parser) + + python_source = conditions[0].return_value.body.body.source + assert python_source == "call_concept(__o_00__, x=x)" + + resolved_objects = {k: v.id for k, v in conditions[0].objects.items()} + resolved_expected_objects = {k: my_map[v].id for k, v in {"__o_00__": "isa"}.items()} + assert resolved_objects == resolved_expected_objects + def test_i_can_parse_when_variables_are_missing(self): sheerka, context = self.initialize_test() expression = "x equals 1" diff --git a/tests/sheerkapython/test_PythonExprVisitor.py b/tests/sheerkapython/test_PythonExprVisitor.py index a0cd41b..3adac14 100644 --- a/tests/sheerkapython/test_PythonExprVisitor.py +++ b/tests/sheerkapython/test_PythonExprVisitor.py @@ -17,7 +17,7 @@ class TestPythonExprVisitor(TestUsingMemoryBasedSheerka): @staticmethod def get_expr_node(context, expression, parser=None): - parser = parser or ExpressionParser(old_style=False) + parser = parser or ExpressionParser() error_sink = ErrorSink() parser_input = ParserInput(expression) parser.reset_parser_input(parser_input, error_sink)