diff --git a/_concepts_full.txt b/_concepts_full.txt index 86b66da..3ecd19f 100644 --- a/_concepts_full.txt +++ b/_concepts_full.txt @@ -93,4 +93,6 @@ set_is_greater_than(__PRECEDENCE, divided, plus) set_is_greater_than(__PRECEDENCE, multiplied, minus) set_is_greater_than(__PRECEDENCE, divided, minus) +activate return values processing + diff --git a/src/core/sheerka/services/SheerkaDebugManager.py b/src/core/sheerka/services/SheerkaDebugManager.py index dab2d90..f6f3c3b 100644 --- a/src/core/sheerka/services/SheerkaDebugManager.py +++ b/src/core/sheerka/services/SheerkaDebugManager.py @@ -28,19 +28,22 @@ class BaseDebugLogger: BaseDebugLogger.ids[hint] = 0 return BaseDebugLogger.ids[hint] - def __init__(self, debug_manager, who, method_name, context_id, debug_id): + def __init__(self, debug_manager, context, who, method_name, debug_id): pass def debug_entering(self, **kwargs): pass + def debug_log(self, text, is_error=False): + pass + def debug_var(self, name, value, is_error=False, hint=None): pass def debug_rule(self, rule, results): pass - def debug_log(self, text, is_error=False): + def debug_concept(self, concept, text=None, **kwargs): pass def is_enabled(self): @@ -57,12 +60,12 @@ class NullDebugLogger(BaseDebugLogger): class ConsoleDebugLogger(BaseDebugLogger): - def __init__(self, debug_manager, service_name, method_name, context_id, debug_id): - BaseDebugLogger.__init__(self, debug_manager, service_name, method_name, context_id, debug_id) + def __init__(self, debug_manager, context, service_name, method_name, debug_id): + BaseDebugLogger.__init__(self, debug_manager, context, service_name, method_name, debug_id) self.debug_manager = debug_manager self.service_name = service_name self.method_name = method_name - self.context_id = context_id + self.context = context self.debug_id = debug_id self.is_highlighted = "" @@ -80,12 +83,16 @@ class ConsoleDebugLogger(BaseDebugLogger): self.debug_manager.debug(self.prefix() + str_text) self.debug_manager.debug(self.prefix() + str_vars) + def debug_log(self, text, is_error=False): + color = 'red' if is_error else 'blue' + self.debug_manager.debug(self.prefix() + f"{CCM[color]}..{text}{CCM['reset']}") + def debug_var(self, name, value, is_error=False, hint=None): - enabled = is_error or self.debug_manager.compute_var_debug(self.service_name, + enabled = is_error or self.debug_manager.compute_debug_var(self.context, + self.service_name, self.method_name, - self.context_id, name, - self.context_id) + self.debug_id) if enabled == False: return @@ -93,37 +100,51 @@ class ConsoleDebugLogger(BaseDebugLogger): hint_str = f"({hint})" if hint is not None else "" str_text = f"{CCM[color]}..{name}{hint_str}={CCM['reset']}" str_vars = "" if isinstance(enabled, str) else pp.pformat(value) - if "\n" not in str(str_vars): - self.debug_manager.debug(self.prefix() + str_text + str_vars) - else: - self.debug_manager.debug(self.prefix() + str_text) - self.debug_manager.debug(self.prefix() + str_vars) + self.debug(str_text, str_vars) def debug_rule(self, rule, results): - if not self.debug_manager.compute_debug_rule(rule.id, self.context_id, self.debug_id): + if not self.debug_manager.compute_debug_rule(self.context, + self.service_name, + self.method_name, + rule.id, + self.debug_id): return str_text = f"{CCM['green']}..results({rule.id})={CCM['reset']}" str_vars = pp.pformat(results) + self.debug(str_text, str_vars) + + def debug_concept(self, concept, text=None, **kwargs): + raw = kwargs.pop('raw', None) + if not self.debug_manager.compute_debug_concept(self.context, + self.service_name, + self.method_name, + concept.id, + self.debug_id): + return + str_vars = raw if raw else pp.pformat(kwargs) if kwargs else "" + text = " - " + text if text is not None else "" + colon = ": " if str_vars else "" + str_text = f"{CCM['cyan']}..concept#{concept.id}{text}{colon} {CCM['reset']}" + + self.debug(str_text, str_vars) + + def debug(self, str_text, str_vars): if "\n" not in str(str_vars): self.debug_manager.debug(self.prefix() + str_text + str_vars) else: self.debug_manager.debug(self.prefix() + str_text) self.debug_manager.debug(self.prefix() + str_vars) - def debug_log(self, text, is_error=False): - color = 'red' if is_error else 'blue' - self.debug_manager.debug(self.prefix() + f"{CCM[color]}..{text}{CCM['reset']}") - def prefix(self): - return f"[{self.context_id:2}][{self.debug_id:2}] {self.is_highlighted}" + return f"[{self.context.id:2}][{self.debug_id:2}] {self.is_highlighted}" @dataclass -class DebugVarSetting: +class DebugItem: + item: str service_name: str method_name: str - variable_name: str context_id: int context_children: bool debug_id: int @@ -132,15 +153,6 @@ class DebugVarSetting: enabled: bool -@dataclass -class DebugRuleSetting: - rule_id: str - context_id: int - debug_id: int - - enabled: bool - - class SheerkaDebugManager(BaseService): NAME = "Debug" PREFIX = "debug." @@ -155,23 +167,28 @@ class SheerkaDebugManager(BaseService): self.variable_cache = set() # debug for specific variable self.debug_vars_settings = [] self.debug_rules_settings = [] + self.debug_concepts_settings = [] def initialize(self): - self.sheerka.bind_service_method(self.set_debug, True) + # TO REMOVE ??? self.sheerka.bind_service_method(self.set_explicit, True) self.sheerka.bind_service_method(self.activate_debug_for, True) self.sheerka.bind_service_method(self.deactivate_debug_for, True) self.sheerka.bind_service_method(self.debug_activated, False) self.sheerka.bind_service_method(self.debug_activated_for, False) self.sheerka.bind_service_method(self.get_context_debug_mode, False) - self.sheerka.bind_service_method(self.debug_rule, True) self.sheerka.bind_service_method(self.debug_rule_activated, False) - self.sheerka.bind_service_method(self.inspect, False) self.sheerka.bind_service_method(self.debug, False, visible=False) + + self.sheerka.bind_service_method(self.set_debug, True) + self.sheerka.bind_service_method(self.inspect, False) self.sheerka.bind_service_method(self.get_debugger, False) - self.sheerka.bind_service_method(self.debug_var, False) self.sheerka.bind_service_method(self.reset_debug, False) - self.sheerka.bind_service_method(self.get_debug_settings, False, as_name="debug_settings") + self.sheerka.bind_service_method(self.debug_var, True) + self.sheerka.bind_service_method(self.debug_rule, True) + self.sheerka.bind_service_method(self.debug_concept, True) + + # self.sheerka.bind_service_method(self.get_debug_settings, False, as_name="debug_settings") def initialize_deferred(self, context, is_first_time): self.restore_values("activated", @@ -179,7 +196,8 @@ class SheerkaDebugManager(BaseService): "context_cache", "variable_cache", "debug_vars_settings", - "debug_rules_settings") + "debug_rules_settings", + "debug_concepts_settings") def reset(self): """ @@ -191,6 +209,7 @@ class SheerkaDebugManager(BaseService): self.variable_cache.clear() self.debug_vars_settings.clear() self.debug_rules_settings.clear() + self.debug_concepts_settings.clear() def set_debug(self, context, value=True): self.activated = value @@ -289,26 +308,30 @@ class SheerkaDebugManager(BaseService): print(*args, **kwargs) def get_debugger(self, context, who, method_name): - if self.compute_debug(who, method_name, context): + if self.compute_debug(context, who, method_name): debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id)) - return ConsoleDebugLogger(self, who, method_name, context.id, debug_id) + return ConsoleDebugLogger(self, context, who, method_name, debug_id) return NullDebugLogger() - def debug_var(self, context, - service=None, - method=None, - variable=None, - context_id=None, - context_children=False, - debug_id=None, - debug_children=False, - enabled=True): + def add_or_update_debug_item(self, context, + item_type, + item=None, + service=None, + method=None, + context_id=None, + context_children=False, + debug_id=None, + debug_children=False, + enabled=True): - for setting in self.debug_vars_settings: - if setting.service_name == service and \ + # if the setting already exist, update it + item_type_full_name = self.container_name(item_type) + items_container = getattr(self, item_type_full_name) + for setting in items_container: + if setting.item == item and \ + setting.service_name == service and \ setting.method_name == method and \ - setting.variable_name == variable and \ setting.context_id == context_id and \ setting.context_children == context_children and \ setting.debug_id == debug_id and \ @@ -316,64 +339,77 @@ class SheerkaDebugManager(BaseService): setting.enabled = enabled break else: - self.debug_vars_settings.append(DebugVarSetting(service, - method, - variable, - context_id, - context_children, - debug_id, - debug_children, - enabled)) + items_container.append(DebugItem(item, + service, + method, + context_id, + context_children, + debug_id, + debug_children, + enabled)) - self.sheerka.record_var(context, self.NAME, "debug_vars_settings", self.debug_vars_settings) + self.sheerka.record_var(context, self.NAME, item_type_full_name, items_container) return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) - def reset_debug(self, context): - self.debug_vars_settings.clear() - self.debug_rules_settings.clear() - self.sheerka.record_var(context, self.NAME, "debug_vars_settings", self.debug_vars_settings) - self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_vars_settings) - return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) - - def compute_debug(self, service_name, method_name, context): + def compute_debug(self, context, service_name, method_name): + """ + Using the debug info, tells if the debug is active for a given service, method, context (and debug_id) + :param context: + :param service_name: + :param method_name: + :return: + """ if not self.activated: return False selected = [] - for setting in self.debug_vars_settings: - if setting.service_name is None and setting.method_name is None and setting.context_id is None: - continue + for item_type in ["vars", "rules", "concepts"]: + for setting in getattr(self, self.container_name(item_type)): + if setting.service_name is None and setting.method_name is None and setting.context_id is None: + continue - if (setting.service_name is None or setting.service_name == service_name) and \ - (setting.method_name is None or setting.method_name == method_name) and \ - (setting.context_id is None or setting.context_id == context.id or ( - setting.context_children and context.has_parent(setting.context_id))): - selected.append(setting.enabled) + if (setting.service_name is None or setting.service_name == service_name) and \ + (setting.method_name is None or setting.method_name == method_name) and \ + (setting.context_id is None or setting.context_id == context.id or ( + setting.context_children and context.has_parent(setting.context_id))): + selected.append(setting.enabled) if len(selected) == 0: return False res = selected[0] for enabled in selected[1:]: - res &= enabled + res |= enabled return res - def compute_var_debug(self, service_name, method_name, context_id, variable_name, debug_id): + def compute_debug_item(self, item_type, context, service_name, method_name, item, debug_id): + """ + Using the debug info, tells if debug is activated for a given item + :param context: + :param item_type: + :param service_name: + :param method_name: + :param item: + :param debug_id: + :return: + """ if not self.activated: return False selected = [] - for setting in self.debug_vars_settings: - if setting.variable_name is None and setting.debug_id is None: + + for setting in getattr(self, self.container_name(item_type)): + if setting.item is None and setting.debug_id is None: continue if (setting.service_name is None or setting.service_name == service_name) and \ (setting.method_name is None or setting.method_name == method_name) and \ - (setting.context_id is None or setting.context_id == context_id) and \ - (setting.variable_name is None or - setting.variable_name == "*" or - setting.variable_name == variable_name) and \ + (setting.context_id is None or setting.context_id == context.id or ( + setting.context_children and context.has_parent(setting.context_id))) and \ + (setting.item is None or + setting.item == "*" or + setting.item == item) and \ (setting.debug_id is None or setting.debug_id == debug_id): selected.append(setting.enabled) @@ -392,56 +428,246 @@ class SheerkaDebugManager(BaseService): return res - def debug_rule(self, context, rule=None, context_id=None, debug_id=None, enabled=True): + def reset_debug(self, context): + for item_type in ["vars", "rules", "concepts"]: + setting_name = self.container_name(item_type) + settings = getattr(self, setting_name) + settings.clear() + self.sheerka.record_var(context, self.NAME, setting_name, settings) + + return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + + def debug_var(self, context, *args, **kwargs): """ - Add a debug rule request + Adds debug item for variables + debug_var(.., [+], ) + with service, method and vat that can be '*' :param context: - :param rule: - :param context_id: - :param debug_id: - :param enabled: + :param args: + :param kwargs: :return: """ - rule = str(rule) if rule is not None else None - for setting in self.debug_rules_settings: - if setting.rule_id == rule and \ - setting.context_id == context_id and \ - setting.debug_id == debug_id: - setting.enabled = enabled - break - else: - self.debug_rules_settings.append(DebugRuleSetting(rule, - context_id, - debug_id, - enabled)) + i, s, m, c_id, c_children, d, e = self.parse_debug_args("variable", *args, **kwargs) + return self.add_or_update_debug_item(context, "vars", i, s, m, c_id, c_children, d, False, e) - self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings) - return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + def debug_rule(self, context, *args, **kwargs): + """ + Adds debug item for rules + debug_var(.., [+], ) + with service, method and rule that can be '*' + debug_var(rule_id, [+], ) + :param context: + :param args: + :param kwargs: + :return: + """ + i, s, m, c_id, c_children, d, e = self.parse_debug_args("rule", *args, **kwargs) + return self.add_or_update_debug_item(context, "rules", i, s, m, c_id, c_children, d, False, e) - def compute_debug_rule(self, rule_id, context_id, debug_id): - if not self.activated: - return False + def debug_concept(self, context, *args, **kwargs): + """ + Adds debug item for concepts + debug_var(.., [+], ) + with service, method and vat that can be '*' + debug_var(concept_id, [+], ) + :param context: + :param args: + :param kwargs: + :return: + """ + i, s, m, c_id, c_children, d, e = self.parse_debug_args("concept", *args, **kwargs) + return self.add_or_update_debug_item(context, "concepts", i, s, m, c_id, c_children, d, False, e) - selected = [] - for setting in self.debug_rules_settings: - if (setting.rule_id is None or setting.rule_id == rule_id) and \ - (setting.context_id is None or setting.context_id == context_id) and \ - (setting.debug_id is None or setting.debug_id == debug_id): - selected.append(setting.enabled) + def compute_debug_var(self, context, service_name, method_name, item, debug_id): + return self.compute_debug_item("vars", context, service_name, method_name, item, debug_id) - if len(selected) == 0: - return False + def compute_debug_concept(self, context, service_name, method_name, item, debug_id): + return self.compute_debug_item("concepts", context, service_name, method_name, item, debug_id) - res = selected[0] - for enabled in selected[1:]: - res &= enabled + def compute_debug_rule(self, context, service_name, method_name, item, debug_id): + return self.compute_debug_item("rules", context, service_name, method_name, item, debug_id) - return res + @staticmethod + def container_name(item_type): + return f"debug_{item_type}_settings" - def reset_debug_rules(self, context): - self.debug_rules_settings.clear() - self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings) - return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + @staticmethod + def parse_debug_args(item_name, *args, **kwargs): + service, method_name, context_id, context_children, item, debug_id, enabled = None, None, None, False, None, None, True + if len(args) > 0: + if args[0] is None or args[0] == "": + pass + elif isinstance(args[0], int): + item = str(args[0]) + else: + parts = args[0].split(".") + service = None if parts[0] == "*" else parts[0] + if len(parts) > 1: + method_name = None if parts[1] == "*" else parts[1] + if len(parts) > 2: + item = parts[2] - def get_debug_settings(self): - return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.debug_vars_settings) + if len(args) > 1: + context_part = args[1] + if isinstance(context_part, int): + context_id = context_part + if isinstance(context_part, str): + m = SheerkaDebugManager.children_activation_regex.match(context_part) + if m: + context_id = int(m.group(1)) + context_children = True + else: + try: + context_id = int(context_part) + except ValueError: + pass + + if len(args) > 2: + debug_id = args[2] + + service = kwargs.get("service", service) + method_name = kwargs.get("method", method_name) + context_id = kwargs.get("context_id", context_id) + context_children = kwargs.get("context_children", context_children) + item = kwargs.get(item_name, item) + debug_id = kwargs.get("debug_id", debug_id) + enabled = kwargs.get("enabled", enabled) + + return item, service, method_name, context_id, context_children, debug_id, enabled + + # def debug_rule(self, context, rule=None, context_id=None, debug_id=None, enabled=True): + # """ + # Add a debug rule request + # :param context: + # :param rule: + # :param context_id: + # :param debug_id: + # :param enabled: + # :return: + # """ + # rule = str(rule) if rule is not None else None + # for setting in self.debug_rules_settings: + # if setting.rule_id == rule and \ + # setting.context_id == context_id and \ + # setting.debug_id == debug_id: + # setting.enabled = enabled + # break + # else: + # self.debug_rules_settings.append(DebugRuleSetting(rule, + # context_id, + # debug_id, + # enabled)) + # + # self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings) + # return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + # + # def debug_concept(self, context, concept=None, context_id=None, debug_id=None, enabled=True): + # """ + # Add a debug rule request + # :param context: + # :param concept: + # :param context_id: + # :param debug_id: + # :param enabled: + # :return: + # """ + # concept_id = concept.id if isinstance(concept, Concept) else str(concept) if concept is not None else None + # for setting in self.debug_concepts_settings: + # if setting.concept_id == concept_id and \ + # setting.context_id == context_id and \ + # setting.debug_id == debug_id: + # setting.enabled = enabled + # break + # else: + # self.debug_concepts_settings.append(DebugConceptSetting(concept_id, + # context_id, + # debug_id, + # enabled)) + # + # self.sheerka.record_var(context, self.NAME, "debug_concepts_settings", self.debug_concepts_settings) + # return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + # + + # + + # def compute_debug_var(self, service_name, method_name, context_id, variable_name, debug_id): + # if not self.activated: + # return False + # + # selected = [] + # for setting in self.debug_vars_settings: + # if setting.variable_name is None and setting.debug_id is None: + # continue + # + # if (setting.service_name is None or setting.service_name == service_name) and \ + # (setting.method_name is None or setting.method_name == method_name) and \ + # (setting.context_id is None or setting.context_id == context_id) and \ + # (setting.variable_name is None or + # setting.variable_name == "*" or + # setting.variable_name == variable_name) and \ + # (setting.debug_id is None or setting.debug_id == debug_id): + # selected.append(setting.enabled) + # + # if len(selected) == 0: + # return False + # + # res = selected[0] + # for enabled in selected[1:]: + # if res == False or enabled == False: + # return False + # + # if isinstance(res, str): + # continue + # + # res = enabled + # + # return res + # + # def compute_debug_rule(self, rule_id, context_id, debug_id): + # if not self.activated: + # return False + # + # selected = [] + # for setting in self.debug_rules_settings: + # if (setting.rule_id is None or setting.rule_id == rule_id) and \ + # (setting.context_id is None or setting.context_id == context_id) and \ + # (setting.debug_id is None or setting.debug_id == debug_id): + # selected.append(setting.enabled) + # + # if len(selected) == 0: + # return False + # + # res = selected[0] + # for enabled in selected[1:]: + # res &= enabled + # + # return res + # + # def compute_debug_concept(self, concept_id, context_id, debug_id): + # if not self.activated: + # return False + # + # selected = [] + # for setting in self.debug_concepts_settings: + # if (setting.concept_id is None or setting.concept_id == concept_id) and \ + # (setting.context_id is None or setting.context_id == context_id) and \ + # (setting.debug_id is None or setting.debug_id == debug_id): + # selected.append(setting.enabled) + # + # if len(selected) == 0: + # return False + # + # res = selected[0] + # for enabled in selected[1:]: + # res &= enabled + # + # return res + # + # def reset_debug_rules(self, context): + # self.debug_rules_settings.clear() + # self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings) + # return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + # + # def get_debug_settings(self): + # lst = self.debug_vars_settings + self.debug_concepts_settings + self.debug_rules_settings + # return self.sheerka.new(BuiltinConcepts.TO_LIST, body=lst) diff --git a/src/core/sheerka/services/SheerkaModifyConcept.py b/src/core/sheerka/services/SheerkaModifyConcept.py index e34cd25..2a7be6c 100644 --- a/src/core/sheerka/services/SheerkaModifyConcept.py +++ b/src/core/sheerka/services/SheerkaModifyConcept.py @@ -2,7 +2,6 @@ from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import ensure_concept from core.concept import NotInit, freeze_concept_attrs, Concept from core.sheerka.services.sheerka_service import BaseService -from parsers.BnfDefinitionParser import BnfDefinitionParser class SheerkaModifyConcept(BaseService): @@ -74,11 +73,6 @@ class SheerkaModifyConcept(BaseService): return for concept_id in refs: - concept = self.sheerka.get_by_id(concept_id) - - if concept.get_bnf() is not None: - BnfDefinitionParser.update_recurse_id(context, concept_id, concept.get_bnf()) - # remove the grammar entry so that it can be recreated self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id) diff --git a/src/core/sheerka/services/SheerkaOut.py b/src/core/sheerka/services/SheerkaOut.py index fea958c..2db4bb7 100644 --- a/src/core/sheerka/services/SheerkaOut.py +++ b/src/core/sheerka/services/SheerkaOut.py @@ -10,7 +10,7 @@ class SheerkaOut(BaseService): def __init__(self, sheerka): super().__init__(sheerka) - self.out_visitors = [ConsoleVisitor()] + self.out_visitors = [ConsoleVisitor(expand_mode="all_but_first")] def initialize(self): self.sheerka.bind_service_method(self.process_return_values, False) diff --git a/src/core/sheerka/services/SheerkaRuleManager.py b/src/core/sheerka/services/SheerkaRuleManager.py index 211af5f..17b4a0e 100644 --- a/src/core/sheerka/services/SheerkaRuleManager.py +++ b/src/core/sheerka/services/SheerkaRuleManager.py @@ -644,6 +644,7 @@ class SheerkaRuleManager(BaseService): self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[2], RULE_COMPARISON_CONTEXT) self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[3], RULE_COMPARISON_CONTEXT) self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[5], RULE_COMPARISON_CONTEXT) + self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[6], RULE_COMPARISON_CONTEXT) self.sheerka.set_is_greatest(context, BuiltinConcepts.PRECEDENCE, rules[0], RULE_COMPARISON_CONTEXT) def get_rule_by_id(self, rule_id): diff --git a/src/core/sheerka/services/SheerkaSetsManager.py b/src/core/sheerka/services/SheerkaSetsManager.py index 5717b8c..e9ccea3 100644 --- a/src/core/sheerka/services/SheerkaSetsManager.py +++ b/src/core/sheerka/services/SheerkaSetsManager.py @@ -89,6 +89,9 @@ class SheerkaSetsManager(BaseService): # update concept_set references self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set) + # remove the grammar entry so that it can be recreated + self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_set.id) + return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) def add_concepts_to_set(self, context, concepts, concept_set): diff --git a/src/out/ConsoleVisistor.py b/src/out/ConsoleVisistor.py index b6d2079..e1c7a41 100644 --- a/src/out/ConsoleVisistor.py +++ b/src/out/ConsoleVisistor.py @@ -7,6 +7,14 @@ class ConsoleVisitor(AsStrVisitor): """ def __init__(self, expand_mode="auto"): + """ + expand_mode: + auto: not the first dict, the sub ones depends of the width + always: all the dicts (and all the lists) + all_but_first: not the first one, but force the sub dict + never: never expand + :param expand_mode: + """ super().__init__() self.out = print self.expand_mode = expand_mode diff --git a/src/parsers/BnfDefinitionParser.py b/src/parsers/BnfDefinitionParser.py index 99096a2..1c4b57f 100644 --- a/src/parsers/BnfDefinitionParser.py +++ b/src/parsers/BnfDefinitionParser.py @@ -294,27 +294,8 @@ class BnfDefinitionParser(BaseParser): expression.rule_name = token.value self.next_token() - if BnfDefinitionParser.is_expression_a_set(self.context, expression): - root_concept = self.context.search(start_with_self=True, - predicate=lambda ec: ec.action == BuiltinConcepts.INIT_BNF, - get_obj=lambda ec: ec.action_context, - stop=lambda ec: ec.action == BuiltinConcepts.INIT_BNF) - root_concept = list(root_concept) - if root_concept and hasattr(root_concept[0], "id"): - expression.recurse_id = expression.get_recurse_id(root_concept[0].id, - expression.concept.id, - expression.rule_name) - return expression @staticmethod def is_expression_a_set(context, expression): return isinstance(expression, ConceptExpression) and context.sheerka.isaset(context, expression.concept) - - @staticmethod - def update_recurse_id(context, concept_id, expression): - if BnfDefinitionParser.is_expression_a_set(context, expression): - expression.recurse_id = expression.get_recurse_id(concept_id, expression.concept.id, expression.rule_name) - - for element in expression.elements: - BnfDefinitionParser.update_recurse_id(context, concept_id, element) diff --git a/src/parsers/BnfNodeParser.py b/src/parsers/BnfNodeParser.py index 9fc163b..6fdf940 100644 --- a/src/parsers/BnfNodeParser.py +++ b/src/parsers/BnfNodeParser.py @@ -17,8 +17,8 @@ from core.builtin_concepts import BuiltinConcepts from core.concept import DEFINITION_TYPE_BNF, DoNotResolve, ConceptParts, Concept from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer, TokenKind, Token +from core.utils import CONSOLE_COLORS_MAP as CCM from parsers.BaseNodeParser import BaseNodeParser, GrammarErrorNode, UnrecognizedTokensNode, ConceptNode, LexerNode -from parsers.BaseParser import BaseParser PARSERS = ["Sequence", "Sya", "Python"] @@ -50,6 +50,10 @@ class ParsingContext: res.append(self.clone()) return res + def __repr__(self): + res = f"ParsingContext('{self.node.get_debug()}', pos={self.pos})" + return res + class NonTerminalNode(LexerNode): """ @@ -86,6 +90,10 @@ class NonTerminalNode(LexerNode): clone = NonTerminalNode(self.parsing_expression, self.start, self.end, self.tokens, self.children.copy()) return clone + def get_debug(self): + res = f"{self.parsing_expression.concept}=>" if isinstance(self.parsing_expression, ConceptExpression) else "" + return res + ".".join([c.get_debug() for c in self.children]) + class TerminalNode(LexerNode): """ @@ -118,6 +126,9 @@ class TerminalNode(LexerNode): clone = TerminalNode(self.parsing_expression, self.start, self.end, self.value) return clone + def get_debug(self): + return self.value + class MultiNode: """" @@ -155,8 +166,6 @@ class ParsingExpression: def __init__(self, *args, **kwargs): self.elements = args - self.debug_enabled = False - self._has_unordered_choice = None nodes = kwargs.get('nodes', []) or [] if not hasattr(nodes, '__iter__'): @@ -184,45 +193,19 @@ class ParsingExpression: def __hash__(self): return hash((self.rule_name, self.elements)) - def parse(self, parser): + def parse(self, parser_helper): # TODO : add memoization - if self.debug_enabled: - self.debug(f">> {parser.pos:3d} : {self}") + # parser_helper.debugger.debug_log(f">> {parser_helper.pos:3d} : {self}") + # if self.debug_enabled: + # self.debug(f">> {parser_helper.pos:3d} : {self}") - res = self._parse(parser) + res = self._parse(parser_helper) return res def add_rule_name_if_needed(self, text): return text + "=" + self.rule_name if self.rule_name else text - def has_unordered_choice(self): - if self._has_unordered_choice is None: - visitor = HasUnorderedChoiceVisitor() - visitor.visit(self) - self._has_unordered_choice = visitor.value - - return self._has_unordered_choice - - def debug(self, msg): - self.log_sink.append((id(self), msg)) - - def get_debug(self): - if not self.debug_enabled: - return None - - # search for the first debug line for the current pexpression - id_self = id(self) - for i, line in enumerate(self.log_sink): - if line[0] == id_self: - break - else: - return "" - - n, debug = self.inner_get_debug(i, "") - self.log_sink.clear() - return debug - def inner_get_debug(self, n, tab=""): """ @@ -275,6 +258,13 @@ class ParsingExpression: return n, debug + @staticmethod + def debug_prefix(self_name, parser_helper): + current_rule_name = parser_helper.get_current_rule_name() + current_concept = parser_helper.concepts[-1] + str_rule_name = f":{current_rule_name}" if current_rule_name not in (None, current_concept.name) else "" + return f"{self_name}({current_concept}{str_rule_name})" + class ConceptExpression(ParsingExpression): """ @@ -284,10 +274,9 @@ class ConceptExpression(ParsingExpression): When the grammar is created, it is replaced by the actual concept """ - def __init__(self, concept, rule_name="", recurse_id=None, nodes=None): + def __init__(self, concept, rule_name="", nodes=None): super().__init__(rule_name=rule_name, nodes=nodes) self.concept = concept - self.recurse_id = recurse_id def __repr__(self): return self.add_rule_name_if_needed(f"{self.concept}") @@ -299,10 +288,6 @@ class ConceptExpression(ParsingExpression): if not isinstance(other, ConceptExpression): return False - # TODO : enable self.recurse_id when it will be correctly implemented - # if self.recurse_id != other.recurse_id: - # return False - if isinstance(self.concept, Concept): return self.concept.id == other.concept.id @@ -313,7 +298,17 @@ class ConceptExpression(ParsingExpression): return hash((self.concept, self.rule_name)) def _parse(self, parser_helper): + parser_helper.rules_names.append(self.rule_name) + parser_helper.push_concept(self.concept) + # parser_helper.debug_concept(self.debug_prefix("ConceptExpression", parser_helper) + "=start") + node = self.nodes[0].parse(parser_helper) + + # parser_helper.debug_concept(self.debug_prefix("ConceptExpression", parser_helper) + "=end") + + parser_helper.pop_concept() + parser_helper.rules_names.pop() + if node is None: return None @@ -327,7 +322,7 @@ class ConceptExpression(ParsingExpression): [node]) @staticmethod - def get_recurse_id(parent_id, concept_id, rule_name): + def get_recursion_id(parent_id, concept_id, rule_name): return f"{parent_id}#{concept_id}({rule_name})" @@ -340,6 +335,9 @@ class Sequence(ParsingExpression): init_pos = parser_helper.pos end_pos = parser_helper.pos + if parser_helper.debugger.is_enabled(): + debug_prefix = self.debug_prefix("Sequence", parser_helper) + parser_helper.debug_concept(debug_prefix, nodes=self.nodes) ntn = NonTerminalNode(self, init_pos, end_pos, @@ -351,10 +349,14 @@ class Sequence(ParsingExpression): for e in self.nodes: for pcontext in parsing_contexts: + if parser_helper.debugger.is_enabled(): + parser_helper.debug_concept(debug_prefix, node=e, pcontext=pcontext) + parser_helper.seek(pcontext.pos) node = e.parse(parser_helper) if node is None: to_remove.append(pcontext) + elif isinstance(node, MultiNode): clones = pcontext * len(node.results) # clones pcontext (but first item is pcontext) to_append.extend(clones[1:]) @@ -373,8 +375,8 @@ class Sequence(ParsingExpression): parsing_contexts.extend(to_append) if len(parsing_contexts) == 0: - if self.debug_enabled: - self.debug(f"<< Failed matching {e}") + if parser_helper.debugger.is_enabled(): + parser_helper.debug_concept(debug_prefix + " All pcontexts are failed. Sequence failed") return None to_append.clear() @@ -388,12 +390,10 @@ class Sequence(ParsingExpression): pcontext.fix_tokens(parser_helper) if len(parsing_contexts) == 1: - if self.debug_enabled: - self.debug(f"<< Found match '{parsing_contexts[0].node.source}'") + # parser_helper.debugger.debug_log(f"<< Found match '{parsing_contexts[0].node.source}'") return parsing_contexts[0].node - if self.debug_enabled: - self.debug(f"<< Found matches {[r.node.source for r in parsing_contexts]}") + # parser_helper.debugger.debug_log(f"<< Found matches {[r.node.source for r in parsing_contexts]}") return MultiNode(parsing_contexts) def __repr__(self): @@ -440,9 +440,18 @@ class UnOrderedChoice(ParsingExpression): init_pos = parser_helper.pos parsing_contexts = [] + if parser_helper.debugger.is_enabled(): + debug_prefix = self.debug_prefix("UnOrderedChoice", parser_helper) + parser_helper.debug_concept(debug_prefix) + debug_text = "" for e in self.nodes: + if isinstance(e, ConceptExpression) and e.concept.id in parser_helper.get_concepts_ids(): + # avoid circular reference + continue + node = e.parse(parser_helper) if node: + debug_text += CCM["green"] + str(e) + CCM["reset"] + ", " if isinstance(node, MultiNode): node.combine(self) parsing_contexts.extend(node.results) @@ -453,8 +462,13 @@ class UnOrderedChoice(ParsingExpression): parser_helper.parser.parser_input.tokens[init_pos: node.end + 1], [node]) parsing_contexts.append(ParsingContext(tn, parser_helper.pos)) + else: + debug_text += f"{e}, " parser_helper.seek(init_pos) # backtrack + if parser_helper.debugger.is_enabled(): + parser_helper.debug_concept(debug_prefix, raw=f"[{debug_text}]") + if len(parsing_contexts) == 0: return None @@ -675,18 +689,22 @@ class StrMatch(Match): def _parse(self, parser_helper): token = parser_helper.get_token() + if parser_helper.debugger.is_enabled(): + debug_prefix = self.debug_prefix("StrMatch", parser_helper) + debug_text = f"pos={parser_helper.pos}, to_match={self.to_match}, token={token.str_value}" + m = token.str_value.lower() == self.to_match.lower() if self.ignore_case \ else token.strip_quote == self.to_match if m: - if self.debug_enabled: - self.debug(f"pos={parser_helper.pos}, token={token.str_value}, to_match={self.to_match} => Matched") + if parser_helper.debugger.is_enabled(): + parser_helper.debug_concept(debug_prefix, raw=f"{CCM['green']}{debug_text}{CCM['reset']}") node = TerminalNode(self, parser_helper.pos, parser_helper.pos, token.str_value) parser_helper.next_token(self.skip_white_space) return node - if self.debug_enabled: - self.debug(f"pos={parser_helper.pos}, token={token.str_value}, to_match={self.to_match} => No Match") + if parser_helper.debugger.is_enabled(): + parser_helper.debug_concept(debug_prefix, raw=f"{CCM['red']}{debug_text}{CCM['reset']}") return None @@ -839,28 +857,17 @@ class BnfNodeConceptExpressionVisitor(ParsingExpressionVisitor): self.references.append(pe.concept) -class HasUnorderedChoiceVisitor(ParsingExpressionVisitor): - def __init__(self): - super().__init__(lambda pe: pe.nodes, circular_ref_strategy="skip") - self.value = False - - def __repr__(self): - return f"HasUnorderedChoiceVisitor(={self.value})" - - def reset(self): - self.value = False - - def visit_UnOrderedChoice(self, parsing_expression): - self.value = True - return ParsingExpressionVisitor.STOP - - class BnfConceptParserHelper: - def __init__(self, parser): + def __init__(self, parser, debugger): self.parser = parser - self.debug = [] - self.errors = [] - self.sequence = [] + self.debugger = debugger + self.debug = [] # keep track of the tokens + self.errors = [] # sink of errors + self.sequence = [] # output. List of lexer nodes correctly parsed + self.concepts = [] # stack of concepts being processed (fed by ConceptExpression) + self.concepts_ids = [] # ids if the concept to increase speed + self.rules_names = [] # stack of concepts rules names + self.concept_depth = 0 # depth of concept (+1 for each ConceptExpression which is not an OrderedChoice) self.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, []) self.has_unrecognized = False @@ -872,7 +879,8 @@ class BnfConceptParserHelper: self.pos = -1 def __repr__(self): - return f"BnfConceptParserHelper({self.sequence})" + concepts = [item.concept if isinstance(item, ConceptNode) else "***" for item in self.sequence] + return f"BnfConceptParserHelper({concepts})" def __eq__(self, other): if id(self) == id(other): @@ -886,6 +894,26 @@ class BnfConceptParserHelper: def __hash__(self): return len(self.sequence) + len(self.errors) + def debug_concept(self, text, **kwargs): + if len(self.concepts) <= 2: + self.debugger.debug_concept(self.concepts[0], text, **kwargs) + + def get_current_rule_name(self): + for rule_name in reversed(self.rules_names): + if rule_name: + return rule_name + + def push_concept(self, concept): + self.concepts.append(concept) + self.concepts_ids.append(concept.id) + + def pop_concept(self): + self.concepts.pop() + self.concepts_ids.pop() + + def get_concepts_ids(self): + return self.concepts_ids + def get_token(self) -> Token: return self.token @@ -917,39 +945,45 @@ class BnfConceptParserHelper: if self.is_locked(): return - self.debug.append(concept) - self.manage_unrecognized() - for forked in self.forked: - # manage the fact that some clone may have been forked - forked.eat_concept(concept, token) - - # init - parsing_expression = self.parser.get_parsing_expression(self.parser.context, concept) - if not isinstance(parsing_expression, ParsingExpression): + try: + self.push_concept(concept) self.debug.append(concept) - error_msg = f"Failed to parse concept '{concept}'" - if parsing_expression is not None: - error_msg += f". Reason: '{parsing_expression}'" - self.errors.append(GrammarErrorNode(error_msg)) - return - self.pos = self.parser.parser_input.pos - self.token = self.parser.parser_input.tokens[self.pos] + self.manage_unrecognized() + for forked in self.forked: + # manage the fact that some clone may have been forked + forked.eat_concept(concept, token) - # parse - node = parsing_expression.parse(self) + # init + parsing_expression = self.parser.get_parsing_expression(self.parser.context, concept) + if not isinstance(parsing_expression, ParsingExpression): + self.debug.append(concept) + error_msg = f"Failed to parse concept '{concept}'" + if parsing_expression is not None: + error_msg += f". Reason: '{parsing_expression}'" + self.errors.append(GrammarErrorNode(error_msg)) + return - if isinstance(node, MultiNode): - # when multiple choices are found, use the longest result - node = node.results[0].node - if node is not None and node.end != -1: - self.sequence.append(self.create_concept_node(concept, node)) - self.pos = node.end - self.bnf_parsed = True - else: - self.debug.append(("Rewind", token)) - self.unrecognized_tokens.add_token(token, self.parser.parser_input.pos) - self.pos = self.parser.parser_input.pos # reset position + self.pos = self.parser.parser_input.pos + self.token = self.parser.parser_input.tokens[self.pos] + + # parse + self.debugger.debug_concept(concept, parsing_expression=parsing_expression) + node = parsing_expression.parse(self) + + if isinstance(node, MultiNode): + # when multiple choices are found, use the longest result + node = node.results[0].node + if node is not None and node.end != -1: + self.sequence.append(self.create_concept_node(concept, node)) + self.pos = node.end + self.bnf_parsed = True + else: + self.debug.append(("Rewind", token)) + self.unrecognized_tokens.add_token(token, self.parser.parser_input.pos) + self.pos = self.parser.parser_input.pos # reset position + finally: + self.concepts.pop() def eat_unrecognized(self, token): if self.is_locked(): @@ -998,7 +1032,7 @@ class BnfConceptParserHelper: self.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, []) def clone(self): - clone = BnfConceptParserHelper(self.parser) + clone = BnfConceptParserHelper(self.parser, self.debugger) clone.debug = self.debug[:] self.errors = self.errors[:] clone.sequence = self.sequence[:] @@ -1148,7 +1182,6 @@ class ToUpdate: class BnfNodeParser(BaseNodeParser): - NAME = "Bnf" def __init__(self, **kwargs): @@ -1215,7 +1248,7 @@ class BnfNodeParser(BaseNodeParser): return res[0] if len(res) == 1 else Sequence(*res) - def get_concepts_sequences(self): + def get_concepts_sequences(self, context): """ Main method that parses the tokens and extract the concepts :return: @@ -1241,23 +1274,36 @@ class BnfNodeParser(BaseNodeParser): return by_end_pos[max(by_end_pos)] forked = [] - - concept_parser_helpers = [BnfConceptParserHelper(self)] + debugger = context.get_debugger(self.NAME, "parse") + debugger.debug_entering(source=self.parser_input.as_text()) + concept_parser_helpers = [BnfConceptParserHelper(self, debugger)] while self.parser_input.next_token(False): token = self.parser_input.token + if debugger.is_enabled(): + debug_prefix = f"pos={self.parser_input.pos}, {token=}, {len(concept_parser_helpers)} parser(s)" try: + not_locked = [p for p in concept_parser_helpers if not p.is_locked()] + if len(not_locked) == 0: + if debugger.is_enabled(): + debugger.debug_log(debug_prefix + ", all parsers are locked. Nothing to do.") + continue + concepts = self.get_concepts(token, self._is_eligible, strip_quotes=False) if not concepts: - for concept_parser in concept_parser_helpers: + if debugger.is_enabled(): + debugger.debug_log(debug_prefix + ", no concept found.") + for concept_parser in not_locked: concept_parser.eat_unrecognized(token) continue + if debugger.is_enabled(): + debugger.debug_log(debug_prefix + f", concept(s) found={concepts}") if len(concepts) == 1: - for concept_parser in concept_parser_helpers: + for concept_parser in not_locked: concept_parser.eat_concept(concepts[0], token) continue @@ -1274,9 +1320,13 @@ class BnfNodeParser(BaseNodeParser): clone = concept_parser.clone() temp_res.append(clone) clone.eat_concept(concept, token) + if debugger.is_enabled(): + debugger.debug_log(f"..{concept}, parsed={clone.bnf_parsed}, length={clone.pos}") # only keep the longest concept_parser_helpers = _get_longest(temp_res) + if debugger.is_enabled() and len(temp_res) > 1: + debugger.debug_log(f"Only keep longest -> {len(concept_parser_helpers)} parser(s) left") finally: _add_forked_to_concept_parser_helpers() @@ -1286,6 +1336,7 @@ class BnfNodeParser(BaseNodeParser): concept_parser.finalize() _add_forked_to_concept_parser_helpers() + debugger.debug_var("result", concept_parser_helpers) return concept_parser_helpers def fix_infinite_recursions(self, context, grammar, concept_id, parsing_expression): @@ -1306,7 +1357,7 @@ class BnfNodeParser(BaseNodeParser): for node_id in path_: expression_ = expression_.nodes[0] if isinstance(expression_, ConceptExpression) else expression_ for i, node in [(i, n) for i, n in enumerate(expression_.nodes) if isinstance(n, ConceptExpression)]: - if node_id in (node.recurse_id, node.concept.id): + if node_id == node.concept.id: index_ = i parent_ = expression_ expression_ = node # take the child of the ConceptExpression found @@ -1336,7 +1387,6 @@ class BnfNodeParser(BaseNodeParser): expression_update.rule_name, new_grammar, set()) new = ConceptExpression(expression_update.concept, rule_name=expression_update.rule_name, - recurse_id=expression_update.recurse_id, nodes=new_nodes) parent.nodes[index] = new @@ -1358,12 +1408,12 @@ class BnfNodeParser(BaseNodeParser): def check_for_infinite_recursion(self, parsing_expression, already_found, in_recursion, only_first=False): if isinstance(parsing_expression, ConceptExpression): - id_to_use = parsing_expression.recurse_id or parsing_expression.concept.id - if id_to_use in already_found: - already_found.append(id_to_use) # add the id again, to know where the cycle starts + if parsing_expression.concept.id in already_found: + already_found.append(parsing_expression.concept.id) # add the id again, to know where the cycle starts in_recursion.extend(already_found) return True - already_found.append(id_to_use) + + already_found.append(parsing_expression.concept.id) return self.check_for_infinite_recursion(parsing_expression.nodes[0], already_found, in_recursion, @@ -1396,13 +1446,13 @@ class BnfNodeParser(BaseNodeParser): return False return False - if isinstance(parsing_expression, UnOrderedChoice): - for node in parsing_expression.nodes: - already_found_for_current_node.clear() - already_found_for_current_node.extend(already_found.copy()) - if self.check_for_infinite_recursion(node, already_found_for_current_node, in_recursion, True): - return True - return False + # if isinstance(parsing_expression, UnOrderedChoice): + # for node in parsing_expression.nodes: + # already_found_for_current_node.clear() + # already_found_for_current_node.extend(already_found.copy()) + # if self.check_for_infinite_recursion(node, already_found_for_current_node, in_recursion, True): + # return True + # return False return False @@ -1429,50 +1479,34 @@ class BnfNodeParser(BaseNodeParser): desc=desc) as sub_context: # get the parsing expression to_skip = {concept.id} - ret = self.resolve_concept_parsing_expression(sub_context, concept, None, grammar, to_skip, to_update) + presult = self.resolve_concept_parsing_expression(sub_context, concept, None, grammar, to_skip, to_update) # check and update parsing expression that are still under construction - # Note that we only update the concept that will update concepts_grammars - # because pe.node may be large for item in to_update: pe = item.parsing_expression for i, node in enumerate(pe.nodes): if isinstance(node, UnderConstruction): pe.nodes[i] = grammar.get(node.concept_id) - # KSI 20200826 - # To be rewritten into get_infinite_recursions - # I have changed resolve_concept_parsing_expression() to directly avoid obvious circular references - # So it's no longer need to search and fix them - concepts_in_recursion = self.fix_infinite_recursions(context, grammar, concept.id, ret) + # check for infinite recursion definitions + already_seen = [concept.id] + in_recursion = [] # there may be cases where in_recursion is less than already_seen + concepts_in_recursion = self.check_for_infinite_recursion(presult, already_seen, in_recursion) if concepts_in_recursion: - chicken_anf_egg = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_recursion) - for concept_id in concepts_in_recursion: + chicken_anf_egg = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=in_recursion) + for concept_id in in_recursion: grammar[concept_id] = chicken_anf_egg - # update, in case of infinite circular recursion - ret = grammar[concept.id] + # update, in case of infinite recursion + presult = grammar[concept.id] - # finally, update the list of the known pexpression (self.concepts_grammars) - # We do not add pexpressions that contain UnOrderedChoice because the choices always depend on the current - # concept. - # For example, the pexpression for 'twenties' found under the concept 'hundreds' won't be the same than - # the pexpression 'twenties' under the concept 'thousand' or even the pexpression 'twenties' without any - # context. + # finally, update the list of the known pexpression (self.concepts_grammars) for latter use for k, v in grammar.items(): - if k == concept.id: - self.concepts_grammars.put(k, v) - elif context.sheerka.isinstance(v, BuiltinConcepts.CHICKEN_AND_EGG): - # not quite sure that it is a good idea. - # Why do we want to corrupt previous valid entries ? - self.concepts_grammars.put(k, v) - else: - if not v.has_unordered_choice(): - self.concepts_grammars.put(k, v) + self.concepts_grammars.put(k, v) - sub_context.add_values(return_values=ret) + sub_context.add_values(return_values=presult) - return ret + return presult def resolve_concept_parsing_expression(self, context, concept, name, grammar, to_skip, to_update): """ @@ -1487,16 +1521,17 @@ class BnfNodeParser(BaseNodeParser): """ sheerka = context.sheerka - if sheerka.isaset(context, concept) and hasattr(context, "obj"): - key_to_use = ConceptExpression.get_recurse_id(context.obj.id, concept.id, name) - else: - key_to_use = concept.id + # if sheerka.isaset(context, concept) and hasattr(context, "obj"): + # key_to_use = ConceptExpression.get_recursion_id(context.obj.id, concept.id, name) + # else: + # key_to_use = concept.id + key_to_use = concept.id if key_to_use in self.concepts_grammars: - # Use the global pexpression only if it does not contains UnOrderedChoice - pe = self.concepts_grammars.get(key_to_use) - if not pe.has_unordered_choice(): - return self.concepts_grammars.get(key_to_use) + return self.concepts_grammars.get(key_to_use) + # # Use the global pexpression only if it does not contains UnOrderedChoice + # pe = self.concepts_grammars.get(key_to_use) + # if not pe.has_unordered_choice(): if key_to_use in grammar: # under construction entry return grammar.get(key_to_use) @@ -1522,20 +1557,12 @@ class BnfNodeParser(BaseNodeParser): ssc.add_inputs(concept=concept) concepts_in_group = self.sheerka.get_set_elements(ssc, concept) - valid_concepts = [c for c in concepts_in_group if c.id not in to_skip] - # for c in concepts_in_group: - # if c.id == context.obj.id: - # continue - # - # if hasattr(context, "concepts_to_skip") and c.id in context.concepts_to_skip: - # continue - # - # valid_concepts.append(c) + # valid_concepts = [c for c in concepts_in_group if c.id not in to_skip] + valid_concepts = concepts_in_group nodes = [] for c in valid_concepts: - c_recurse_id = f"{c.id}#{c.name}#{concept.id}" if self.sheerka.isaset(context, c) else None - nodes.append(ConceptExpression(c, rule_name=c.name, recurse_id=c_recurse_id)) + nodes.append(ConceptExpression(c, rule_name=c.name)) resolved = self.resolve_parsing_expression(ssc, UnOrderedChoice(*nodes), @@ -1664,7 +1691,7 @@ class BnfNodeParser(BaseNodeParser): False, context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink)) - sequences = self.get_concepts_sequences() + sequences = self.get_concepts_sequences(context) valid_parser_helpers = self.get_valid(sequences) if valid_parser_helpers is None: # token error diff --git a/src/parsers/PythonParser.py b/src/parsers/PythonParser.py index f1dc9aa..b683eb7 100644 --- a/src/parsers/PythonParser.py +++ b/src/parsers/PythonParser.py @@ -40,10 +40,12 @@ class PythonNode(Node): self.ast_ = ast_ # if ast_ else ast.parse(source, mode="eval") if source else None self.objects = objects or {} # when objects (mainly concepts or rules) are recognized in the expression self.compiled = None + self.ast_str = self.get_dump(self.ast_) def init_ast(self): if self.ast_ is None and self.source: self.ast_ = ast.parse(self.source, mode="eval") + self.ast_str = self.get_dump(self.ast_) return self def get_compiled(self): @@ -77,6 +79,8 @@ class PythonNode(Node): @staticmethod def get_dump(ast_): + if not ast_: + return None dump = ast.dump(ast_) for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]: dump = dump.replace(to_remove, "") diff --git a/src/sheerkapickle/sheerka_handlers.py b/src/sheerkapickle/sheerka_handlers.py index 3468d7e..3289df7 100644 --- a/src/sheerkapickle/sheerka_handlers.py +++ b/src/sheerkapickle/sheerka_handlers.py @@ -198,6 +198,7 @@ class PythonNodeHandler(BaseHandler): pickler = self.context data["source"] = obj.source + data["ast_str"] = obj.ast_str data["objects"] = pickler.flatten(obj.objects) return data @@ -208,6 +209,7 @@ class PythonNodeHandler(BaseHandler): pickler = self.context instance.__init__(data["source"], objects=pickler.restore(data["objects"])) + instance.ast_str = data["ast_str"] return instance diff --git a/tests/core/test_SheerkaDebugManager.py b/tests/core/test_SheerkaDebugManager.py index b3b589b..c618606 100644 --- a/tests/core/test_SheerkaDebugManager.py +++ b/tests/core/test_SheerkaDebugManager.py @@ -1,7 +1,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.sheerka.ExecutionContext import ExecutionContext -from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugVarSetting, DebugRuleSetting +from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugItem from sdp.sheerkaDataProvider import Event from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -224,6 +224,66 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): assert sub_sub_context.debug_enabled assert not root_context.debug_enabled + @pytest.mark.parametrize("item_type", [ + "vars", "rules", "concepts" + ]) + def test_i_can_add_a_debug_item(self, item_type): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + + service_name = "service_name" + method_name = "method_name" + item = "id" + context_id = 1 + context_children = True + debug_id = 1 + debug_children = True + + service.add_or_update_debug_item(context, + item_type, + item, + service_name, + method_name, + context_id, + context_children, + debug_id, + debug_children, + True) + + item_container = f"debug_{item_type}_settings" + assert getattr(service, item_container) == [DebugItem( + item, + service_name, + method_name, + context_id, + context_children, + debug_id, + debug_children, + True + )] + + def test_i_manage_debug_item_default_values(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + + service.add_or_update_debug_item(context, "vars", item="item") + assert service.debug_vars_settings == [ + DebugItem("item", None, None, None, False, None, False, True) + ] + + def test_i_can_update_debug_item(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + + service.add_or_update_debug_item(context, "vars", "item", "service_name", "method_name", enabled=True) + service.add_or_update_debug_item(context, "vars", "item2", "service_name", "method_name", enabled=True) + service.add_or_update_debug_item(context, "vars", "item", "service_name", "method_name", enabled=False) + + assert service.debug_vars_settings == [ + DebugItem("item", "service_name", "method_name", None, False, None, False, False), + DebugItem("item2", "service_name", "method_name", None, False, None, False, True), + ] + @pytest.mark.parametrize("settings, expected", [ ({"service": "my_service"}, True), ({"service": "other_service"}, False), @@ -235,8 +295,8 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaDebugManager.NAME] service.set_debug(context, True) - service.debug_var(context, **settings) - assert service.compute_debug("my_service", "my_method", context) == expected + service.add_or_update_debug_item(context, "vars", **settings) + assert service.compute_debug(context, "my_service", "my_method") == expected def test_i_can_compute_debug_for_context(self): sheerka, root_context = self.init_concepts() @@ -245,109 +305,16 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): context1 = root_context.push(BuiltinConcepts.TESTING, None) context2 = root_context.push(BuiltinConcepts.TESTING, None) - service.debug_var(root_context, context_id=context1.id) + service.add_or_update_debug_item(root_context, "vars", context_id=context1.id) - assert service.compute_debug("my_service", "my_method", context1) - assert not service.compute_debug("my_service", "my_method", context2) - - def test_i_can_disable_debug_setting(self): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaDebugManager.NAME] - - service.debug_var(context, service="my_service", enabled=False) - assert not service.compute_debug("my_service", "my_method", context) - - def test_i_can_compute_combination_of_debug_settings(self): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaDebugManager.NAME] - service.set_debug(context, True) - - service.debug_var(context, service="my_service", enabled=True) - service.debug_var(context, method="my_method", enabled=False) - service.debug_var(context, variable="xxx", enabled=False) # is not used - service.debug_var(context, debug_id=1, enabled=False) # is not used - - assert not service.compute_debug("my_service", "my_method", context) # False > True - assert service.compute_debug("my_service", "another_method", context) - assert not service.compute_debug("another_service", "my_method", context) - - def test_variable_debug_is_deactivated_by_default(self): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaDebugManager.NAME] - service.set_debug(context, True) - - assert not service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) - - def test_i_can_activate_variable_debug_for_all_variables(self): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaDebugManager.NAME] - service.set_debug(context, True) - - service.debug_var(context, variable="*") - - assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == True - - def test_i_can_activate_variable_display_for_all_variables(self): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaDebugManager.NAME] - service.set_debug(context, True) - - service.debug_var(context, variable="*", enabled='list') - - assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == 'list' - - def test_i_can_disable_variable_debug(self): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaDebugManager.NAME] - service.set_debug(context, True) - - service.debug_var(context, variable="my_variable", enabled=False) - assert not service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) - - @pytest.mark.parametrize("enabled_1, enabled_2, expected", [ - (False, False, False), - (False, True, False), - (False, 'list', False), - (True, False, False), - (True, True, True), - (True, 'list', 'list'), - ('list', False, False), - ('list', True, 'list'), - ('list', 'list', 'list'), - ]) - def test_i_can_compute_combination_of_debug_var_settings(self, enabled_1, enabled_2, expected): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaDebugManager.NAME] - service.set_debug(context, True) - - service.debug_var(context, variable="my_variable", enabled=enabled_1) - service.debug_var(context, variable="*", enabled=enabled_2) - - assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == expected - - @pytest.mark.parametrize("settings, expected", [ - ({"service": "my_service"}, True), - ({"service": "other_service"}, False), - ({"method": "my_method"}, True), - ({"method": "other_method"}, False), - ({"context_id": 0}, True), - ({"context_id": 1}, False), - ]) - def test_service_and_method_and_context_id_are_tested_before_disabling_a_variable(self, settings, expected): - sheerka, context = self.init_concepts() - service = sheerka.services[SheerkaDebugManager.NAME] - service.set_debug(context, True) - - settings["variable"] = "my_variable" - - service.debug_var(context, **settings, enabled=True) - assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == expected + assert service.compute_debug(context1, "my_service", "my_method") + assert not service.compute_debug(context2, "my_service", "my_method") @pytest.mark.parametrize("context_children, expected", [ (True, True), (False, False), ]) - def test_i_can_compute_debug_var_for_context_children(self, context_children, expected): + def test_i_can_compute_debug_for_context_children(self, context_children, expected): sheerka, context = self.init_concepts() service = sheerka.services[SheerkaDebugManager.NAME] service.set_debug(context, True) @@ -355,49 +322,223 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): sub_context = context.push(BuiltinConcepts.TESTING, None) sub_sub_context = sub_context.push(BuiltinConcepts.TESTING, None) - service.debug_var(context, context_id=sub_context.id, context_children=context_children) - assert service.compute_debug("my_service", "my_method", sub_sub_context) == expected + service.add_or_update_debug_item(context, "vars", context_id=sub_context.id, context_children=context_children) + assert service.compute_debug(sub_sub_context, "my_service", "my_method") == expected - def test_i_can_update_debug_setting(self): + def test_compute_debug_returns_true_in_case_of_conflict(self): sheerka, context = self.init_concepts() service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) - service.debug_var(context, service="my_service", enabled=True) - service.debug_var(context, service="my_service", enabled=False) + service.add_or_update_debug_item(context, "vars", service="my_service", enabled=True) + service.add_or_update_debug_item(context, "rules", service="my_service", enabled=False) - assert len(service.debug_vars_settings) == 1 - assert not service.debug_vars_settings[0].enabled + assert service.compute_debug(context, "my_service", "my_method") - service.debug_var(context, service="my_service", enabled=True) - assert len(service.debug_vars_settings) == 1 - assert service.debug_vars_settings[0].enabled + @pytest.mark.parametrize("settings, expected", [ + ({"service": "my_service"}, False), # by default debug item is False if item is not specified + ({"service": "other_service"}, False), + ({"method": "my_method"}, False), # by default debug item is False if item is not specified + ({"method": "other_method"}, False), + ({"item": "my_item"}, True), + ({"item": "other_item"}, False), + ({"debug_id": 10}, True), + ({"debug_id": 0}, False), + ]) + def test_i_can_compute_debug_item(self, settings, expected): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "rules", **settings) + assert service.compute_debug_item("rules", context, "my_service", "my_method", "my_item", 10) == expected + + def test_i_can_compute_debug_item_using_the_correct_service(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "rules", service="service1", item="item", enabled=True) + service.add_or_update_debug_item(context, "rules", service="service2", item="item", enabled=False) + + assert service.compute_debug_item("rules", context, "service1", "my_method", "item", 10) + assert not service.compute_debug_item("rules", context, "service2", "my_method", "item", 10) + + def test_i_can_compute_debug_item_using_the_correct_method(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "rules", method="method1", item="item", enabled=True) + service.add_or_update_debug_item(context, "rules", method="method2", item="item", enabled=False) + + assert service.compute_debug_item("rules", context, "my_service", "method1", "item", 10) + assert not service.compute_debug_item("rules", context, "my_service", "method2", "item", 10) + + def test_i_can_compute_debug_item_using_the_correct_context(self): + sheerka, context = self.init_concepts() + another_context = context.push(BuiltinConcepts.TESTING, None) + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "rules", context_id=context.id, item="item", enabled=True) + service.add_or_update_debug_item(context, "rules", context_id=999, item="item", enabled=False) + + assert service.compute_debug_item("rules", context, "my_service", "my_method", "item", 10) + assert not service.compute_debug_item("rules", another_context, "my_service", "my_method", "item", 10) + + def test_i_can_compute_debug_item_using_the_correct_sub_context(self): + sheerka, context = self.init_concepts() + sub_context = context.push(BuiltinConcepts.TESTING, None) + another_context = self.get_context(sheerka) + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "rules", context_id=context.id, item="item", context_children=True) + + assert service.compute_debug_item("rules", context, "my_service", "my_method", "item", 10) + assert service.compute_debug_item("rules", sub_context, "my_service", "my_method", "item", 10) + assert not service.compute_debug_item("rules", another_context, "my_service", "my_method", "item", 10) + + def test_compute_debug_item_returns_false_in_case_of_conflict(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "concepts", service="my_service", item="item", enabled=True) + service.add_or_update_debug_item(context, "concepts", method="my_method", item="item", enabled=False) + + assert not service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10) + + def test_compute_debug_item_returns_false_in_case_of_conflict_2(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "concepts", item="item", enabled=True) + service.add_or_update_debug_item(context, "concepts", item="*", enabled=False) + + assert not service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10) + + @pytest.mark.parametrize("settings", [ + {"service": "my_service"}, + {"method": "my_method"}, + ]) + def test_by_default_item_is_disabled(self, settings): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "concepts", **settings) + + assert not service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10) + + def test_i_can_activate_debug_for_all_items(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "vars", item="*") + + assert service.compute_debug_item("vars", context, "my_service", "my_method", "whatever variable name", 0) + + def test_disabled_is_the_highest_priority(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "concepts", service="my_service", item="item", enabled=True) + service.add_or_update_debug_item(context, "concepts", method="my_method", item="item", enabled=False) + service.add_or_update_debug_item(context, "concepts", context_id=context.id, item="item", enabled="value") + + assert not service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10) + + def test_string_value_is_the_second_best_priority(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + service.set_debug(context, True) + + service.add_or_update_debug_item(context, "concepts", service="my_service", item="item", enabled=True) + service.add_or_update_debug_item(context, "concepts", context_id=context.id, item="item", enabled="value") + + assert service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10) == "value" def test_i_can_reset_debug_settings(self): sheerka, context = self.init_concepts() service = sheerka.services[SheerkaDebugManager.NAME] service.set_debug(context, True) - service.debug_var(context, service="my_service") - assert service.compute_debug("my_service", "my_method", context) + service.add_or_update_debug_item(context, "concepts", service="my_service") + service.add_or_update_debug_item(context, "vars", service="my_service") + service.add_or_update_debug_item(context, "rules", service="my_service") - service.reset_debug(context) - assert not service.compute_debug("my_service", "my_method", context) + sheerka.reset_debug(context) - @pytest.mark.parametrize("settings, expected", [ - ({"rule": "1"}, True), - ({"rule": "2"}, False), - ({"context_id": 0}, True), - ({"context_id": 1}, False), - ({"debug_id": 0}, True), - ({"debug_id": 1}, False), + assert len(service.debug_vars_settings) == 0 + assert len(service.debug_rules_settings) == 0 + assert len(service.debug_concepts_settings) == 0 + + @pytest.mark.parametrize("args, kwargs, expected", [ + (["my_service.my_method.my_var"], {}, ("my_var", "my_service", "my_method", None, False, None, True)), + (["*.*.my_var"], {}, ("my_var", None, None, None, False, None, True)), + (["my_service"], {}, (None, "my_service", None, None, False, None, True)), + (["my_service.my_method"], {}, (None, "my_service", "my_method", None, False, None, True)), + (["*.*.*"], {}, ("*", None, None, None, False, None, True)), + ([1], {}, ("1", None, None, None, False, None, True)), + (["", 1], {}, (None, None, None, 1, False, None, True)), + ([None, 1], {}, (None, None, None, 1, False, None, True)), + ([None, "1+"], {}, (None, None, None, 1, True, None, True)), + ([None, "xxx"], {}, (None, None, None, None, False, None, True)), + (["s.m.v", "1+"], {}, ("v", "s", "m", 1, True, None, True)), + ([None, None, 1], {}, (None, None, None, None, False, 1, True)), + ([None, "", 1], {}, (None, None, None, None, False, 1, True)), + (["s.m.v", "1+", 10], {}, ("v", "s", "m", 1, True, 10, True)), + (["s.m.v", "1+", 10], {"variable": "my_var"}, ("my_var", "s", "m", 1, True, 10, True)), + (["s.m.v", "1+", 10], {"service": "my_service"}, ("v", "my_service", "m", 1, True, 10, True)), + (["s.m.v", "1+", 10], {"method": "my_method"}, ("v", "s", "my_method", 1, True, 10, True)), + (["s.m.v", "1+", 10], {"context_id": 155}, ("v", "s", "m", 155, True, 10, True)), + (["s.m.v", "1+", 10], {"context_children": False}, ("v", "s", "m", 1, False, 10, True)), + (["s.m.v", "1+", 10], {"debug_id": 155}, ("v", "s", "m", 1, True, 155, True)), + (["s.m.v", "1+", 10], {"enabled": False}, ("v", "s", "m", 1, True, 10, False)), ]) - def test_i_can_compute_rules_debug_settings(self, settings, expected): + def test_i_can_parse_args(self, args, kwargs, expected): sheerka, context = self.init_concepts() service = sheerka.services[SheerkaDebugManager.NAME] - service.set_debug(context, True) - service.debug_rule(context, **settings) - assert service.compute_debug_rule("1", 0, 0) == expected + assert service.parse_debug_args("variable", *args, **kwargs) == expected + + def test_i_can_debug_var(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.debug_var(context, "s.m.v", "1+", 10, variable="my_var") + assert service.debug_vars_settings == [ + DebugItem("my_var", "s", "m", 1, True, 10, False, True) + ] + assert service.debug_concepts_settings == [] + assert service.debug_rules_settings == [] + + def test_i_can_debug_rule(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.debug_rule(context, "s.m.v", "1+", 10, rule="my_rule") + assert service.debug_rules_settings == [ + DebugItem("my_rule", "s", "m", 1, True, 10, False, True) + ] + assert service.debug_concepts_settings == [] + assert service.debug_vars_settings == [] + + def test_i_can_debug_concept(self): + sheerka, context = self.init_concepts() + service = sheerka.services[SheerkaDebugManager.NAME] + + sheerka.debug_concept(context, "s.m.v", "1+", 10, concept="my_concept") + assert service.debug_concepts_settings == [ + DebugItem("my_concept", "s", "m", 1, True, 10, False, True) + ] + assert service.debug_rules_settings == [] + assert service.debug_vars_settings == [] def test_state_is_saved_and_restored(self): sheerka = self.get_sheerka() @@ -408,8 +549,9 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): sheerka.set_explicit(root_context, False) sheerka.activate_debug_for(root_context, 1, children=True) sheerka.activate_debug_for(root_context, "SomeVar") - sheerka.debug_rule(root_context, "1", 10, 15) - sheerka.debug_var(root_context, service="service_name") + sheerka.debug_var(root_context, "service_name.*.var") + sheerka.debug_rule(root_context, 1) + sheerka.debug_concept(root_context, 1001) another_service = SheerkaDebugManager(sheerka) another_service.initialize_deferred(root_context, True) @@ -418,8 +560,9 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): assert not another_service.explicit assert another_service.context_cache == {1, "1+"} assert another_service.variable_cache == {"SomeVar"} - assert another_service.debug_rules_settings == [ - DebugRuleSetting("1", 10, 15, True) - ] assert another_service.debug_vars_settings == [ - DebugVarSetting('service_name', None, None, None, False, None, False, True)] + DebugItem('var', 'service_name', None, None, False, None, False, True)] + assert another_service.debug_rules_settings == [ + DebugItem('1', None, None, None, False, None, False, True)] + assert another_service.debug_concepts_settings == [ + DebugItem('1001', None, None, None, False, None, False, True)] diff --git a/tests/core/test_SheerkaSetsManager.py b/tests/core/test_SheerkaSetsManager.py index 91e7480..8d5d90e 100644 --- a/tests/core/test_SheerkaSetsManager.py +++ b/tests/core/test_SheerkaSetsManager.py @@ -260,21 +260,6 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka): assert sheerka.isa(sheerka.new("one"), number) # sanity assert not sheerka.isa(another_one, number) # Correct this misbehaviour when BuiltinConcepts.IS is implemented - def test_concept_expression_recurse_id_is_updated(self): - sheerka, context, one, number, twenties = self.init_concepts( - "one", - "number", - Concept("twenties", definition="'twenty' number").def_var("number"), - create_new=True - ) - - assert twenties.get_bnf().elements[1].recurse_id is None - - # update number - sheerka.set_isa(context, sheerka.new("one"), number) - - assert twenties.get_bnf().elements[1].recurse_id == "1003#1002(number)" - def test_concepts_in_group_cache_is_updated(self): sheerka, context, one, two, number = self.init_concepts("one", "two", "number") @@ -286,6 +271,9 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka): concepts_in_cache = sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_IN_GROUPS_ENTRY, number.id) assert [c.id for c in concepts_in_cache] == [one.id] + # pretend that number has been updated in sheerka.concepts_grammar + sheerka.concepts_grammars.put(number.id, "some parsing expression with 'one' only") + # add another element to number sheerka.set_isa(context, sheerka.new("two"), number) elements = sheerka.get_set_elements(context, number) @@ -294,6 +282,9 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka): concepts_in_cache = sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_IN_GROUPS_ENTRY, number.id) assert {c.id for c in concepts_in_cache} == {one.id, two.id} + # make sure the bnf definition is also updated + assert number.id not in sheerka.concepts_grammars + class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): def test_i_can_add_concept_to_set_and_retrieve_it_in_another_session(self): diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index 423b4b2..d1beced 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -1156,7 +1156,7 @@ as: sheerka = self.init_scenario(init) res = sheerka.evaluate_user_input("twenty one") - assert len(res) > 1 + assert len(res) > 1 # not recognized sheerka.evaluate_user_input("set_isa(one, number)") res = sheerka.evaluate_user_input("twenty one") diff --git a/tests/parsers/test_BnfNodeParser.py b/tests/parsers/test_BnfNodeParser.py index 80cd003..0ff135a 100644 --- a/tests/parsers/test_BnfNodeParser.py +++ b/tests/parsers/test_BnfNodeParser.py @@ -3,9 +3,9 @@ from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF, NotInit from core.sheerka.services.SheerkaExecute import ParserInput from parsers.BaseNodeParser import CNC, UTN, CN +from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \ Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser -from parsers.BnfDefinitionParser import BnfDefinitionParser import tests.parsers.parsers_utils from tests.BaseTest import BaseTest @@ -172,7 +172,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): parser.init_from_concepts(context, updated) parser.reset_parser(context, ParserInput(text)) - bnf_parsers_helpers = parser.get_concepts_sequences() + bnf_parsers_helpers = parser.get_concepts_sequences(context) assert len(bnf_parsers_helpers) == len(expected_array) for parser_helper, expected_sequence in zip(bnf_parsers_helpers, expected_array): @@ -263,14 +263,14 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): expected_array = compute_expected_array(my_map, text, expected) parser.reset_parser(context, ParserInput(text)) - bnf_parsers_helpers = parser.get_concepts_sequences() + bnf_parsers_helpers = parser.get_concepts_sequences(context) assert bnf_parsers_helpers[0].sequence == expected_array assert not bnf_parsers_helpers[0].has_unrecognized # but I cannot parse text = "- - filter" parser.reset_parser(context, ParserInput(text)) - bnf_parsers_helpers = parser.get_concepts_sequences() + bnf_parsers_helpers = parser.get_concepts_sequences(context) assert bnf_parsers_helpers[0].has_unrecognized def test_i_can_match_multiple_sequences(self): @@ -788,7 +788,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): concept_foo = sequences[0].concept assert concept_foo.body == NotInit assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]), - ConceptParts.BODY: DoNotResolve(value='twenty two')} + ConceptParts.BODY: DoNotResolve(value='twenty two')} text = "twenty one" expected = [CN("foo", source="twenty one")] @@ -798,13 +798,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): concept_foo = sequences[0].concept assert concept_foo.body == NotInit assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]), - ConceptParts.BODY: DoNotResolve(value='twenty one')} + ConceptParts.BODY: DoNotResolve(value='twenty one')} @pytest.mark.parametrize("bar_expr, expected", [ (ConceptExpression("foo"), {}), (OrderedChoice(ConceptExpression("foo"), StrMatch("one")), {'one': ['1002']}), (Sequence(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")), {'one': ['1001', '1002']}), - # (UnOrderedChoice(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")), {'one': ['1001', '1002']}) ]) def test_i_can_detect_infinite_recursion(self, bar_expr, expected): my_map = { @@ -850,6 +849,33 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert parser.concepts_grammars.get(my_map["foo"].id).body == ["1001", "1002", "1003", "1004", "1001"] + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG) + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG) + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG) + + def test_i_can_detect_partial_infinite_recursion(self): + my_map = { + "foo": self.bnf_concept("foo", ConceptExpression("bar")), + "bar": self.bnf_concept("bar", ConceptExpression("baz")), + "baz": self.bnf_concept("baz", ConceptExpression("qux")), + "qux": self.bnf_concept("qux", ConceptExpression("baz")), + } + + sheerka, context, parser = self.init_parser(my_map, singleton=True) + parser.context = context + parser.sheerka = sheerka + + # every obvious cyclic recursion are removed from concept_by_first_keyword dict + parser.init_from_concepts(context, my_map.values()) + assert parser.concepts_by_first_keyword == {} + + parsing_expression = parser.get_parsing_expression(context, my_map["foo"]) + assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) + assert parser.concepts_grammars.get(my_map["foo"].id).body == ["1001", "1002", "1003", "1004", "1003"] + + assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG) @@ -904,8 +930,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes - assert my_map["number"].id not in parser.concepts_grammars - def test_i_can_get_parsing_expression_when_starting_by_isa_concept(self): my_map = { "one": Concept("one"), @@ -1015,7 +1039,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): parser.init_from_concepts(context, my_map.values()) parser.reset_parser(context, ParserInput("one three")) - sequences = parser.get_concepts_sequences() + sequences = parser.get_concepts_sequences(context) sequence = parser.get_valid(sequences) assert len(sequence) == 1 @@ -1122,7 +1146,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array - def test_i_can_parse_one_thousand(self): + def test_i_can_parse_when_starting_by_isa_concept(self): """ Test of simple number + 'thousand' :return: @@ -1403,15 +1427,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert parser.parse(context, ParserInput("foo foo foo bar")).status assert not parser.parse(context, ParserInput("foo baz")).status - def test_i_only_get_the_requested_parsing_expression(self): - sheerka, context, parser = self.init_parser(init_from_sheerka=True) - parser.context = context - parser.sheerka = sheerka - sheerka.concepts_grammars.clear() # to simulate restart - - parser.get_parsing_expression(context, sheerka.resolve("thirties")) - assert len(parser.concepts_grammars) == 9 # requested concept + concepts that do not contains UnorderedChoice - @pytest.mark.parametrize("name, expected", [ (None, []), ("", []), diff --git a/tests/parsers/test_BnfParser.py b/tests/parsers/test_BnfParser.py index 1db659f..0660bc9 100644 --- a/tests/parsers/test_BnfParser.py +++ b/tests/parsers/test_BnfParser.py @@ -2,13 +2,13 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, DEFINITION_TYPE_BNF from core.sheerka.services.SheerkaExecute import ParserInput -from core.tokenizer import Tokenizer, TokenKind, LexerError, Token +from core.tokenizer import Tokenizer, TokenKind, LexerError from parsers.BaseNodeParser import cnode from parsers.BaseParser import UnexpectedTokenErrorNode +from parsers.BnfDefinitionParser import BnfDefinitionParser, UnexpectedEndOfFileError +from parsers.BnfNodeParser import BnfNodeParser from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \ OneOrMore, ConceptExpression -from parsers.BnfNodeParser import BnfNodeParser -from parsers.BnfDefinitionParser import BnfDefinitionParser, UnexpectedEndOfFileError from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -236,4 +236,3 @@ class TestBnfParser(TestUsingMemoryBasedSheerka): assert res.status pexpression = res.value.value assert pexpression == Sequence(StrMatch('twenty'), ConceptExpression(number, "n1")) - assert pexpression.elements[1].recurse_id == "1004#1003(n1)"