kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bug when evaluating numbers several times

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-11-27 11:01:02 +01:00
parent cac732bd93
commit 4f899280c4
16 changed files with 887 additions and 491 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_concepts_full.txt
+2
View File
@@ -93,4 +93,6 @@ set_is_greater_than(__PRECEDENCE, divided, plus)
set_is_greater_than(__PRECEDENCE, multiplied, minus)
set_is_greater_than(__PRECEDENCE, divided, minus)
activate return values processing
src/core/sheerka/services/SheerkaDebugManager.py
+330 -104
View File
@@ -28,19 +28,22 @@ class BaseDebugLogger:
BaseDebugLogger.ids[hint] = 0
return BaseDebugLogger.ids[hint]
def __init__(self, debug_manager, who, method_name, context_id, debug_id):
def __init__(self, debug_manager, context, who, method_name, debug_id):
pass
def debug_entering(self, **kwargs):
pass
def debug_log(self, text, is_error=False):
pass
def debug_var(self, name, value, is_error=False, hint=None):
pass
def debug_rule(self, rule, results):
pass
def debug_log(self, text, is_error=False):
def debug_concept(self, concept, text=None, **kwargs):
pass
def is_enabled(self):
@@ -57,12 +60,12 @@ class NullDebugLogger(BaseDebugLogger):
class ConsoleDebugLogger(BaseDebugLogger):
def __init__(self, debug_manager, service_name, method_name, context_id, debug_id):
BaseDebugLogger.__init__(self, debug_manager, service_name, method_name, context_id, debug_id)
def __init__(self, debug_manager, context, service_name, method_name, debug_id):
BaseDebugLogger.__init__(self, debug_manager, context, service_name, method_name, debug_id)
self.debug_manager = debug_manager
self.service_name = service_name
self.method_name = method_name
self.context_id = context_id
self.context = context
self.debug_id = debug_id
self.is_highlighted = ""
@@ -80,12 +83,16 @@ class ConsoleDebugLogger(BaseDebugLogger):
self.debug_manager.debug(self.prefix() + str_text)
self.debug_manager.debug(self.prefix() + str_vars)
def debug_log(self, text, is_error=False):
color = 'red' if is_error else 'blue'
self.debug_manager.debug(self.prefix() + f"{CCM[color]}..{text}{CCM['reset']}")
def debug_var(self, name, value, is_error=False, hint=None):
enabled = is_error or self.debug_manager.compute_var_debug(self.service_name,
enabled = is_error or self.debug_manager.compute_debug_var(self.context,
self.service_name,
self.method_name,
self.context_id,
name,
self.context_id)
self.debug_id)
if enabled == False:
return
@@ -93,37 +100,51 @@ class ConsoleDebugLogger(BaseDebugLogger):
hint_str = f"({hint})" if hint is not None else ""
str_text = f"{CCM[color]}..{name}{hint_str}={CCM['reset']}"
str_vars = "" if isinstance(enabled, str) else pp.pformat(value)
if "\n" not in str(str_vars):
self.debug_manager.debug(self.prefix() + str_text + str_vars)
else:
self.debug_manager.debug(self.prefix() + str_text)
self.debug_manager.debug(self.prefix() + str_vars)
self.debug(str_text, str_vars)
def debug_rule(self, rule, results):
if not self.debug_manager.compute_debug_rule(rule.id, self.context_id, self.debug_id):
if not self.debug_manager.compute_debug_rule(self.context,
self.service_name,
self.method_name,
rule.id,
self.debug_id):
return
str_text = f"{CCM['green']}..results({rule.id})={CCM['reset']}"
str_vars = pp.pformat(results)
self.debug(str_text, str_vars)
def debug_concept(self, concept, text=None, **kwargs):
raw = kwargs.pop('raw', None)
if not self.debug_manager.compute_debug_concept(self.context,
self.service_name,
self.method_name,
concept.id,
self.debug_id):
return
str_vars = raw if raw else pp.pformat(kwargs) if kwargs else ""
text = " - " + text if text is not None else ""
colon = ": " if str_vars else ""
str_text = f"{CCM['cyan']}..concept#{concept.id}{text}{colon} {CCM['reset']}"
self.debug(str_text, str_vars)
def debug(self, str_text, str_vars):
if "\n" not in str(str_vars):
self.debug_manager.debug(self.prefix() + str_text + str_vars)
else:
self.debug_manager.debug(self.prefix() + str_text)
self.debug_manager.debug(self.prefix() + str_vars)
def debug_log(self, text, is_error=False):
color = 'red' if is_error else 'blue'
self.debug_manager.debug(self.prefix() + f"{CCM[color]}..{text}{CCM['reset']}")
def prefix(self):
return f"[{self.context_id:2}][{self.debug_id:2}] {self.is_highlighted}"
return f"[{self.context.id:2}][{self.debug_id:2}] {self.is_highlighted}"
@dataclass
class DebugVarSetting:
class DebugItem:
item: str
service_name: str
method_name: str
variable_name: str
context_id: int
context_children: bool
debug_id: int
@@ -132,15 +153,6 @@ class DebugVarSetting:
enabled: bool
@dataclass
class DebugRuleSetting:
rule_id: str
context_id: int
debug_id: int
enabled: bool
class SheerkaDebugManager(BaseService):
NAME = "Debug"
PREFIX = "debug."
@@ -155,23 +167,28 @@ class SheerkaDebugManager(BaseService):
self.variable_cache = set() # debug for specific variable
self.debug_vars_settings = []
self.debug_rules_settings = []
self.debug_concepts_settings = []
def initialize(self):
self.sheerka.bind_service_method(self.set_debug, True)
# TO REMOVE ???
self.sheerka.bind_service_method(self.set_explicit, True)
self.sheerka.bind_service_method(self.activate_debug_for, True)
self.sheerka.bind_service_method(self.deactivate_debug_for, True)
self.sheerka.bind_service_method(self.debug_activated, False)
self.sheerka.bind_service_method(self.debug_activated_for, False)
self.sheerka.bind_service_method(self.get_context_debug_mode, False)
self.sheerka.bind_service_method(self.debug_rule, True)
self.sheerka.bind_service_method(self.debug_rule_activated, False)
self.sheerka.bind_service_method(self.inspect, False)
self.sheerka.bind_service_method(self.debug, False, visible=False)
self.sheerka.bind_service_method(self.set_debug, True)
self.sheerka.bind_service_method(self.inspect, False)
self.sheerka.bind_service_method(self.get_debugger, False)
self.sheerka.bind_service_method(self.debug_var, False)
self.sheerka.bind_service_method(self.reset_debug, False)
self.sheerka.bind_service_method(self.get_debug_settings, False, as_name="debug_settings")
self.sheerka.bind_service_method(self.debug_var, True)
self.sheerka.bind_service_method(self.debug_rule, True)
self.sheerka.bind_service_method(self.debug_concept, True)
# self.sheerka.bind_service_method(self.get_debug_settings, False, as_name="debug_settings")
def initialize_deferred(self, context, is_first_time):
self.restore_values("activated",
@@ -179,7 +196,8 @@ class SheerkaDebugManager(BaseService):
"context_cache",
"variable_cache",
"debug_vars_settings",
"debug_rules_settings")
"debug_rules_settings",
"debug_concepts_settings")
def reset(self):
"""
@@ -191,6 +209,7 @@ class SheerkaDebugManager(BaseService):
self.variable_cache.clear()
self.debug_vars_settings.clear()
self.debug_rules_settings.clear()
self.debug_concepts_settings.clear()
def set_debug(self, context, value=True):
self.activated = value
@@ -289,26 +308,30 @@ class SheerkaDebugManager(BaseService):
print(*args, **kwargs)
def get_debugger(self, context, who, method_name):
if self.compute_debug(who, method_name, context):
if self.compute_debug(context, who, method_name):
debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id))
return ConsoleDebugLogger(self, who, method_name, context.id, debug_id)
return ConsoleDebugLogger(self, context, who, method_name, debug_id)
return NullDebugLogger()
def debug_var(self, context,
def add_or_update_debug_item(self, context,
item_type,
item=None,
service=None,
method=None,
variable=None,
context_id=None,
context_children=False,
debug_id=None,
debug_children=False,
enabled=True):
for setting in self.debug_vars_settings:
if setting.service_name == service and \
# if the setting already exist, update it
item_type_full_name = self.container_name(item_type)
items_container = getattr(self, item_type_full_name)
for setting in items_container:
if setting.item == item and \
setting.service_name == service and \
setting.method_name == method and \
setting.variable_name == variable and \
setting.context_id == context_id and \
setting.context_children == context_children and \
setting.debug_id == debug_id and \
@@ -316,31 +339,32 @@ class SheerkaDebugManager(BaseService):
setting.enabled = enabled
break
else:
self.debug_vars_settings.append(DebugVarSetting(service,
items_container.append(DebugItem(item,
service,
method,
variable,
context_id,
context_children,
debug_id,
debug_children,
enabled))
self.sheerka.record_var(context, self.NAME, "debug_vars_settings", self.debug_vars_settings)
self.sheerka.record_var(context, self.NAME, item_type_full_name, items_container)
return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def reset_debug(self, context):
self.debug_vars_settings.clear()
self.debug_rules_settings.clear()
self.sheerka.record_var(context, self.NAME, "debug_vars_settings", self.debug_vars_settings)
self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_vars_settings)
return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def compute_debug(self, service_name, method_name, context):
def compute_debug(self, context, service_name, method_name):
"""
Using the debug info, tells if the debug is active for a given service, method, context (and debug_id)
:param context:
:param service_name:
:param method_name:
:return:
"""
if not self.activated:
return False
selected = []
for setting in self.debug_vars_settings:
for item_type in ["vars", "rules", "concepts"]:
for setting in getattr(self, self.container_name(item_type)):
if setting.service_name is None and setting.method_name is None and setting.context_id is None:
continue
@@ -355,25 +379,37 @@ class SheerkaDebugManager(BaseService):
res = selected[0]
for enabled in selected[1:]:
res &= enabled
res |= enabled
return res
def compute_var_debug(self, service_name, method_name, context_id, variable_name, debug_id):
def compute_debug_item(self, item_type, context, service_name, method_name, item, debug_id):
"""
Using the debug info, tells if debug is activated for a given item
:param context:
:param item_type:
:param service_name:
:param method_name:
:param item:
:param debug_id:
:return:
"""
if not self.activated:
return False
selected = []
for setting in self.debug_vars_settings:
if setting.variable_name is None and setting.debug_id is None:
for setting in getattr(self, self.container_name(item_type)):
if setting.item is None and setting.debug_id is None:
continue
if (setting.service_name is None or setting.service_name == service_name) and \
(setting.method_name is None or setting.method_name == method_name) and \
(setting.context_id is None or setting.context_id == context_id) and \
(setting.variable_name is None or
setting.variable_name == "*" or
setting.variable_name == variable_name) and \
(setting.context_id is None or setting.context_id == context.id or (
setting.context_children and context.has_parent(setting.context_id))) and \
(setting.item is None or
setting.item == "*" or
setting.item == item) and \
(setting.debug_id is None or setting.debug_id == debug_id):
selected.append(setting.enabled)
@@ -392,56 +428,246 @@ class SheerkaDebugManager(BaseService):
return res
def debug_rule(self, context, rule=None, context_id=None, debug_id=None, enabled=True):
def reset_debug(self, context):
for item_type in ["vars", "rules", "concepts"]:
setting_name = self.container_name(item_type)
settings = getattr(self, setting_name)
settings.clear()
self.sheerka.record_var(context, self.NAME, setting_name, settings)
return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def debug_var(self, context, *args, **kwargs):
"""
Add a debug rule request
Adds debug item for variables
debug_var(<service>.<method>.<var>, <context_id>[+], <debug_id>)
with service, method and vat that can be '*'
:param context:
:param rule:
:param context_id:
:param debug_id:
:param enabled:
:param args:
:param kwargs:
:return:
"""
rule = str(rule) if rule is not None else None
for setting in self.debug_rules_settings:
if setting.rule_id == rule and \
setting.context_id == context_id and \
setting.debug_id == debug_id:
setting.enabled = enabled
break
i, s, m, c_id, c_children, d, e = self.parse_debug_args("variable", *args, **kwargs)
return self.add_or_update_debug_item(context, "vars", i, s, m, c_id, c_children, d, False, e)
def debug_rule(self, context, *args, **kwargs):
"""
Adds debug item for rules
debug_var(<service>.<method>.<rule>, <context_id>[+], <debug_id>)
with service, method and rule that can be '*'
debug_var(rule_id, <context_id>[+], <debug_id>)
:param context:
:param args:
:param kwargs:
:return:
"""
i, s, m, c_id, c_children, d, e = self.parse_debug_args("rule", *args, **kwargs)
return self.add_or_update_debug_item(context, "rules", i, s, m, c_id, c_children, d, False, e)
def debug_concept(self, context, *args, **kwargs):
"""
Adds debug item for concepts
debug_var(<service>.<method>.<concept>, <context_id>[+], <debug_id>)
with service, method and vat that can be '*'
debug_var(concept_id, <context_id>[+], <debug_id>)
:param context:
:param args:
:param kwargs:
:return:
"""
i, s, m, c_id, c_children, d, e = self.parse_debug_args("concept", *args, **kwargs)
return self.add_or_update_debug_item(context, "concepts", i, s, m, c_id, c_children, d, False, e)
def compute_debug_var(self, context, service_name, method_name, item, debug_id):
return self.compute_debug_item("vars", context, service_name, method_name, item, debug_id)
def compute_debug_concept(self, context, service_name, method_name, item, debug_id):
return self.compute_debug_item("concepts", context, service_name, method_name, item, debug_id)
def compute_debug_rule(self, context, service_name, method_name, item, debug_id):
return self.compute_debug_item("rules", context, service_name, method_name, item, debug_id)
@staticmethod
def container_name(item_type):
return f"debug_{item_type}_settings"
@staticmethod
def parse_debug_args(item_name, *args, **kwargs):
service, method_name, context_id, context_children, item, debug_id, enabled = None, None, None, False, None, None, True
if len(args) > 0:
if args[0] is None or args[0] == "":
pass
elif isinstance(args[0], int):
item = str(args[0])
else:
self.debug_rules_settings.append(DebugRuleSetting(rule,
context_id,
debug_id,
enabled))
parts = args[0].split(".")
service = None if parts[0] == "*" else parts[0]
if len(parts) > 1:
method_name = None if parts[1] == "*" else parts[1]
if len(parts) > 2:
item = parts[2]
self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings)
return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
if len(args) > 1:
context_part = args[1]
if isinstance(context_part, int):
context_id = context_part
if isinstance(context_part, str):
m = SheerkaDebugManager.children_activation_regex.match(context_part)
if m:
context_id = int(m.group(1))
context_children = True
else:
try:
context_id = int(context_part)
except ValueError:
pass
def compute_debug_rule(self, rule_id, context_id, debug_id):
if not self.activated:
return False
if len(args) > 2:
debug_id = args[2]
selected = []
for setting in self.debug_rules_settings:
if (setting.rule_id is None or setting.rule_id == rule_id) and \
(setting.context_id is None or setting.context_id == context_id) and \
(setting.debug_id is None or setting.debug_id == debug_id):
selected.append(setting.enabled)
service = kwargs.get("service", service)
method_name = kwargs.get("method", method_name)
context_id = kwargs.get("context_id", context_id)
context_children = kwargs.get("context_children", context_children)
item = kwargs.get(item_name, item)
debug_id = kwargs.get("debug_id", debug_id)
enabled = kwargs.get("enabled", enabled)
if len(selected) == 0:
return False
return item, service, method_name, context_id, context_children, debug_id, enabled
res = selected[0]
for enabled in selected[1:]:
res &= enabled
# def debug_rule(self, context, rule=None, context_id=None, debug_id=None, enabled=True):
# """
# Add a debug rule request
# :param context:
# :param rule:
# :param context_id:
# :param debug_id:
# :param enabled:
# :return:
# """
# rule = str(rule) if rule is not None else None
# for setting in self.debug_rules_settings:
# if setting.rule_id == rule and \
# setting.context_id == context_id and \
# setting.debug_id == debug_id:
# setting.enabled = enabled
# break
# else:
# self.debug_rules_settings.append(DebugRuleSetting(rule,
# context_id,
# debug_id,
# enabled))
#
# self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings)
# return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
#
# def debug_concept(self, context, concept=None, context_id=None, debug_id=None, enabled=True):
# """
# Add a debug rule request
# :param context:
# :param concept:
# :param context_id:
# :param debug_id:
# :param enabled:
# :return:
# """
# concept_id = concept.id if isinstance(concept, Concept) else str(concept) if concept is not None else None
# for setting in self.debug_concepts_settings:
# if setting.concept_id == concept_id and \
# setting.context_id == context_id and \
# setting.debug_id == debug_id:
# setting.enabled = enabled
# break
# else:
# self.debug_concepts_settings.append(DebugConceptSetting(concept_id,
# context_id,
# debug_id,
# enabled))
#
# self.sheerka.record_var(context, self.NAME, "debug_concepts_settings", self.debug_concepts_settings)
# return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
#
return res
#
def reset_debug_rules(self, context):
self.debug_rules_settings.clear()
self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings)
return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def get_debug_settings(self):
return self.sheerka.new(BuiltinConcepts.TO_LIST, body=self.debug_vars_settings)
# def compute_debug_var(self, service_name, method_name, context_id, variable_name, debug_id):
# if not self.activated:
# return False
#
# selected = []
# for setting in self.debug_vars_settings:
# if setting.variable_name is None and setting.debug_id is None:
# continue
#
# if (setting.service_name is None or setting.service_name == service_name) and \
# (setting.method_name is None or setting.method_name == method_name) and \
# (setting.context_id is None or setting.context_id == context_id) and \
# (setting.variable_name is None or
# setting.variable_name == "*" or
# setting.variable_name == variable_name) and \
# (setting.debug_id is None or setting.debug_id == debug_id):
# selected.append(setting.enabled)
#
# if len(selected) == 0:
# return False
#
# res = selected[0]
# for enabled in selected[1:]:
# if res == False or enabled == False:
# return False
#
# if isinstance(res, str):
# continue
#
# res = enabled
#
# return res
#
# def compute_debug_rule(self, rule_id, context_id, debug_id):
# if not self.activated:
# return False
#
# selected = []
# for setting in self.debug_rules_settings:
# if (setting.rule_id is None or setting.rule_id == rule_id) and \
# (setting.context_id is None or setting.context_id == context_id) and \
# (setting.debug_id is None or setting.debug_id == debug_id):
# selected.append(setting.enabled)
#
# if len(selected) == 0:
# return False
#
# res = selected[0]
# for enabled in selected[1:]:
# res &= enabled
#
# return res
#
# def compute_debug_concept(self, concept_id, context_id, debug_id):
# if not self.activated:
# return False
#
# selected = []
# for setting in self.debug_concepts_settings:
# if (setting.concept_id is None or setting.concept_id == concept_id) and \
# (setting.context_id is None or setting.context_id == context_id) and \
# (setting.debug_id is None or setting.debug_id == debug_id):
# selected.append(setting.enabled)
#
# if len(selected) == 0:
# return False
#
# res = selected[0]
# for enabled in selected[1:]:
# res &= enabled
#
# return res
#
# def reset_debug_rules(self, context):
# self.debug_rules_settings.clear()
# self.sheerka.record_var(context, self.NAME, "debug_rules_settings", self.debug_rules_settings)
# return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
#
# def get_debug_settings(self):
# lst = self.debug_vars_settings + self.debug_concepts_settings + self.debug_rules_settings
# return self.sheerka.new(BuiltinConcepts.TO_LIST, body=lst)
src/core/sheerka/services/SheerkaModifyConcept.py
-6
View File
@@ -2,7 +2,6 @@ from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import ensure_concept
from core.concept import NotInit, freeze_concept_attrs, Concept
from core.sheerka.services.sheerka_service import BaseService
from parsers.BnfDefinitionParser import BnfDefinitionParser
class SheerkaModifyConcept(BaseService):
@@ -74,11 +73,6 @@ class SheerkaModifyConcept(BaseService):
return
for concept_id in refs:
concept = self.sheerka.get_by_id(concept_id)
if concept.get_bnf() is not None:
BnfDefinitionParser.update_recurse_id(context, concept_id, concept.get_bnf())
# remove the grammar entry so that it can be recreated
self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_id)
src/core/sheerka/services/SheerkaOut.py
+1 -1
View File
@@ -10,7 +10,7 @@ class SheerkaOut(BaseService):
def __init__(self, sheerka):
super().__init__(sheerka)
self.out_visitors = [ConsoleVisitor()]
self.out_visitors = [ConsoleVisitor(expand_mode="all_but_first")]
def initialize(self):
self.sheerka.bind_service_method(self.process_return_values, False)
src/core/sheerka/services/SheerkaRuleManager.py
+1
View File
@@ -644,6 +644,7 @@ class SheerkaRuleManager(BaseService):
self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[2], RULE_COMPARISON_CONTEXT)
self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[3], RULE_COMPARISON_CONTEXT)
self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[5], RULE_COMPARISON_CONTEXT)
self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[6], RULE_COMPARISON_CONTEXT)
self.sheerka.set_is_greatest(context, BuiltinConcepts.PRECEDENCE, rules[0], RULE_COMPARISON_CONTEXT)
def get_rule_by_id(self, rule_id):
src/core/sheerka/services/SheerkaSetsManager.py
+3
View File
@@ -89,6 +89,9 @@ class SheerkaSetsManager(BaseService):
# update concept_set references
self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
# remove the grammar entry so that it can be recreated
self.sheerka.cache_manager.delete(self.sheerka.CONCEPTS_GRAMMARS_ENTRY, concept_set.id)
return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
def add_concepts_to_set(self, context, concepts, concept_set):
src/out/ConsoleVisistor.py
+8
View File
@@ -7,6 +7,14 @@ class ConsoleVisitor(AsStrVisitor):
"""
def __init__(self, expand_mode="auto"):
"""
expand_mode:
auto: not the first dict, the sub ones depends of the width
always: all the dicts (and all the lists)
all_but_first: not the first one, but force the sub dict
never: never expand
:param expand_mode:
"""
super().__init__()
self.out = print
self.expand_mode = expand_mode
src/parsers/BnfDefinitionParser.py
-19
View File
@@ -294,27 +294,8 @@ class BnfDefinitionParser(BaseParser):
expression.rule_name = token.value
self.next_token()
if BnfDefinitionParser.is_expression_a_set(self.context, expression):
root_concept = self.context.search(start_with_self=True,
predicate=lambda ec: ec.action == BuiltinConcepts.INIT_BNF,
get_obj=lambda ec: ec.action_context,
stop=lambda ec: ec.action == BuiltinConcepts.INIT_BNF)
root_concept = list(root_concept)
if root_concept and hasattr(root_concept[0], "id"):
expression.recurse_id = expression.get_recurse_id(root_concept[0].id,
expression.concept.id,
expression.rule_name)
return expression
@staticmethod
def is_expression_a_set(context, expression):
return isinstance(expression, ConceptExpression) and context.sheerka.isaset(context, expression.concept)
@staticmethod
def update_recurse_id(context, concept_id, expression):
if BnfDefinitionParser.is_expression_a_set(context, expression):
expression.recurse_id = expression.get_recurse_id(concept_id, expression.concept.id, expression.rule_name)
for element in expression.elements:
BnfDefinitionParser.update_recurse_id(context, concept_id, element)
src/parsers/BnfNodeParser.py
+165 -138
View File
@@ -17,8 +17,8 @@ from core.builtin_concepts import BuiltinConcepts
from core.concept import DEFINITION_TYPE_BNF, DoNotResolve, ConceptParts, Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, TokenKind, Token
from core.utils import CONSOLE_COLORS_MAP as CCM
from parsers.BaseNodeParser import BaseNodeParser, GrammarErrorNode, UnrecognizedTokensNode, ConceptNode, LexerNode
from parsers.BaseParser import BaseParser
PARSERS = ["Sequence", "Sya", "Python"]
@@ -50,6 +50,10 @@ class ParsingContext:
res.append(self.clone())
return res
def __repr__(self):
res = f"ParsingContext('{self.node.get_debug()}', pos={self.pos})"
return res
class NonTerminalNode(LexerNode):
"""
@@ -86,6 +90,10 @@ class NonTerminalNode(LexerNode):
clone = NonTerminalNode(self.parsing_expression, self.start, self.end, self.tokens, self.children.copy())
return clone
def get_debug(self):
res = f"{self.parsing_expression.concept}=>" if isinstance(self.parsing_expression, ConceptExpression) else ""
return res + ".".join([c.get_debug() for c in self.children])
class TerminalNode(LexerNode):
"""
@@ -118,6 +126,9 @@ class TerminalNode(LexerNode):
clone = TerminalNode(self.parsing_expression, self.start, self.end, self.value)
return clone
def get_debug(self):
return self.value
class MultiNode:
""""
@@ -155,8 +166,6 @@ class ParsingExpression:
def __init__(self, *args, **kwargs):
self.elements = args
self.debug_enabled = False
self._has_unordered_choice = None
nodes = kwargs.get('nodes', []) or []
if not hasattr(nodes, '__iter__'):
@@ -184,45 +193,19 @@ class ParsingExpression:
def __hash__(self):
return hash((self.rule_name, self.elements))
def parse(self, parser):
def parse(self, parser_helper):
# TODO : add memoization
if self.debug_enabled:
self.debug(f">> {parser.pos:3d} : {self}")
# parser_helper.debugger.debug_log(f">> {parser_helper.pos:3d} : {self}")
# if self.debug_enabled:
# self.debug(f">> {parser_helper.pos:3d} : {self}")
res = self._parse(parser)
res = self._parse(parser_helper)
return res
def add_rule_name_if_needed(self, text):
return text + "=" + self.rule_name if self.rule_name else text
def has_unordered_choice(self):
if self._has_unordered_choice is None:
visitor = HasUnorderedChoiceVisitor()
visitor.visit(self)
self._has_unordered_choice = visitor.value
return self._has_unordered_choice
def debug(self, msg):
self.log_sink.append((id(self), msg))
def get_debug(self):
if not self.debug_enabled:
return None
# search for the first debug line for the current pexpression
id_self = id(self)
for i, line in enumerate(self.log_sink):
if line[0] == id_self:
break
else:
return ""
n, debug = self.inner_get_debug(i, "")
self.log_sink.clear()
return debug
def inner_get_debug(self, n, tab=""):
"""
@@ -275,6 +258,13 @@ class ParsingExpression:
return n, debug
@staticmethod
def debug_prefix(self_name, parser_helper):
current_rule_name = parser_helper.get_current_rule_name()
current_concept = parser_helper.concepts[-1]
str_rule_name = f":{current_rule_name}" if current_rule_name not in (None, current_concept.name) else ""
return f"{self_name}({current_concept}{str_rule_name})"
class ConceptExpression(ParsingExpression):
"""
@@ -284,10 +274,9 @@ class ConceptExpression(ParsingExpression):
When the grammar is created, it is replaced by the actual concept
"""
def __init__(self, concept, rule_name="", recurse_id=None, nodes=None):
def __init__(self, concept, rule_name="", nodes=None):
super().__init__(rule_name=rule_name, nodes=nodes)
self.concept = concept
self.recurse_id = recurse_id
def __repr__(self):
return self.add_rule_name_if_needed(f"{self.concept}")
@@ -299,10 +288,6 @@ class ConceptExpression(ParsingExpression):
if not isinstance(other, ConceptExpression):
return False
# TODO : enable self.recurse_id when it will be correctly implemented
# if self.recurse_id != other.recurse_id:
# return False
if isinstance(self.concept, Concept):
return self.concept.id == other.concept.id
@@ -313,7 +298,17 @@ class ConceptExpression(ParsingExpression):
return hash((self.concept, self.rule_name))
def _parse(self, parser_helper):
parser_helper.rules_names.append(self.rule_name)
parser_helper.push_concept(self.concept)
# parser_helper.debug_concept(self.debug_prefix("ConceptExpression", parser_helper) + "=start")
node = self.nodes[0].parse(parser_helper)
# parser_helper.debug_concept(self.debug_prefix("ConceptExpression", parser_helper) + "=end")
parser_helper.pop_concept()
parser_helper.rules_names.pop()
if node is None:
return None
@@ -327,7 +322,7 @@ class ConceptExpression(ParsingExpression):
[node])
@staticmethod
def get_recurse_id(parent_id, concept_id, rule_name):
def get_recursion_id(parent_id, concept_id, rule_name):
return f"{parent_id}#{concept_id}({rule_name})"
@@ -340,6 +335,9 @@ class Sequence(ParsingExpression):
init_pos = parser_helper.pos
end_pos = parser_helper.pos
if parser_helper.debugger.is_enabled():
debug_prefix = self.debug_prefix("Sequence", parser_helper)
parser_helper.debug_concept(debug_prefix, nodes=self.nodes)
ntn = NonTerminalNode(self,
init_pos,
end_pos,
@@ -351,10 +349,14 @@ class Sequence(ParsingExpression):
for e in self.nodes:
for pcontext in parsing_contexts:
if parser_helper.debugger.is_enabled():
parser_helper.debug_concept(debug_prefix, node=e, pcontext=pcontext)
parser_helper.seek(pcontext.pos)
node = e.parse(parser_helper)
if node is None:
to_remove.append(pcontext)
elif isinstance(node, MultiNode):
clones = pcontext * len(node.results) # clones pcontext (but first item is pcontext)
to_append.extend(clones[1:])
@@ -373,8 +375,8 @@ class Sequence(ParsingExpression):
parsing_contexts.extend(to_append)
if len(parsing_contexts) == 0:
if self.debug_enabled:
self.debug(f"<< Failed matching {e}")
if parser_helper.debugger.is_enabled():
parser_helper.debug_concept(debug_prefix + " All pcontexts are failed. Sequence failed")
return None
to_append.clear()
@@ -388,12 +390,10 @@ class Sequence(ParsingExpression):
pcontext.fix_tokens(parser_helper)
if len(parsing_contexts) == 1:
if self.debug_enabled:
self.debug(f"<< Found match '{parsing_contexts[0].node.source}'")
# parser_helper.debugger.debug_log(f"<< Found match '{parsing_contexts[0].node.source}'")
return parsing_contexts[0].node
if self.debug_enabled:
self.debug(f"<< Found matches {[r.node.source for r in parsing_contexts]}")
# parser_helper.debugger.debug_log(f"<< Found matches {[r.node.source for r in parsing_contexts]}")
return MultiNode(parsing_contexts)
def __repr__(self):
@@ -440,9 +440,18 @@ class UnOrderedChoice(ParsingExpression):
init_pos = parser_helper.pos
parsing_contexts = []
if parser_helper.debugger.is_enabled():
debug_prefix = self.debug_prefix("UnOrderedChoice", parser_helper)
parser_helper.debug_concept(debug_prefix)
debug_text = ""
for e in self.nodes:
if isinstance(e, ConceptExpression) and e.concept.id in parser_helper.get_concepts_ids():
# avoid circular reference
continue
node = e.parse(parser_helper)
if node:
debug_text += CCM["green"] + str(e) + CCM["reset"] + ", "
if isinstance(node, MultiNode):
node.combine(self)
parsing_contexts.extend(node.results)
@@ -453,8 +462,13 @@ class UnOrderedChoice(ParsingExpression):
parser_helper.parser.parser_input.tokens[init_pos: node.end + 1],
[node])
parsing_contexts.append(ParsingContext(tn, parser_helper.pos))
else:
debug_text += f"{e}, "
parser_helper.seek(init_pos) # backtrack
if parser_helper.debugger.is_enabled():
parser_helper.debug_concept(debug_prefix, raw=f"[{debug_text}]")
if len(parsing_contexts) == 0:
return None
@@ -675,18 +689,22 @@ class StrMatch(Match):
def _parse(self, parser_helper):
token = parser_helper.get_token()
if parser_helper.debugger.is_enabled():
debug_prefix = self.debug_prefix("StrMatch", parser_helper)
debug_text = f"pos={parser_helper.pos}, to_match={self.to_match}, token={token.str_value}"
m = token.str_value.lower() == self.to_match.lower() if self.ignore_case \
else token.strip_quote == self.to_match
if m:
if self.debug_enabled:
self.debug(f"pos={parser_helper.pos}, token={token.str_value}, to_match={self.to_match} => Matched")
if parser_helper.debugger.is_enabled():
parser_helper.debug_concept(debug_prefix, raw=f"{CCM['green']}{debug_text}{CCM['reset']}")
node = TerminalNode(self, parser_helper.pos, parser_helper.pos, token.str_value)
parser_helper.next_token(self.skip_white_space)
return node
if self.debug_enabled:
self.debug(f"pos={parser_helper.pos}, token={token.str_value}, to_match={self.to_match} => No Match")
if parser_helper.debugger.is_enabled():
parser_helper.debug_concept(debug_prefix, raw=f"{CCM['red']}{debug_text}{CCM['reset']}")
return None
@@ -839,28 +857,17 @@ class BnfNodeConceptExpressionVisitor(ParsingExpressionVisitor):
self.references.append(pe.concept)
class HasUnorderedChoiceVisitor(ParsingExpressionVisitor):
def __init__(self):
super().__init__(lambda pe: pe.nodes, circular_ref_strategy="skip")
self.value = False
def __repr__(self):
return f"HasUnorderedChoiceVisitor(={self.value})"
def reset(self):
self.value = False
def visit_UnOrderedChoice(self, parsing_expression):
self.value = True
return ParsingExpressionVisitor.STOP
class BnfConceptParserHelper:
def __init__(self, parser):
def __init__(self, parser, debugger):
self.parser = parser
self.debug = []
self.errors = []
self.sequence = []
self.debugger = debugger
self.debug = [] # keep track of the tokens
self.errors = [] # sink of errors
self.sequence = [] # output. List of lexer nodes correctly parsed
self.concepts = [] # stack of concepts being processed (fed by ConceptExpression)
self.concepts_ids = [] # ids if the concept to increase speed
self.rules_names = [] # stack of concepts rules names
self.concept_depth = 0 # depth of concept (+1 for each ConceptExpression which is not an OrderedChoice)
self.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, [])
self.has_unrecognized = False
@@ -872,7 +879,8 @@ class BnfConceptParserHelper:
self.pos = -1
def __repr__(self):
return f"BnfConceptParserHelper({self.sequence})"
concepts = [item.concept if isinstance(item, ConceptNode) else "***" for item in self.sequence]
return f"BnfConceptParserHelper({concepts})"
def __eq__(self, other):
if id(self) == id(other):
@@ -886,6 +894,26 @@ class BnfConceptParserHelper:
def __hash__(self):
return len(self.sequence) + len(self.errors)
def debug_concept(self, text, **kwargs):
if len(self.concepts) <= 2:
self.debugger.debug_concept(self.concepts[0], text, **kwargs)
def get_current_rule_name(self):
for rule_name in reversed(self.rules_names):
if rule_name:
return rule_name
def push_concept(self, concept):
self.concepts.append(concept)
self.concepts_ids.append(concept.id)
def pop_concept(self):
self.concepts.pop()
self.concepts_ids.pop()
def get_concepts_ids(self):
return self.concepts_ids
def get_token(self) -> Token:
return self.token
@@ -917,7 +945,10 @@ class BnfConceptParserHelper:
if self.is_locked():
return
try:
self.push_concept(concept)
self.debug.append(concept)
self.manage_unrecognized()
for forked in self.forked:
# manage the fact that some clone may have been forked
@@ -937,6 +968,7 @@ class BnfConceptParserHelper:
self.token = self.parser.parser_input.tokens[self.pos]
# parse
self.debugger.debug_concept(concept, parsing_expression=parsing_expression)
node = parsing_expression.parse(self)
if isinstance(node, MultiNode):
@@ -950,6 +982,8 @@ class BnfConceptParserHelper:
self.debug.append(("Rewind", token))
self.unrecognized_tokens.add_token(token, self.parser.parser_input.pos)
self.pos = self.parser.parser_input.pos # reset position
finally:
self.concepts.pop()
def eat_unrecognized(self, token):
if self.is_locked():
@@ -998,7 +1032,7 @@ class BnfConceptParserHelper:
self.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, [])
def clone(self):
clone = BnfConceptParserHelper(self.parser)
clone = BnfConceptParserHelper(self.parser, self.debugger)
clone.debug = self.debug[:]
self.errors = self.errors[:]
clone.sequence = self.sequence[:]
@@ -1148,7 +1182,6 @@ class ToUpdate:
class BnfNodeParser(BaseNodeParser):
NAME = "Bnf"
def __init__(self, **kwargs):
@@ -1215,7 +1248,7 @@ class BnfNodeParser(BaseNodeParser):
return res[0] if len(res) == 1 else Sequence(*res)
def get_concepts_sequences(self):
def get_concepts_sequences(self, context):
"""
Main method that parses the tokens and extract the concepts
:return:
@@ -1241,23 +1274,36 @@ class BnfNodeParser(BaseNodeParser):
return by_end_pos[max(by_end_pos)]
forked = []
concept_parser_helpers = [BnfConceptParserHelper(self)]
debugger = context.get_debugger(self.NAME, "parse")
debugger.debug_entering(source=self.parser_input.as_text())
concept_parser_helpers = [BnfConceptParserHelper(self, debugger)]
while self.parser_input.next_token(False):
token = self.parser_input.token
if debugger.is_enabled():
debug_prefix = f"pos={self.parser_input.pos}, {token=}, {len(concept_parser_helpers)} parser(s)"
try:
not_locked = [p for p in concept_parser_helpers if not p.is_locked()]
if len(not_locked) == 0:
if debugger.is_enabled():
debugger.debug_log(debug_prefix + ", all parsers are locked. Nothing to do.")
continue
concepts = self.get_concepts(token, self._is_eligible, strip_quotes=False)
if not concepts:
for concept_parser in concept_parser_helpers:
if debugger.is_enabled():
debugger.debug_log(debug_prefix + ", no concept found.")
for concept_parser in not_locked:
concept_parser.eat_unrecognized(token)
continue
if debugger.is_enabled():
debugger.debug_log(debug_prefix + f", concept(s) found={concepts}")
if len(concepts) == 1:
for concept_parser in concept_parser_helpers:
for concept_parser in not_locked:
concept_parser.eat_concept(concepts[0], token)
continue
@@ -1274,9 +1320,13 @@ class BnfNodeParser(BaseNodeParser):
clone = concept_parser.clone()
temp_res.append(clone)
clone.eat_concept(concept, token)
if debugger.is_enabled():
debugger.debug_log(f"..{concept}, parsed={clone.bnf_parsed}, length={clone.pos}")
# only keep the longest
concept_parser_helpers = _get_longest(temp_res)
if debugger.is_enabled() and len(temp_res) > 1:
debugger.debug_log(f"Only keep longest -> {len(concept_parser_helpers)} parser(s) left")
finally:
_add_forked_to_concept_parser_helpers()
@@ -1286,6 +1336,7 @@ class BnfNodeParser(BaseNodeParser):
concept_parser.finalize()
_add_forked_to_concept_parser_helpers()
debugger.debug_var("result", concept_parser_helpers)
return concept_parser_helpers
def fix_infinite_recursions(self, context, grammar, concept_id, parsing_expression):
@@ -1306,7 +1357,7 @@ class BnfNodeParser(BaseNodeParser):
for node_id in path_:
expression_ = expression_.nodes[0] if isinstance(expression_, ConceptExpression) else expression_
for i, node in [(i, n) for i, n in enumerate(expression_.nodes) if isinstance(n, ConceptExpression)]:
if node_id in (node.recurse_id, node.concept.id):
if node_id == node.concept.id:
index_ = i
parent_ = expression_
expression_ = node # take the child of the ConceptExpression found
@@ -1336,7 +1387,6 @@ class BnfNodeParser(BaseNodeParser):
expression_update.rule_name, new_grammar, set())
new = ConceptExpression(expression_update.concept,
rule_name=expression_update.rule_name,
recurse_id=expression_update.recurse_id,
nodes=new_nodes)
parent.nodes[index] = new
@@ -1358,12 +1408,12 @@ class BnfNodeParser(BaseNodeParser):
def check_for_infinite_recursion(self, parsing_expression, already_found, in_recursion, only_first=False):
if isinstance(parsing_expression, ConceptExpression):
id_to_use = parsing_expression.recurse_id or parsing_expression.concept.id
if id_to_use in already_found:
already_found.append(id_to_use) # add the id again, to know where the cycle starts
if parsing_expression.concept.id in already_found:
already_found.append(parsing_expression.concept.id) # add the id again, to know where the cycle starts
in_recursion.extend(already_found)
return True
already_found.append(id_to_use)
already_found.append(parsing_expression.concept.id)
return self.check_for_infinite_recursion(parsing_expression.nodes[0],
already_found,
in_recursion,
@@ -1396,13 +1446,13 @@ class BnfNodeParser(BaseNodeParser):
return False
return False
if isinstance(parsing_expression, UnOrderedChoice):
for node in parsing_expression.nodes:
already_found_for_current_node.clear()
already_found_for_current_node.extend(already_found.copy())
if self.check_for_infinite_recursion(node, already_found_for_current_node, in_recursion, True):
return True
return False
# if isinstance(parsing_expression, UnOrderedChoice):
# for node in parsing_expression.nodes:
# already_found_for_current_node.clear()
# already_found_for_current_node.extend(already_found.copy())
# if self.check_for_infinite_recursion(node, already_found_for_current_node, in_recursion, True):
# return True
# return False
return False
@@ -1429,50 +1479,34 @@ class BnfNodeParser(BaseNodeParser):
desc=desc) as sub_context:
# get the parsing expression
to_skip = {concept.id}
ret = self.resolve_concept_parsing_expression(sub_context, concept, None, grammar, to_skip, to_update)
presult = self.resolve_concept_parsing_expression(sub_context, concept, None, grammar, to_skip, to_update)
# check and update parsing expression that are still under construction
# Note that we only update the concept that will update concepts_grammars
# because pe.node may be large
for item in to_update:
pe = item.parsing_expression
for i, node in enumerate(pe.nodes):
if isinstance(node, UnderConstruction):
pe.nodes[i] = grammar.get(node.concept_id)
# KSI 20200826
# To be rewritten into get_infinite_recursions
# I have changed resolve_concept_parsing_expression() to directly avoid obvious circular references
# So it's no longer need to search and fix them
concepts_in_recursion = self.fix_infinite_recursions(context, grammar, concept.id, ret)
# check for infinite recursion definitions
already_seen = [concept.id]
in_recursion = [] # there may be cases where in_recursion is less than already_seen
concepts_in_recursion = self.check_for_infinite_recursion(presult, already_seen, in_recursion)
if concepts_in_recursion:
chicken_anf_egg = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_recursion)
for concept_id in concepts_in_recursion:
chicken_anf_egg = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=in_recursion)
for concept_id in in_recursion:
grammar[concept_id] = chicken_anf_egg
# update, in case of infinite circular recursion
ret = grammar[concept.id]
# update, in case of infinite recursion
presult = grammar[concept.id]
# finally, update the list of the known pexpression (self.concepts_grammars)
# We do not add pexpressions that contain UnOrderedChoice because the choices always depend on the current
# concept.
# For example, the pexpression for 'twenties' found under the concept 'hundreds' won't be the same than
# the pexpression 'twenties' under the concept 'thousand' or even the pexpression 'twenties' without any
# context.
# finally, update the list of the known pexpression (self.concepts_grammars) for latter use
for k, v in grammar.items():
if k == concept.id:
self.concepts_grammars.put(k, v)
elif context.sheerka.isinstance(v, BuiltinConcepts.CHICKEN_AND_EGG):
# not quite sure that it is a good idea.
# Why do we want to corrupt previous valid entries ?
self.concepts_grammars.put(k, v)
else:
if not v.has_unordered_choice():
self.concepts_grammars.put(k, v)
sub_context.add_values(return_values=ret)
sub_context.add_values(return_values=presult)
return ret
return presult
def resolve_concept_parsing_expression(self, context, concept, name, grammar, to_skip, to_update):
"""
@@ -1487,16 +1521,17 @@ class BnfNodeParser(BaseNodeParser):
"""
sheerka = context.sheerka
if sheerka.isaset(context, concept) and hasattr(context, "obj"):
key_to_use = ConceptExpression.get_recurse_id(context.obj.id, concept.id, name)
else:
# if sheerka.isaset(context, concept) and hasattr(context, "obj"):
# key_to_use = ConceptExpression.get_recursion_id(context.obj.id, concept.id, name)
# else:
# key_to_use = concept.id
key_to_use = concept.id
if key_to_use in self.concepts_grammars:
# Use the global pexpression only if it does not contains UnOrderedChoice
pe = self.concepts_grammars.get(key_to_use)
if not pe.has_unordered_choice():
return self.concepts_grammars.get(key_to_use)
# # Use the global pexpression only if it does not contains UnOrderedChoice
# pe = self.concepts_grammars.get(key_to_use)
# if not pe.has_unordered_choice():
if key_to_use in grammar: # under construction entry
return grammar.get(key_to_use)
@@ -1522,20 +1557,12 @@ class BnfNodeParser(BaseNodeParser):
ssc.add_inputs(concept=concept)
concepts_in_group = self.sheerka.get_set_elements(ssc, concept)
valid_concepts = [c for c in concepts_in_group if c.id not in to_skip]
# for c in concepts_in_group:
# if c.id == context.obj.id:
# continue
#
# if hasattr(context, "concepts_to_skip") and c.id in context.concepts_to_skip:
# continue
#
# valid_concepts.append(c)
# valid_concepts = [c for c in concepts_in_group if c.id not in to_skip]
valid_concepts = concepts_in_group
nodes = []
for c in valid_concepts:
c_recurse_id = f"{c.id}#{c.name}#{concept.id}" if self.sheerka.isaset(context, c) else None
nodes.append(ConceptExpression(c, rule_name=c.name, recurse_id=c_recurse_id))
nodes.append(ConceptExpression(c, rule_name=c.name))
resolved = self.resolve_parsing_expression(ssc,
UnOrderedChoice(*nodes),
@@ -1664,7 +1691,7 @@ class BnfNodeParser(BaseNodeParser):
False,
context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))
sequences = self.get_concepts_sequences()
sequences = self.get_concepts_sequences(context)
valid_parser_helpers = self.get_valid(sequences)
if valid_parser_helpers is None:
# token error
src/parsers/PythonParser.py
+4
View File
@@ -40,10 +40,12 @@ class PythonNode(Node):
self.ast_ = ast_ # if ast_ else ast.parse(source, mode="eval") if source else None
self.objects = objects or {} # when objects (mainly concepts or rules) are recognized in the expression
self.compiled = None
self.ast_str = self.get_dump(self.ast_)
def init_ast(self):
if self.ast_ is None and self.source:
self.ast_ = ast.parse(self.source, mode="eval")
self.ast_str = self.get_dump(self.ast_)
return self
def get_compiled(self):
@@ -77,6 +79,8 @@ class PythonNode(Node):
@staticmethod
def get_dump(ast_):
if not ast_:
return None
dump = ast.dump(ast_)
for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
dump = dump.replace(to_remove, "")
src/sheerkapickle/sheerka_handlers.py
+2
View File
@@ -198,6 +198,7 @@ class PythonNodeHandler(BaseHandler):
pickler = self.context
data["source"] = obj.source
data["ast_str"] = obj.ast_str
data["objects"] = pickler.flatten(obj.objects)
return data
@@ -208,6 +209,7 @@ class PythonNodeHandler(BaseHandler):
pickler = self.context
instance.__init__(data["source"], objects=pickler.restore(data["objects"]))
instance.ast_str = data["ast_str"]
return instance
tests/core/test_SheerkaDebugManager.py
+274 -131
View File
@@ -1,7 +1,7 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.sheerka.ExecutionContext import ExecutionContext
from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugVarSetting, DebugRuleSetting
from core.sheerka.services.SheerkaDebugManager import SheerkaDebugManager, DebugItem
from sdp.sheerkaDataProvider import Event
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -224,6 +224,66 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
assert sub_sub_context.debug_enabled
assert not root_context.debug_enabled
@pytest.mark.parametrize("item_type", [
"vars", "rules", "concepts"
])
def test_i_can_add_a_debug_item(self, item_type):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service_name = "service_name"
method_name = "method_name"
item = "id"
context_id = 1
context_children = True
debug_id = 1
debug_children = True
service.add_or_update_debug_item(context,
item_type,
item,
service_name,
method_name,
context_id,
context_children,
debug_id,
debug_children,
True)
item_container = f"debug_{item_type}_settings"
assert getattr(service, item_container) == [DebugItem(
item,
service_name,
method_name,
context_id,
context_children,
debug_id,
debug_children,
True
)]
def test_i_manage_debug_item_default_values(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.add_or_update_debug_item(context, "vars", item="item")
assert service.debug_vars_settings == [
DebugItem("item", None, None, None, False, None, False, True)
]
def test_i_can_update_debug_item(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.add_or_update_debug_item(context, "vars", "item", "service_name", "method_name", enabled=True)
service.add_or_update_debug_item(context, "vars", "item2", "service_name", "method_name", enabled=True)
service.add_or_update_debug_item(context, "vars", "item", "service_name", "method_name", enabled=False)
assert service.debug_vars_settings == [
DebugItem("item", "service_name", "method_name", None, False, None, False, False),
DebugItem("item2", "service_name", "method_name", None, False, None, False, True),
]
@pytest.mark.parametrize("settings, expected", [
({"service": "my_service"}, True),
({"service": "other_service"}, False),
@@ -235,8 +295,8 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.debug_var(context, **settings)
assert service.compute_debug("my_service", "my_method", context) == expected
service.add_or_update_debug_item(context, "vars", **settings)
assert service.compute_debug(context, "my_service", "my_method") == expected
def test_i_can_compute_debug_for_context(self):
sheerka, root_context = self.init_concepts()
@@ -245,109 +305,16 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
context1 = root_context.push(BuiltinConcepts.TESTING, None)
context2 = root_context.push(BuiltinConcepts.TESTING, None)
service.debug_var(root_context, context_id=context1.id)
service.add_or_update_debug_item(root_context, "vars", context_id=context1.id)
assert service.compute_debug("my_service", "my_method", context1)
assert not service.compute_debug("my_service", "my_method", context2)
def test_i_can_disable_debug_setting(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.debug_var(context, service="my_service", enabled=False)
assert not service.compute_debug("my_service", "my_method", context)
def test_i_can_compute_combination_of_debug_settings(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.debug_var(context, service="my_service", enabled=True)
service.debug_var(context, method="my_method", enabled=False)
service.debug_var(context, variable="xxx", enabled=False) # is not used
service.debug_var(context, debug_id=1, enabled=False) # is not used
assert not service.compute_debug("my_service", "my_method", context) # False > True
assert service.compute_debug("my_service", "another_method", context)
assert not service.compute_debug("another_service", "my_method", context)
def test_variable_debug_is_deactivated_by_default(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
assert not service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0)
def test_i_can_activate_variable_debug_for_all_variables(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.debug_var(context, variable="*")
assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == True
def test_i_can_activate_variable_display_for_all_variables(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.debug_var(context, variable="*", enabled='list')
assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == 'list'
def test_i_can_disable_variable_debug(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.debug_var(context, variable="my_variable", enabled=False)
assert not service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0)
@pytest.mark.parametrize("enabled_1, enabled_2, expected", [
(False, False, False),
(False, True, False),
(False, 'list', False),
(True, False, False),
(True, True, True),
(True, 'list', 'list'),
('list', False, False),
('list', True, 'list'),
('list', 'list', 'list'),
])
def test_i_can_compute_combination_of_debug_var_settings(self, enabled_1, enabled_2, expected):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.debug_var(context, variable="my_variable", enabled=enabled_1)
service.debug_var(context, variable="*", enabled=enabled_2)
assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == expected
@pytest.mark.parametrize("settings, expected", [
({"service": "my_service"}, True),
({"service": "other_service"}, False),
({"method": "my_method"}, True),
({"method": "other_method"}, False),
({"context_id": 0}, True),
({"context_id": 1}, False),
])
def test_service_and_method_and_context_id_are_tested_before_disabling_a_variable(self, settings, expected):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
settings["variable"] = "my_variable"
service.debug_var(context, **settings, enabled=True)
assert service.compute_var_debug("my_service", "my_method", 0, "my_variable", 0) == expected
assert service.compute_debug(context1, "my_service", "my_method")
assert not service.compute_debug(context2, "my_service", "my_method")
@pytest.mark.parametrize("context_children, expected", [
(True, True),
(False, False),
])
def test_i_can_compute_debug_var_for_context_children(self, context_children, expected):
def test_i_can_compute_debug_for_context_children(self, context_children, expected):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
@@ -355,49 +322,223 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
sub_context = context.push(BuiltinConcepts.TESTING, None)
sub_sub_context = sub_context.push(BuiltinConcepts.TESTING, None)
service.debug_var(context, context_id=sub_context.id, context_children=context_children)
assert service.compute_debug("my_service", "my_method", sub_sub_context) == expected
service.add_or_update_debug_item(context, "vars", context_id=sub_context.id, context_children=context_children)
assert service.compute_debug(sub_sub_context, "my_service", "my_method") == expected
def test_i_can_update_debug_setting(self):
def test_compute_debug_returns_true_in_case_of_conflict(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.debug_var(context, service="my_service", enabled=True)
service.debug_var(context, service="my_service", enabled=False)
service.add_or_update_debug_item(context, "vars", service="my_service", enabled=True)
service.add_or_update_debug_item(context, "rules", service="my_service", enabled=False)
assert len(service.debug_vars_settings) == 1
assert not service.debug_vars_settings[0].enabled
assert service.compute_debug(context, "my_service", "my_method")
service.debug_var(context, service="my_service", enabled=True)
assert len(service.debug_vars_settings) == 1
assert service.debug_vars_settings[0].enabled
@pytest.mark.parametrize("settings, expected", [
({"service": "my_service"}, False), # by default debug item is False if item is not specified
({"service": "other_service"}, False),
({"method": "my_method"}, False), # by default debug item is False if item is not specified
({"method": "other_method"}, False),
({"item": "my_item"}, True),
({"item": "other_item"}, False),
({"debug_id": 10}, True),
({"debug_id": 0}, False),
])
def test_i_can_compute_debug_item(self, settings, expected):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "rules", **settings)
assert service.compute_debug_item("rules", context, "my_service", "my_method", "my_item", 10) == expected
def test_i_can_compute_debug_item_using_the_correct_service(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "rules", service="service1", item="item", enabled=True)
service.add_or_update_debug_item(context, "rules", service="service2", item="item", enabled=False)
assert service.compute_debug_item("rules", context, "service1", "my_method", "item", 10)
assert not service.compute_debug_item("rules", context, "service2", "my_method", "item", 10)
def test_i_can_compute_debug_item_using_the_correct_method(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "rules", method="method1", item="item", enabled=True)
service.add_or_update_debug_item(context, "rules", method="method2", item="item", enabled=False)
assert service.compute_debug_item("rules", context, "my_service", "method1", "item", 10)
assert not service.compute_debug_item("rules", context, "my_service", "method2", "item", 10)
def test_i_can_compute_debug_item_using_the_correct_context(self):
sheerka, context = self.init_concepts()
another_context = context.push(BuiltinConcepts.TESTING, None)
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "rules", context_id=context.id, item="item", enabled=True)
service.add_or_update_debug_item(context, "rules", context_id=999, item="item", enabled=False)
assert service.compute_debug_item("rules", context, "my_service", "my_method", "item", 10)
assert not service.compute_debug_item("rules", another_context, "my_service", "my_method", "item", 10)
def test_i_can_compute_debug_item_using_the_correct_sub_context(self):
sheerka, context = self.init_concepts()
sub_context = context.push(BuiltinConcepts.TESTING, None)
another_context = self.get_context(sheerka)
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "rules", context_id=context.id, item="item", context_children=True)
assert service.compute_debug_item("rules", context, "my_service", "my_method", "item", 10)
assert service.compute_debug_item("rules", sub_context, "my_service", "my_method", "item", 10)
assert not service.compute_debug_item("rules", another_context, "my_service", "my_method", "item", 10)
def test_compute_debug_item_returns_false_in_case_of_conflict(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "concepts", service="my_service", item="item", enabled=True)
service.add_or_update_debug_item(context, "concepts", method="my_method", item="item", enabled=False)
assert not service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10)
def test_compute_debug_item_returns_false_in_case_of_conflict_2(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "concepts", item="item", enabled=True)
service.add_or_update_debug_item(context, "concepts", item="*", enabled=False)
assert not service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10)
@pytest.mark.parametrize("settings", [
{"service": "my_service"},
{"method": "my_method"},
])
def test_by_default_item_is_disabled(self, settings):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "concepts", **settings)
assert not service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10)
def test_i_can_activate_debug_for_all_items(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "vars", item="*")
assert service.compute_debug_item("vars", context, "my_service", "my_method", "whatever variable name", 0)
def test_disabled_is_the_highest_priority(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "concepts", service="my_service", item="item", enabled=True)
service.add_or_update_debug_item(context, "concepts", method="my_method", item="item", enabled=False)
service.add_or_update_debug_item(context, "concepts", context_id=context.id, item="item", enabled="value")
assert not service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10)
def test_string_value_is_the_second_best_priority(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.add_or_update_debug_item(context, "concepts", service="my_service", item="item", enabled=True)
service.add_or_update_debug_item(context, "concepts", context_id=context.id, item="item", enabled="value")
assert service.compute_debug_item("concepts", context, "my_service", "my_method", "item", 10) == "value"
def test_i_can_reset_debug_settings(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.debug_var(context, service="my_service")
assert service.compute_debug("my_service", "my_method", context)
service.add_or_update_debug_item(context, "concepts", service="my_service")
service.add_or_update_debug_item(context, "vars", service="my_service")
service.add_or_update_debug_item(context, "rules", service="my_service")
service.reset_debug(context)
assert not service.compute_debug("my_service", "my_method", context)
sheerka.reset_debug(context)
@pytest.mark.parametrize("settings, expected", [
({"rule": "1"}, True),
({"rule": "2"}, False),
({"context_id": 0}, True),
({"context_id": 1}, False),
({"debug_id": 0}, True),
({"debug_id": 1}, False),
assert len(service.debug_vars_settings) == 0
assert len(service.debug_rules_settings) == 0
assert len(service.debug_concepts_settings) == 0
@pytest.mark.parametrize("args, kwargs, expected", [
(["my_service.my_method.my_var"], {}, ("my_var", "my_service", "my_method", None, False, None, True)),
(["*.*.my_var"], {}, ("my_var", None, None, None, False, None, True)),
(["my_service"], {}, (None, "my_service", None, None, False, None, True)),
(["my_service.my_method"], {}, (None, "my_service", "my_method", None, False, None, True)),
(["*.*.*"], {}, ("*", None, None, None, False, None, True)),
([1], {}, ("1", None, None, None, False, None, True)),
(["", 1], {}, (None, None, None, 1, False, None, True)),
([None, 1], {}, (None, None, None, 1, False, None, True)),
([None, "1+"], {}, (None, None, None, 1, True, None, True)),
([None, "xxx"], {}, (None, None, None, None, False, None, True)),
(["s.m.v", "1+"], {}, ("v", "s", "m", 1, True, None, True)),
([None, None, 1], {}, (None, None, None, None, False, 1, True)),
([None, "", 1], {}, (None, None, None, None, False, 1, True)),
(["s.m.v", "1+", 10], {}, ("v", "s", "m", 1, True, 10, True)),
(["s.m.v", "1+", 10], {"variable": "my_var"}, ("my_var", "s", "m", 1, True, 10, True)),
(["s.m.v", "1+", 10], {"service": "my_service"}, ("v", "my_service", "m", 1, True, 10, True)),
(["s.m.v", "1+", 10], {"method": "my_method"}, ("v", "s", "my_method", 1, True, 10, True)),
(["s.m.v", "1+", 10], {"context_id": 155}, ("v", "s", "m", 155, True, 10, True)),
(["s.m.v", "1+", 10], {"context_children": False}, ("v", "s", "m", 1, False, 10, True)),
(["s.m.v", "1+", 10], {"debug_id": 155}, ("v", "s", "m", 1, True, 155, True)),
(["s.m.v", "1+", 10], {"enabled": False}, ("v", "s", "m", 1, True, 10, False)),
])
def test_i_can_compute_rules_debug_settings(self, settings, expected):
def test_i_can_parse_args(self, args, kwargs, expected):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
service.set_debug(context, True)
service.debug_rule(context, **settings)
assert service.compute_debug_rule("1", 0, 0) == expected
assert service.parse_debug_args("variable", *args, **kwargs) == expected
def test_i_can_debug_var(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
sheerka.debug_var(context, "s.m.v", "1+", 10, variable="my_var")
assert service.debug_vars_settings == [
DebugItem("my_var", "s", "m", 1, True, 10, False, True)
]
assert service.debug_concepts_settings == []
assert service.debug_rules_settings == []
def test_i_can_debug_rule(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
sheerka.debug_rule(context, "s.m.v", "1+", 10, rule="my_rule")
assert service.debug_rules_settings == [
DebugItem("my_rule", "s", "m", 1, True, 10, False, True)
]
assert service.debug_concepts_settings == []
assert service.debug_vars_settings == []
def test_i_can_debug_concept(self):
sheerka, context = self.init_concepts()
service = sheerka.services[SheerkaDebugManager.NAME]
sheerka.debug_concept(context, "s.m.v", "1+", 10, concept="my_concept")
assert service.debug_concepts_settings == [
DebugItem("my_concept", "s", "m", 1, True, 10, False, True)
]
assert service.debug_rules_settings == []
assert service.debug_vars_settings == []
def test_state_is_saved_and_restored(self):
sheerka = self.get_sheerka()
@@ -408,8 +549,9 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
sheerka.set_explicit(root_context, False)
sheerka.activate_debug_for(root_context, 1, children=True)
sheerka.activate_debug_for(root_context, "SomeVar")
sheerka.debug_rule(root_context, "1", 10, 15)
sheerka.debug_var(root_context, service="service_name")
sheerka.debug_var(root_context, "service_name.*.var")
sheerka.debug_rule(root_context, 1)
sheerka.debug_concept(root_context, 1001)
another_service = SheerkaDebugManager(sheerka)
another_service.initialize_deferred(root_context, True)
@@ -418,8 +560,9 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
assert not another_service.explicit
assert another_service.context_cache == {1, "1+"}
assert another_service.variable_cache == {"SomeVar"}
assert another_service.debug_rules_settings == [
DebugRuleSetting("1", 10, 15, True)
]
assert another_service.debug_vars_settings == [
DebugVarSetting('service_name', None, None, None, False, None, False, True)]
DebugItem('var', 'service_name', None, None, False, None, False, True)]
assert another_service.debug_rules_settings == [
DebugItem('1', None, None, None, False, None, False, True)]
assert another_service.debug_concepts_settings == [
DebugItem('1001', None, None, None, False, None, False, True)]
tests/core/test_SheerkaSetsManager.py
+6 -15
View File
@@ -260,21 +260,6 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
assert sheerka.isa(sheerka.new("one"), number) # sanity
assert not sheerka.isa(another_one, number) # Correct this misbehaviour when BuiltinConcepts.IS is implemented
def test_concept_expression_recurse_id_is_updated(self):
sheerka, context, one, number, twenties = self.init_concepts(
"one",
"number",
Concept("twenties", definition="'twenty' number").def_var("number"),
create_new=True
)
assert twenties.get_bnf().elements[1].recurse_id is None
# update number
sheerka.set_isa(context, sheerka.new("one"), number)
assert twenties.get_bnf().elements[1].recurse_id == "1003#1002(number)"
def test_concepts_in_group_cache_is_updated(self):
sheerka, context, one, two, number = self.init_concepts("one", "two", "number")
@@ -286,6 +271,9 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
concepts_in_cache = sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_IN_GROUPS_ENTRY, number.id)
assert [c.id for c in concepts_in_cache] == [one.id]
# pretend that number has been updated in sheerka.concepts_grammar
sheerka.concepts_grammars.put(number.id, "some parsing expression with 'one' only")
# add another element to number
sheerka.set_isa(context, sheerka.new("two"), number)
elements = sheerka.get_set_elements(context, number)
@@ -294,6 +282,9 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka):
concepts_in_cache = sheerka.cache_manager.get(SheerkaSetsManager.CONCEPTS_IN_GROUPS_ENTRY, number.id)
assert {c.id for c in concepts_in_cache} == {one.id, two.id}
# make sure the bnf definition is also updated
assert number.id not in sheerka.concepts_grammars
class TestSheerkaSetsManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
def test_i_can_add_concept_to_set_and_retrieve_it_in_another_session(self):
tests/non_reg/test_sheerka_non_reg.py
+1 -1
View File
@@ -1156,7 +1156,7 @@ as:
sheerka = self.init_scenario(init)
res = sheerka.evaluate_user_input("twenty one")
assert len(res) > 1
assert len(res) > 1 # not recognized
sheerka.evaluate_user_input("set_isa(one, number)")
res = sheerka.evaluate_user_input("twenty one")
tests/parsers/test_BnfNodeParser.py
+33 -18
View File
@@ -3,9 +3,9 @@ from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF, NotInit
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import CNC, UTN, CN
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser
from parsers.BnfDefinitionParser import BnfDefinitionParser
import tests.parsers.parsers_utils
from tests.BaseTest import BaseTest
@@ -172,7 +172,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
parser.init_from_concepts(context, updated)
parser.reset_parser(context, ParserInput(text))
bnf_parsers_helpers = parser.get_concepts_sequences()
bnf_parsers_helpers = parser.get_concepts_sequences(context)
assert len(bnf_parsers_helpers) == len(expected_array)
for parser_helper, expected_sequence in zip(bnf_parsers_helpers, expected_array):
@@ -263,14 +263,14 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(my_map, text, expected)
parser.reset_parser(context, ParserInput(text))
bnf_parsers_helpers = parser.get_concepts_sequences()
bnf_parsers_helpers = parser.get_concepts_sequences(context)
assert bnf_parsers_helpers[0].sequence == expected_array
assert not bnf_parsers_helpers[0].has_unrecognized
# but I cannot parse
text = "- - filter"
parser.reset_parser(context, ParserInput(text))
bnf_parsers_helpers = parser.get_concepts_sequences()
bnf_parsers_helpers = parser.get_concepts_sequences(context)
assert bnf_parsers_helpers[0].has_unrecognized
def test_i_can_match_multiple_sequences(self):
@@ -804,7 +804,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
(ConceptExpression("foo"), {}),
(OrderedChoice(ConceptExpression("foo"), StrMatch("one")), {'one': ['1002']}),
(Sequence(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")), {'one': ['1001', '1002']}),
# (UnOrderedChoice(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")), {'one': ['1001', '1002']})
])
def test_i_can_detect_infinite_recursion(self, bar_expr, expected):
my_map = {
@@ -850,6 +849,33 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert parser.concepts_grammars.get(my_map["foo"].id).body == ["1001", "1002", "1003", "1004", "1001"]
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG)
def test_i_can_detect_partial_infinite_recursion(self):
my_map = {
"foo": self.bnf_concept("foo", ConceptExpression("bar")),
"bar": self.bnf_concept("bar", ConceptExpression("baz")),
"baz": self.bnf_concept("baz", ConceptExpression("qux")),
"qux": self.bnf_concept("qux", ConceptExpression("baz")),
}
sheerka, context, parser = self.init_parser(my_map, singleton=True)
parser.context = context
parser.sheerka = sheerka
# every obvious cyclic recursion are removed from concept_by_first_keyword dict
parser.init_from_concepts(context, my_map.values())
assert parser.concepts_by_first_keyword == {}
parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert parser.concepts_grammars.get(my_map["foo"].id).body == ["1001", "1002", "1003", "1004", "1003"]
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG)
@@ -904,8 +930,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes
assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes
assert my_map["number"].id not in parser.concepts_grammars
def test_i_can_get_parsing_expression_when_starting_by_isa_concept(self):
my_map = {
"one": Concept("one"),
@@ -1015,7 +1039,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
parser.init_from_concepts(context, my_map.values())
parser.reset_parser(context, ParserInput("one three"))
sequences = parser.get_concepts_sequences()
sequences = parser.get_concepts_sequences(context)
sequence = parser.get_valid(sequences)
assert len(sequence) == 1
@@ -1122,7 +1146,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
def test_i_can_parse_one_thousand(self):
def test_i_can_parse_when_starting_by_isa_concept(self):
"""
Test of simple number + 'thousand'
:return:
@@ -1403,15 +1427,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert parser.parse(context, ParserInput("foo foo foo bar")).status
assert not parser.parse(context, ParserInput("foo baz")).status
def test_i_only_get_the_requested_parsing_expression(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
parser.context = context
parser.sheerka = sheerka
sheerka.concepts_grammars.clear() # to simulate restart
parser.get_parsing_expression(context, sheerka.resolve("thirties"))
assert len(parser.concepts_grammars) == 9 # requested concept + concepts that do not contains UnorderedChoice
@pytest.mark.parametrize("name, expected", [
(None, []),
("", []),
tests/parsers/test_BnfParser.py
+3 -4
View File
@@ -2,13 +2,13 @@ import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_BNF
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
from core.tokenizer import Tokenizer, TokenKind, LexerError
from parsers.BaseNodeParser import cnode
from parsers.BaseParser import UnexpectedTokenErrorNode
from parsers.BnfDefinitionParser import BnfDefinitionParser, UnexpectedEndOfFileError
from parsers.BnfNodeParser import BnfNodeParser
from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \
OneOrMore, ConceptExpression
from parsers.BnfNodeParser import BnfNodeParser
from parsers.BnfDefinitionParser import BnfDefinitionParser, UnexpectedEndOfFileError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -236,4 +236,3 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
assert res.status
pexpression = res.value.value
assert pexpression == Sequence(StrMatch('twenty'), ConceptExpression(number, "n1"))
assert pexpression.elements[1].recurse_id == "1004#1003(n1)"