Resolved some remaining chicken and egg when parsing BNF nodes
This commit is contained in:
@@ -373,15 +373,15 @@ class ExecutionContext:
|
||||
:param predicate: what execution context to keep
|
||||
:param get_obj: lambda to compute what to return
|
||||
:param start_with_self: include the current execution context in the search
|
||||
:param stop: stop the search if matched
|
||||
:param stop: condition to stop
|
||||
:return:
|
||||
"""
|
||||
current = self if start_with_self else self._parent
|
||||
while current:
|
||||
if stop and stop(current):
|
||||
break
|
||||
|
||||
if predicate is None or predicate(current):
|
||||
yield current if get_obj is None else get_obj(current)
|
||||
|
||||
if stop and stop(current):
|
||||
break
|
||||
|
||||
current = current._parent
|
||||
|
||||
@@ -8,6 +8,7 @@ from cache.CacheManager import CacheManager
|
||||
from cache.DictionaryCache import DictionaryCache
|
||||
from cache.IncCache import IncCache
|
||||
from cache.ListIfNeededCache import ListIfNeededCache
|
||||
from cache.SetCache import SetCache
|
||||
from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
|
||||
UnknownConcept
|
||||
from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
|
||||
@@ -31,6 +32,8 @@ class Sheerka(Concept):
|
||||
CONCEPTS_BY_NAME_ENTRY = "Concepts_By_Name"
|
||||
CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash" # store hash of concepts definitions (not values)
|
||||
|
||||
CONCEPTS_REFERENCES = "Concepts_References" # tracks references between concepts
|
||||
|
||||
CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
|
||||
RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Resolved_Concepts_By_First_Keyword"
|
||||
CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
|
||||
@@ -215,6 +218,9 @@ class Sheerka(Concept):
|
||||
cache = ListIfNeededCache(**params(self.CONCEPTS_BY_HASH_ENTRY))
|
||||
register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.get_definition_hash(), True)
|
||||
|
||||
cache = SetCache(default=lambda k: self.sdp.get(self.CONCEPTS_REFERENCES, k))
|
||||
self.cache_manager.register_cache(self.CONCEPTS_REFERENCES, cache)
|
||||
|
||||
cache = DictionaryCache(default=lambda k: self.sdp.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, k))
|
||||
self.cache_manager.register_cache(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, cache)
|
||||
self.cache_manager.get(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, None) # to init from sdp
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import core.utils
|
||||
from core.builtin_concepts import BuiltinConcepts, ErrorConcept
|
||||
from core.concept import Concept, DEFINITION_TYPE_DEF, ensure_concept
|
||||
from core.concept import Concept, DEFINITION_TYPE_DEF, ensure_concept, DEFINITION_TYPE_BNF
|
||||
from core.sheerka.services.sheerka_service import BaseService
|
||||
from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
|
||||
|
||||
@@ -64,7 +64,6 @@ class SheerkaCreateNewConcept(BaseService):
|
||||
resolved_concepts_by_first_keyword = init_ret_value.body
|
||||
|
||||
# if everything is fine
|
||||
|
||||
concept.freeze_definition_hash()
|
||||
cache_manager.add_concept(concept)
|
||||
cache_manager.put(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, concepts_by_first_keyword)
|
||||
@@ -74,9 +73,36 @@ class SheerkaCreateNewConcept(BaseService):
|
||||
# allow search by definition when definition relevant
|
||||
cache_manager.put(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.metadata.definition, concept)
|
||||
|
||||
# update references
|
||||
for ref in self.compute_references(concept):
|
||||
cache_manager.put(sheerka.CONCEPTS_REFERENCES, ref, concept.id)
|
||||
|
||||
# TODO : this line seems to be useless
|
||||
# The grammar is never reset
|
||||
if concept.bnf and init_bnf_ret_value is not None and init_bnf_ret_value.status:
|
||||
sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
|
||||
|
||||
# process the return if needed
|
||||
ret = sheerka.ret(self.NAME, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
|
||||
return ret
|
||||
|
||||
def compute_references(self, concept):
|
||||
"""
|
||||
We need to keep a track of all concepts used by the current concept
|
||||
So that if one of these are modified, we can modify the current concept accordingly
|
||||
:param concept:
|
||||
:return:
|
||||
"""
|
||||
refs = set()
|
||||
|
||||
if concept.metadata.definition_type == DEFINITION_TYPE_BNF:
|
||||
from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
|
||||
other_concepts_visitor = BnfNodeConceptExpressionVisitor()
|
||||
other_concepts_visitor.visit(concept.bnf)
|
||||
|
||||
for concept in other_concepts_visitor.references:
|
||||
if isinstance(concept, str):
|
||||
concept = self.sheerka.get_by_key(concept)
|
||||
refs.add(concept.id)
|
||||
|
||||
return refs
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.sheerka.services.sheerka_service import BaseService
|
||||
from parsers.BnfParser import BnfParser
|
||||
|
||||
|
||||
class SheerkaModifyConcept(BaseService):
|
||||
@@ -36,6 +37,10 @@ class SheerkaModifyConcept(BaseService):
|
||||
BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
|
||||
body=concept))
|
||||
|
||||
old_references = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_REFERENCES, concept.id)
|
||||
if old_references:
|
||||
old_references = old_references.copy()
|
||||
|
||||
self.sheerka.cache_manager.update_concept(old_version, concept)
|
||||
|
||||
# TODO : update concept by first keyword
|
||||
@@ -45,3 +50,21 @@ class SheerkaModifyConcept(BaseService):
|
||||
|
||||
ret = self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
|
||||
return ret
|
||||
|
||||
def update_references(self, context, concept):
|
||||
"""
|
||||
Updates all the concepts that reference concept
|
||||
:param context:
|
||||
:param concept:
|
||||
:return:
|
||||
"""
|
||||
|
||||
refs = self.sheerka.cache_manager.get(self.sheerka.CONCEPTS_REFERENCES, concept.id)
|
||||
if not refs:
|
||||
return
|
||||
|
||||
for concept_id in refs:
|
||||
concept = self.sheerka.get_by_id(concept_id)
|
||||
|
||||
if concept.bnf is not None:
|
||||
BnfParser.update_recurse_id(context, concept_id, concept.bnf)
|
||||
|
||||
@@ -4,6 +4,7 @@ from cache.SetCache import SetCache
|
||||
from core.ast.nodes import python_to_concept
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.concept import Concept, ConceptParts, ensure_concept, DEFINITION_TYPE_BNF
|
||||
from core.sheerka.services.SheerkaModifyConcept import SheerkaModifyConcept
|
||||
from core.sheerka.services.sheerka_service import BaseService
|
||||
|
||||
GROUP_PREFIX = 'All_'
|
||||
@@ -54,7 +55,13 @@ class SheerkaSetsManager(BaseService):
|
||||
if not res.status:
|
||||
return res
|
||||
|
||||
return self.add_concept_to_set(context, concept, concept_set)
|
||||
res = self.add_concept_to_set(context, concept, concept_set)
|
||||
|
||||
# update concept_set references
|
||||
self.sheerka.services[SheerkaModifyConcept.NAME].update_references(context, concept_set)
|
||||
self.concepts_in_set.delete(concept_set.id)
|
||||
|
||||
return res
|
||||
|
||||
def add_concept_to_set(self, context, concept, concept_set):
|
||||
"""
|
||||
|
||||
+117
-25
@@ -158,7 +158,7 @@ class ParsingExpression:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.elements = args
|
||||
|
||||
nodes = kwargs.get('nodes', [])
|
||||
nodes = kwargs.get('nodes', []) or []
|
||||
if not hasattr(nodes, '__iter__'):
|
||||
nodes = [nodes]
|
||||
self.nodes = nodes
|
||||
@@ -201,8 +201,8 @@ class ConceptExpression(ParsingExpression):
|
||||
When the grammar is created, it is replaced by the actual concept
|
||||
"""
|
||||
|
||||
def __init__(self, concept, rule_name="", recurse_id=None):
|
||||
super().__init__(rule_name=rule_name)
|
||||
def __init__(self, concept, rule_name="", recurse_id=None, nodes=None):
|
||||
super().__init__(rule_name=rule_name, nodes=nodes)
|
||||
self.concept = concept
|
||||
self.recurse_id = recurse_id
|
||||
|
||||
@@ -216,6 +216,10 @@ class ConceptExpression(ParsingExpression):
|
||||
if not isinstance(other, ConceptExpression):
|
||||
return False
|
||||
|
||||
# TODO : enable self.recurse_id when it will be correctly implemented
|
||||
# if self.recurse_id != other.recurse_id:
|
||||
# return False
|
||||
|
||||
if isinstance(self.concept, Concept):
|
||||
return self.concept.id == other.concept.id
|
||||
|
||||
@@ -425,6 +429,12 @@ class Repetition(ParsingExpression):
|
||||
super(Repetition, self).__init__(*elements, **kwargs)
|
||||
self.sep = kwargs.get('sep', None)
|
||||
|
||||
def clone(self):
|
||||
return Repetition(self.elements,
|
||||
rule_name=self.rule_name,
|
||||
nodes=self.nodes,
|
||||
sep=self.sep)
|
||||
|
||||
|
||||
class ZeroOrMore(Repetition):
|
||||
"""
|
||||
@@ -711,6 +721,14 @@ class BnfNodeFirstTokenVisitor(ParsingExpressionVisitor):
|
||||
return self.STOP
|
||||
|
||||
|
||||
class BnfNodeConceptExpressionVisitor(ParsingExpressionVisitor):
|
||||
def __init__(self):
|
||||
self.references = []
|
||||
|
||||
def visit_ConceptExpression(self, pe):
|
||||
self.references.append(pe.concept)
|
||||
|
||||
|
||||
class BnfConceptParserHelper:
|
||||
def __init__(self, parser):
|
||||
self.parser = parser
|
||||
@@ -1137,14 +1155,84 @@ class BnfNodeParser(BaseNodeParser):
|
||||
|
||||
return concept_parser_helpers
|
||||
|
||||
def check_for_infinite_recursion(self, parsing_expression, already_found, only_first=False):
|
||||
def fix_infinite_recursions(self, context, grammar, concept_id, parsing_expression):
|
||||
"""
|
||||
Check the newly created parsing expresion
|
||||
Some infinite recursion can be resolved, simply by removing the pexpression that causes the loop
|
||||
Let's look for that
|
||||
:param context:
|
||||
:param grammar:
|
||||
:param concept_id:
|
||||
:param parsing_expression:
|
||||
:return:
|
||||
"""
|
||||
|
||||
def _find(expression_, path_):
|
||||
index_ = -1
|
||||
parent_ = None
|
||||
for node_id in path_:
|
||||
expression_ = expression_.nodes[0] if isinstance(expression_, ConceptExpression) else expression_
|
||||
for i, node in [(i, n) for i, n in enumerate(expression_.nodes) if isinstance(n, ConceptExpression)]:
|
||||
if node.recurse_id == node_id or node.concept.id == node_id:
|
||||
index_ = i
|
||||
parent_ = expression_
|
||||
expression_ = node # take the child of the ConceptExpression found
|
||||
break
|
||||
else:
|
||||
raise IndexError(f"path {path_} cannot be found in '{expression_}'")
|
||||
|
||||
return parent_, index_, expression_
|
||||
|
||||
def _fix_node(expression, path):
|
||||
parent, index, expression_update = _find(expression, path[1:-2])
|
||||
|
||||
assert isinstance(expression_update, ConceptExpression)
|
||||
|
||||
desc = f"Fixing circular reference {path}"
|
||||
with context.push(BuiltinConcepts.INIT_BNF,
|
||||
expression_update.concept,
|
||||
who=self.name,
|
||||
obj=expression_update.concept,
|
||||
concepts_to_skip=[concept_id],
|
||||
desc=desc) as sub_context:
|
||||
new_grammar = grammar.copy()
|
||||
for node_id in path[-2:]:
|
||||
del new_grammar[node_id]
|
||||
new_nodes = self.resolve_concept_parsing_expression(sub_context,
|
||||
expression_update.concept,
|
||||
expression_update.rule_name, new_grammar, set())
|
||||
new = ConceptExpression(expression_update.concept,
|
||||
rule_name=expression_update.rule_name,
|
||||
recurse_id=expression_update.recurse_id,
|
||||
nodes=new_nodes)
|
||||
|
||||
parent.nodes[index] = new
|
||||
|
||||
while True:
|
||||
already_found = [concept_id]
|
||||
concepts_in_recursion = []
|
||||
if self.check_for_infinite_recursion(parsing_expression, already_found, concepts_in_recursion):
|
||||
if "#" in concepts_in_recursion[-2]:
|
||||
# means that it's isaset concept
|
||||
_fix_node(parsing_expression, concepts_in_recursion[:-1])
|
||||
else:
|
||||
break
|
||||
else:
|
||||
break
|
||||
|
||||
return concepts_in_recursion
|
||||
|
||||
def check_for_infinite_recursion(self, parsing_expression, already_found, in_recursion, only_first=False):
|
||||
|
||||
if isinstance(parsing_expression, ConceptExpression):
|
||||
id_to_use = parsing_expression.recurse_id or parsing_expression.concept.id
|
||||
if id_to_use in already_found:
|
||||
already_found.append(id_to_use) # add the id again, to know where the cycle starts
|
||||
in_recursion.extend(already_found)
|
||||
return True
|
||||
already_found.add(id_to_use)
|
||||
return self.check_for_infinite_recursion(parsing_expression.nodes[0], already_found, only_first)
|
||||
already_found.append(id_to_use)
|
||||
return self.check_for_infinite_recursion(
|
||||
parsing_expression.nodes[0], already_found, in_recursion, only_first)
|
||||
|
||||
if isinstance(parsing_expression, Sequence):
|
||||
# for sequence, we need to check all nodes
|
||||
@@ -1154,8 +1242,7 @@ class BnfNodeParser(BaseNodeParser):
|
||||
nodes = parsing_expression.nodes
|
||||
for node in nodes:
|
||||
already_found_for_current_node = already_found.copy()
|
||||
if self.check_for_infinite_recursion(node, already_found_for_current_node, False):
|
||||
already_found.update(already_found_for_current_node)
|
||||
if self.check_for_infinite_recursion(node, already_found_for_current_node, in_recursion, False):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -1164,8 +1251,7 @@ class BnfNodeParser(BaseNodeParser):
|
||||
# we are safe
|
||||
for node in parsing_expression.nodes:
|
||||
already_found_for_current_node = already_found.copy()
|
||||
if self.check_for_infinite_recursion(node, already_found, True):
|
||||
already_found.update(already_found_for_current_node)
|
||||
if self.check_for_infinite_recursion(node, already_found_for_current_node, in_recursion, True):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
@@ -1174,8 +1260,7 @@ class BnfNodeParser(BaseNodeParser):
|
||||
if isinstance(parsing_expression, UnOrderedChoice):
|
||||
for node in parsing_expression.nodes:
|
||||
already_found_for_current_node = already_found.copy()
|
||||
if self.check_for_infinite_recursion(node, already_found_for_current_node, True):
|
||||
already_found.update(already_found_for_current_node)
|
||||
if self.check_for_infinite_recursion(node, already_found_for_current_node, in_recursion, True):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -1215,13 +1300,20 @@ class BnfNodeParser(BaseNodeParser):
|
||||
if isinstance(node, UnderConstruction):
|
||||
pe.nodes[i] = grammar.get(node.concept_id)
|
||||
|
||||
# check for infinite recursion.
|
||||
# We are adding a new concept. Does it create an infinite recursion ?
|
||||
concepts_in_recursion = set()
|
||||
if self.check_for_infinite_recursion(ret, concepts_in_recursion):
|
||||
cycle = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_recursion)
|
||||
# # check for infinite recursions.
|
||||
# # and try to fix them when possible
|
||||
# already_found = [concept.id]
|
||||
# concepts_in_recursion = []
|
||||
# if self.check_for_infinite_recursion(ret, already_found, concepts_in_recursion):
|
||||
# chicken_anf_egg = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_recursion)
|
||||
# for concept_id in concepts_in_recursion:
|
||||
# grammar[concept_id] = chicken_anf_egg
|
||||
|
||||
concepts_in_recursion = self.fix_infinite_recursions(context, grammar, concept.id, ret)
|
||||
if concepts_in_recursion:
|
||||
chicken_anf_egg = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_recursion)
|
||||
for concept_id in concepts_in_recursion:
|
||||
grammar[concept_id] = cycle
|
||||
grammar[concept_id] = chicken_anf_egg
|
||||
|
||||
# update, in case of infinite circular recursion
|
||||
ret = grammar[concept.id]
|
||||
@@ -1287,13 +1379,16 @@ class BnfNodeParser(BaseNodeParser):
|
||||
if c.id == context.obj.id:
|
||||
continue
|
||||
|
||||
# c_pe = self.resolve_concept_parsing_expression(context, c, None, grammar, to_update, to_keep)
|
||||
# if self.check_for_infinite_recursion(c_pe, {concept.id}, True):
|
||||
# continue
|
||||
if hasattr(context, "concepts_to_skip") and c.id in context.concepts_to_skip:
|
||||
continue
|
||||
|
||||
valid_concepts.append(c)
|
||||
|
||||
nodes = [ConceptExpression(c, rule_name=c.name, recurse_id=key_to_use) for c in valid_concepts]
|
||||
nodes = []
|
||||
for c in valid_concepts:
|
||||
c_recurse_id = f"{c.id}#{c.name}#{concept.id}" if self.sheerka.isaset(context, c) else None
|
||||
nodes.append(ConceptExpression(c, rule_name=c.name, recurse_id=c_recurse_id))
|
||||
|
||||
resolved = self.resolve_parsing_expression(ssc,
|
||||
UnOrderedChoice(*nodes),
|
||||
grammar,
|
||||
@@ -1341,9 +1436,6 @@ class BnfNodeParser(BaseNodeParser):
|
||||
|
||||
if not isinstance(pe, (ParsingExpression, UnderConstruction)):
|
||||
return pe # an error is detected, escalate it
|
||||
#
|
||||
# if isinstance(pe, UnderConstruction) and expression.concept.id == pe.concept_id:
|
||||
# return pe # we are looking for ourself, just return it
|
||||
|
||||
if isinstance(pe, UnderConstruction):
|
||||
to_update.add(ToUpdate(id(expression), expression))
|
||||
|
||||
+31
-10
@@ -35,7 +35,7 @@ class BnfParser(BaseParser):
|
||||
|
||||
self.lexer_iter = None
|
||||
self._current = None
|
||||
self.after_current = None
|
||||
self.after_current = None # one look ahead
|
||||
self.nb_open_par = 0
|
||||
self.context = None
|
||||
self.source = ""
|
||||
@@ -283,16 +283,37 @@ class BnfParser(BaseParser):
|
||||
|
||||
def eat_rule_name_if_needed(self, expression):
|
||||
token = self.get_token()
|
||||
if token is None or token.type != TokenKind.EQUALS:
|
||||
return expression
|
||||
if token is not None and token.type == TokenKind.EQUALS:
|
||||
self.next_token() # eat equals
|
||||
token = self.get_token()
|
||||
|
||||
self.next_token() # eat equals
|
||||
token = self.get_token()
|
||||
if token is None or token.type != TokenKind.IDENTIFIER:
|
||||
return self.add_error(
|
||||
UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, [TokenKind.IDENTIFIER]))
|
||||
|
||||
if token is None or token.type != TokenKind.IDENTIFIER:
|
||||
return self.add_error(
|
||||
UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, [TokenKind.IDENTIFIER]))
|
||||
expression.rule_name = token.value
|
||||
self.next_token()
|
||||
|
||||
if BnfParser.is_expression_a_set(self.context, expression):
|
||||
root_concept = self.context.search(
|
||||
start_with_self=True,
|
||||
predicate=lambda ec: ec.action == BuiltinConcepts.INIT_BNF,
|
||||
get_obj=lambda ec: ec.action_context,
|
||||
stop=lambda ec: ec.action == BuiltinConcepts.INIT_BNF)
|
||||
root_concept = list(root_concept)
|
||||
if root_concept and hasattr(root_concept[0], "id"):
|
||||
expression.recurse_id = f"{expression.concept.id}#{expression.rule_name}#{root_concept[0].id}"
|
||||
|
||||
expression.rule_name = token.value
|
||||
self.next_token()
|
||||
return expression
|
||||
|
||||
@staticmethod
|
||||
def is_expression_a_set(context, expression):
|
||||
return isinstance(expression, ConceptExpression) and context.sheerka.isaset(context, expression.concept)
|
||||
|
||||
@staticmethod
|
||||
def update_recurse_id(context, concept_id, expression):
|
||||
if BnfParser.is_expression_a_set(context, expression):
|
||||
expression.recurse_id = f"{expression.concept.id}#{expression.rule_name}#{concept_id}"
|
||||
|
||||
for element in expression.elements:
|
||||
BnfParser.update_recurse_id(context, concept_id, element)
|
||||
|
||||
@@ -355,14 +355,14 @@ class DefaultParser(BaseParser):
|
||||
self.add_error(SyntaxErrorNode([definition_tokens[1]], "Empty declaration"), False)
|
||||
return None, NotInitializedNode()
|
||||
|
||||
regex_parser = BnfParser()
|
||||
bnf_regex_parser = BnfParser()
|
||||
desc = f"Resolving BNF {current_concept_def.definition}"
|
||||
with self.context.push(BuiltinConcepts.INIT_BNF,
|
||||
current_concept_def,
|
||||
who=self.name,
|
||||
obj=current_concept_def,
|
||||
desc=desc) as sub_context:
|
||||
parsing_result = regex_parser.parse(sub_context, tokens)
|
||||
parsing_result = bnf_regex_parser.parse(sub_context, tokens)
|
||||
sub_context.add_values(return_values=parsing_result)
|
||||
|
||||
if not parsing_result.status:
|
||||
|
||||
Reference in New Issue
Block a user