Fixed infinite recursion when parsing complex BNF node
This commit is contained in:
+256
-91
@@ -154,18 +154,6 @@ class ConceptExpression(ParsingExpression):
|
||||
[node])
|
||||
|
||||
|
||||
# class ConceptGroupExpression(ConceptExpression):
|
||||
# def _parse(self, parser_helper):
|
||||
# node = self.nodes[0].parse(parser_helper)
|
||||
# if node is None:
|
||||
# return None
|
||||
# return NonTerminalNode(self,
|
||||
# node.start,
|
||||
# node.end,
|
||||
# node.tokens, # node is an OrderedChoice
|
||||
# [node])
|
||||
|
||||
|
||||
class Sequence(ParsingExpression):
|
||||
"""
|
||||
Will match sequence of parser expressions in exact order they are defined.
|
||||
@@ -422,6 +410,69 @@ class StrMatch(Match):
|
||||
return None
|
||||
|
||||
|
||||
# class RegExMatch(Match):
|
||||
# '''
|
||||
# This Match class will perform input matching based on Regular Expressions.
|
||||
#
|
||||
# Args:
|
||||
# to_match (regex string): A regular expression string to match.
|
||||
# It will be used to create regular expression using re.compile.
|
||||
# ignore_case(bool): If case insensitive match is needed.
|
||||
# Default is None to support propagation from global parser setting.
|
||||
# multiline(bool): allow regex to works on multiple lines
|
||||
# (re.DOTALL flag). Default is None to support propagation from
|
||||
# global parser setting.
|
||||
# str_repr(str): A string that is used to represent this regex.
|
||||
# re_flags: flags parameter for re.compile if neither ignore_case
|
||||
# or multiple are set.
|
||||
#
|
||||
# '''
|
||||
# def __init__(self, to_match, rule_name='', root=False, ignore_case=None,
|
||||
# multiline=None, str_repr=None, re_flags=re.MULTILINE):
|
||||
# super(RegExMatch, self).__init__(rule_name, root)
|
||||
# self.to_match_regex = to_match
|
||||
# self.ignore_case = ignore_case
|
||||
# self.multiline = multiline
|
||||
# self.explicit_flags = re_flags
|
||||
#
|
||||
# self.to_match = str_repr if str_repr is not None else to_match
|
||||
#
|
||||
# def compile(self):
|
||||
# flags = self.explicit_flags
|
||||
# if self.multiline is True:
|
||||
# flags |= re.DOTALL
|
||||
# if self.multiline is False and flags & re.DOTALL:
|
||||
# flags -= re.DOTALL
|
||||
# if self.ignore_case is True:
|
||||
# flags |= re.IGNORECASE
|
||||
# if self.ignore_case is False and flags & re.IGNORECASE:
|
||||
# flags -= re.IGNORECASE
|
||||
# self.regex = re.compile(self.to_match_regex, flags)
|
||||
#
|
||||
# def __str__(self):
|
||||
# return self.to_match
|
||||
#
|
||||
# def __unicode__(self):
|
||||
# return self.__str__()
|
||||
#
|
||||
# def _parse(self, parser):
|
||||
# c_pos = parser.position
|
||||
# m = self.regex.match(parser.input, c_pos)
|
||||
# if m:
|
||||
# matched = m.group()
|
||||
# if parser.debug:
|
||||
# parser.dprint(
|
||||
# "++ Match '%s' at %d => '%s'" %
|
||||
# (matched, c_pos, parser.context(len(matched))))
|
||||
# parser.position += len(matched)
|
||||
# if matched:
|
||||
# return Terminal(self, c_pos, matched, extra_info=m)
|
||||
# else:
|
||||
# if parser.debug:
|
||||
# parser.dprint("-- NoMatch at {}".format(c_pos))
|
||||
# parser._nm_raise(self, c_pos, parser)
|
||||
|
||||
|
||||
class ParsingExpressionVisitor:
|
||||
"""
|
||||
visit ParsingExpression
|
||||
@@ -550,7 +601,7 @@ class BnfConceptParserHelper:
|
||||
forked.eat_concept(concept, token)
|
||||
|
||||
# init
|
||||
parsing_expression = self.parser.get_parsing_expression(concept)
|
||||
parsing_expression = self.parser.get_parsing_expression(self.parser.context, concept)
|
||||
if not isinstance(parsing_expression, ParsingExpression):
|
||||
self.debug.append(concept)
|
||||
error_msg = f"Failed to parse concept '{concept}'"
|
||||
@@ -733,6 +784,11 @@ class BnfConceptParserHelper:
|
||||
return concept
|
||||
|
||||
|
||||
@dataclass
|
||||
class UnderConstruction:
|
||||
concept_id: str
|
||||
|
||||
|
||||
class BnfNodeParser(BaseNodeParser):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__("BnfNode", 50, **kwargs)
|
||||
@@ -769,6 +825,11 @@ class BnfNodeParser(BaseNodeParser):
|
||||
return valid_parser_helpers
|
||||
|
||||
def get_concepts_sequences(self):
|
||||
"""
|
||||
Main method that parses the tokens and extract the concepts
|
||||
:return:
|
||||
"""
|
||||
|
||||
def _add_forked_to_concept_parser_helpers():
|
||||
# check that if some new InfixToPostfix are created
|
||||
for parser in concept_parser_helpers:
|
||||
@@ -836,110 +897,214 @@ class BnfNodeParser(BaseNodeParser):
|
||||
|
||||
return concept_parser_helpers
|
||||
|
||||
def get_parsing_expression(self, concept, already_seen=None):
|
||||
def check_for_infinite_recursion(self, parsing_expression, already_found, only_first=False):
|
||||
|
||||
if isinstance(parsing_expression, ConceptExpression):
|
||||
if parsing_expression.concept in already_found:
|
||||
return True
|
||||
already_found.add(parsing_expression.concept)
|
||||
return self.check_for_infinite_recursion(parsing_expression.nodes[0], already_found, False)
|
||||
|
||||
if isinstance(parsing_expression, Sequence):
|
||||
# for sequence, we need to check all nodes
|
||||
if only_first:
|
||||
nodes = [] if len(parsing_expression.nodes) == 0 else [parsing_expression.nodes[0]]
|
||||
else:
|
||||
nodes = parsing_expression.nodes
|
||||
for node in nodes:
|
||||
already_found_for_current_node = already_found.copy()
|
||||
if self.check_for_infinite_recursion(node, already_found_for_current_node, False):
|
||||
already_found.update(already_found_for_current_node)
|
||||
return True
|
||||
return False
|
||||
|
||||
if isinstance(parsing_expression, OrderedChoice):
|
||||
# for ordered choice, if there is at least one node that does not resolved to a recursion
|
||||
# we are safe
|
||||
for node in parsing_expression.nodes:
|
||||
already_found_for_current_node = already_found.copy()
|
||||
if self.check_for_infinite_recursion(node, already_found, True):
|
||||
already_found.update(already_found_for_current_node)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
def get_parsing_expression(self, context, concept):
|
||||
if concept.id in self.concepts_grammars:
|
||||
return self.concepts_grammars.get(concept.id)
|
||||
|
||||
if not concept.bnf:
|
||||
BaseNodeParser.ensure_bnf(self.context, concept, self.name)
|
||||
grammar = self.concepts_grammars.copy()
|
||||
to_resolve = {} # the key is the instance id of the parsing expression
|
||||
isa_concepts = set()
|
||||
self.resolve_concept_parsing_expression(context, concept, grammar, to_resolve, isa_concepts)
|
||||
|
||||
expression = concept.bnf
|
||||
desc = f"Resolving parsing expression {expression}"
|
||||
with self.context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context:
|
||||
sub_context.add_inputs(expression=expression)
|
||||
resolved = self.resolve_parsing_expression(expression, already_seen or set())
|
||||
sub_context.add_values(return_values=resolved)
|
||||
for _id, pe in to_resolve.items():
|
||||
for i, node in enumerate(pe.nodes):
|
||||
if isinstance(node, UnderConstruction):
|
||||
pe.nodes[i] = grammar.get(node.concept_id)
|
||||
|
||||
self.concepts_grammars.put(concept.id, resolved)
|
||||
concepts_in_recursion = set()
|
||||
if self.check_for_infinite_recursion(pe, concepts_in_recursion):
|
||||
cycle = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c.id for c in concepts_in_recursion})
|
||||
for concept in concepts_in_recursion:
|
||||
grammar[concept.id] = cycle
|
||||
|
||||
if self.has_error:
|
||||
return None
|
||||
# Make sure you do not put isa concepts in cache
|
||||
# why :
|
||||
# twenties = 'twenty' number where number < 10
|
||||
# hundreds = number 'hundred' where number < 99
|
||||
# the concept of number depends on its utilisation
|
||||
for concept_id in [c for c in grammar if c not in isa_concepts]:
|
||||
self.concepts_grammars.put(concept_id, grammar[concept_id])
|
||||
|
||||
return self.concepts_grammars.get(concept.id)
|
||||
|
||||
def resolve_parsing_expression(self, parsing_expression, already_seen):
|
||||
def resolve_concept_parsing_expression(self, context, concept, grammar, to_resolve, isa_concepts):
|
||||
if concept.id in grammar:
|
||||
return grammar.get(concept.id)
|
||||
|
||||
def inner_resolve(expression, inner_already_seen):
|
||||
# if isinstance(expression, Concept):
|
||||
# if self.sheerka.isaset(self.context, expression):
|
||||
# ret = ConceptGroupExpression(expression, rule_name=expression.name)
|
||||
# else:
|
||||
# ret = ConceptExpression(expression, rule_name=expression.name)
|
||||
# possible_recursion.add(expression)
|
||||
if isinstance(expression, str):
|
||||
ret = StrMatch(expression, ignore_case=self.ignore_case)
|
||||
desc = f"Get parsing expression for '{concept}'"
|
||||
with context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context:
|
||||
if not concept.bnf: # to save a function call. Not sure it worth it.
|
||||
BaseNodeParser.ensure_bnf(sub_context, concept, self.name)
|
||||
|
||||
elif not isinstance(expression, ParsingExpression):
|
||||
return expression # escalate the error
|
||||
grammar[concept.id] = UnderConstruction(concept.id)
|
||||
sheerka = context.sheerka
|
||||
|
||||
elif isinstance(expression, ConceptExpression):
|
||||
concept = self.get_concept(expression.concept)
|
||||
if concept in inner_already_seen:
|
||||
return self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concept)
|
||||
expression.concept = concept
|
||||
inner_already_seen.add(concept)
|
||||
if concept.metadata.definition_type == DEFINITION_TYPE_BNF:
|
||||
expression = concept.bnf
|
||||
desc = f"Bnf concept detected. Resolving parsing expression '{expression}'"
|
||||
with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc:
|
||||
ssc.add_inputs(expression=expression)
|
||||
resolved = self.resolve_parsing_expression(ssc, expression, grammar, to_resolve, isa_concepts)
|
||||
ssc.add_values(return_values=resolved)
|
||||
|
||||
if not self.sheerka.is_known(concept):
|
||||
unknown_concept = self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept)
|
||||
return self.add_error(unknown_concept)
|
||||
elif sheerka.isaset(context, concept):
|
||||
desc = f"Concept is a group. Resolving parsing expression using 'isa'"
|
||||
with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc:
|
||||
ssc.add_inputs(concept=concept)
|
||||
isa_concepts.add(concept.id)
|
||||
concepts_in_group = self.sheerka.get_set_elements(ssc, concept)
|
||||
|
||||
# bnf concept
|
||||
elif concept.metadata.definition_type == DEFINITION_TYPE_BNF:
|
||||
pe = self.get_parsing_expression(concept, inner_already_seen)
|
||||
# concepts_in_group comes from a set, so the order of its elements is not guaranteed
|
||||
# to avoid random failure (ie random CHICKEN_AND_EGG), we need to rearrange
|
||||
# We also remove the root concept (the one from get_parsing_expression())
|
||||
|
||||
elif self.sheerka.isaset(self.context, concept):
|
||||
concepts_in_group = self.sheerka.get_set_elements(self.context, concept)
|
||||
nodes = [ConceptExpression(c, rule_name=c.name) for c in concepts_in_group]
|
||||
pe = inner_resolve(OrderedChoice(*nodes), inner_already_seen)
|
||||
root_concept_as_set = set(context.search(
|
||||
predicate=lambda ec: ec.action == BuiltinConcepts.INIT_BNF,
|
||||
get_obj=lambda ec: ec.obj,
|
||||
stop=lambda ec: ec.action != BuiltinConcepts.INIT_BNF)) # there only one item in the set
|
||||
root_concept = list(root_concept_as_set)[0]
|
||||
reordered = []
|
||||
for c in concepts_in_group:
|
||||
if c.id == root_concept.id:
|
||||
continue
|
||||
|
||||
else:
|
||||
# regular concepts
|
||||
tokens = Tokenizer(concept.name)
|
||||
nodes = [StrMatch(token.strip_quote) for token in list(tokens)[:-1]]
|
||||
pe = inner_resolve(nodes[0] if len(nodes) == 1 else Sequence(nodes), inner_already_seen)
|
||||
# I do not guaranty the same order every time, but I minimize the ChickenAndEgg random issue
|
||||
if c.metadata.definition_type == DEFINITION_TYPE_BNF or sheerka.isaset(ssc, c):
|
||||
reordered.append(c)
|
||||
else:
|
||||
reordered.insert(0, c)
|
||||
|
||||
if not isinstance(pe, ParsingExpression):
|
||||
return pe
|
||||
expression.nodes = [pe]
|
||||
expression.rule_name = expression.rule_name or concept.name
|
||||
ret = expression
|
||||
|
||||
elif isinstance(expression, StrMatch):
|
||||
ret = expression
|
||||
if ret.ignore_case is None:
|
||||
ret.ignore_case = self.ignore_case
|
||||
|
||||
elif isinstance(expression, Sequence) or \
|
||||
isinstance(expression, OrderedChoice) or \
|
||||
isinstance(expression, ZeroOrMore) or \
|
||||
isinstance(expression, OneOrMore) or \
|
||||
isinstance(expression, Optional):
|
||||
ret = expression
|
||||
ret.nodes = []
|
||||
for e in ret.elements:
|
||||
pe = inner_resolve(e, already_seen.copy())
|
||||
if not isinstance(pe, ParsingExpression):
|
||||
return pe
|
||||
ret.nodes.append(pe)
|
||||
nodes = [ConceptExpression(c, rule_name=c.name) for c in reordered]
|
||||
resolved = self.resolve_parsing_expression(ssc,
|
||||
OrderedChoice(*nodes),
|
||||
grammar,
|
||||
to_resolve,
|
||||
isa_concepts)
|
||||
ssc.add_values(concepts_in_group=concepts_in_group)
|
||||
ssc.add_values(return_values=resolved)
|
||||
|
||||
else:
|
||||
ret = self.add_error(GrammarErrorNode(f"Unrecognized grammar element '{expression}'."), False)
|
||||
desc = f"Concept is a simple concept."
|
||||
with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc:
|
||||
tokens = Tokenizer(concept.name, yield_eof=False)
|
||||
nodes = [StrMatch(token.strip_quote) for token in tokens]
|
||||
expression = nodes[0] if len(nodes) == 1 else Sequence(nodes)
|
||||
resolved = self.resolve_parsing_expression(ssc, expression, grammar, to_resolve, isa_concepts)
|
||||
|
||||
# Translate separator expression.
|
||||
if isinstance(expression, Repetition) and expression.sep:
|
||||
expression.sep = inner_resolve(expression.sep, already_seen)
|
||||
grammar[concept.id] = resolved
|
||||
|
||||
return ret
|
||||
if self.has_error:
|
||||
sub_context.add_values(errors=self.error_sink)
|
||||
return None
|
||||
|
||||
parsing_expression = inner_resolve(parsing_expression, already_seen)
|
||||
return parsing_expression
|
||||
sub_context.add_values(return_values=resolved)
|
||||
return resolved
|
||||
|
||||
def get_concept(self, concept):
|
||||
def resolve_parsing_expression(self, context, expression, grammar, to_resolve, isa_concepts):
|
||||
|
||||
if isinstance(expression, str):
|
||||
ret = StrMatch(expression, ignore_case=self.ignore_case)
|
||||
|
||||
elif not isinstance(expression, ParsingExpression):
|
||||
return expression # escalate the error
|
||||
|
||||
elif isinstance(expression, ConceptExpression):
|
||||
concept = self.get_concept(context, expression.concept)
|
||||
expression.concept = concept
|
||||
|
||||
if not self.sheerka.is_known(concept):
|
||||
unknown_concept = self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept)
|
||||
return self.add_error(unknown_concept)
|
||||
|
||||
pe = self.resolve_concept_parsing_expression(context, concept, grammar, to_resolve, isa_concepts)
|
||||
|
||||
if not isinstance(pe, (ParsingExpression, UnderConstruction)):
|
||||
return pe # an error is detected, escalate it
|
||||
#
|
||||
# if isinstance(pe, UnderConstruction) and expression.concept.id == pe.concept_id:
|
||||
# return pe # we are looking for ourself, just return it
|
||||
|
||||
if isinstance(pe, UnderConstruction):
|
||||
to_resolve[id(expression)] = expression
|
||||
|
||||
expression.nodes = [pe]
|
||||
expression.rule_name = expression.rule_name or concept.name
|
||||
ret = expression
|
||||
|
||||
elif isinstance(expression, StrMatch):
|
||||
ret = expression
|
||||
if ret.ignore_case is None:
|
||||
ret.ignore_case = self.ignore_case
|
||||
|
||||
elif isinstance(expression, Sequence) or \
|
||||
isinstance(expression, OrderedChoice) or \
|
||||
isinstance(expression, ZeroOrMore) or \
|
||||
isinstance(expression, OneOrMore) or \
|
||||
isinstance(expression, Optional):
|
||||
ret = expression
|
||||
ret.nodes = []
|
||||
for e in ret.elements:
|
||||
pe = self.resolve_parsing_expression(context, e, grammar, to_resolve, isa_concepts)
|
||||
if not isinstance(pe, (ParsingExpression, UnderConstruction)):
|
||||
return pe # an error is detected, escalate it
|
||||
if isinstance(pe, UnderConstruction):
|
||||
to_resolve[id(ret)] = ret # remember that there is an unresolved parsing expression
|
||||
ret.nodes.append(pe)
|
||||
|
||||
else:
|
||||
ret = self.add_error(GrammarErrorNode(f"Unrecognized grammar element '{expression}'."), False)
|
||||
|
||||
# Translate separator expression.
|
||||
if isinstance(ret, Repetition) and expression.sep:
|
||||
expression.sep = self.resolve_parsing_expression(context,
|
||||
expression.sep,
|
||||
grammar,
|
||||
to_resolve,
|
||||
isa_concepts)
|
||||
|
||||
return ret
|
||||
|
||||
def get_concept(self, context, concept):
|
||||
if isinstance(concept, Concept):
|
||||
return concept
|
||||
|
||||
if concept in self.context.concepts:
|
||||
return self.context.concepts[concept]
|
||||
if concept in context.concepts:
|
||||
return context.concepts[concept]
|
||||
return self.sheerka.get_by_key(concept)
|
||||
|
||||
def parse(self, context, parser_input: ParserInput):
|
||||
|
||||
Reference in New Issue
Block a user