Renamed ConceptMatch into ConceptExpression and added unit tests
This commit is contained in:
@@ -5,7 +5,7 @@ from core.builtin_concepts import BuiltinConcepts
|
||||
from core.sheerka import ExecutionContext
|
||||
from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
|
||||
from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode
|
||||
from parsers.ConceptLexerParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, ConceptMatch, StrMatch
|
||||
from parsers.ConceptLexerParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, ConceptExpression, StrMatch
|
||||
|
||||
|
||||
@dataclass()
|
||||
@@ -231,7 +231,7 @@ class BnfParser(BaseParser):
|
||||
|
||||
if token.type == TokenKind.IDENTIFIER:
|
||||
self.next_token()
|
||||
return ConceptMatch(token.value)
|
||||
return ConceptExpression(token.value)
|
||||
# concept = self.sheerka.get(str(token.value))
|
||||
# if hasattr(concept, "__iter__") or self.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
|
||||
# self.add_error(CannotResolveConceptNode(str(token.value)))
|
||||
|
||||
@@ -244,6 +244,64 @@ class ParsingExpression:
|
||||
return self._parse(parser)
|
||||
|
||||
|
||||
class ConceptExpression(ParsingExpression):
|
||||
"""
|
||||
Will match a concept
|
||||
It used only for rule definition
|
||||
|
||||
When the grammar is created, it is replaced by the actual concept
|
||||
"""
|
||||
|
||||
def __init__(self, concept, rule_name=""):
|
||||
super().__init__(rule_name=rule_name)
|
||||
self.concept = concept
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.concept}"
|
||||
|
||||
def __eq__(self, other):
|
||||
if not super().__eq__(other):
|
||||
return False
|
||||
|
||||
if not isinstance(other, ConceptExpression):
|
||||
return False
|
||||
|
||||
if isinstance(self.concept, Concept):
|
||||
return self.concept.name == other.concept.name
|
||||
|
||||
return self.concept == other.concept
|
||||
|
||||
@staticmethod
|
||||
def get_parsing_expression_from_name(name):
|
||||
tokens = Tokenizer(name)
|
||||
nodes = [StrMatch(core.utils.strip_quotes(token.value)) for token in list(tokens)[:-1]]
|
||||
if len(nodes) == 1:
|
||||
return nodes[0]
|
||||
else:
|
||||
sequence = Sequence(nodes)
|
||||
sequence.nodes = nodes
|
||||
return sequence
|
||||
|
||||
def _parse(self, parser):
|
||||
to_match = parser.get_concept(self.concept) if isinstance(self.concept, str) else self.concept
|
||||
if parser.sheerka.isinstance(to_match, BuiltinConcepts.UNKNOWN_CONCEPT):
|
||||
return None
|
||||
|
||||
self.concept = to_match # Memoize
|
||||
|
||||
if to_match not in parser.concepts_grammars:
|
||||
# Try to match the concept using its name
|
||||
expr = self.get_parsing_expression_from_name(to_match.name)
|
||||
node = expr.parse(parser)
|
||||
else:
|
||||
node = parser.concepts_grammars[to_match].parse(parser)
|
||||
|
||||
if node is None:
|
||||
return None
|
||||
|
||||
return NonTerminalNode(self, node.start, node.end, parser.tokens[node.start: node.end + 1], [node])
|
||||
|
||||
|
||||
class Sequence(ParsingExpression):
|
||||
"""
|
||||
Will match sequence of parser expressions in exact order they are defined.
|
||||
@@ -486,64 +544,6 @@ class StrMatch(Match):
|
||||
return None
|
||||
|
||||
|
||||
class ConceptMatch(Match):
|
||||
"""
|
||||
Will match a concept
|
||||
It used only for rule definition
|
||||
|
||||
When the grammar is created, it is replaced by the actual concept
|
||||
"""
|
||||
|
||||
def __init__(self, concept, rule_name=""):
|
||||
super(Match, self).__init__(rule_name=rule_name)
|
||||
self.concept = concept
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.concept}"
|
||||
|
||||
def __eq__(self, other):
|
||||
if not super().__eq__(other):
|
||||
return False
|
||||
|
||||
if not isinstance(other, ConceptMatch):
|
||||
return False
|
||||
|
||||
if isinstance(self.concept, Concept):
|
||||
return self.concept.name == other.concept.name
|
||||
|
||||
return self.concept == other.concept
|
||||
|
||||
@staticmethod
|
||||
def get_parsing_expression_from_name(name):
|
||||
tokens = Tokenizer(name)
|
||||
nodes = [StrMatch(core.utils.strip_quotes(token.value)) for token in list(tokens)[:-1]]
|
||||
if len(nodes) == 1:
|
||||
return nodes[0]
|
||||
else:
|
||||
sequence = Sequence(nodes)
|
||||
sequence.nodes = nodes
|
||||
return sequence
|
||||
|
||||
def _parse(self, parser):
|
||||
to_match = parser.get_concept(self.concept) if isinstance(self.concept, str) else self.concept
|
||||
if parser.sheerka.isinstance(to_match, BuiltinConcepts.UNKNOWN_CONCEPT):
|
||||
return None
|
||||
|
||||
self.concept = to_match # Memoize
|
||||
|
||||
if to_match not in parser.concepts_grammars:
|
||||
# Try to match the concept using its name
|
||||
expr = self.get_parsing_expression_from_name(to_match.name)
|
||||
node = expr.parse(parser)
|
||||
else:
|
||||
node = parser.concepts_grammars[to_match].parse(parser)
|
||||
|
||||
if node is None:
|
||||
return None
|
||||
|
||||
return NonTerminalNode(self, node.start, node.end, parser.tokens[node.start: node.end + 1], [node])
|
||||
|
||||
|
||||
class ConceptLexerParser(BaseParser):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__("ConceptLexer", 50)
|
||||
@@ -667,9 +667,9 @@ class ConceptLexerParser(BaseParser):
|
||||
# A copy must be created
|
||||
def inner_get_model(expression):
|
||||
if isinstance(expression, Concept):
|
||||
ret = ConceptMatch(expression, rule_name=expression.name)
|
||||
ret = ConceptExpression(expression, rule_name=expression.name)
|
||||
concepts_to_resolve.add(expression)
|
||||
elif isinstance(expression, ConceptMatch):
|
||||
elif isinstance(expression, ConceptExpression):
|
||||
if expression.rule_name is None or expression.rule_name == "":
|
||||
expression.rule_name = expression.concept.name if isinstance(expression.concept, Concept) \
|
||||
else expression.concept
|
||||
@@ -705,7 +705,7 @@ class ConceptLexerParser(BaseParser):
|
||||
|
||||
# infinite recursion matcher
|
||||
def _is_infinite_recursion(ref_concept, node):
|
||||
if isinstance(node, ConceptMatch):
|
||||
if isinstance(node, ConceptExpression):
|
||||
if node.concept == ref_concept:
|
||||
return True
|
||||
|
||||
@@ -856,7 +856,7 @@ class ConceptLexerParser(BaseParser):
|
||||
Goes in recursion if the property is a concept
|
||||
"""
|
||||
|
||||
# this cache is to make sure that we return the same concept for the same ConceptMatch
|
||||
# this cache is to make sure that we return the same concept for the same ConceptExpression
|
||||
_underlying_value_cache = {}
|
||||
|
||||
def _add_prop(_concept, prop_name, value):
|
||||
@@ -877,7 +877,7 @@ class ConceptLexerParser(BaseParser):
|
||||
_concept.cached_asts[prop_name] = new_value
|
||||
|
||||
def _look_for_concept_match(_underlying):
|
||||
if isinstance(_underlying.parsing_expression, ConceptMatch):
|
||||
if isinstance(_underlying.parsing_expression, ConceptExpression):
|
||||
return _underlying
|
||||
|
||||
if not isinstance(_underlying, NonTerminalNode):
|
||||
@@ -957,7 +957,7 @@ class ParsingExpressionVisitor:
|
||||
|
||||
for node in parsing_expression.elements:
|
||||
if isinstance(node, Concept):
|
||||
self.visit(ConceptMatch(node.key or node.name))
|
||||
self.visit(ConceptExpression(node.key or node.name))
|
||||
elif isinstance(node, str):
|
||||
self.visit(StrMatch(node))
|
||||
else:
|
||||
|
||||
@@ -21,9 +21,9 @@ class PythonErrorNode(ErrorNode):
|
||||
|
||||
class PythonNode(Node):
|
||||
|
||||
def __init__(self, source, ast_, concepts=None):
|
||||
def __init__(self, source, ast_=None, concepts=None):
|
||||
self.source = source
|
||||
self.ast_ = ast_
|
||||
self.ast_ = ast_ if ast_ else ast.parse(source, mode="eval") if source else None
|
||||
self.concepts = concepts or {} # when concepts are recognized in the expression
|
||||
|
||||
# def __repr__(self):
|
||||
|
||||
Reference in New Issue
Block a user