1075 lines
34 KiB
Python
1075 lines
34 KiB
Python
from collections import namedtuple
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
from typing import Set
|
|
|
|
import core.utils
|
|
from core.builtin_concepts import BuiltinConcepts
|
|
from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, ConceptParts
|
|
from core.rule import Rule
|
|
from core.tokenizer import TokenKind, Token
|
|
from parsers.BaseParser import Node, BaseParser, ParsingError
|
|
|
|
DEBUG_COMPILED = True
|
|
|
|
|
|
@dataclass
|
|
class ChickenAndEggError(Exception):
|
|
concepts: Set[str]
|
|
|
|
|
|
@dataclass
|
|
class NoFirstTokenError(ParsingError):
|
|
concept: Concept
|
|
key: str
|
|
|
|
|
|
@dataclass()
|
|
class LexerNode(Node):
|
|
start: int # starting index in the tokens list
|
|
end: int # ending index in the tokens list
|
|
tokens: list = None # tokens
|
|
source: str = None # string representation of what was parsed
|
|
|
|
def __post_init__(self):
|
|
if self.source is None:
|
|
self.source = core.utils.get_text_from_tokens(self.tokens)
|
|
|
|
def __eq__(self, other):
|
|
if not isinstance(other, LexerNode):
|
|
return False
|
|
|
|
return self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source and \
|
|
self.tokens == other.tokens
|
|
|
|
def fix_source(self, force=True):
|
|
if force or self.source is None:
|
|
self.source = core.utils.get_text_from_tokens(self.tokens)
|
|
return self
|
|
|
|
def clone(self):
|
|
pass
|
|
|
|
def to_short_str(self):
|
|
raise NotImplementedError
|
|
|
|
|
|
class UnrecognizedTokensNode(LexerNode):
|
|
def __init__(self, start, end, tokens):
|
|
super().__init__(start, end, tokens)
|
|
self.is_frozen = False # TODO: Remove as it seems to now be useless
|
|
self.parenthesis_count = 0
|
|
|
|
def freeze(self):
|
|
# TODO: Remove as it seems to now be useless
|
|
self.is_frozen = True
|
|
|
|
def reset(self):
|
|
self.start = self.end = -1
|
|
self.tokens.clear()
|
|
self.is_frozen = False
|
|
self.parenthesis_count = 0
|
|
self.source = ""
|
|
|
|
def add_token(self, token, pos):
|
|
if self.is_frozen:
|
|
raise Exception("The node is frozen")
|
|
|
|
if self.end != -1 and pos == self.end + 2:
|
|
# add the missing whitespace
|
|
p = self.tokens[-1] # previous token
|
|
self.tokens.append(Token(TokenKind.WHITESPACE, " ", p.index + 1, p.line, p.column + 1))
|
|
|
|
self.tokens.append(token)
|
|
self.end = pos
|
|
if self.start == -1:
|
|
self.start = pos
|
|
|
|
if token.type == TokenKind.LPAR:
|
|
self.parenthesis_count += 1
|
|
|
|
if token.type == TokenKind.RPAR:
|
|
self.parenthesis_count -= 1
|
|
|
|
return self
|
|
|
|
def pop(self, token_kind):
|
|
if self.is_frozen:
|
|
raise Exception("The node is frozen")
|
|
|
|
if len(self.tokens) > 0 and self.tokens[-1].type == token_kind:
|
|
self.tokens.pop()
|
|
if len(self.tokens) == 0:
|
|
self.reset()
|
|
else:
|
|
self.end -= 1
|
|
|
|
def has_open_paren(self):
|
|
return self.parenthesis_count > 0
|
|
|
|
def not_whitespace(self):
|
|
return not self.is_whitespace()
|
|
|
|
def is_whitespace(self):
|
|
for t in self.tokens:
|
|
if t.type not in (TokenKind.WHITESPACE, TokenKind.NEWLINE):
|
|
return False
|
|
return True
|
|
|
|
def is_empty(self):
|
|
return len(self.tokens) == 0
|
|
|
|
def last_token_type(self):
|
|
if len(self.tokens) == 0:
|
|
return None
|
|
return self.tokens[-1].type
|
|
|
|
def __eq__(self, other):
|
|
if isinstance(other, utnode):
|
|
return self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
if isinstance(other, UTN):
|
|
return other == self
|
|
|
|
if not isinstance(other, UnrecognizedTokensNode):
|
|
return False
|
|
|
|
return self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
def __hash__(self):
|
|
return hash((self.start, self.end, self.source))
|
|
|
|
def __repr__(self):
|
|
return f"UnrecognizedTokensNode(source='{self.source}', start={self.start}, end={self.end})"
|
|
|
|
def clone(self):
|
|
clone = UnrecognizedTokensNode(self.start, self.end, self.tokens[:])
|
|
clone.is_frozen = self.is_frozen
|
|
clone.parenthesis_count = self.parenthesis_count
|
|
return clone
|
|
|
|
def to_short_str(self):
|
|
return f"UTN('{self.source}')"
|
|
|
|
|
|
class RuleNode(LexerNode):
|
|
def __init__(self, rule, start, end, tokens=None, source=None):
|
|
super().__init__(start, end, tokens, source)
|
|
self.rule = rule
|
|
self.fix_source(False)
|
|
|
|
def __eq__(self, other):
|
|
if id(self) == id(other):
|
|
return True
|
|
|
|
if isinstance(other, RN):
|
|
return other == self
|
|
|
|
if not isinstance(other, RuleNode):
|
|
return False
|
|
|
|
return self.rule == other.rule and \
|
|
self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
def __hash__(self):
|
|
return hash((self.rule, self.start, self.end, self.source))
|
|
|
|
def __repr__(self):
|
|
return f"RuleNode(rule='{self.rule}', source='{self.source}', start={self.start}, end={self.end})"
|
|
|
|
def clone(self):
|
|
return RuleNode(self.rule, self.start, self.end, self.tokens, self.source)
|
|
|
|
def to_short_str(self):
|
|
return f'RN({self.rule})'
|
|
|
|
|
|
class ConceptNode(LexerNode):
|
|
"""
|
|
Returned by the BnfNodeParser
|
|
It represents a recognized concept
|
|
"""
|
|
|
|
def __init__(self, concept, start, end, tokens=None, source=None, underlying=None):
|
|
super().__init__(start, end, tokens, source)
|
|
self.concept = concept
|
|
self.underlying = underlying
|
|
self.fix_source(False)
|
|
|
|
def __eq__(self, other):
|
|
if id(self) == id(other):
|
|
return True
|
|
|
|
if isinstance(other, (CN, CNC)):
|
|
return other == self
|
|
|
|
if isinstance(other, cnode):
|
|
return self.concept.key == other.concept_key and \
|
|
self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
if isinstance(other, short_cnode):
|
|
return self.concept.key == other.concept_key and self.source == other.source
|
|
|
|
if not isinstance(other, ConceptNode):
|
|
return False
|
|
|
|
return self.concept == other.concept and \
|
|
self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source and \
|
|
self.underlying == other.underlying
|
|
|
|
def __hash__(self):
|
|
return hash((self.concept, self.start, self.end, self.source, self.underlying))
|
|
|
|
def __repr__(self):
|
|
text = f"ConceptNode(concept='{self.concept}', source='{self.source}', start={self.start}, end={self.end}"
|
|
if DEBUG_COMPILED:
|
|
for k, v in self.concept.get_compiled().items():
|
|
text += f", {k}='{v}'"
|
|
return text + ")"
|
|
|
|
def clone(self):
|
|
# do we need to clone the concept as well ?
|
|
clone = ConceptNode(self.concept, self.start, self.end, self.tokens, self.source, self.underlying)
|
|
return clone
|
|
|
|
def as_bag(self):
|
|
"""
|
|
Creates a dictionary with the useful properties of the ConceptNode
|
|
see Concept.as_bag() for extra informations
|
|
"""
|
|
bag = {}
|
|
for k, v in self.__dict__.items():
|
|
bag[k] = v
|
|
|
|
# if isinstance(self.concept, Concept):
|
|
# bag["compiled"] = self.concept.get_compiled()
|
|
return bag
|
|
|
|
def to_short_str(self):
|
|
return f'CN({self.concept})'
|
|
|
|
|
|
class SourceCodeNode(LexerNode):
|
|
"""
|
|
Returned when some source code (like Python source code is recognized)
|
|
"""
|
|
|
|
def __init__(self, start, end, tokens=None, source=None, python_node=None, return_value=None):
|
|
"""
|
|
|
|
:param start: start position (index of the first token)
|
|
:param end: end position (index of the last token)
|
|
:param tokens:
|
|
:param source: tokens as string
|
|
:param python_node: PythonNode found (when the SourceCodeNode is validated)
|
|
:param return_value: ReturnValueConcept returned when the source was validated
|
|
|
|
When return_value is provided,
|
|
You should have return_value.body.body == node
|
|
"""
|
|
super().__init__(start, end, tokens, source)
|
|
self.python_node = python_node # The PythonNode (or whatever language node) that is found
|
|
self.return_value = return_value # original result of the parsing
|
|
|
|
def __eq__(self, other):
|
|
if isinstance(other, scnode):
|
|
return self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
if isinstance(other, SCN):
|
|
return other == self
|
|
|
|
if not isinstance(other, SourceCodeNode):
|
|
return False
|
|
|
|
return self.python_node == other.python_node and \
|
|
self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
def __hash__(self):
|
|
return hash((self.start, self.end, self.source))
|
|
|
|
def __repr__(self):
|
|
return f"SourceCodeNode(start={self.start}, end={self.end}, source='{self.source}')"
|
|
|
|
def to_short_str(self):
|
|
return f"SCN('{self.source}')"
|
|
|
|
|
|
class SourceCodeWithConceptNode(LexerNode):
|
|
"""
|
|
Kind of temporary version for SourceCodeNode
|
|
I know that there is some code,
|
|
I know that there are some concepts
|
|
I just don't want to make the glue yet
|
|
|
|
So I push all the nodes into one big bag
|
|
"""
|
|
|
|
def __init__(self, first_node, last_node, content_nodes=None, has_unrecognized=False):
|
|
super().__init__(9999, -1, None) # why not sys.maxint ?
|
|
self.first = first_node
|
|
self.last = last_node
|
|
self.nodes = content_nodes or []
|
|
self.has_unrecognized = has_unrecognized
|
|
self._all_nodes = None
|
|
self.fix_all_pos()
|
|
|
|
self.python_node = None # if the source code node is validated against a python parse, here is the PythonNode
|
|
self.return_value = None # return_value that produced the PythonNode
|
|
|
|
def add_node(self, node):
|
|
self.nodes.append(node)
|
|
self.fix_pos(node)
|
|
self._all_nodes = None
|
|
|
|
return self
|
|
|
|
def __eq__(self, other):
|
|
if id(self) == id(other):
|
|
return True
|
|
|
|
if isinstance(other, SCWC):
|
|
return other == self
|
|
|
|
if not isinstance(other, SourceCodeWithConceptNode):
|
|
return False
|
|
|
|
if self.start != other.start or self.end != other.end:
|
|
return False
|
|
|
|
if self.first != other.first:
|
|
return False
|
|
|
|
if self.last != other.last:
|
|
return False
|
|
|
|
if len(self.nodes) != len(other.nodes):
|
|
return False
|
|
|
|
for self_node, other_node in zip(self.nodes, other.nodes):
|
|
if self_node != other_node:
|
|
return False
|
|
|
|
# at last
|
|
return True
|
|
|
|
def __hash__(self):
|
|
return hash((self.first, self.last, len(self.nodes)))
|
|
|
|
def __repr__(self):
|
|
return f"SourceCodeWithConceptNode(start={self.start}, end={self.end}, source='{self.source}')"
|
|
|
|
def fix_all_pos(self):
|
|
if self.first is None: # to ease some unit test where only the python_node is necessary
|
|
return
|
|
|
|
for n in [self.first, self.last] + self.nodes:
|
|
self.fix_pos(n)
|
|
|
|
def fix_pos(self, node):
|
|
if hasattr(node, "start") and node.start is not None:
|
|
if node.start < self.start:
|
|
self.start = node.start
|
|
|
|
if hasattr(node, "end") and node.end is not None:
|
|
if node.end > self.end:
|
|
self.end = node.end
|
|
return self
|
|
|
|
def pseudo_fix_source(self):
|
|
"""
|
|
pseudo because the code is not that clean !
|
|
:return:
|
|
"""
|
|
self.source = self.first.source
|
|
for n in self.nodes:
|
|
self.source += " "
|
|
if hasattr(n, "source"):
|
|
self.source += n.source
|
|
elif hasattr(n, "concept"):
|
|
self.source += str(n.concept)
|
|
else:
|
|
self.source += " unknown"
|
|
self.source += self.last.source
|
|
return self
|
|
|
|
def get_all_nodes(self):
|
|
if self._all_nodes:
|
|
return self._all_nodes
|
|
|
|
self._all_nodes = [self.first, *self.nodes, self.last]
|
|
return self._all_nodes
|
|
|
|
def clone(self):
|
|
clone = SourceCodeWithConceptNode(self.first, self.last, self.nodes.copy(), self.has_unrecognized)
|
|
return clone
|
|
|
|
def to_short_str(self):
|
|
return f"SCWC({self.first}" + ", ".join(n.to_short_str for n in self.nodes) + f"{self.last})"
|
|
|
|
|
|
@dataclass()
|
|
class GrammarErrorNode(ParsingError):
|
|
message: str
|
|
|
|
|
|
class SyaAssociativity(Enum):
|
|
Left = "left"
|
|
Right = "right"
|
|
No = "No"
|
|
|
|
def __repr__(self):
|
|
return self.value
|
|
|
|
|
|
cnode = namedtuple("ConceptNode", "concept_key start end source")
|
|
short_cnode = namedtuple("ConceptNode", "concept_key source")
|
|
utnode = namedtuple("utnode", "start end source")
|
|
scnode = namedtuple("scnode", "start end source")
|
|
|
|
|
|
class HelperWithPos:
|
|
def __init__(self, start=None, end=None):
|
|
self.start = start
|
|
self.end = end
|
|
|
|
self.start_is_fixed = start is not None
|
|
self.end_is_fixed = end is not None
|
|
|
|
def fix_pos(self, node):
|
|
if not self.start_is_fixed:
|
|
start = node.start if hasattr(node, "start") else \
|
|
node[0] if isinstance(node, tuple) else None
|
|
|
|
if start is not None and (self.start is None or start < self.start):
|
|
self.start = start
|
|
|
|
if not self.end_is_fixed:
|
|
end = node.end if hasattr(node, "end") else \
|
|
node[1] if isinstance(node, tuple) else None
|
|
|
|
if end is not None and (self.end is None or end > self.end):
|
|
self.end = end
|
|
return self
|
|
|
|
|
|
class SCN(HelperWithPos):
|
|
"""
|
|
SourceCodeNode tester class
|
|
It matches with SourceCodeNode but with less constraints
|
|
|
|
SCN == SourceCodeNode if source, start, end (start and end are not validated when None)
|
|
"""
|
|
|
|
def __init__(self, source, start=None, end=None):
|
|
super().__init__(start, end)
|
|
self.source = source
|
|
|
|
def __eq__(self, other):
|
|
if id(self) == id(other):
|
|
return True
|
|
|
|
if isinstance(other, SourceCodeNode):
|
|
if self.source != other.source:
|
|
return False
|
|
if self.start is not None and self.start != other.start:
|
|
return False
|
|
if self.end is not None and self.end != other.end:
|
|
return False
|
|
|
|
return True
|
|
|
|
if not isinstance(other, CN):
|
|
return False
|
|
|
|
return self.source == other.source and \
|
|
self.start == other.start and \
|
|
self.end == other.end
|
|
|
|
def __hash__(self):
|
|
return hash((self.source, self.start, self.end))
|
|
|
|
def __repr__(self):
|
|
txt = f"SCN(source='{self.source}'"
|
|
if self.start is not None:
|
|
txt += f", start={self.start}"
|
|
if self.end is not None:
|
|
txt += f", end={self.end}"
|
|
return txt + ")"
|
|
|
|
|
|
class SCWC(HelperWithPos):
|
|
"""
|
|
SourceNodeWithConcept tester class
|
|
It matches with a SourceNodeWithConcept
|
|
but it's easier to instantiate during the tests
|
|
"""
|
|
|
|
def __init__(self, first, last, *args):
|
|
super().__init__(None, None)
|
|
self.first = first
|
|
self.last = last
|
|
self.content = args
|
|
|
|
def __eq__(self, other):
|
|
if id(self) == id(other):
|
|
return True
|
|
|
|
if isinstance(other, SourceCodeWithConceptNode):
|
|
if self.first != other.first:
|
|
return False
|
|
|
|
if self.last != other.last:
|
|
return False
|
|
|
|
if len(self.content) != len(other.nodes):
|
|
return False
|
|
|
|
for self_node, other_node in zip(self.content, other.nodes):
|
|
if self_node != other_node:
|
|
return False
|
|
|
|
# at last
|
|
return True
|
|
|
|
def __repr__(self):
|
|
txt = "SCWC("
|
|
if self.start is not None:
|
|
txt += f"start={self.start}"
|
|
if self.end is not None:
|
|
txt += f", end={self.end}"
|
|
txt += f", source='{self.source}'"
|
|
return txt + ")"
|
|
|
|
@property
|
|
def source(self):
|
|
"""
|
|
this code is a copy and paste from SourceCodeWithConceptNode.pseudo_fix_source
|
|
TODO: create a common function or whatever...
|
|
:return:
|
|
"""
|
|
source = self.first.source if hasattr(self.first, "source") else self.first
|
|
for n in self.content:
|
|
source += " "
|
|
if hasattr(n, "source"):
|
|
source += n.source
|
|
elif hasattr(n, "concept"):
|
|
source += str(n.concept)
|
|
else:
|
|
source += " unknown"
|
|
source += self.last.source if hasattr(self.last, "source") else self.last
|
|
return source
|
|
|
|
|
|
class CN(HelperWithPos):
|
|
"""
|
|
ConceptNode tester class
|
|
It matches with ConceptNode but with less constraints
|
|
|
|
CN == ConceptNode if concept key, start, end and source are the same
|
|
"""
|
|
|
|
def __init__(self, concept, start=None, end=None, source=None):
|
|
"""
|
|
|
|
:param concept: Concept or concept_key (only the key is used anyway)
|
|
:param start:
|
|
:param end:
|
|
:param source:
|
|
"""
|
|
super().__init__(start, end)
|
|
self.concept_key = concept.key if isinstance(concept, Concept) else concept
|
|
self.source = source
|
|
self.concept = concept if isinstance(concept, Concept) else None
|
|
|
|
def fix_source(self, str_tokens):
|
|
self.source = "".join(str_tokens)
|
|
return self
|
|
|
|
def __eq__(self, other):
|
|
if id(self) == id(other):
|
|
return True
|
|
|
|
if isinstance(other, ConceptNode):
|
|
if other.concept is None:
|
|
return False
|
|
if other.concept.key != self.concept_key:
|
|
return False
|
|
if self.start is not None and self.start != other.start:
|
|
return False
|
|
if self.end is not None and self.end != other.end:
|
|
return False
|
|
if self.source is not None and self.source != other.source:
|
|
return False
|
|
return True
|
|
|
|
if not isinstance(other, CN):
|
|
return False
|
|
|
|
return self.concept_key == other.concept_key and \
|
|
self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
def __hash__(self):
|
|
return hash((self.concept_key, self.start, self.end, self.source))
|
|
|
|
def __repr__(self):
|
|
if self.concept:
|
|
txt = f"CN(concept='{self.concept}'"
|
|
else:
|
|
txt = f"CN(concept_key='{self.concept_key}'"
|
|
txt += f", source='{self.source}'"
|
|
if self.start is not None:
|
|
txt += f", start={self.start}"
|
|
if self.end is not None:
|
|
txt += f", end={self.end}"
|
|
return txt + ")"
|
|
|
|
|
|
class CNC(CN):
|
|
"""
|
|
ConceptNode for Compiled tester class
|
|
It matches with ConceptNode
|
|
But focuses on the 'compiled' property of the concept
|
|
|
|
CNC == ConceptNode if CNC.get_compiled() == ConceptNode.concept.get_compiled()
|
|
"""
|
|
|
|
def __init__(self, concept_key, start=None, end=None, source=None, exclude_body=False, **kwargs):
|
|
super().__init__(concept_key, start, end, source)
|
|
self.compiled = kwargs
|
|
self.exclude_body = exclude_body
|
|
if "body" in self.compiled:
|
|
self.compiled[ConceptParts.BODY] = self.compiled["body"]
|
|
del self.compiled["body"]
|
|
|
|
def __eq__(self, other):
|
|
if id(self) == id(other):
|
|
return True
|
|
|
|
if isinstance(other, ConceptNode):
|
|
if other.concept is None:
|
|
return False
|
|
if other.concept.key != self.concept_key:
|
|
return False
|
|
if self.start is not None and self.start != other.start:
|
|
return False
|
|
if self.end is not None and self.end != other.end:
|
|
return False
|
|
if self.source is not None and self.source != other.source:
|
|
return False
|
|
if self.exclude_body:
|
|
to_compare = {k: v for k, v in other.concept.get_compiled().items() if k != ConceptParts.BODY}
|
|
else:
|
|
to_compare = other.concept.get_compiled()
|
|
if self.compiled == to_compare: # expanded form to ease the debug
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
if not isinstance(other, CNC):
|
|
return False
|
|
|
|
return self.concept_key == other.concept_key and \
|
|
self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source and \
|
|
self.compiled == other.compiled
|
|
|
|
def __repr__(self):
|
|
if self.concept:
|
|
txt = f"CNC(concept='{self.concept}'"
|
|
else:
|
|
txt = f"CNC(concept_key='{self.concept_key}'"
|
|
txt += f", source='{self.source}'"
|
|
if self.start is not None:
|
|
txt += f", start={self.start}"
|
|
if self.end is not None:
|
|
txt += f", end={self.end}"
|
|
|
|
for k, v in self.compiled.items():
|
|
txt += f", {k}='{v}'"
|
|
return txt + ")"
|
|
|
|
|
|
class UTN(HelperWithPos):
|
|
"""
|
|
Tester class for UnrecognizedTokenNode
|
|
compare the source, and start, end if defined
|
|
"""
|
|
|
|
def __init__(self, source, start=None, end=None):
|
|
"""
|
|
:param source:
|
|
:param start:
|
|
:param end:
|
|
"""
|
|
super().__init__(start, end)
|
|
self.source = source
|
|
|
|
def __eq__(self, other):
|
|
if id(self) == id(other):
|
|
return True
|
|
|
|
if isinstance(other, UnrecognizedTokensNode):
|
|
return self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
if not isinstance(other, UTN):
|
|
return False
|
|
|
|
return self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
def __hash__(self):
|
|
return hash((self.source, self.start, self.end))
|
|
|
|
def __repr__(self):
|
|
txt = f"UTN(source='{self.source}'"
|
|
if self.start is not None:
|
|
txt += f", start={self.start}"
|
|
if self.end is not None:
|
|
txt += f", end={self.end}"
|
|
return txt + ")"
|
|
|
|
|
|
class RN(HelperWithPos):
|
|
"""
|
|
Helper class to test RuleNode
|
|
"""
|
|
|
|
def __init__(self, rule, start=None, end=None, source=None):
|
|
"""
|
|
|
|
:param concept: Concept or concept_key (only the key is used anyway)
|
|
:param start:
|
|
:param end:
|
|
:param source:
|
|
"""
|
|
super().__init__(start, end)
|
|
self.rule_id = rule.id if isinstance(rule, Rule) else rule
|
|
self.source = source or core.utils.str_concept((None, self.rule_id), prefix="r:")
|
|
self.rule = rule if isinstance(rule, Rule) else None
|
|
|
|
def __eq__(self, other):
|
|
if id(self) == id(other):
|
|
return True
|
|
|
|
if isinstance(other, RuleNode):
|
|
if other.rule is None:
|
|
return False
|
|
if other.rule.id != self.rule_id:
|
|
return False
|
|
if self.start is not None and self.start != other.start:
|
|
return False
|
|
if self.end is not None and self.end != other.end:
|
|
return False
|
|
if self.source is not None and self.source != other.source:
|
|
return False
|
|
return True
|
|
|
|
if not isinstance(other, RN):
|
|
return False
|
|
|
|
return self.rule_id == other.rule_id and \
|
|
self.start == other.start and \
|
|
self.end == other.end and \
|
|
self.source == other.source
|
|
|
|
def __hash__(self):
|
|
return hash((self.rule_id, self.start, self.end, self.source))
|
|
|
|
def __repr__(self):
|
|
if self.rule:
|
|
txt = f"RN(rule='{self.rule}'"
|
|
else:
|
|
txt = f"RN(rule_id='{self.rule_id}'"
|
|
txt += f", source='{self.source}'"
|
|
if self.start is not None:
|
|
txt += f", start={self.start}"
|
|
if self.end is not None:
|
|
txt += f", end={self.end}"
|
|
return txt + ")"
|
|
|
|
|
|
class BaseNodeParser(BaseParser):
|
|
"""
|
|
Parser that return LexerNode
|
|
"""
|
|
|
|
def __init__(self, name, priority, **kwargs):
|
|
super().__init__(name, priority, yield_eof=True)
|
|
if 'sheerka' in kwargs:
|
|
sheerka = kwargs.get("sheerka")
|
|
self.concepts_by_first_keyword = sheerka.resolved_concepts_by_first_keyword
|
|
|
|
else:
|
|
self.concepts_by_first_keyword = None
|
|
|
|
def init_from_concepts(self, context, concepts, **kwargs):
|
|
"""
|
|
Initialize the parser with a list of concepts
|
|
For unit tests convenience
|
|
:param context
|
|
:param concepts
|
|
:return:
|
|
"""
|
|
concepts_by_first_keyword = self.get_concepts_by_first_token(context, concepts).body
|
|
self.concepts_by_first_keyword = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword).body
|
|
|
|
def get_concepts(self, token, to_keep, custom=None, to_map=None, strip_quotes=False):
|
|
"""
|
|
Tries to find if there are concepts that match the value of the token
|
|
Caution: Returns the actual cache, not a copy
|
|
:param token:
|
|
:param to_keep: predicate to tell if the concept is eligible
|
|
:param custom: lambda name -> List[Concepts] that gives extra concepts, according to the name
|
|
:param to_map:
|
|
:param strip_quotes: Remove quotes from strings
|
|
:return:
|
|
"""
|
|
|
|
if token.type == TokenKind.WHITESPACE:
|
|
return None
|
|
|
|
if token.type == TokenKind.STRING:
|
|
name = token.value[1:-1] if strip_quotes else token.value
|
|
else:
|
|
name = token.value
|
|
|
|
custom_concepts = custom(name) if custom else [] # to get extra concepts using an alternative method
|
|
|
|
result = []
|
|
if name in self.concepts_by_first_keyword:
|
|
for concept_id in self.concepts_by_first_keyword.get(name):
|
|
|
|
concept = self.sheerka.get_by_id(concept_id)
|
|
|
|
if not to_keep(concept):
|
|
continue
|
|
|
|
concept = to_map(concept, self, self.sheerka) if to_map else concept
|
|
result.append(concept)
|
|
|
|
return core.utils.make_unique(result + custom_concepts,
|
|
lambda c: c.concept.id if hasattr(c, "concept") else c.id)
|
|
|
|
return custom_concepts if custom else None
|
|
|
|
@staticmethod
|
|
def get_concepts_by_first_token(context, concepts, use_sheerka=False, previous_entries=None):
|
|
"""
|
|
Create the map describing the first token expected by a concept
|
|
:param context:
|
|
:param concepts: lists of concepts to parse
|
|
:param use_sheerka: if True, update concepts_by_first_keyword from sheerka
|
|
:param previous_entries:
|
|
:return:
|
|
"""
|
|
sheerka = context.sheerka
|
|
res = sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) if use_sheerka else (previous_entries or {})
|
|
for concept in concepts:
|
|
keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
|
|
|
|
if keywords is None:
|
|
# no first token found for a concept ?
|
|
return sheerka.ret(sheerka.name, False, NoFirstTokenError(concept, concept.key))
|
|
|
|
for keyword in keywords:
|
|
res.setdefault(keyword, []).append(concept.id)
|
|
|
|
# 'uniquify' the lists
|
|
for k, v in res.items():
|
|
res[k] = core.utils.make_unique(v)
|
|
|
|
return sheerka.ret("BaseNodeParser", True, res)
|
|
|
|
@staticmethod
|
|
def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword, modified_concepts=None):
|
|
sheerka = context.sheerka
|
|
res = {}
|
|
|
|
def get_by_id(c_id):
|
|
if modified_concepts and c_id in modified_concepts:
|
|
return modified_concepts[c_id]
|
|
return sheerka.get_by_id(c_id)
|
|
|
|
def resolve_concepts(concept_str):
|
|
c_key, c_id = core.utils.unstr_concept(concept_str)
|
|
if c_id in already_seen:
|
|
return ChickenAndEggError(already_seen)
|
|
|
|
already_seen.add(c_id)
|
|
|
|
resolved = set()
|
|
to_resolve = set()
|
|
chicken_and_egg = set()
|
|
|
|
concept = get_by_id(c_id)
|
|
|
|
if sheerka.isaset(context, concept):
|
|
concepts = sheerka.get_set_elements(context, concept)
|
|
else:
|
|
concepts = [concept]
|
|
|
|
for concept in concepts:
|
|
BaseNodeParser.ensure_bnf(context, concept) # need to make sure that it cannot fail
|
|
keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
|
|
for keyword in keywords:
|
|
(to_resolve if keyword.startswith("c:|") else resolved).add(keyword)
|
|
|
|
for concept_to_resolve_str in to_resolve:
|
|
res = resolve_concepts(concept_to_resolve_str)
|
|
if isinstance(res, ChickenAndEggError):
|
|
chicken_and_egg |= res.concepts
|
|
else:
|
|
resolved |= res
|
|
to_resolve.clear()
|
|
|
|
if len(resolved) == 0 and len(chicken_and_egg) > 0:
|
|
raise ChickenAndEggError(chicken_and_egg)
|
|
else:
|
|
return resolved
|
|
|
|
for k, v in concepts_by_first_keyword.items():
|
|
if k.startswith("c:|"):
|
|
try:
|
|
already_seen = set()
|
|
resolved_keywords = resolve_concepts(k)
|
|
for resolved in resolved_keywords:
|
|
res.setdefault(resolved, []).extend(v)
|
|
except ChickenAndEggError as ex:
|
|
context.log(f"Chicken and egg detected for {k}, concepts={ex.concepts}")
|
|
concepts_in_recursion = ex.concepts
|
|
# make sure to have all the parents
|
|
for parent in v:
|
|
concepts_in_recursion.add(parent)
|
|
|
|
for concept_id in concepts_in_recursion:
|
|
# make sure we keep the longest chain
|
|
old = sheerka.chicken_and_eggs.get(concept_id)
|
|
if old is None or len(old) < len(ex.concepts):
|
|
sheerka.chicken_and_eggs.put(concept_id, concepts_in_recursion)
|
|
else:
|
|
res.setdefault(k, []).extend(v)
|
|
|
|
# 'uniquify' the lists
|
|
for k, v in res.items():
|
|
res[k] = core.utils.make_unique(v)
|
|
|
|
return sheerka.ret("BaseNodeParser", True, res)
|
|
|
|
@staticmethod
|
|
def get_referenced_concepts(context, concept_id, already_seen):
|
|
"""
|
|
Gets all the tokens that may allow to recognize concept concept_id
|
|
Basically, it returns all the starting tokens for concept concept_id
|
|
CHICKEN_AND_EGG is returned when a circular references are found
|
|
:param context:
|
|
:param concept_id:
|
|
:param already_seen:
|
|
:return:
|
|
"""
|
|
if concept_id in already_seen:
|
|
return ChickenAndEggError(already_seen)
|
|
|
|
already_seen.add(concept_id)
|
|
|
|
resolved = set()
|
|
to_resolve = set()
|
|
chicken_and_egg = set()
|
|
sheerka = context.sheerka
|
|
concept = sheerka.get_by_id(concept_id)
|
|
|
|
if sheerka.isaset(context, concept):
|
|
concepts = sheerka.get_set_elements(context, concept)
|
|
else:
|
|
concepts = [concept]
|
|
|
|
for concept in concepts:
|
|
BaseNodeParser.ensure_bnf(context, concept) # need to make sure that it cannot fail
|
|
keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
|
|
for keyword in keywords:
|
|
(to_resolve if keyword.startswith("c:|") else resolved).add(keyword)
|
|
|
|
for concept_to_resolve_str in to_resolve:
|
|
c_key, c_id = core.utils.unstr_concept(concept_to_resolve_str)
|
|
res = BaseNodeParser.get_referenced_concepts(context, c_id, already_seen)
|
|
if isinstance(res, ChickenAndEggError):
|
|
chicken_and_egg |= res.concepts
|
|
else:
|
|
resolved |= res
|
|
to_resolve.clear()
|
|
|
|
if len(resolved) == 0 and len(chicken_and_egg) > 0:
|
|
raise ChickenAndEggError(chicken_and_egg)
|
|
else:
|
|
return resolved
|
|
|
|
@staticmethod
|
|
def resolve_sya_associativity_and_precedence(context, sya):
|
|
pass
|
|
|
|
@staticmethod
|
|
def get_first_tokens(sheerka, concept):
|
|
"""
|
|
|
|
:param sheerka:
|
|
:param concept:
|
|
:return:
|
|
"""
|
|
if concept.get_bnf():
|
|
from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor
|
|
bnf_visitor = BnfNodeFirstTokenVisitor(sheerka)
|
|
bnf_visitor.visit(concept.get_bnf())
|
|
return bnf_visitor.first_tokens
|
|
else:
|
|
keywords = concept.key.split()
|
|
for keyword in keywords:
|
|
if keyword.startswith(VARIABLE_PREFIX):
|
|
continue
|
|
|
|
return [keyword]
|
|
|
|
return None
|
|
|
|
@staticmethod
|
|
def ensure_bnf(context, concept, parser_name="BaseNodeParser"):
|
|
if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf():
|
|
from parsers.BnfDefinitionParser import BnfDefinitionParser
|
|
regex_parser = BnfDefinitionParser()
|
|
desc = f"Resolving BNF '{concept.get_metadata().definition}'"
|
|
with context.push(BuiltinConcepts.INIT_BNF,
|
|
concept,
|
|
who=parser_name,
|
|
obj=concept,
|
|
desc=desc) as sub_context:
|
|
sub_context.add_inputs(parser_input=concept.get_metadata().definition)
|
|
bnf_parsing_ret_val = regex_parser.parse(sub_context, concept.get_metadata().definition)
|
|
sub_context.add_values(return_values=bnf_parsing_ret_val)
|
|
|
|
if not bnf_parsing_ret_val.status:
|
|
raise Exception(bnf_parsing_ret_val.value)
|
|
|
|
concept.set_bnf(bnf_parsing_ret_val.body.body)
|
|
if concept.id:
|
|
context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf()) # update bnf in cache
|