Fixed #109 : Mix python and concept. List comprehension
Fixed #110 : SheerkaDebugManager: add list_debug_settings Fixed #111 : SheerkaDebugManager: Implement ListDebugLogger Fixed #112 : SyaNodeParser: rewrite this parser Fixed #113 : Sheerka.: Add enable_parser_caching to disable parsers caching Fixed #114 : SyaNodeParser : Implement fast cache to resolve unrecognized tokens requests Fixed #115 : BnfNodeParser : Implement fast cache to resolve unrecognized tokens requests Fixed #116 : SequenceNodeParser : Implement fast cache to resolve unrecognized tokens requests Fixed #117 : ResolveMultiplePluralAmbiguityEvaluator: Resolve Multiple plural ambiguity
This commit is contained in:
@@ -1,15 +1,15 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
from core import builtin_helpers
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.builtin_helpers import update_concepts_hints
|
||||
from core.concept import DEFINITION_TYPE_BNF, Concept
|
||||
from core.builtin_helpers import debug_nodes, update_concepts_hints
|
||||
from core.concept import Concept, DEFINITION_TYPE_BNF
|
||||
from core.global_symbols import NotFound
|
||||
from core.sheerka.services.SheerkaExecute import ParserInput
|
||||
from core.tokenizer import Tokenizer, TokenKind
|
||||
from core.utils import strip_tokens, make_unique
|
||||
from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, UnrecognizedTokensNode, SourceCodeNode
|
||||
from parsers.BaseParser import UnexpectedTokenParsingError, ParsingError
|
||||
from core.tokenizer import TokenKind, Tokenizer
|
||||
from core.utils import make_unique, strip_tokens
|
||||
from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, SourceCodeNode, UnrecognizedTokensCache, \
|
||||
UnrecognizedTokensNode
|
||||
from parsers.BaseParser import ParsingError, UnexpectedTokenParsingError
|
||||
from parsers.BnfNodeParser import BnfNodeParser
|
||||
from parsers.SyaNodeParser import SyaNodeParser
|
||||
|
||||
@@ -47,9 +47,10 @@ class TokensNodeFoundError(ParsingError):
|
||||
|
||||
|
||||
class AtomConceptParserHelper:
|
||||
def __init__(self, context):
|
||||
def __init__(self, parser):
|
||||
|
||||
self.context = context
|
||||
self.parser = parser
|
||||
self.context = parser.context
|
||||
self.debug = []
|
||||
self.sequence = [] # sequence of concepts already found found
|
||||
self.current_concept: ConceptNode = None # concept being parsed
|
||||
@@ -80,7 +81,7 @@ class AtomConceptParserHelper:
|
||||
return hash(len(self.sequence))
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.sequence}"
|
||||
return f"{debug_nodes(self.sequence)}"
|
||||
|
||||
def lock(self):
|
||||
self.is_locked = True
|
||||
@@ -146,10 +147,7 @@ class AtomConceptParserHelper:
|
||||
self.unrecognized_tokens.fix_source()
|
||||
|
||||
# try to recognize concepts
|
||||
nodes_sequences = builtin_helpers.get_lexer_nodes_from_unrecognized(
|
||||
self.context,
|
||||
self.unrecognized_tokens,
|
||||
PARSERS)
|
||||
nodes_sequences = self.parser.cache.get_lexer_nodes_from_unrecognized(self.context, self.unrecognized_tokens)
|
||||
|
||||
if nodes_sequences:
|
||||
instances = [self]
|
||||
@@ -162,7 +160,7 @@ class AtomConceptParserHelper:
|
||||
for node in node_sequence:
|
||||
instance.sequence.append(node)
|
||||
if isinstance(node, (UnrecognizedTokensNode, SourceCodeNode)) or \
|
||||
hasattr(node, "unrecognized_tokens") and node.unrecognized_tokens:
|
||||
hasattr(node, "unrecognized_tokens") and node.unrecognized_tokens:
|
||||
instance.has_unrecognized = True
|
||||
instance.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, [])
|
||||
|
||||
@@ -191,7 +189,7 @@ class AtomConceptParserHelper:
|
||||
self.errors.append(TokensNodeFoundError(self.expected_tokens))
|
||||
|
||||
def clone(self):
|
||||
clone = AtomConceptParserHelper(self.context)
|
||||
clone = AtomConceptParserHelper(self.parser)
|
||||
clone.debug = self.debug[:]
|
||||
clone.sequence = self.sequence[:]
|
||||
clone.current_concept = self.current_concept.clone() if self.current_concept else None
|
||||
@@ -224,6 +222,7 @@ class SequenceNodeParser(BaseNodeParser):
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(SequenceNodeParser.NAME, 50, **kwargs)
|
||||
self.cache = UnrecognizedTokensCache(PARSERS)
|
||||
|
||||
@staticmethod
|
||||
def _is_eligible(concept):
|
||||
@@ -278,7 +277,7 @@ class SequenceNodeParser(BaseNodeParser):
|
||||
concept_parser_helpers.extend(forked)
|
||||
forked.clear()
|
||||
|
||||
concept_parser_helpers = [AtomConceptParserHelper(self.context)]
|
||||
concept_parser_helpers = [AtomConceptParserHelper(self)]
|
||||
|
||||
while self.parser_input.next_token(False):
|
||||
for concept_parser in concept_parser_helpers:
|
||||
@@ -355,7 +354,7 @@ class SequenceNodeParser(BaseNodeParser):
|
||||
res = []
|
||||
start, end = self.get_tokens_boundaries(self.parser_input.as_tokens())
|
||||
for concept in concepts:
|
||||
parser_helper = AtomConceptParserHelper(None)
|
||||
parser_helper = AtomConceptParserHelper(self)
|
||||
parser_helper.sequence.append(ConceptNode(concept,
|
||||
start,
|
||||
end,
|
||||
@@ -419,6 +418,9 @@ class SequenceNodeParser(BaseNodeParser):
|
||||
False,
|
||||
context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))
|
||||
|
||||
debugger = context.get_debugger(self.NAME, "parse")
|
||||
debugger.debug_entering(source=self.parser_input.as_text())
|
||||
|
||||
sequences = self.get_concepts_sequences()
|
||||
if by_name := self.get_by_name():
|
||||
# note that concepts by names must be appended, not prepended
|
||||
@@ -427,6 +429,10 @@ class SequenceNodeParser(BaseNodeParser):
|
||||
|
||||
parser_helpers = self.get_valid(sequences)
|
||||
|
||||
if debugger.is_enabled():
|
||||
debugger.debug_var("stats", self.cache.to_dict())
|
||||
debugger.debug_leaving(result=parser_helpers)
|
||||
|
||||
if len(parser_helpers):
|
||||
ret = []
|
||||
for parser_helper in parser_helpers:
|
||||
@@ -471,8 +477,18 @@ class SequenceNodeParser(BaseNodeParser):
|
||||
if not eligible:
|
||||
return None
|
||||
|
||||
return [self.sheerka.new_dynamic(c, BuiltinConcepts.PLURAL, name=token.value, props={BuiltinConcepts.PLURAL: c})
|
||||
for c in concepts]
|
||||
plural_concepts = [self.sheerka.new_dynamic(c,
|
||||
BuiltinConcepts.PLURAL,
|
||||
name=token.value,
|
||||
props={BuiltinConcepts.PLURAL: c})
|
||||
for c in concepts]
|
||||
|
||||
for concept in plural_concepts:
|
||||
underlying_concept = concept.get_prop(BuiltinConcepts.PLURAL)
|
||||
if self.sheerka.isaset(self.context, underlying_concept):
|
||||
concept.get_metadata().body = f"get_set_elements(c:|{underlying_concept.id}:)"
|
||||
|
||||
return plural_concepts
|
||||
|
||||
@staticmethod
|
||||
def as_list(obj):
|
||||
|
||||
Reference in New Issue
Block a user