kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #109 : Mix python and concept. List comprehension

Browse Source 
Fixed #110 : SheerkaDebugManager: add list_debug_settings
Fixed #111 : SheerkaDebugManager: Implement ListDebugLogger
Fixed #112 : SyaNodeParser: rewrite this parser
Fixed #113 : Sheerka.: Add enable_parser_caching to disable parsers caching
Fixed #114 : SyaNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #115 : BnfNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #116 : SequenceNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #117 : ResolveMultiplePluralAmbiguityEvaluator: Resolve Multiple plural ambiguity
This commit is contained in:
Kodjo Sossouvi
2021-09-06 11:51:50 +02:00
parent 71d1b1d1ca
commit 54e5681c5a
57 changed files with 5179 additions and 3125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/parsers/BnfNodeParser.py
+21 -15
View File
@@ -16,13 +16,13 @@ import core.builtin_helpers
import core.utils
from cache.Cache import Cache
from core.builtin_concepts import BuiltinConcepts
from core.concept import DEFINITION_TYPE_BNF, DoNotResolve, ConceptParts, Concept
from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, DoNotResolve
from core.global_symbols import NotFound
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, TokenKind, Token
from core.tokenizer import Token, TokenKind, Tokenizer
from core.utils import CONSOLE_COLORS_MAP as CCM
from parsers.BaseNodeParser import BaseNodeParser, GrammarErrorNode, UnrecognizedTokensNode, ConceptNode, \
NoMatchingTokenError, RuleNode, SourceCodeNode, SourceCodeWithConceptNode
from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, GrammarErrorNode, NoMatchingTokenError, RuleNode, \
SourceCodeNode, SourceCodeWithConceptNode, UnrecognizedTokensCache, UnrecognizedTokensNode
PARSERS = ["Sequence", "Sya", "Python"]
VARIABLE_EXPR_PARSER = ["Sequence", "Sya", "Python", "Bnf"]
@@ -618,9 +618,8 @@ class VariableExpression(ParsingExpression):
return None
utn = UnrecognizedTokensNode(start, end, tokens)
nodes_sequences = core.builtin_helpers.get_lexer_nodes_from_unrecognized(parser_helper.parser.context,
utn,
VARIABLE_EXPR_PARSER)
nodes_sequences = parser_helper.parser.cache2.get_lexer_nodes_from_unrecognized(parser_helper.parser.context,
utn)
return nodes_sequences
@staticmethod
@@ -1264,8 +1263,8 @@ class BnfConceptParserHelper:
self.pos = -1
def __repr__(self):
concepts = [item.concept if isinstance(item, ConceptNode) else "***" for item in self.sequence]
return f"BnfConceptParserHelper({concepts})"
nodes = core.builtin_helpers.debug_nodes(self.sequence)
return f"BnfConceptParserHelper({nodes})"
def __eq__(self, other):
if id(self) == id(other):
@@ -1443,9 +1442,8 @@ class BnfConceptParserHelper:
self.unrecognized_tokens.fix_source()
# try to recognize concepts
nodes_sequences = core.builtin_helpers.get_lexer_nodes_from_unrecognized(self.parser.context,
self.unrecognized_tokens,
PARSERS)
nodes_sequences = self.parser.cache.get_lexer_nodes_from_unrecognized(self.parser.context,
self.unrecognized_tokens)
if nodes_sequences:
instances = [self]
@@ -1458,7 +1456,7 @@ class BnfConceptParserHelper:
for node in node_sequence:
instance.sequence.append(node)
if isinstance(node, UnrecognizedTokensNode) or \
hasattr(node, "unrecognized_tokens") and node.unrecognized_tokens:
hasattr(node, "unrecognized_tokens") and node.unrecognized_tokens:
instance.has_unrecognized = True
instance.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, [])
@@ -1568,7 +1566,7 @@ class BnfConceptParserHelper:
if _underlying.parsing_expression.rule_name:
# make sure VariableExpression are only added once
if (not isinstance(_underlying.parsing_expression, VariableExpression) or
_underlying.parsing_expression.rule_name not in _concept.get_compiled()):
_underlying.parsing_expression.rule_name not in _concept.get_compiled()):
var_value = _get_underlying_value(_underlying)
_add_compiled(_concept, _underlying.parsing_expression.rule_name, var_value)
_concept.get_hints().need_validation = True
@@ -1638,6 +1636,8 @@ class BnfNodeParser(BaseNodeParser):
else:
self.concepts_grammars = Cache()
self.cache = UnrecognizedTokensCache(PARSERS)
self.cache2 = UnrecognizedTokensCache(VARIABLE_EXPR_PARSER)
self.ignore_case = True
@staticmethod
@@ -1963,7 +1963,7 @@ class BnfNodeParser(BaseNodeParser):
nodes = []
for c in valid_concepts:
nodes.append(ConceptExpression(c, rule_name=c.name))
nodes.append(ConceptExpression(c, rule_name=c.key))
resolved = self.resolve_parsing_expression(ssc,
UnOrderedChoice(*nodes),
@@ -2116,6 +2116,12 @@ class BnfNodeParser(BaseNodeParser):
sequences = self.get_concepts_sequences(context)
valid_parser_helpers = self.get_valid(sequences)
debugger = context.get_debugger(self.NAME, "parse")
if debugger.is_enabled:
debugger.debug_var("stats", self.cache.to_dict())
#debugger.debug_var("stats", self.cache2.to_dict())
if valid_parser_helpers is None:
return self.sheerka.ret(
self.name,