Added SyaNodeParser (finally, after one month)

This commit is contained in:
2020-04-09 15:42:36 +02:00
parent c9acfa99a1
commit 6c7c529016
56 changed files with 5322 additions and 404 deletions
+12 -13
View File
@@ -1,18 +1,20 @@
# to be replaced by SyaNodeParser
import ast
from core.builtin_concepts import BuiltinConcepts
from core.tokenizer import TokenKind
from parsers.BaseNodeParser import SourceCodeNode
from parsers.BaseParser import BaseParser
from parsers.ConceptLexerParser import ConceptLexerParser, UnrecognizedTokensNode, ConceptNode, SourceCodeNode
from parsers.BnfNodeParser import BnfNodeParser, UnrecognizedTokensNode, ConceptNode
import core.utils
from parsers.PythonParser import PythonParser
concept_lexer_parser = ConceptLexerParser()
concept_lexer_parser = BnfNodeParser()
class MultipleConceptsParser(BaseParser):
"""
Parser that will take the result of ConceptLexerParser and
Parser that will take the result of BnfNodeParser and
try to resolve the unrecognized tokens token by token
It is a success when it returns a list ConceptNode exclusively
@@ -20,6 +22,7 @@ class MultipleConceptsParser(BaseParser):
def __init__(self, **kwargs):
BaseParser.__init__(self, "MultipleConcepts", 45)
self.enabled = False
@staticmethod
def finalize(nodes_found, unrecognized_tokens):
@@ -40,16 +43,12 @@ class MultipleConceptsParser(BaseParser):
unrecognized_tokens = UnrecognizedTokensNode(index, index, [token])
return unrecognized_tokens
def parse(self, context, text):
def parse(self, context, parser_input):
sheerka = context.sheerka
if not sheerka.isinstance(text, BuiltinConcepts.PARSER_RESULT):
nodes = self.get_input_as_lexer_nodes(parser_input, concept_lexer_parser)
if not nodes:
return None
if not text.parser == concept_lexer_parser:
return None
sheerka = context.sheerka
nodes = text.value
nodes_found = [[]]
concepts_only = True
@@ -97,16 +96,16 @@ class MultipleConceptsParser(BaseParser):
sheerka.new(
BuiltinConcepts.PARSER_RESULT,
parser=self,
source=text.source,
source=parser_input.source,
body=choice,
try_parsed=None))
)
if len(ret) == 1:
self.log_result(context, text.source, ret[0])
self.log_result(context, parser_input.source, ret[0])
return ret[0]
else:
self.log_multiple_results(context, text.source, ret)
self.log_multiple_results(context, parser_input.source, ret)
return ret
@staticmethod