kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-05-12 17:21:10 +02:00
parent 7d3a490bc5
commit 6e343ba996
110 changed files with 13865 additions and 7540 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parsers/parsers_utils.py
+63 -14
View File
@@ -1,6 +1,7 @@
from core.concept import CC, Concept
from core.concept import CC, Concept, ConceptParts, DoNotResolve
from core.tokenizer import Tokenizer, TokenKind, Token
from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN
from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN, \
SCN
from parsers.SyaNodeParser import SyaConceptParserHelper
@@ -43,7 +44,16 @@ def compute_debug_array(res):
return to_compare
def get_node(concepts_map, expression_as_tokens, sub_expr, concept_key=None, skip=0, is_bnf=False, sya=False):
def get_node(
concepts_map,
expression_as_tokens,
sub_expr,
concept_key=None,
skip=0,
is_bnf=False,
sya=False,
init_empty_body=False,
exclude_body=False):
"""
Tries to find sub in expression
When found, transform it to its correct type
@@ -54,6 +64,8 @@ def get_node(concepts_map, expression_as_tokens, sub_expr, concept_key=None, ski
:param skip: number of occurrences of sub_expr to skip
:param is_bnf: True if the concept to search is a bnf definition
:param sya: Return SyaConceptParserHelper instead of a ConceptNode when needed
:param init_empty_body: if True adds the source in the body (actually in compiled.BODY)
:param exclude_body: Ask to not compare body
:return:
"""
if sub_expr == "')'":
@@ -80,23 +92,38 @@ def get_node(concepts_map, expression_as_tokens, sub_expr, concept_key=None, ski
content = [get_node(concepts_map, expression_as_tokens, c, sya=sya) for c in sub_expr.content]
return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
if isinstance(sub_expr, SCN):
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya)
sub_expr.fix_pos(node)
return sub_expr
if isinstance(sub_expr, (CNC, CC, CN)):
concept_node = get_node(
concepts_map,
expression_as_tokens,
sub_expr.source or sub_expr.concept_key,
sub_expr.concept_key, sya=sya)
if not hasattr(concept_node, "concept"):
raise Exception(f"'{sub_expr.concept_key}' is not a concept. Check your map.")
concept_found = concept_node.concept
sub_expr.concept_key = concept_found.key
sub_expr.concept = concept_found
sub_expr.fix_pos((concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start))
if hasattr(sub_expr, "compiled"):
for k, v in sub_expr.compiled.items():
node = get_node(concepts_map, expression_as_tokens, v, sya=sya) # need to get start and end positions
new_value = CC(Concept().update_from(concepts_map[v])) if (isinstance(v, str) and v in concepts_map) \
else node
node = get_node(concepts_map, expression_as_tokens, v, sya=sya, exclude_body=exclude_body) # need to get start and end positions
if isinstance(v, str) and v in concepts_map:
new_value_concept = concepts_map[v]
new_value = CC(Concept().update_from(new_value_concept), exclude_body=exclude_body)
if init_empty_body:
init_body(new_value, concept_found, v)
else:
new_value = node
sub_expr.compiled[k] = new_value
sub_expr.fix_pos(node)
if init_empty_body:
init_body(sub_expr, concept_found, sub_expr.source)
if hasattr(sub_expr, "fix_source"):
sub_expr.fix_source(expression_as_tokens[sub_expr.start: sub_expr.end + 1])
@@ -119,32 +146,54 @@ def get_node(concepts_map, expression_as_tokens, sub_expr, concept_key=None, ski
# special case of python source code
if "+" in sub_expr and sub_expr.strip() != "+":
return scnode(start, start + length - 1, sub_expr)
return SCN(sub_expr, start, start + length - 1)
# try to match one of the concept from the map
concept_key = concept_key or sub_expr
concept_found = concepts_map.get(concept_key, None)
if concept_found:
concept_found = Concept().update_from(concept_found) # make a copy when massively used in tests
if not sya or len(concept_found.metadata.props) == 0 or is_bnf:
# if it's an atom, then return a ConceptNode
return CN(concept_found, start, start + length - 1, source=sub_expr)
else:
# else return a ParserHelper
if sya and len(concept_found.metadata.variables) > 0 and not is_bnf:
return SyaConceptParserHelper(concept_found, start)
elif init_empty_body:
node = CNC(concept_found, start, start + length - 1, source=sub_expr, exclude_body=exclude_body)
init_body(node, concept_found, sub_expr)
return node
else:
return CN(concept_found, start, start + length - 1, source=sub_expr)
else:
# else an UnrecognizedTokensNode
return utnode(start, start + length - 1, sub_expr)
def compute_expected_array(concepts_map, expression, expected, sya=False):
def init_body(item, concept, value):
if "body" in item.compiled:
item.compiled[ConceptParts.BODY] = item.compiled["body"]
del (item.compiled["body"])
return
if not concept or concept.metadata.body or ConceptParts.BODY in item.compiled:
return
item.compiled[ConceptParts.BODY] = DoNotResolve(value)
def compute_expected_array(concepts_map, expression, expected, sya=False, init_empty_body=False, exclude_body=False):
"""
Computes a simple but sufficient version of the result of infix_to_postfix()
:param concepts_map:
:param expression:
:param expected:
:param sya: if true, generate an SyaConceptParserHelper instead of a cnode
:param init_empty_body: if True adds the source in the body (actually in compiled.BODY)
:param exclude_body: do not include ConceptParts.BODY in comparison
:return:
"""
expression_as_tokens = [token.value for token in Tokenizer(expression) if token.type != TokenKind.EOF]
return [get_node(concepts_map, expression_as_tokens, sub_expr, sya=sya) for sub_expr in expected]
return [get_node(
concepts_map,
expression_as_tokens,
sub_expr,
sya=sya,
init_empty_body=init_empty_body,
exclude_body=exclude_body) for sub_expr in expected]