203 lines
7.8 KiB
Python
203 lines
7.8 KiB
Python
from core.concept import CC, Concept, ConceptParts, DoNotResolve
|
|
from core.tokenizer import Tokenizer, TokenKind, Token
|
|
from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN, \
|
|
SCN
|
|
from parsers.SyaNodeParser import SyaConceptParserHelper
|
|
|
|
|
|
def _index(tokens, expr, index):
|
|
"""
|
|
Finds a sub list in a bigger list
|
|
:param tokens:
|
|
:param expr:
|
|
:param index:
|
|
:return:
|
|
"""
|
|
expected = [token.value for token in Tokenizer(expr) if token.type != TokenKind.EOF]
|
|
for i in range(0, len(tokens) - len(expected) + 1):
|
|
for j in range(len(expected)):
|
|
if tokens[i + j] != expected[j]:
|
|
break
|
|
else:
|
|
if index == 0:
|
|
return i, len(expected)
|
|
else:
|
|
index -= 1
|
|
|
|
raise ValueError(f"substring '{expr}' not found")
|
|
|
|
|
|
def compute_debug_array(res):
|
|
to_compare = []
|
|
for r in res:
|
|
res_debug = []
|
|
for token in r.debug:
|
|
if isinstance(token, Token):
|
|
if token.type == TokenKind.WHITESPACE:
|
|
continue
|
|
else:
|
|
res_debug.append("T(" + token.value + ")")
|
|
else:
|
|
res_debug.append("C(" + token.concept.name + ")")
|
|
to_compare.append(res_debug)
|
|
|
|
return to_compare
|
|
|
|
|
|
def get_node(
|
|
concepts_map,
|
|
expression_as_tokens,
|
|
sub_expr,
|
|
concept_key=None,
|
|
skip=0,
|
|
is_bnf=False,
|
|
sya=False,
|
|
init_empty_body=False,
|
|
exclude_body=False):
|
|
"""
|
|
Tries to find sub in expression
|
|
When found, transform it to its correct type
|
|
:param expression_as_tokens: full expression
|
|
:param sub_expr: sub expression to search in the full expression
|
|
:param concepts_map: hash of the known concepts
|
|
:param concept_key: key of the concept if different from sub_expr
|
|
:param skip: number of occurrences of sub_expr to skip
|
|
:param is_bnf: True if the concept to search is a bnf definition
|
|
:param sya: Return SyaConceptParserHelper instead of a ConceptNode when needed
|
|
:param init_empty_body: if True adds the source in the body (actually in compiled.BODY)
|
|
:param exclude_body: Ask to not compare body
|
|
:return:
|
|
"""
|
|
if sub_expr == "')'":
|
|
return ")"
|
|
|
|
if isinstance(sub_expr, (scnode, utnode, DoNotResolve)):
|
|
return sub_expr
|
|
|
|
if isinstance(sub_expr, cnode):
|
|
# for cnode, map the concept key to the one from concepts_maps if needed
|
|
if sub_expr.concept_key.startswith("#"):
|
|
return cnode(
|
|
concepts_map[sub_expr.concept_key[1:]].key,
|
|
sub_expr.start,
|
|
sub_expr.end,
|
|
sub_expr.source
|
|
)
|
|
else:
|
|
return sub_expr
|
|
|
|
if isinstance(sub_expr, SCWC):
|
|
sub_expr.first = get_node(concepts_map, expression_as_tokens, sub_expr.first, sya=sya)
|
|
sub_expr.last = get_node(concepts_map, expression_as_tokens, sub_expr.last, sya=sya)
|
|
sub_expr.content = [get_node(concepts_map, expression_as_tokens, c, sya=sya) for c in sub_expr.content]
|
|
sub_expr.fix_pos(sub_expr.first)
|
|
sub_expr.fix_pos(sub_expr.last)
|
|
return sub_expr
|
|
#return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
|
|
|
|
if isinstance(sub_expr, SCN):
|
|
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya)
|
|
sub_expr.fix_pos(node)
|
|
return sub_expr
|
|
|
|
if isinstance(sub_expr, (CNC, CC, CN)):
|
|
concept_node = get_node(
|
|
concepts_map,
|
|
expression_as_tokens,
|
|
sub_expr.source or sub_expr.concept_key,
|
|
sub_expr.concept_key, sya=sya)
|
|
if not hasattr(concept_node, "concept"):
|
|
raise Exception(f"'{sub_expr.concept_key}' is not a concept. Check your map.")
|
|
concept_found = concept_node.concept
|
|
sub_expr.concept_key = concept_found.key
|
|
sub_expr.concept = concept_found
|
|
sub_expr.fix_pos((concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start))
|
|
if hasattr(sub_expr, "compiled"):
|
|
for k, v in sub_expr.compiled.items():
|
|
node = get_node(concepts_map, expression_as_tokens, v, sya=sya, exclude_body=exclude_body) # need to get start and end positions
|
|
if isinstance(v, str) and v in concepts_map:
|
|
new_value_concept = concepts_map[v]
|
|
new_value = CC(Concept().update_from(new_value_concept), exclude_body=exclude_body)
|
|
if init_empty_body:
|
|
init_body(new_value, concept_found, v)
|
|
else:
|
|
new_value = node
|
|
|
|
sub_expr.compiled[k] = new_value
|
|
sub_expr.fix_pos(node)
|
|
if init_empty_body:
|
|
init_body(sub_expr, concept_found, sub_expr.source)
|
|
|
|
if hasattr(sub_expr, "fix_source"):
|
|
sub_expr.fix_source(expression_as_tokens[sub_expr.start: sub_expr.end + 1])
|
|
return sub_expr
|
|
|
|
if isinstance(sub_expr, UTN):
|
|
node = get_node(concepts_map, expression_as_tokens, sub_expr.source)
|
|
sub_expr.fix_pos(node)
|
|
return sub_expr
|
|
|
|
if isinstance(sub_expr, short_cnode):
|
|
return get_node(concepts_map, expression_as_tokens, sub_expr.source,
|
|
concept_key=sub_expr.concept_key, skip=skip, is_bnf=True, sya=sya)
|
|
|
|
if isinstance(sub_expr, tuple):
|
|
return get_node(concepts_map, expression_as_tokens, sub_expr[0],
|
|
concept_key=concept_key, skip=sub_expr[1], is_bnf=is_bnf, sya=sya)
|
|
|
|
start, length = _index(expression_as_tokens, sub_expr, skip)
|
|
|
|
# special case of python source code
|
|
if "+" in sub_expr and sub_expr.strip() != "+":
|
|
return SCN(sub_expr, start, start + length - 1)
|
|
|
|
# try to match one of the concept from the map
|
|
concept_key = concept_key or sub_expr
|
|
concept_found = concepts_map.get(concept_key, None)
|
|
if concept_found:
|
|
concept_found = Concept().update_from(concept_found) # make a copy when massively used in tests
|
|
if sya and len(concept_found.metadata.variables) > 0 and not is_bnf:
|
|
return SyaConceptParserHelper(concept_found, start, start + length - 1)
|
|
elif init_empty_body:
|
|
node = CNC(concept_found, start, start + length - 1, source=sub_expr, exclude_body=exclude_body)
|
|
init_body(node, concept_found, sub_expr)
|
|
return node
|
|
else:
|
|
return CN(concept_found, start, start + length - 1, source=sub_expr)
|
|
else:
|
|
# else an UnrecognizedTokensNode
|
|
return utnode(start, start + length - 1, sub_expr)
|
|
|
|
|
|
def init_body(item, concept, value):
|
|
if "body" in item.compiled:
|
|
item.compiled[ConceptParts.BODY] = item.compiled["body"]
|
|
del (item.compiled["body"])
|
|
return
|
|
|
|
if not concept or concept.metadata.body or ConceptParts.BODY in item.compiled:
|
|
return
|
|
|
|
item.compiled[ConceptParts.BODY] = DoNotResolve(value)
|
|
|
|
|
|
def compute_expected_array(concepts_map, expression, expected, sya=False, init_empty_body=False, exclude_body=False):
|
|
"""
|
|
Computes a simple but sufficient version of the result of infix_to_postfix()
|
|
:param concepts_map:
|
|
:param expression:
|
|
:param expected:
|
|
:param sya: if true, generate an SyaConceptParserHelper instead of a cnode
|
|
:param init_empty_body: if True adds the source in the body (actually in compiled.BODY)
|
|
:param exclude_body: do not include ConceptParts.BODY in comparison
|
|
:return:
|
|
"""
|
|
expression_as_tokens = [token.value for token in Tokenizer(expression) if token.type != TokenKind.EOF]
|
|
return [get_node(
|
|
concepts_map,
|
|
expression_as_tokens,
|
|
sub_expr,
|
|
sya=sya,
|
|
init_empty_body=init_empty_body,
|
|
exclude_body=exclude_body) for sub_expr in expected]
|