from core.concept import CC, Concept, ConceptParts, DoNotResolve, CIO from core.tokenizer import Tokenizer, TokenKind, Token from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN, \ SCN, RN from parsers.SyaNodeParser import SyaConceptParserHelper def _index(tokens, expr, index): """ Finds a sub list in a bigger list :param tokens: :param expr: :param index: :return: """ expected = [token.str_value for token in Tokenizer(expr) if token.type != TokenKind.EOF] for i in range(0, len(tokens) - len(expected) + 1): for j in range(len(expected)): if tokens[i + j] != expected[j]: break else: if index == 0: return i, len(expected) else: index -= 1 raise ValueError(f"substring '{expr}' not found") def compute_debug_array(res): to_compare = [] for r in res: res_debug = [] for token in r.debug: if isinstance(token, Token): if token.type == TokenKind.WHITESPACE: continue else: res_debug.append("T(" + token.value + ")") else: res_debug.append("C(" + token.concept.name + ")") to_compare.append(res_debug) return to_compare def get_node( concepts_map, expression_as_tokens, sub_expr, concept_key=None, skip=0, is_bnf=False, sya=False, init_empty_body=False, exclude_body=False): """ Tries to find sub in expression When found, transform it to its correct type :param expression_as_tokens: full expression :param sub_expr: sub expression to search in the full expression :param concepts_map: hash of the known concepts :param concept_key: key of the concept if different from sub_expr :param skip: number of occurrences of sub_expr to skip :param is_bnf: True if the concept to search is a bnf definition :param sya: Return SyaConceptParserHelper instead of a ConceptNode when needed :param init_empty_body: if True adds the source in the body (actually in compiled.BODY) :param exclude_body: Ask to not compare body :return: """ if sub_expr == "')'": return ")" if isinstance(sub_expr, (scnode, utnode, DoNotResolve)): return sub_expr if isinstance(sub_expr, CIO): sub_expr.set_concept(concepts_map[sub_expr.concept_name]) if sub_expr.source: node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya) sub_expr.start = node.start sub_expr.end = node.end return sub_expr if isinstance(sub_expr, cnode): # for cnode, map the concept key to the one from concepts_maps if needed if sub_expr.concept_key.startswith("#"): return cnode( concepts_map[sub_expr.concept_key[1:]].key, sub_expr.start, sub_expr.end, sub_expr.source ) else: return sub_expr if isinstance(sub_expr, SCWC): sub_expr.first = get_node(concepts_map, expression_as_tokens, sub_expr.first, sya=sya) sub_expr.last = get_node(concepts_map, expression_as_tokens, sub_expr.last, sya=sya) sub_expr.content = [get_node(concepts_map, expression_as_tokens, c, sya=sya) for c in sub_expr.content] sub_expr.fix_pos(sub_expr.first) sub_expr.fix_pos(sub_expr.last) return sub_expr #return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source() if isinstance(sub_expr, SCN): node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya) sub_expr.fix_pos(node) return sub_expr if isinstance(sub_expr, RN): start, length = _index(expression_as_tokens, sub_expr.source, skip) sub_expr.start = start sub_expr.end = start + length - 1 return sub_expr if isinstance(sub_expr, (CNC, CC, CN)): concept_node = get_node( concepts_map, expression_as_tokens, sub_expr.source or sub_expr.concept_key, sub_expr.concept_key, sya=sya) if not hasattr(concept_node, "concept"): raise Exception(f"'{sub_expr.concept_key}' is not a concept. Check your map.") concept_found = concept_node.concept sub_expr.concept_key = concept_found.key sub_expr.concept = concept_found sub_expr.fix_pos((concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start)) if hasattr(sub_expr, "compiled"): for k, v in sub_expr.compiled.items(): node = get_node(concepts_map, expression_as_tokens, v, sya=sya, exclude_body=exclude_body) # need to get start and end positions if isinstance(v, str) and v in concepts_map: new_value_concept = concepts_map[v] new_value = CC(Concept().update_from(new_value_concept), exclude_body=exclude_body) if init_empty_body: init_body(new_value, concept_found, v) else: new_value = node sub_expr.compiled[k] = new_value sub_expr.fix_pos(node) if init_empty_body: init_body(sub_expr, concept_found, sub_expr.source) if hasattr(sub_expr, "fix_source"): sub_expr.fix_source(expression_as_tokens[sub_expr.start: sub_expr.end + 1]) return sub_expr if isinstance(sub_expr, UTN): node = get_node(concepts_map, expression_as_tokens, sub_expr.source) sub_expr.fix_pos(node) return sub_expr if isinstance(sub_expr, short_cnode): return get_node(concepts_map, expression_as_tokens, sub_expr.source, concept_key=sub_expr.concept_key, skip=skip, is_bnf=True, sya=sya) if isinstance(sub_expr, tuple): return get_node(concepts_map, expression_as_tokens, sub_expr[0], concept_key=concept_key, skip=sub_expr[1], is_bnf=is_bnf, sya=sya) start, length = _index(expression_as_tokens, sub_expr, skip) # special case of python source code if "+" in sub_expr and sub_expr.strip() != "+": return SCN(sub_expr, start, start + length - 1) # try to match one of the concept from the map concept_key = concept_key or sub_expr concept_found = concepts_map.get(concept_key, None) if concept_found: concept_found = Concept().update_from(concept_found) # make a copy when massively used in tests if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf: return SyaConceptParserHelper(concept_found, start, start + length - 1) elif init_empty_body: node = CNC(concept_found, start, start + length - 1, source=sub_expr, exclude_body=exclude_body) init_body(node, concept_found, sub_expr) return node else: return CN(concept_found, start, start + length - 1, source=sub_expr) else: # else an UnrecognizedTokensNode return utnode(start, start + length - 1, sub_expr) def init_body(item, concept, value): if "body" in item.compiled: item.compiled[ConceptParts.BODY] = item.compiled["body"] del (item.compiled["body"]) return if not concept or concept.get_metadata().body or ConceptParts.BODY in item.compiled: return item.compiled[ConceptParts.BODY] = DoNotResolve(value) def compute_expected_array(concepts_map, expression, expected, sya=False, init_empty_body=False, exclude_body=False): """ Computes a simple but sufficient version of the result of infix_to_postfix() :param concepts_map: :param expression: :param expected: :param sya: if true, generate an SyaConceptParserHelper instead of a cnode :param init_empty_body: if True adds the source in the body (actually in compiled.BODY) :param exclude_body: do not include ConceptParts.BODY in comparison :return: """ expression_as_tokens = [token.str_value for token in Tokenizer(expression) if token.type != TokenKind.EOF] return [get_node( concepts_map, expression_as_tokens, sub_expr, sya=sya, init_empty_body=init_empty_body, exclude_body=exclude_body) for sub_expr in expected]