kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #127 : SyaNodeParser : Allow tokens parsing

Browse Source 
Fixed #128 : parser_utils.get_node() : Refactor
This commit is contained in:
Kodjo Sossouvi
2021-09-12 11:26:14 +02:00
parent 945807b375
commit a61a1c0d2b
12 changed files with 327 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parsers/parsers_utils.py
+205 -185
View File
@@ -2,7 +2,7 @@ import ast
from dataclasses import dataclass
from typing import List, Union
from core.builtin_concepts import ReturnValueConcept
from core.builtin_concepts import ParserResultConcept, ReturnValueConcept
from core.builtin_helpers import CreateObjectIdentifiers
from core.concept import AllConceptParts, Concept, ConceptParts, DoNotResolve
from core.rule import Rule
@@ -20,14 +20,6 @@ from sheerkarete.common import V
from sheerkarete.conditions import AndConditions, Condition, NegatedCondition, NegatedConjunctiveConditions
@dataclass
class Obj:
prop_a: object
prop_b: object = None
prop_c: object = None
parent: object = None
class ExprTestObj:
@staticmethod
def get_pos(nodes):
@@ -276,6 +268,130 @@ class LC(ExprTestObj): # for List Comprehension node
return ListComprehensionNode(start, end, full_text_as_tokens[start: end + 1], element, comprehensions)
class FN(ExprTestObj):
"""
Test class only
It matches with FunctionNode but with less constraints
Thereby,
FN("first", "last", ["param1," ...]) can be compared to
FunctionNode(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
Note that FunctionParameter can easily be defined with a single string
* "param" -> FunctionParameter(NameExprNode("param"), None)
* "param, " -> FunctionParameter(NameExprNode("param"), NameExprNode(", "))
For more complicated situations, you can use a tuple (value, sep) to define the value part and the separator part
"""
def __init__(self, first, last, parameters):
self.first = first
self.last = last
self.parameters = []
for param in parameters:
if isinstance(param, tuple):
self.parameters.append(param)
elif isinstance(param, str) and (pos := param.find(",")) != -1:
self.parameters.append((param[:pos], param[pos:]))
else:
self.parameters.append((param, None))
def __repr__(self):
res = self.first
for param in self.parameters:
if param[1]:
res += f"{param[0]}{param[1]} "
else:
res += f"{param[0]}"
return res + self.last
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, FN):
return self.first == other.first and self.last == other.last and self.parameters == other.parameters
return False
def __hash__(self):
return hash((self.first, self.last, self.parameters))
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, FN):
return other
if isinstance(other, FunctionNode):
params = []
for self_parameter, other_parameter in zip(self.parameters, other.parameters):
if isinstance(self_parameter[0], str):
value = other_parameter.value.value
else:
value = get_test_obj_delegate(other_parameter.value, self_parameter[0])
sep = other_parameter.separator.value if other_parameter.separator else None
params.append((value, sep))
return FN(other.first.value, other.last.value, params)
raise Exception(f"Expecting FunctionNode but received {other=}")
def get_expr_node(self, full_text_as_tokens=None):
start, end = self.get_pos_from_source(self.first, full_text_as_tokens)
first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
start, end = self.get_pos_from_source(self.last, full_text_as_tokens)
last = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
parameters = []
for param_value, sep in self.parameters:
if isinstance(param_value, str):
start, end = self.get_pos_from_source(param_value, full_text_as_tokens)
param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
param_as_expr_node = param_value.get_expr_node(full_text_as_tokens)
if sep:
sep_tokens = Tokenizer(sep, yield_eof=False)
start = param_as_expr_node.end + 1
end = start + len(list(sep_tokens)) - 1
sep_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
sep_as_expr_node = None
parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
start, end = first.start, last.end
return FunctionNode(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
class HelperWithPos:
def __init__(self, start=None, end=None):
self.start = start
self.end = end
self.start_is_fixed = start is not None
self.end_is_fixed = end is not None
def fix_pos(self, node):
"""
:param node: an object or a tuple
:return:
"""
if hasattr(node, "start"):
target_start, target_end = node.start, node.end
elif isinstance(node, tuple):
target_start, target_end = node
else:
target_start, target_end = None, None
if not self.start_is_fixed:
if target_start is not None and (self.start is None or target_start < self.start):
self.start = target_start
if not self.end_is_fixed:
if target_end is not None and (self.end is None or target_end > self.end):
self.end = target_end
return self
class CC:
"""
Concept class for test purpose
@@ -578,29 +694,60 @@ class CIO:
raise Exception(f"Expecting Concept but received {other=}")
class HelperWithPos:
def __init__(self, start=None, end=None):
self.start = start
self.end = end
class RETVAL:
"""
Class helper for return value for parser result
"""
self.start_is_fixed = start is not None
self.end_is_fixed = end is not None
def __init__(self, source, who=None, parser=None):
self.source = source
self.who = who
self.parser = parser
def fix_pos(self, node):
if not self.start_is_fixed:
start = node.start if hasattr(node, "start") else \
node[0] if isinstance(node, tuple) else None
def __eq__(self, other):
if id(self) == id(other):
return True
if start is not None and (self.start is None or start < self.start):
self.start = start
if not isinstance(other, RETVAL):
return False
if not self.end_is_fixed:
end = node.end if hasattr(node, "end") else \
node[1] if isinstance(node, tuple) else None
return (self.source == other.source and
self.who == other.who and
self.parser == other.parser)
if end is not None and (self.end is None or end > self.end):
self.end = end
return self
def __hash__(self):
return hash((self.source, self.who))
def __repr__(self):
txt = f"RV(source='{self.source}'"
if self.who is not None:
txt += f", who={self.who}"
if self.parser is not None:
txt += f", parser={self.parser}"
return txt + ")"
def transform_real_obj(self, other, get_test_obj_delegate):
"""
Transform other into CNC, to ease the comparison
:param other:
:param get_test_obj_delegate:
:return:
"""
if isinstance(other, RETVAL):
return other
if isinstance(other, ReturnValueConcept):
if not isinstance(other.body, ParserResultConcept):
raise Exception(f"ParserResultConcept not found body={other.body}")
parser_result = other.body
return RETVAL(parser_result.source,
other.who if self.who is not None else None,
parser_result.parser if self.parser is not None else None)
raise Exception(f"Expecting ReturnValueConcept but received {other=}")
class SCN(HelperWithPos):
@@ -992,10 +1139,10 @@ class RN(HelperWithPos):
if not isinstance(other, RN):
return False
return self.rule_id == other.rule_id and \
self.start == other.start and \
self.end == other.end and \
self.source == other.source
return (self.rule_id == other.rule_id and
self.start == other.start and
self.end == other.end and
self.source == other.source)
def __hash__(self):
return hash((self.rule_id, self.start, self.end, self.source))
@@ -1032,99 +1179,6 @@ class RN(HelperWithPos):
raise Exception(f"Expecting RuleNode but received {other=}")
class FN(ExprTestObj):
"""
Test class only
It matches with FunctionNode but with less constraints
Thereby,
FN("first", "last", ["param1," ...]) can be compared to
FunctionNode(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
Note that FunctionParameter can easily be defined with a single string
* "param" -> FunctionParameter(NameExprNode("param"), None)
* "param, " -> FunctionParameter(NameExprNode("param"), NameExprNode(", "))
For more complicated situations, you can use a tuple (value, sep) to define the value part and the separator part
"""
def __init__(self, first, last, parameters):
self.first = first
self.last = last
self.parameters = []
for param in parameters:
if isinstance(param, tuple):
self.parameters.append(param)
elif isinstance(param, str) and (pos := param.find(",")) != -1:
self.parameters.append((param[:pos], param[pos:]))
else:
self.parameters.append((param, None))
def __repr__(self):
res = self.first
for param in self.parameters:
if param[1]:
res += f"{param[0]}{param[1]} "
else:
res += f"{param[0]}"
return res + self.last
def __eq__(self, other):
if id(self) == id(other):
return True
if isinstance(other, FN):
return self.first == other.first and self.last == other.last and self.parameters == other.parameters
return False
def __hash__(self):
return hash((self.first, self.last, self.parameters))
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, FN):
return other
if isinstance(other, FunctionNode):
params = []
for self_parameter, other_parameter in zip(self.parameters, other.parameters):
if isinstance(self_parameter[0], str):
value = other_parameter.value.value
else:
value = get_test_obj_delegate(other_parameter.value, self_parameter[0])
sep = other_parameter.separator.value if other_parameter.separator else None
params.append((value, sep))
return FN(other.first.value, other.last.value, params)
raise Exception(f"Expecting FunctionNode but received {other=}")
def get_expr_node(self, full_text_as_tokens=None):
start, end = self.get_pos_from_source(self.first, full_text_as_tokens)
first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
start, end = self.get_pos_from_source(self.last, full_text_as_tokens)
last = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
parameters = []
for param_value, sep in self.parameters:
if isinstance(param_value, str):
start, end = self.get_pos_from_source(param_value, full_text_as_tokens)
param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
param_as_expr_node = param_value.get_expr_node(full_text_as_tokens)
if sep:
sep_tokens = Tokenizer(sep, yield_eof=False)
start = param_as_expr_node.end + 1
end = start + len(list(sep_tokens)) - 1
sep_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
sep_as_expr_node = None
parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
start, end = first.start, last.end
return FunctionNode(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
@dataclass()
class NEGCOND:
"""
@@ -1207,8 +1261,6 @@ def get_node(
sub_expr,
concept_key=None,
skip=0,
is_bnf=False,
sya=False,
init_empty_body=False,
exclude_body=False):
"""
@@ -1219,41 +1271,41 @@ def get_node(
:param concepts_map: hash of the known concepts
:param concept_key: key of the concept if different from sub_expr
:param skip: number of occurrences of sub_expr to skip
:param is_bnf: True if the concept to search is a bnf definition
:param sya: Return SyaConceptParserHelper instead of a ConceptNode when needed
:param init_empty_body: if True adds the source in the body (actually in compiled.BODY)
:param exclude_body: Ask to not compare body
:return:
"""
if sub_expr == "')'":
return ")"
if isinstance(sub_expr, list):
return [get_node(concepts_map,
expression_as_tokens,
s,
concept_key,
skip,
init_empty_body,
exclude_body) for s in sub_expr]
if isinstance(sub_expr, ReturnValueConcept):
return sub_expr
if isinstance(sub_expr, tuple):
return get_node(concepts_map,
expression_as_tokens,
sub_expr[0],
concept_key,
sub_expr[1],
init_empty_body,
exclude_body)
if isinstance(sub_expr, DoNotResolve):
return sub_expr
if isinstance(sub_expr, CIO):
sub_expr.set_concept(concepts_map[sub_expr.concept_name])
source = sub_expr.source or sub_expr.concept_name
if source:
node = get_node(concepts_map, expression_as_tokens, source, sya=sya)
sub_expr.start = node.start
sub_expr.end = node.end
if isinstance(sub_expr, (DoNotResolve, ReturnValueConcept, RETVAL)):
return sub_expr
if isinstance(sub_expr, SCWC):
sub_expr.first = get_node(concepts_map, expression_as_tokens, sub_expr.first, sya=sya)
sub_expr.last = get_node(concepts_map, expression_as_tokens, sub_expr.last, sya=sya)
sub_expr.content = [get_node(concepts_map, expression_as_tokens, c, sya=sya) for c in sub_expr.content]
sub_expr.first = get_node(concepts_map, expression_as_tokens, sub_expr.first, skip=skip)
sub_expr.last = get_node(concepts_map, expression_as_tokens, sub_expr.last, skip=skip)
sub_expr.content = [get_node(concepts_map, expression_as_tokens, c, skip=skip) for c in sub_expr.content]
sub_expr.fix_pos(sub_expr.first)
sub_expr.fix_pos(sub_expr.last)
return sub_expr
# return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
if isinstance(sub_expr, SCN):
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya)
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, skip=skip)
sub_expr.fix_pos(node)
return sub_expr
@@ -1263,13 +1315,14 @@ def get_node(
sub_expr.end = start + length - 1
return sub_expr
if isinstance(sub_expr, (CNC, CC, CN, CMV)):
if isinstance(sub_expr, (CNC, CC, CN, CMV, CIO)):
if sub_expr.concept is None or sub_expr.start is None or sub_expr.end is None:
concept_node = get_node(
concepts_map,
expression_as_tokens,
sub_expr.source or sub_expr.concept_key,
sub_expr.concept_key, sya=sya)
sub_expr.concept_key,
skip)
if not hasattr(concept_node, "concept"):
raise Exception(f"'{sub_expr.concept_key}' is not a concept. Check your map.")
concept_found = concept_node.concept
@@ -1279,7 +1332,7 @@ def get_node(
(concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start))
if hasattr(sub_expr, "compiled"):
for k, v in sub_expr.compiled.items():
node = get_node(concepts_map, expression_as_tokens, v, sya=sya,
node = get_node(concepts_map, expression_as_tokens, v,
exclude_body=exclude_body) # need to get start and end positions
if isinstance(v, str) and v in concepts_map:
new_value_concept = concepts_map[v]
@@ -1299,27 +1352,17 @@ def get_node(
return sub_expr
if isinstance(sub_expr, UTN):
node = get_node(concepts_map, expression_as_tokens, sub_expr.source)
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, skip=skip)
sub_expr.fix_pos(node)
return sub_expr
if isinstance(sub_expr, tuple):
return get_node(concepts_map, expression_as_tokens, sub_expr[0],
concept_key=concept_key, skip=sub_expr[1], is_bnf=is_bnf, sya=sya)
start, length = _index(expression_as_tokens, sub_expr, skip)
# special case of python source code
if "+" in sub_expr and sub_expr.strip() != "+":
return SCN(sub_expr, start, start + length - 1)
# try to match one of the concept from the map
concept_key = concept_key or sub_expr
concept_found = concepts_map.get(concept_key, None)
if concept_found:
concept_found = Concept().update_from(concept_found) # make a copy when massively used in tests
# if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf:
# return SyaConceptParserHelper(concept_found, start, start + length - 1)
if init_empty_body:
node = CNC(concept_found, sub_expr, start, start + length - 1, exclude_body=exclude_body)
init_body(node, concept_found, sub_expr)
@@ -1343,13 +1386,12 @@ def init_body(item, concept, value):
item.compiled[ConceptParts.BODY] = DoNotResolve(value)
def compute_expected_array(concepts_map, expression, expected, sya=False, init_empty_body=False, exclude_body=False):
def compute_expected_array(concepts_map, expression, expected, init_empty_body=False, exclude_body=False):
"""
Computes a simple but sufficient version of the result of infix_to_postfix()
:param concepts_map:
:param expression:
:param expected:
:param sya: if true, generate an SyaConceptParserHelper instead of a cnode
:param init_empty_body: if True adds the source in the body (actually in compiled.BODY)
:param exclude_body: do not include ConceptParts.BODY in comparison
:return:
@@ -1359,7 +1401,6 @@ def compute_expected_array(concepts_map, expression, expected, sya=False, init_e
concepts_map,
expression_as_tokens,
sub_expr,
sya=sya,
init_empty_body=init_empty_body,
exclude_body=exclude_body) for sub_expr in expected]
@@ -1398,27 +1439,6 @@ def get_source_code_node(start, text, concepts_map, id_manager=None):
return SourceCodeNode(start, start + len(tokens) - 1, tokens, text, python_node)
def resolve_test_concept(concept_map, hint):
if isinstance(hint, str):
return concept_map[hint]
if isinstance(hint, CC):
concept = concept_map[hint.concept_key]
compiled = {k: resolve_test_concept(concept_map, v) for k, v in hint.compiled.items()}
return CC(concept, source=hint.source, exclude_body=hint.exclude_body, **compiled)
if isinstance(hint, CMV):
concept = concept_map[hint.concept_key]
return CMV(concept, **hint.variables)
# CV
#
# CMV
#
# CIO
raise NotImplementedError()
def get_rete_conditions(*conditions):
"""
Transform a list of string into a list of Condition (Rete conditions)