kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #131 : Implement ExprToConditions

Browse Source 
Fixed #130 : ArithmeticOperatorParser
Fixed #129 : python_wrapper : create_namespace
Fixed #128 : ExpressionParser: Cannot parse func(x) infixed concept 'xxx'
This commit is contained in:
Kodjo Sossouvi
2021-10-13 16:06:57 +02:00
parent a61a1c0d2b
commit 89e1f20975
76 changed files with 5867 additions and 3206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parsers/parsers_utils.py
+236 -60
View File
@@ -8,12 +8,14 @@ from core.concept import AllConceptParts, Concept, ConceptParts, DoNotResolve
from core.rule import Rule
from core.tokenizer import Token, TokenKind, Tokenizer
from core.utils import get_text_from_tokens, str_concept, tokens_index
from parsers.BaseExpressionParser import AndNode, ComparisonNode, ComparisonType, Comprehension, FunctionParameter, \
from parsers.BaseExpressionParser import AndNode, BinaryNode, ComparisonNode, ComparisonType, Comprehension, \
FunctionNode, \
FunctionParameter, \
ListComprehensionNode, ListNode, NameExprNode, \
NotNode, OrNode, VariableNode, comma
NotNode, OrNode, SequenceNode, VariableNode, t_comma
from parsers.BaseNodeParser import ConceptNode, RuleNode, SourceCodeNode, SourceCodeWithConceptNode, \
UnrecognizedTokensNode
from parsers.FunctionParser import FunctionNode
from parsers.FunctionParserOld import FunctionNodeOld
from parsers.PythonParser import PythonNode
from sheerkapython.python_wrapper import sheerka_globals
from sheerkarete.common import V
@@ -52,16 +54,29 @@ class ExprTestObj:
return list(Tokenizer(source, yield_eof=False)), to_skip
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
raise NotImplementedError()
@staticmethod
def safe_get_expr_node(obj, full_text_as_tokens):
def safe_get_expr_node(obj, full_text_as_tokens, default_expr_obj):
if obj is None:
return None
obj = EXPR(obj) if isinstance(obj, (str, tuple)) else obj
return obj.get_expr_node(full_text_as_tokens)
obj = default_expr_obj(obj) if isinstance(obj, (str, tuple)) else obj
return obj.get_expr_node(full_text_as_tokens, default_expr_obj)
class SEQ(ExprTestObj):
""" Test class for SequenceNode"""
def __init__(self, *parts, source=None):
self.parts = parts
self.source = source
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
parts = [self.safe_get_expr_node(part, full_text_as_tokens, default_expr_obj) for part in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
return SequenceNode(start, end, full_text_as_tokens[start: end + 1], *parts)
class AND(ExprTestObj):
@@ -71,8 +86,8 @@ class AND(ExprTestObj):
self.parts = parts
self.source = source
def get_expr_node(self, full_text_as_tokens=None):
parts = [part.get_expr_node(full_text_as_tokens) for part in self.parts]
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
parts = [self.safe_get_expr_node(part, full_text_as_tokens, default_expr_obj) for part in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
return AndNode(start, end, full_text_as_tokens[start: end + 1], *parts)
@@ -84,8 +99,8 @@ class OR(ExprTestObj):
self.parts = parts
self.source = source
def get_expr_node(self, full_text_as_tokens=None):
parts = [part.get_expr_node(full_text_as_tokens) for part in self.parts]
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
parts = [self.safe_get_expr_node(part, full_text_as_tokens, default_expr_obj) for part in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
return OrNode(start, end, full_text_as_tokens[start: end + 1], *parts)
@@ -96,8 +111,8 @@ class NOT(ExprTestObj):
expr: ExprTestObj
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
part = self.expr.get_expr_node(full_text_as_tokens)
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
part = self.safe_get_expr_node(self.expr, full_text_as_tokens, default_expr_obj)
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else (
part.start - 2, part.end)
return NotNode(start, end, full_text_as_tokens[start: end + 1], part)
@@ -108,7 +123,7 @@ class EXPR(ExprTestObj):
"""Test class for NameNode"""
source: str
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
value_as_tokens, to_skip = self.as_tokens(self.source)
start = tokens_index(full_text_as_tokens, value_as_tokens, to_skip)
end = start + len(value_as_tokens) - 1
@@ -122,7 +137,7 @@ class VAR(ExprTestObj):
full_name: str
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
value_as_tokens = list(Tokenizer(self.source or self.full_name, yield_eof=False))
start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
end = start + len(value_as_tokens) - 1
@@ -133,19 +148,77 @@ class VAR(ExprTestObj):
return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0], *parts[1:])
class BinaryExprTestObj(ExprTestObj):
def __init__(self, *parts: Union[ExprTestObj, str], source=None):
self.parts = parts
self.source = source
@staticmethod
def get_op_from_name(name):
if name == "ADD":
return Token(TokenKind.PLUS, "+", -1, -1, -1)
elif name == "SUB":
return Token(TokenKind.MINUS, "-", -1, -1, -1)
elif name == "MULT":
return Token(TokenKind.STAR, "*", -1, -1, -1)
elif name == "DIV":
return Token(TokenKind.SLASH, "/", -1, -1, -1)
elif name == "POW":
return Token(TokenKind.STARSTAR, "**", -1, -1, -1)
elif name == "FDIV":
return Token(TokenKind.SLASHSLASH, "//", -1, -1, -1)
elif name == "MOD":
return Token(TokenKind.PERCENT, "%", -1, -1, -1)
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
op = self.get_op_from_name(type(self).__name__)
parts_as_node = [self.safe_get_expr_node(p, full_text_as_tokens, default_expr_obj) for p in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else \
self.get_pos(parts_as_node)
return BinaryNode(start, end, full_text_as_tokens[start: end + 1], op, *parts_as_node)
class ADD(BinaryExprTestObj):
pass
class SUB(BinaryExprTestObj):
pass
class MULT(BinaryExprTestObj):
pass
class DIV(BinaryExprTestObj):
pass
class FDIV(BinaryExprTestObj):
pass
class MOD(BinaryExprTestObj):
pass
class POW(BinaryExprTestObj):
pass
@dataclass
class CompExprTestObj(ExprTestObj):
"""
Test object for comparison ==, <=, ...
"""
left: ExprTestObj
right: ExprTestObj
left: Union[ExprTestObj, str]
right: Union[ExprTestObj, str]
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
node_type = comparison_type_mapping[type(self).__name__]
left_node = self.left.get_expr_node(full_text_as_tokens)
right_node = self.right.get_expr_node(full_text_as_tokens)
left_node = self.safe_get_expr_node(self.left, full_text_as_tokens, default_expr_obj)
right_node = self.safe_get_expr_node(self.right, full_text_as_tokens, default_expr_obj)
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else \
self.get_pos([left_node, right_node])
return ComparisonNode(start, end, full_text_as_tokens[start: end + 1], node_type, left_node, right_node)
@@ -202,20 +275,20 @@ class L_EXPR(ExprTestObj):
self.first = first
self.last = last
self.items = items
self.sep = sep or comma
self.sep = sep or t_comma
self.source = source
def get_expr_node(self, full_text_as_tokens=None):
first = self.safe_get_expr_node(self.first, full_text_as_tokens)
last = self.safe_get_expr_node(self.last, full_text_as_tokens)
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
first = self.safe_get_expr_node(self.first, full_text_as_tokens, EXPR)
last = self.safe_get_expr_node(self.last, full_text_as_tokens, EXPR)
items = [self.safe_get_expr_node(item, full_text_as_tokens) for item in self.items]
items = [self.safe_get_expr_node(item, full_text_as_tokens, default_expr_obj) for item in self.items]
if self.source is None:
source = self.first if self.first else ""
source = first.get_source() if self.first else ""
source += f"{self.sep.value} ".join(item.get_source() for item in items)
if self.last:
source += self.last
if last:
source += last.get_source()
else:
source = self.source
@@ -223,6 +296,52 @@ class L_EXPR(ExprTestObj):
return ListNode(start, end, full_text_as_tokens[start: end + 1], first, last, items, self.sep)
class FN(ExprTestObj):
"""
Test class only
It matches with FunctionNode
"""
def __init__(self, name, *parameters, source=None):
self.name = name
self.parameters = parameters
self.source = source
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
start, end = -1, -1
if self.parameters and isinstance(self.parameters[0], L_EXPR):
params_as_test_obj = self.parameters[0]
else:
nb_left_parenthesis = 0
nb_right_parenthesis = 0
if self.source:
start, end = self.get_pos_from_source(self.source, full_text_as_tokens)
for i in range(start):
if full_text_as_tokens[i].type == TokenKind.LPAR:
nb_left_parenthesis += 1
for i in range(end):
if full_text_as_tokens[i].type == TokenKind.RPAR:
nb_right_parenthesis += 1
else:
for p in self.parameters:
if isinstance(p, str):
nb_right_parenthesis += p.count(")")
first = ("(", nb_left_parenthesis) if nb_left_parenthesis > 0 else "("
last = (")", nb_right_parenthesis) if nb_right_parenthesis > 0 else ")"
params_as_test_obj = L_EXPR(first, last, *self.parameters)
name_as_expr = self.safe_get_expr_node(self.name, full_text_as_tokens, default_expr_obj)
params_as_expr = self.safe_get_expr_node(params_as_test_obj, full_text_as_tokens, default_expr_obj)
if start < 1:
start, end = self.get_pos([name_as_expr, params_as_expr.last])
return FunctionNode(start, end, full_text_as_tokens[start:end + 1], name_as_expr, params_as_expr)
@dataclass
class LCC:
"""
@@ -239,7 +358,7 @@ class LC(ExprTestObj): # for List Comprehension node
generators: list
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
# first transform str into NameExprTestObj (ie EXPR)
if isinstance(self.element, str):
self.element = EXPR(self.element)
@@ -254,13 +373,13 @@ class LC(ExprTestObj): # for List Comprehension node
self.generators = comprehensions
# then transform into ListComprehensionNode
element = self.element.get_expr_node(full_text_as_tokens)
element = self.element.get_expr_node(full_text_as_tokens, default_expr_obj)
nodes.append(element)
comprehensions = []
for comp in self.generators:
target = comp.target.get_expr_node(full_text_as_tokens)
iterable = comp.iterable.get_expr_node(full_text_as_tokens)
if_expr = comp.if_expr.get_expr_node(full_text_as_tokens) if comp.if_expr else None
target = comp.target.get_expr_node(full_text_as_tokens, default_expr_obj)
iterable = comp.iterable.get_expr_node(full_text_as_tokens, default_expr_obj)
if_expr = comp.if_expr.get_expr_node(full_text_as_tokens, default_expr_obj) if comp.if_expr else None
comprehensions.append(Comprehension(target, iterable, if_expr))
nodes.extend([target, iterable, if_expr])
@@ -268,14 +387,14 @@ class LC(ExprTestObj): # for List Comprehension node
return ListComprehensionNode(start, end, full_text_as_tokens[start: end + 1], element, comprehensions)
class FN(ExprTestObj):
class FNOld(ExprTestObj):
"""
Test class only
It matches with FunctionNode but with less constraints
It matches with FunctionNodeOld but with less constraints
Thereby,
FN("first", "last", ["param1," ...]) can be compared to
FunctionNode(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
FNOld("first", "last", ["param1," ...]) can be compared to
FunctionNodeOld(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
Note that FunctionParameter can easily be defined with a single string
* "param" -> FunctionParameter(NameExprNode("param"), None)
@@ -308,7 +427,7 @@ class FN(ExprTestObj):
if id(self) == id(other):
return True
if isinstance(other, FN):
if isinstance(other, FNOld):
return self.first == other.first and self.last == other.last and self.parameters == other.parameters
return False
@@ -317,10 +436,10 @@ class FN(ExprTestObj):
return hash((self.first, self.last, self.parameters))
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, FN):
if isinstance(other, FNOld):
return other
if isinstance(other, FunctionNode):
if isinstance(other, FunctionNodeOld):
params = []
for self_parameter, other_parameter in zip(self.parameters, other.parameters):
if isinstance(self_parameter[0], str):
@@ -330,11 +449,11 @@ class FN(ExprTestObj):
sep = other_parameter.separator.value if other_parameter.separator else None
params.append((value, sep))
return FN(other.first.value, other.last.value, params)
return FNOld(other.first.value, other.last.value, params)
raise Exception(f"Expecting FunctionNode but received {other=}")
raise Exception(f"Expecting FunctionNodeOld but received {other=}")
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
start, end = self.get_pos_from_source(self.first, full_text_as_tokens)
first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
start, end = self.get_pos_from_source(self.last, full_text_as_tokens)
@@ -345,7 +464,7 @@ class FN(ExprTestObj):
start, end = self.get_pos_from_source(param_value, full_text_as_tokens)
param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
param_as_expr_node = param_value.get_expr_node(full_text_as_tokens)
param_as_expr_node = param_value.get_expr_node(full_text_as_tokens, default_expr_obj)
if sep:
sep_tokens = Tokenizer(sep, yield_eof=False)
@@ -358,7 +477,7 @@ class FN(ExprTestObj):
parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
start, end = first.start, last.end
return FunctionNode(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
return FunctionNodeOld(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
class HelperWithPos:
@@ -697,12 +816,16 @@ class CIO:
class RETVAL:
"""
Class helper for return value for parser result
Note that it is designed for ParserResultConcept only
Please create another RETVAL or rename this one if extra functionalities are needed
"""
def __init__(self, source, who=None, parser=None):
def __init__(self, text, source=None, who=None, parser=None, objects=None):
self.text = text
self.source = source
self.who = who
self.parser = parser
self.objects = objects
def __eq__(self, other):
if id(self) == id(other):
@@ -711,15 +834,19 @@ class RETVAL:
if not isinstance(other, RETVAL):
return False
return (self.source == other.source and
return (self.text == other.text and
self.source == other.source and
self.who == other.who and
self.parser == other.parser)
self.parser == other.parser and
self.objects == other.objects)
def __hash__(self):
return hash((self.source, self.who))
return hash((self.text, self.source, self.who))
def __repr__(self):
txt = f"RV(source='{self.source}'"
txt = f"RV(text='{self.text}'"
if self.source is not None:
txt += f", source={self.source}"
if self.who is not None:
txt += f", who={self.who}"
if self.parser is not None:
@@ -738,14 +865,33 @@ class RETVAL:
return other
if isinstance(other, ReturnValueConcept):
if not isinstance(other.body, ParserResultConcept):
raise Exception(f"ParserResultConcept not found body={other.body}")
parser_result = other.body
if not isinstance(parser_result.body, PythonNode):
raise Exception(f"PythonNode not found parser_result.body={parser_result.body}")
python_node = parser_result.body
other_objects = None
if self.objects:
other_objects = {}
if len(self.objects) != len(python_node.objects):
raise Exception(f"objects have different size {self.objects=}, other.objects={python_node.objects}")
try:
for k, v in self.objects.items():
other_objects[k] = get_test_obj_delegate(python_node.objects[k], v, get_test_obj_delegate)
except KeyError as err:
raise Exception(f"object {err} is expected but not found")
return RETVAL(parser_result.source,
python_node.source if self.source is not None else None,
other.who if self.who is not None else None,
parser_result.parser if self.parser is not None else None)
parser_result.parser if self.parser is not None else None,
other_objects)
raise Exception(f"Expecting ReturnValueConcept but received {other=}")
@@ -758,9 +904,10 @@ class SCN(HelperWithPos):
SCN == SourceCodeNode if source, start, end (start and end are not validated when None)
"""
def __init__(self, source, start=None, end=None):
def __init__(self, source, objects=None, start=None, end=None):
super().__init__(start, end)
self.source = source
self.objects = objects
def __eq__(self, other):
if id(self) == id(other):
@@ -779,9 +926,10 @@ class SCN(HelperWithPos):
if not isinstance(other, SCN):
return False
return self.source == other.source and \
self.start == other.start and \
self.end == other.end
return (self.source == other.source and
self.objects == other.objects and
self.start == other.start and
self.end == other.end)
def __hash__(self):
return hash((self.source, self.start, self.end))
@@ -806,7 +954,18 @@ class SCN(HelperWithPos):
return other
if isinstance(other, SourceCodeNode):
other_objects = None
if self.objects:
other_objects = []
if not other.python_node:
Exception(f"no python node found in SourceCodeNode {other=}")
if len(self.objects) != len(other.python_node.objects):
Exception(f"objects have different size {self.objects=}, other.objects={other.python_node.objects}")
for self_obj, other_obj in zip(self.objects, other.python_node.objects.values()):
other_objects.append(to_compare_delegate(other_obj, self_obj, to_compare_delegate))
return SCN(other.source,
other_objects,
other.start if self.start is not None else None,
other.end if self.end is not None else None)
@@ -1208,12 +1367,12 @@ comparison_type_mapping = {
}
def get_expr_node_from_test_node(full_text, test_node):
def get_expr_node_from_test_node(full_text, test_node, default_expr_obj=EXPR):
"""
Returns EXPR, OR, NOT, AND object to ease the comparison with the real ExprNode
"""
full_text_as_tokens = list(Tokenizer(full_text, yield_eof=False))
return test_node.get_expr_node(full_text_as_tokens)
return test_node.get_expr_node(full_text_as_tokens, default_expr_obj)
def _index(tokens, expr, index):
@@ -1293,7 +1452,13 @@ def get_node(
init_empty_body,
exclude_body)
if isinstance(sub_expr, (DoNotResolve, ReturnValueConcept, RETVAL)):
if isinstance(sub_expr, (DoNotResolve, ReturnValueConcept)):
return sub_expr
if isinstance(sub_expr, RETVAL):
if sub_expr.objects:
sub_expr.objects = {k: get_node(concepts_map, expression_as_tokens, v, skip=skip)
for k, v in sub_expr.objects.items()}
return sub_expr
if isinstance(sub_expr, SCWC):
@@ -1307,6 +1472,8 @@ def get_node(
if isinstance(sub_expr, SCN):
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, skip=skip)
sub_expr.fix_pos(node)
if sub_expr.objects:
sub_expr.objects = [get_node(concepts_map, expression_as_tokens, c, skip=skip) for c in sub_expr.objects]
return sub_expr
if isinstance(sub_expr, RN):
@@ -1315,7 +1482,16 @@ def get_node(
sub_expr.end = start + length - 1
return sub_expr
if isinstance(sub_expr, (CNC, CC, CN, CMV, CIO)):
if isinstance(sub_expr, CIO):
sub_expr.set_concept(concepts_map[sub_expr.concept_name])
source = sub_expr.source or sub_expr.concept_name
if source:
node = get_node(concepts_map, expression_as_tokens, source)
sub_expr.start = node.start
sub_expr.end = node.end
return sub_expr
if isinstance(sub_expr, (CNC, CC, CN, CMV)):
if sub_expr.concept is None or sub_expr.start is None or sub_expr.end is None:
concept_node = get_node(
concepts_map,