kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #131 : Implement ExprToConditions

Browse Source 
Fixed #130 : ArithmeticOperatorParser
Fixed #129 : python_wrapper : create_namespace
Fixed #128 : ExpressionParser: Cannot parse func(x) infixed concept 'xxx'
This commit is contained in:
Kodjo Sossouvi
2021-10-13 16:06:57 +02:00
parent a61a1c0d2b
commit 89e1f20975
76 changed files with 5867 additions and 3206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parsers/parsers_utils.py
+236 -60
View File
@@ -8,12 +8,14 @@ from core.concept import AllConceptParts, Concept, ConceptParts, DoNotResolve
from core.rule import Rule
from core.tokenizer import Token, TokenKind, Tokenizer
from core.utils import get_text_from_tokens, str_concept, tokens_index
from parsers.BaseExpressionParser import AndNode, ComparisonNode, ComparisonType, Comprehension, FunctionParameter, \
from parsers.BaseExpressionParser import AndNode, BinaryNode, ComparisonNode, ComparisonType, Comprehension, \
FunctionNode, \
FunctionParameter, \
ListComprehensionNode, ListNode, NameExprNode, \
NotNode, OrNode, VariableNode, comma
NotNode, OrNode, SequenceNode, VariableNode, t_comma
from parsers.BaseNodeParser import ConceptNode, RuleNode, SourceCodeNode, SourceCodeWithConceptNode, \
UnrecognizedTokensNode
from parsers.FunctionParser import FunctionNode
from parsers.FunctionParserOld import FunctionNodeOld
from parsers.PythonParser import PythonNode
from sheerkapython.python_wrapper import sheerka_globals
from sheerkarete.common import V
@@ -52,16 +54,29 @@ class ExprTestObj:
return list(Tokenizer(source, yield_eof=False)), to_skip
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
raise NotImplementedError()
@staticmethod
def safe_get_expr_node(obj, full_text_as_tokens):
def safe_get_expr_node(obj, full_text_as_tokens, default_expr_obj):
if obj is None:
return None
obj = EXPR(obj) if isinstance(obj, (str, tuple)) else obj
return obj.get_expr_node(full_text_as_tokens)
obj = default_expr_obj(obj) if isinstance(obj, (str, tuple)) else obj
return obj.get_expr_node(full_text_as_tokens, default_expr_obj)
class SEQ(ExprTestObj):
""" Test class for SequenceNode"""
def __init__(self, *parts, source=None):
self.parts = parts
self.source = source
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
parts = [self.safe_get_expr_node(part, full_text_as_tokens, default_expr_obj) for part in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
return SequenceNode(start, end, full_text_as_tokens[start: end + 1], *parts)
class AND(ExprTestObj):
@@ -71,8 +86,8 @@ class AND(ExprTestObj):
self.parts = parts
self.source = source
def get_expr_node(self, full_text_as_tokens=None):
parts = [part.get_expr_node(full_text_as_tokens) for part in self.parts]
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
parts = [self.safe_get_expr_node(part, full_text_as_tokens, default_expr_obj) for part in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
return AndNode(start, end, full_text_as_tokens[start: end + 1], *parts)
@@ -84,8 +99,8 @@ class OR(ExprTestObj):
self.parts = parts
self.source = source
def get_expr_node(self, full_text_as_tokens=None):
parts = [part.get_expr_node(full_text_as_tokens) for part in self.parts]
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
parts = [self.safe_get_expr_node(part, full_text_as_tokens, default_expr_obj) for part in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
return OrNode(start, end, full_text_as_tokens[start: end + 1], *parts)
@@ -96,8 +111,8 @@ class NOT(ExprTestObj):
expr: ExprTestObj
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
part = self.expr.get_expr_node(full_text_as_tokens)
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
part = self.safe_get_expr_node(self.expr, full_text_as_tokens, default_expr_obj)
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else (
part.start - 2, part.end)
return NotNode(start, end, full_text_as_tokens[start: end + 1], part)
@@ -108,7 +123,7 @@ class EXPR(ExprTestObj):
"""Test class for NameNode"""
source: str
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
value_as_tokens, to_skip = self.as_tokens(self.source)
start = tokens_index(full_text_as_tokens, value_as_tokens, to_skip)
end = start + len(value_as_tokens) - 1
@@ -122,7 +137,7 @@ class VAR(ExprTestObj):
full_name: str
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
value_as_tokens = list(Tokenizer(self.source or self.full_name, yield_eof=False))
start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
end = start + len(value_as_tokens) - 1
@@ -133,19 +148,77 @@ class VAR(ExprTestObj):
return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0], *parts[1:])
class BinaryExprTestObj(ExprTestObj):
def __init__(self, *parts: Union[ExprTestObj, str], source=None):
self.parts = parts
self.source = source
@staticmethod
def get_op_from_name(name):
if name == "ADD":
return Token(TokenKind.PLUS, "+", -1, -1, -1)
elif name == "SUB":
return Token(TokenKind.MINUS, "-", -1, -1, -1)
elif name == "MULT":
return Token(TokenKind.STAR, "*", -1, -1, -1)
elif name == "DIV":
return Token(TokenKind.SLASH, "/", -1, -1, -1)
elif name == "POW":
return Token(TokenKind.STARSTAR, "**", -1, -1, -1)
elif name == "FDIV":
return Token(TokenKind.SLASHSLASH, "//", -1, -1, -1)
elif name == "MOD":
return Token(TokenKind.PERCENT, "%", -1, -1, -1)
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
op = self.get_op_from_name(type(self).__name__)
parts_as_node = [self.safe_get_expr_node(p, full_text_as_tokens, default_expr_obj) for p in self.parts]
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else \
self.get_pos(parts_as_node)
return BinaryNode(start, end, full_text_as_tokens[start: end + 1], op, *parts_as_node)
class ADD(BinaryExprTestObj):
pass
class SUB(BinaryExprTestObj):
pass
class MULT(BinaryExprTestObj):
pass
class DIV(BinaryExprTestObj):
pass
class FDIV(BinaryExprTestObj):
pass
class MOD(BinaryExprTestObj):
pass
class POW(BinaryExprTestObj):
pass
@dataclass
class CompExprTestObj(ExprTestObj):
"""
Test object for comparison ==, <=, ...
"""
left: ExprTestObj
right: ExprTestObj
left: Union[ExprTestObj, str]
right: Union[ExprTestObj, str]
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
node_type = comparison_type_mapping[type(self).__name__]
left_node = self.left.get_expr_node(full_text_as_tokens)
right_node = self.right.get_expr_node(full_text_as_tokens)
left_node = self.safe_get_expr_node(self.left, full_text_as_tokens, default_expr_obj)
right_node = self.safe_get_expr_node(self.right, full_text_as_tokens, default_expr_obj)
start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else \
self.get_pos([left_node, right_node])
return ComparisonNode(start, end, full_text_as_tokens[start: end + 1], node_type, left_node, right_node)
@@ -202,20 +275,20 @@ class L_EXPR(ExprTestObj):
self.first = first
self.last = last
self.items = items
self.sep = sep or comma
self.sep = sep or t_comma
self.source = source
def get_expr_node(self, full_text_as_tokens=None):
first = self.safe_get_expr_node(self.first, full_text_as_tokens)
last = self.safe_get_expr_node(self.last, full_text_as_tokens)
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
first = self.safe_get_expr_node(self.first, full_text_as_tokens, EXPR)
last = self.safe_get_expr_node(self.last, full_text_as_tokens, EXPR)
items = [self.safe_get_expr_node(item, full_text_as_tokens) for item in self.items]
items = [self.safe_get_expr_node(item, full_text_as_tokens, default_expr_obj) for item in self.items]
if self.source is None:
source = self.first if self.first else ""
source = first.get_source() if self.first else ""
source += f"{self.sep.value} ".join(item.get_source() for item in items)
if self.last:
source += self.last
if last:
source += last.get_source()
else:
source = self.source
@@ -223,6 +296,52 @@ class L_EXPR(ExprTestObj):
return ListNode(start, end, full_text_as_tokens[start: end + 1], first, last, items, self.sep)
class FN(ExprTestObj):
"""
Test class only
It matches with FunctionNode
"""
def __init__(self, name, *parameters, source=None):
self.name = name
self.parameters = parameters
self.source = source
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
start, end = -1, -1
if self.parameters and isinstance(self.parameters[0], L_EXPR):
params_as_test_obj = self.parameters[0]
else:
nb_left_parenthesis = 0
nb_right_parenthesis = 0
if self.source:
start, end = self.get_pos_from_source(self.source, full_text_as_tokens)
for i in range(start):
if full_text_as_tokens[i].type == TokenKind.LPAR:
nb_left_parenthesis += 1
for i in range(end):
if full_text_as_tokens[i].type == TokenKind.RPAR:
nb_right_parenthesis += 1
else:
for p in self.parameters:
if isinstance(p, str):
nb_right_parenthesis += p.count(")")
first = ("(", nb_left_parenthesis) if nb_left_parenthesis > 0 else "("
last = (")", nb_right_parenthesis) if nb_right_parenthesis > 0 else ")"
params_as_test_obj = L_EXPR(first, last, *self.parameters)
name_as_expr = self.safe_get_expr_node(self.name, full_text_as_tokens, default_expr_obj)
params_as_expr = self.safe_get_expr_node(params_as_test_obj, full_text_as_tokens, default_expr_obj)
if start < 1:
start, end = self.get_pos([name_as_expr, params_as_expr.last])
return FunctionNode(start, end, full_text_as_tokens[start:end + 1], name_as_expr, params_as_expr)
@dataclass
class LCC:
"""
@@ -239,7 +358,7 @@ class LC(ExprTestObj): # for List Comprehension node
generators: list
source: str = None
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
# first transform str into NameExprTestObj (ie EXPR)
if isinstance(self.element, str):
self.element = EXPR(self.element)
@@ -254,13 +373,13 @@ class LC(ExprTestObj): # for List Comprehension node
self.generators = comprehensions
# then transform into ListComprehensionNode
element = self.element.get_expr_node(full_text_as_tokens)
element = self.element.get_expr_node(full_text_as_tokens, default_expr_obj)
nodes.append(element)
comprehensions = []
for comp in self.generators:
target = comp.target.get_expr_node(full_text_as_tokens)
iterable = comp.iterable.get_expr_node(full_text_as_tokens)
if_expr = comp.if_expr.get_expr_node(full_text_as_tokens) if comp.if_expr else None
target = comp.target.get_expr_node(full_text_as_tokens, default_expr_obj)
iterable = comp.iterable.get_expr_node(full_text_as_tokens, default_expr_obj)
if_expr = comp.if_expr.get_expr_node(full_text_as_tokens, default_expr_obj) if comp.if_expr else None
comprehensions.append(Comprehension(target, iterable, if_expr))
nodes.extend([target, iterable, if_expr])
@@ -268,14 +387,14 @@ class LC(ExprTestObj): # for List Comprehension node
return ListComprehensionNode(start, end, full_text_as_tokens[start: end + 1], element, comprehensions)
class FN(ExprTestObj):
class FNOld(ExprTestObj):
"""
Test class only
It matches with FunctionNode but with less constraints
It matches with FunctionNodeOld but with less constraints
Thereby,
FN("first", "last", ["param1," ...]) can be compared to
FunctionNode(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
FNOld("first", "last", ["param1," ...]) can be compared to
FunctionNodeOld(NameExprNode("first"), NameExprNode("second"), [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
Note that FunctionParameter can easily be defined with a single string
* "param" -> FunctionParameter(NameExprNode("param"), None)
@@ -308,7 +427,7 @@ class FN(ExprTestObj):
if id(self) == id(other):
return True
if isinstance(other, FN):
if isinstance(other, FNOld):
return self.first == other.first and self.last == other.last and self.parameters == other.parameters
return False
@@ -317,10 +436,10 @@ class FN(ExprTestObj):
return hash((self.first, self.last, self.parameters))
def transform_real_obj(self, other, get_test_obj_delegate):
if isinstance(other, FN):
if isinstance(other, FNOld):
return other
if isinstance(other, FunctionNode):
if isinstance(other, FunctionNodeOld):
params = []
for self_parameter, other_parameter in zip(self.parameters, other.parameters):
if isinstance(self_parameter[0], str):
@@ -330,11 +449,11 @@ class FN(ExprTestObj):
sep = other_parameter.separator.value if other_parameter.separator else None
params.append((value, sep))
return FN(other.first.value, other.last.value, params)
return FNOld(other.first.value, other.last.value, params)
raise Exception(f"Expecting FunctionNode but received {other=}")
raise Exception(f"Expecting FunctionNodeOld but received {other=}")
def get_expr_node(self, full_text_as_tokens=None):
def get_expr_node(self, full_text_as_tokens, default_expr_obj):
start, end = self.get_pos_from_source(self.first, full_text_as_tokens)
first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
start, end = self.get_pos_from_source(self.last, full_text_as_tokens)
@@ -345,7 +464,7 @@ class FN(ExprTestObj):
start, end = self.get_pos_from_source(param_value, full_text_as_tokens)
param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
else:
param_as_expr_node = param_value.get_expr_node(full_text_as_tokens)
param_as_expr_node = param_value.get_expr_node(full_text_as_tokens, default_expr_obj)
if sep:
sep_tokens = Tokenizer(sep, yield_eof=False)
@@ -358,7 +477,7 @@ class FN(ExprTestObj):
parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
start, end = first.start, last.end
return FunctionNode(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
return FunctionNodeOld(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
class HelperWithPos:
@@ -697,12 +816,16 @@ class CIO:
class RETVAL:
"""
Class helper for return value for parser result
Note that it is designed for ParserResultConcept only
Please create another RETVAL or rename this one if extra functionalities are needed
"""
def __init__(self, source, who=None, parser=None):
def __init__(self, text, source=None, who=None, parser=None, objects=None):
self.text = text
self.source = source
self.who = who
self.parser = parser
self.objects = objects
def __eq__(self, other):
if id(self) == id(other):
@@ -711,15 +834,19 @@ class RETVAL:
if not isinstance(other, RETVAL):
return False
return (self.source == other.source and
return (self.text == other.text and
self.source == other.source and
self.who == other.who and
self.parser == other.parser)
self.parser == other.parser and
self.objects == other.objects)
def __hash__(self):
return hash((self.source, self.who))
return hash((self.text, self.source, self.who))
def __repr__(self):
txt = f"RV(source='{self.source}'"
txt = f"RV(text='{self.text}'"
if self.source is not None:
txt += f", source={self.source}"
if self.who is not None:
txt += f", who={self.who}"
if self.parser is not None:
@@ -738,14 +865,33 @@ class RETVAL:
return other
if isinstance(other, ReturnValueConcept):
if not isinstance(other.body, ParserResultConcept):
raise Exception(f"ParserResultConcept not found body={other.body}")
parser_result = other.body
if not isinstance(parser_result.body, PythonNode):
raise Exception(f"PythonNode not found parser_result.body={parser_result.body}")
python_node = parser_result.body
other_objects = None
if self.objects:
other_objects = {}
if len(self.objects) != len(python_node.objects):
raise Exception(f"objects have different size {self.objects=}, other.objects={python_node.objects}")
try:
for k, v in self.objects.items():
other_objects[k] = get_test_obj_delegate(python_node.objects[k], v, get_test_obj_delegate)
except KeyError as err:
raise Exception(f"object {err} is expected but not found")
return RETVAL(parser_result.source,
python_node.source if self.source is not None else None,
other.who if self.who is not None else None,
parser_result.parser if self.parser is not None else None)
parser_result.parser if self.parser is not None else None,
other_objects)
raise Exception(f"Expecting ReturnValueConcept but received {other=}")
@@ -758,9 +904,10 @@ class SCN(HelperWithPos):
SCN == SourceCodeNode if source, start, end (start and end are not validated when None)
"""
def __init__(self, source, start=None, end=None):
def __init__(self, source, objects=None, start=None, end=None):
super().__init__(start, end)
self.source = source
self.objects = objects
def __eq__(self, other):
if id(self) == id(other):
@@ -779,9 +926,10 @@ class SCN(HelperWithPos):
if not isinstance(other, SCN):
return False
return self.source == other.source and \
self.start == other.start and \
self.end == other.end
return (self.source == other.source and
self.objects == other.objects and
self.start == other.start and
self.end == other.end)
def __hash__(self):
return hash((self.source, self.start, self.end))
@@ -806,7 +954,18 @@ class SCN(HelperWithPos):
return other
if isinstance(other, SourceCodeNode):
other_objects = None
if self.objects:
other_objects = []
if not other.python_node:
Exception(f"no python node found in SourceCodeNode {other=}")
if len(self.objects) != len(other.python_node.objects):
Exception(f"objects have different size {self.objects=}, other.objects={other.python_node.objects}")
for self_obj, other_obj in zip(self.objects, other.python_node.objects.values()):
other_objects.append(to_compare_delegate(other_obj, self_obj, to_compare_delegate))
return SCN(other.source,
other_objects,
other.start if self.start is not None else None,
other.end if self.end is not None else None)
@@ -1208,12 +1367,12 @@ comparison_type_mapping = {
}
def get_expr_node_from_test_node(full_text, test_node):
def get_expr_node_from_test_node(full_text, test_node, default_expr_obj=EXPR):
"""
Returns EXPR, OR, NOT, AND object to ease the comparison with the real ExprNode
"""
full_text_as_tokens = list(Tokenizer(full_text, yield_eof=False))
return test_node.get_expr_node(full_text_as_tokens)
return test_node.get_expr_node(full_text_as_tokens, default_expr_obj)
def _index(tokens, expr, index):
@@ -1293,7 +1452,13 @@ def get_node(
init_empty_body,
exclude_body)
if isinstance(sub_expr, (DoNotResolve, ReturnValueConcept, RETVAL)):
if isinstance(sub_expr, (DoNotResolve, ReturnValueConcept)):
return sub_expr
if isinstance(sub_expr, RETVAL):
if sub_expr.objects:
sub_expr.objects = {k: get_node(concepts_map, expression_as_tokens, v, skip=skip)
for k, v in sub_expr.objects.items()}
return sub_expr
if isinstance(sub_expr, SCWC):
@@ -1307,6 +1472,8 @@ def get_node(
if isinstance(sub_expr, SCN):
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, skip=skip)
sub_expr.fix_pos(node)
if sub_expr.objects:
sub_expr.objects = [get_node(concepts_map, expression_as_tokens, c, skip=skip) for c in sub_expr.objects]
return sub_expr
if isinstance(sub_expr, RN):
@@ -1315,7 +1482,16 @@ def get_node(
sub_expr.end = start + length - 1
return sub_expr
if isinstance(sub_expr, (CNC, CC, CN, CMV, CIO)):
if isinstance(sub_expr, CIO):
sub_expr.set_concept(concepts_map[sub_expr.concept_name])
source = sub_expr.source or sub_expr.concept_name
if source:
node = get_node(concepts_map, expression_as_tokens, source)
sub_expr.start = node.start
sub_expr.end = node.end
return sub_expr
if isinstance(sub_expr, (CNC, CC, CN, CMV)):
if sub_expr.concept is None or sub_expr.start is None or sub_expr.end is None:
concept_node = get_node(
concepts_map,
tests/parsers/test_ArithmericOperatorParser.py
+129
View File
@@ -0,0 +1,129 @@
import pytest
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.ArithmericOperatorParser import ArithmeticOperatorParser
from parsers.BaseExpressionParser import LeftPartNotFoundError, ParenthesisMismatchError, RightPartNotFoundError
from parsers.BaseParser import ErrorSink
from parsers.ExpressionParser import ExpressionParser
from parsers.FunctionParser import FunctionParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import ADD, DIV, EXPR, FDIV, FN, LT, MOD, MULT, POW, SEQ, SUB, VAR, \
get_expr_node_from_test_node
class TestArithmeticOperatorParser(TestUsingMemoryBasedSheerka):
def init_parser(self, expr_parser=None, function_parser=None):
sheerka, context = self.init_concepts()
expr_parser = expr_parser or ExpressionParser(auto_compile=False)
parser = ArithmeticOperatorParser(expr_parser=expr_parser, function_parser=function_parser)
return sheerka, context, parser
@pytest.mark.parametrize("expression, expected", [
("one complicated expression", VAR("one complicated expression")),
("a + b", ADD(VAR("a"), VAR("b"))),
("a - b", SUB("a", "b")),
("a * b", MULT("a", "b")),
("a / b", DIV("a", "b")),
("a // b", FDIV("a", "b")),
("a % b", MOD("a", "b")),
("a ** b", POW("a", "b")),
("a + b * c", ADD("a", MULT("b", "c"))),
("a * b + c", ADD(MULT("a", "b"), "c")),
("a * b ** c", MULT("a", POW("b", "c"))),
("a ** b * c", MULT(POW("a", "b"), "c")),
("(a + b) * c", MULT(ADD("a", "b"), "c", source="(a + b) * c")),
("a * (b + c)", MULT("a", ADD("b", "c"), source="a * (b + c)")),
("func(a, b) * 3", MULT(FN(EXPR("func"), EXPR("a"), EXPR("b")), EXPR("3"))),
("3 * func(a, b)", MULT(EXPR("3"), FN(EXPR("func"), EXPR("a"), EXPR("b")))),
("a + b + c", ADD("a", "b", "c")),
("a - b - c", SUB("a", "b", "c")),
("a + b - c - d", SUB(ADD("a", "b"), "c", "d")),
("a * b * c + d + e + f", ADD(MULT("a", "b", "c"), "d", "e", "f")),
("a + b < c", ADD("a", LT("b", "c"))), # this is possible is function_parser is not set
("foo x + y", SEQ(EXPR("foo "), ADD("x", "y"))), # to manager concept call
("long name concept x + y + z", SEQ(EXPR("long name concept "), ADD("x", "y", "z"))),
("long name concept x + y - z", SEQ(EXPR("long name concept "), SUB(ADD("x", "y"), "z"))),
("long name concept x + y * z", SEQ(EXPR("long name concept "), ADD("x", MULT("y", "z")))),
("long name concept x * y + z", SEQ(EXPR("long name concept "), ADD(MULT("x", "y"), "z"))),
("(foo x) + y", ADD("foo x", "y", source="(foo x) + y")),
("a + foo x + y", ADD("a", SEQ(EXPR("foo "), ADD("x", "y")))),
("a + b * foo x * y + z", ADD("a", MULT("b", SEQ(EXPR("foo "), ADD(MULT("x", "y"), "z"))))),
("a + foo b + c + bar e + f", ADD("a", SEQ(EXPR("foo "), ADD("b", "c", SEQ(EXPR("bar "), ADD("e", "f")))))),
("a + foo x", ADD("a", "foo x")),
("a + x y bar + b - c", ADD("a", SEQ(EXPR("x y "), SUB(ADD("bar", "b"), "c")))),
("a + x + y bar + b - c", ADD("a", "x", SEQ(EXPR("y "), SUB(ADD("bar", "b"), "c")))),
("a + x ? y : z + b - c", ADD("a", SEQ(EXPR("x ? y : "), SUB(ADD("z", "b"), "c")))),
("twenty one + 3", SEQ(EXPR("twenty "), ADD("one", EXPR("3")))),
("(foo x) * (bar y)", MULT("foo x", "bar y", source="(foo x) * (bar y)"))
])
def test_i_can_parse_input_expression(self, expression, expected):
sheerka, context, parser = self.init_parser()
parser_input = ParserInput(expression).reset()
parser_input.next_token()
error_sink = ErrorSink()
expr = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
expected = get_expr_node_from_test_node(expression, expected, default_expr_obj=VAR)
assert expr == expected
@pytest.mark.parametrize("expression, expected", [
("one complicated expression", VAR("one complicated expression")),
("a + b", ADD(VAR("a"), VAR("b"))),
("func(a, b) * 3", MULT(FN(EXPR("func"), EXPR("a"), EXPR("b")), EXPR("3"))),
("3 * func(a, b)", MULT(EXPR("3"), FN(EXPR("func"), EXPR("a"), EXPR("b")))),
("a + b < c", ADD("a", EXPR("b < c"))), # the comparison is no longer recognized
("foo x + y", SEQ(EXPR("foo "), ADD("x", "y"))), # to manager concept call
("long name concept x + y + z", SEQ(EXPR("long name concept "), ADD("x", "y", "z"))),
("long name concept x + y - z", SEQ(EXPR("long name concept "), SUB(ADD("x", "y"), "z"))),
("long name concept x + y * z", SEQ(EXPR("long name concept "), ADD("x", MULT("y", "z")))),
("long name concept x * y + z", SEQ(EXPR("long name concept "), ADD(MULT("x", "y"), "z"))),
("(foo x) + y", ADD("foo x", "y", source="(foo x) + y")),
("a + foo x + y", ADD("a", SEQ(EXPR("foo "), ADD("x", "y")))),
("a + b * foo x * y + z", ADD("a", MULT("b", SEQ(EXPR("foo "), ADD(MULT("x", "y"), "z"))))),
("a + foo b + c + bar e + f", ADD("a", SEQ(EXPR("foo "), ADD("b", "c", SEQ(EXPR("bar "), ADD("e", "f")))))),
("a + foo x", ADD("a", "foo x")),
("a + x y bar + b - c", ADD("a", SEQ(EXPR("x y "), SUB(ADD("bar", "b"), "c")))),
("a + x + y bar + b - c", ADD("a", "x", SEQ(EXPR("y "), SUB(ADD("bar", "b"), "c")))),
("a + x ? y : z + b - c", ADD("a", SEQ(EXPR("x ? y : "), SUB(ADD("z", "b"), "c")))),
("(foo x) * (bar y) + 2", ADD(MULT("foo x", "bar y", source="(foo x) * (bar y)"), EXPR("2"))),
])
def test_i_can_parse_input_when_function_parser_is_set(self, expression, expected):
sheerka, context, parser = self.init_parser(function_parser=FunctionParser())
parser_input = ParserInput(expression).reset()
parser_input.next_token()
error_sink = ErrorSink()
expr = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
expected = get_expr_node_from_test_node(expression, expected, default_expr_obj=VAR)
assert expr == expected
@pytest.mark.parametrize("expression, expected_error, expr_is_none", [
("a +", [RightPartNotFoundError], False),
("+ a", [LeftPartNotFoundError], True),
("a ++ b", [RightPartNotFoundError], False),
("(a + b", [ParenthesisMismatchError], False),
# ("a + b)", [ParenthesisMismatchError], False), # only detected is strict is on
])
def test_i_can_detect_errors(self, expression, expected_error, expr_is_none):
sheerka, context, parser = self.init_parser()
parser_input = ParserInput(expression).reset()
parser_input.next_token()
error_sink = ErrorSink()
expr = parser.parse_input(context, parser_input, error_sink)
assert error_sink.has_error
assert (expr is None) == expr_is_none
resolved_errors = [type(e) for e in error_sink.sink]
assert resolved_errors == expected_error
tests/parsers/test_DefConceptParser.py
+2 -2
View File
@@ -16,7 +16,7 @@ from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Se
from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode, CannotHandleParsingError
from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode
from parsers.ExpressionParser import ExpressionParser
from parsers.FunctionParser import FunctionParser
from parsers.FunctionParserOld import FunctionParserOld
from parsers.PythonParser import PythonParser, PythonNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, SCWC, compare_with_test_object, CIO
@@ -77,7 +77,7 @@ def get_concept_part(part, use_expression=False):
status=True,
value=ParserResultConcept(
source=part.source,
parser=FunctionParser(),
parser=FunctionParserOld(),
value=nodes[0],
try_parsed=nodes[0]))
tests/parsers/test_DefRuleParser.py
+1 -1
View File
@@ -50,7 +50,7 @@ class TestDefRuleParser(TestUsingMemoryBasedSheerka):
def test_i_cannot_parse_when_parser_input_is_initialized_from_token(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput("", list(Tokenizer("init from tokens"))))
res = parser.parse(context, ParserInput(None, list(Tokenizer("init from tokens"))))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
tests/parsers/test_ExactConceptParser.py
+35 -4
View File
@@ -1,8 +1,11 @@
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.ExactConceptParser import ExactConceptParser
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_DEF
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind, Tokenizer
from parsers.BaseExpressionParser import NameExprNode
from parsers.ExactConceptParser import ExactConceptParser, NotSupportedElement
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CMV, compare_with_test_object
@@ -189,6 +192,18 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert concept_found.get_hints().need_validation
assert not concept_found.get_hints().is_evaluated
def test_i_cannot_parse_when_expression_contains_expr_token(self):
sheerka, context = self.init_concepts()
tokens = list(Tokenizer("foo bar baz", yield_eof=False))
tokens.append(Token(TokenKind.EXPR, NameExprNode(-1, -1, []), -1, -1, -1))
ret = ExactConceptParser().parse(context, ParserInput(None, tokens))
assert not ret.status
assert sheerka.isinstance(ret.body, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(ret.body.reason, NotSupportedElement)
def test_i_can_manage_unknown_concept(self):
context = self.get_context(self.get_sheerka(singleton=True))
source = "def concept hello" # this is not a concept by itself
@@ -209,6 +224,22 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
assert res.value.reason.body == source
assert res.value.body == source
@pytest.mark.parametrize("text, concept_def", [
("foo x", Concept("foo", definition="foo x", definition_type=DEFINITION_TYPE_DEF).def_var("x")),
("foo", Concept("foo", definition="'foo' ('a'|'b')=x").def_var("x")),
])
def test_i_choose_concept_definition_over_concept_instance(self, text, concept_def):
sheerka, context, foo = self.init_concepts(concept_def, create_new=True)
res = ExactConceptParser().parse(context, ParserInput(text))
assert len(res) == 1
assert res[0].status
concept_found = res[0].body.body
assert concept_found.get_hints().is_evaluated
assert not concept_found.get_hints().is_instance
# def test_i_can_detect_concept_from_tokens(self):
# context = self.get_context(self.get_sheerka(singleton=True))
# concept = get_concept("hello world", [])
tests/parsers/test_ExpressionParser.py
+28 -25
View File
@@ -3,12 +3,12 @@ import pytest
from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from parsers.BaseExpressionParser import VariableNode, ComparisonNode, ExprNode
from parsers.BaseParser import ErrorSink, BaseParser
from parsers.BaseExpressionParser import ComparisonNode, SequenceNode, VariableNode
from parsers.BaseParser import ErrorSink
from parsers.ExpressionParser import ExpressionParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, FN, AND, NOT, OR, GT, GTE, LT, LTE, EQ, \
NEQ, IN, NIN
from tests.parsers.parsers_utils import ADD, AND, EQ, EXPR, FN, GT, GTE, IN, LT, LTE, L_EXPR, MULT, NEQ, NIN, NOT, OR, \
VAR, get_expr_node_from_test_node
class TestExpressionParser(TestUsingMemoryBasedSheerka):
@@ -32,7 +32,7 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
@pytest.mark.parametrize("expression, expected", [
("var1 + var2", EXPR("var1 + var2")),
("var1 + var2", ADD(VAR("var1"), VAR("var2"))),
("variable", VAR("variable")),
("var.attr", VAR("var.attr")),
("var1 and var2", AND(VAR("var1"), VAR("var2"))),
@@ -45,18 +45,22 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
("var1 = var2", EQ(VAR("var1"), VAR("var2"))),
("var1 == var2", EQ(VAR("var1"), VAR("var2"))),
("var1 != var2", NEQ(VAR("var1"), VAR("var2"))),
("var1 in (var2.attr2, var3.attr3)", IN(VAR("var1"), EXPR("var2.attr2, var3.attr3"))),
("var1 not in (var2.attr2, var3.attr3)", NIN(VAR("var1"), EXPR("var2.attr2, var3.attr3"))),
("var1 in (var2.attr2, var3.attr3)", IN(VAR("var1"), L_EXPR("(", ")", "var2.attr2", "var3.attr3"))),
("var1 not in (var2.attr2, var3.attr3)", NIN(VAR("var1"), L_EXPR("(", ")", "var2.attr2", "var3.attr3"))),
("var1 < var2 and var3 > var4", AND(LT(VAR("var1"), VAR("var2")), GT(VAR("var3"), VAR("var4")))),
("func1(one, 1 + 2, func2(3))", FN("func1(", (")", 1), [(VAR("one"), ", "),
(EXPR("1 + 2"), ", "),
FN("func2(", ")", [EXPR("3")])])),
("func(var.attr)", FN("func(", ")", [VAR("var.attr")])),
("func(var1.attr1 and var2.attr2)", FN("func(", ")", [AND(VAR("var1.attr1"), VAR("var2.attr2"))])),
("func(var1.attr1 > var2.attr2)", FN("func(", ")", [GT(VAR("var1.attr1"), VAR("var2.attr2"))])),
("func1(var1) and func2(var2)", AND(FN("func1(", ")", [VAR("var1")]), FN("func2(", (")", 1), [VAR("var2")]))),
("var1 < var2 + var3", LT(VAR("var1"), ADD(VAR("var2"), VAR("var3")))),
("func1(one, 1 + 2, func2(3))", FN("func1", "one", "1 + 2", "func2(3)")),
("func(var.attr)", FN("func", "var.attr")),
("func(var1.attr1 and var2.attr2)", FN("func", "var1.attr1 and var2.attr2")),
("func(var1.attr1 > var2.attr2)", FN("func", "var1.attr1 > var2.attr2")),
("__ret", VAR("__ret")),
# ("func1().func2()", [])
("f(x) is another b(y)", EXPR("f(x) is another b(y)")),
("var1 + var2 * var3", ADD(VAR("var1"), MULT(VAR("var2"), VAR("var3")))),
("(a) + (b)", ADD(VAR("a"), VAR("b"), source="(a) + (b)")),
("(a + b) < (c + d)", LT(ADD(VAR("a"), VAR("b")), ADD(VAR("c"), VAR("d")), source="(a + b) < (c + d)")),
("(a, b, c)", L_EXPR("(", ")", VAR("a"), VAR("b"), VAR("c"))),
("[a, b, c]", L_EXPR("[", "]", VAR("a"), VAR("b"), VAR("c"))),
("{a, b, c}", L_EXPR("{", "}", VAR("a"), VAR("b"), VAR("c"))),
])
def test_i_can_parse_input(self, expression, expected):
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
@@ -90,10 +94,11 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
parsed = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
assert parsed == get_expr_node_from_test_node(expression, EXPR("var1 + var2"))
assert parsed == get_expr_node_from_test_node(expression, ADD("var1", "var2"), default_expr_obj=VAR)
@pytest.mark.parametrize("expression, expected", [
("ret.status in ('a', 1 , func())", "new_var in ('a', 1 , func())"),
("ret.status in ['a', 1 , func()]", "new_var in ['a', 1 , func()]"),
("ret.status not in ('a', 1 , func())", "new_var not in ('a', 1 , func())"),
("ret.status == 10", "new_var == 10"),
("ret.status == 'a'", "new_var == 'a'"),
@@ -114,14 +119,12 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
def test_i_can_compile(self):
sheerka, context, parser = self.init_parser()
def test_i_can_clone(self):
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source("foo x + 1")
expr = parser.parse_input(context, parser_input, error_sink)
text = ParserInput("a > b and c < d")
res = parser.parse(context, text)
assert isinstance(expr, SequenceNode)
assert res.who == BaseParser.PREFIX + ExpressionParser.NAME
assert res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(res.body.body, ExprNode)
assert res.body.body.compiled is not None
clone = expr.clone()
assert expr == clone
tests/parsers/test_FunctionParser.py
+133 -168
View File
@@ -3,13 +3,11 @@ import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import SourceCodeWithConceptNode
from parsers.BaseParser import ErrorSink
from parsers.BaseParser import ErrorSink, UnexpectedTokenParsingError
from parsers.FunctionParser import FunctionParser
from parsers.PythonParser import PythonErrorNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, SCN, SCWC, CN, UTN, CNC, RN, FN, get_test_obj, \
get_expr_node_from_test_node
from tests.parsers.parsers_utils import FN, SCN, SEQ, compute_expected_array, \
get_expr_node_from_test_node, get_test_obj
cmap = {
"one": Concept("one"),
@@ -24,7 +22,7 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
@classmethod
def setup_class(cls):
init_test_helper = cls().init_test(cache_only=False, ontology="#TestFunctionParser#")
init_test_helper = cls().init_test(cache_only=False, ontology="#TestFunctionParserOld#")
sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
for i, concept_name in enumerate(cmap):
cmap[concept_name] = updated[i]
@@ -32,7 +30,7 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
cls.shared_ontology = sheerka.get_ontology(context)
sheerka.pop_ontology(context)
def init_parser(self, my_concepts_map=None, **kwargs):
def init_parser(self, my_concepts_map=None, strict=True, **kwargs):
if my_concepts_map is None:
sheerka, context = self.init_test().unpack()
sheerka.add_ontology(context, self.shared_ontology)
@@ -41,11 +39,12 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
for i, pair in enumerate(my_concepts_map):
my_concepts_map[pair] = updated[i]
parser = FunctionParser()
expr_parser = kwargs.get("expr_parser", None)
parser = FunctionParser(strict, expr_parser=expr_parser)
return sheerka, context, parser
def init_parser_with_source(self, source):
sheerka, context, parser = self.init_parser()
def init_parser_with_source(self, source, strict=True):
sheerka, context, parser = self.init_parser(None, strict)
error_sink = ErrorSink()
parser_input = ParserInput(source)
parser.reset_parser_input(parser_input, error_sink)
@@ -70,180 +69,146 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@pytest.mark.parametrize("expression, expected", [
("func()", FN("func(", ")", [])),
("concept(one)", FN("concept(", ")", ["one"])),
("func(one)", FN("func(", ")", ["one"])),
("func(a long two, 'three', ;:$*)", FN("func(", ")", ["a long two, ", "'three', ", ";:$*"])),
("func(func1(one), two, func2(func3(), func4(three)))", FN("func(", (")", 4), [
(FN("func1(", ")", ["one"]), ", "),
"two, ",
(FN("func2(", (")", 3), [
(FN("func3(", (")", 1), []), ", "),
(FN("func4(", (")", 2), ["three"]), None),
]), None)
])),
("func(r:|1:)", FN("func(", ")", ["r:|1:"]))
("func()", FN("func")),
("concept(one)", FN("concept", "one")),
("func(one)", FN("func", "one")),
("func(a long two, 'three', ;:$*)", FN("func", "a long two", "'three'", ";:$*")),
("func(func1(one), two, func2(func3(), func4(three)))",
FN("func", "func1(one)", "two", "func2(func3(), func4(three))")),
("func(r:|1:)", FN("func", "r:|1:")),
("func(bar fn(x))", FN("func", "bar fn(x)")),
("func([one, two])", FN("func", "[one, two]")),
("func((one, two))", FN("func", "(one, two)")),
("func({one, two})", FN("func", "{one, two}")),
])
def test_i_can_parse_function(self, expression, expected):
def test_i_can_parse_input_function_when_expr_parser_is_none(self, expression, expected):
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
expected = get_expr_node_from_test_node(expression, expected)
parsed = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
assert parsed == expected
@pytest.mark.parametrize("text, expected", [
("func()", SCN("func()")),
(" func()", SCN("func()")),
("func(one)", SCWC("func(", ")", CN("one"))),
("func(one, unknown, two)", SCWC("func(", ")", CN("one"), ", ", UTN("unknown"), (", ", 1), CN("two"))),
("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))),
("func(one plus two, three)", SCWC("func(", ")", CNC("plus", a="one", b="two"), ", ", UTN("three"))),
("func(func1(one), two)", SCWC("func(", (")", 1), SCWC("func1(", ")", "one"), ", ", "two"))
parser.strict = False
parser.reset_parser_input(parser_input, error_sink)
parsed = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
assert parsed == expected
def test_i_can_parse_input_when_prefixed_depending_on_strict_mode(self):
expression = "bar fn(x)"
expected = SEQ("bar ", FN("fn", "x"))
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
expected = get_expr_node_from_test_node(expression, expected)
parsed = parser.parse_input(context, parser_input, error_sink)
assert error_sink.has_error
assert parsed is None
assert isinstance(error_sink.sink[0], UnexpectedTokenParsingError)
parser.strict = False
parser.reset_parser_input(parser_input, error_sink)
parsed = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
assert parsed == expected
def test_i_can_parse_input_when_suffixed_depending_on_strict_mode(self):
expression = "fn(x) bar"
expected = SEQ(FN("fn", "x"), " bar")
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
expected = get_expr_node_from_test_node(expression, expected)
parsed = parser.parse_input(context, parser_input, error_sink)
assert error_sink.has_error
assert parsed is None
assert isinstance(error_sink.sink[0], UnexpectedTokenParsingError)
parser.strict = False
parser.reset_parser_input(parser_input, error_sink)
parsed = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
assert parsed == expected
def test_i_can_parse_input_when_infixed_and_suffixed(self):
expression = "foo fn(x) bar"
expected = SEQ("foo ", FN("fn", "x"), " bar")
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
expected = get_expr_node_from_test_node(expression, expected)
parser.strict = False
parser.reset_parser_input(parser_input, error_sink)
parsed = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
assert parsed == expected
@pytest.mark.parametrize("text, expected, objects", [
("func()", SCN("func()"), {}),
(" func()", SCN("func()"), {}),
("func(one)", SCN("func(one)"), {"__o_00__": "one"}),
("func(one, unknown, two)", SCN("func(one, unknown, two)"), {"__o_00__": "one", "__o_01__": "two"}),
("func(one, twenty two)", SCN("func(one, twenty two)"), {"__o_00__": "one", "__o_01__": "twenties"}),
("func(one plus two, three)", SCN("func(one plus two, three)"), {"__o_00__": "plus"}),
("func(func1(one), two)", SCN("func(func1(one), two)"), {"__o_00__": "one", "__o_01__": "two"}),
])
def test_i_can_parse(self, text, expected):
def test_i_can_parse(self, text, expected, objects):
sheerka, context, parser = self.init_parser()
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
res = parser.parse(context, ParserInput(text))
parser_result = res.body
expression = res.body.body
node = res.body.body
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
transformed_expression = get_test_obj(expression, resolved_expected)
transformed_expression = get_test_obj(node, resolved_expected)
assert transformed_expression == resolved_expected
assert expression.python_node is not None
assert expression.return_value is not None
assert node.python_node is not None
assert node.return_value is not None
def test_i_can_parse_when_multiple_results_when_requested(self):
# the previous output was
# [
# SCWC("func(", ")", "one", ", ", "twenty ", "two"),
# SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))
# ]
# But the first one is now filtered out, as it's not a valid python function call
sheerka, context, parser = self.init_parser()
parser.longest_concepts_only = False
text = "func(one, twenty two)"
expected = [SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))]
resolved_expected = compute_expected_array(cmap, text, expected)
# compare objects
actual_objects = {k: v.id for k, v in node.python_node.objects.items()}
expected_objects = {k: cmap[v].id for k, v in objects.items()}
assert actual_objects == expected_objects
results = parser.parse(context, ParserInput(text))
assert len(results) == 2
res = results[0]
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert len(res.body.body) == 1
assert (res.body.body[0], PythonErrorNode)
res = results[1]
parser_result = res.body
expressions = res.body.body
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
transformed_expressions = get_test_obj(expressions, resolved_expected[0])
assert transformed_expressions == resolved_expected[0]
def test_i_can_parse_when_the_parameter_is_not_a_concept(self):
"""
It's not a concept, but it can be a valid short term memory object
:return:
"""
sheerka, context, parser = self.init_parser()
text = "func(unknown_concept)"
res = parser.parse(context, ParserInput(text))
expected = [SCWC("func(", ")", "unknown_concept")]
resolved_expected = compute_expected_array(cmap, text, expected)
assert res.status
parsed = res.body.body
transformed_parsed = get_test_obj([parsed], resolved_expected)
assert transformed_parsed == resolved_expected
def test_i_can_parse_when_the_concept_is_not_found(self):
"""
We do not check yet if it's a valid concept
If you find a cheap way to do so, simply remove this test
:return:
"""
sheerka, context, parser = self.init_parser()
text = "func(c:|xxx:)"
res = parser.parse(context, ParserInput(text))
assert res.status
def test_i_can_parse_when_rules(self):
sheerka, context, parser = self.init_parser()
text = "func(r:|1:)"
expected = SCWC("func(", ")", RN("1"))
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
res = parser.parse(context, ParserInput(text))
parser_result = res.body
expression = res.body.body
transformed_expression = get_test_obj(expression, resolved_expected)
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert transformed_expression == resolved_expected
assert expression.python_node is not None
assert expression.return_value is not None
def test_i_can_parse_when_the_parameter_is_a_dynamic_concept(self):
sheerka, context, parser = self.init_parser()
text = "func(ones)"
res = parser.parse(context, ParserInput(text))
assert res.status
assert isinstance(res.body.body, SourceCodeWithConceptNode)
assert res.body.body.python_node.source == 'func(__C__ones__1001___PLURAL__C__)'
assert "__C__ones__1001___PLURAL__C__" in res.body.body.python_node.objects
@pytest.mark.parametrize("text, expected_error_type", [
("one", BuiltinConcepts.NOT_FOR_ME), # no function found
("$*!", BuiltinConcepts.NOT_FOR_ME), # no function found
("func(", BuiltinConcepts.ERROR), # function found, but incomplete
("func(one", BuiltinConcepts.ERROR), # function found, but incomplete
("func(one, two, ", BuiltinConcepts.ERROR), # function found, but incomplete
("func(one) and func(two)", BuiltinConcepts.ERROR), # to many function
("one func(one)", BuiltinConcepts.NOT_FOR_ME), # function not found ! (as it is not the first)
("func(a=b, c)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
("func(one two)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
])
def test_i_cannot_parse(self, text, expected_error_type):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, expected_error_type)
@pytest.mark.parametrize("sequence, expected", [
(None, None),
([["a"]], [["a"]]),
([["a"], ["b", "c"]], [["a"]]),
([["b", "c"], ["a"]], [["a"]]),
([["b", "c"], ["a"], ["d", "e"], ["f"]], [["a"], ["f"]]),
])
def test_i_can_get_the_longest_concept_sequence(self, sequence, expected):
assert FunctionParser.get_longest_concepts(sequence) == expected
def test_concepts_found_are_fully_initialized(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput("func(one plus three)"))
concept = res.body.body.nodes[0].concept
assert res.status
assert isinstance(concept.get_compiled()["a"], Concept)
# three is not recognized,
# so it will be transformed into list of ReturnValueConcept that indicate how to recognized it
assert isinstance(concept.get_compiled()["b"], list)
for item in concept.get_compiled()["b"]:
assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)
# @pytest.mark.parametrize("text, expected_error_type", [
# ("one", BuiltinConcepts.NOT_FOR_ME), # no function found
# ("$*!", BuiltinConcepts.NOT_FOR_ME), # no function found
# ("func(", BuiltinConcepts.ERROR), # function found, but incomplete
# ("func(one", BuiltinConcepts.ERROR), # function found, but incomplete
# ("func(one, two, ", BuiltinConcepts.ERROR), # function found, but incomplete
# ("func(one) and func(two)", BuiltinConcepts.ERROR), # to many function
# ("one func(one)", BuiltinConcepts.NOT_FOR_ME), # function not found ! (as it is not the first)
# ("func(a=b, c)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
# ("func(one two)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
# ])
# def test_i_cannot_parse(self, text, expected_error_type):
# sheerka, context, parser = self.init_parser()
#
# res = parser.parse(context, ParserInput(text))
#
# assert not res.status
# assert sheerka.isinstance(res.body, expected_error_type)
#
# @pytest.mark.parametrize("sequence, expected", [
# (None, None),
# ([["a"]], [["a"]]),
# ([["a"], ["b", "c"]], [["a"]]),
# ([["b", "c"], ["a"]], [["a"]]),
# ([["b", "c"], ["a"], ["d", "e"], ["f"]], [["a"], ["f"]]),
# ])
# def test_i_can_get_the_longest_concept_sequence(self, sequence, expected):
# assert FunctionParserOld.get_longest_concepts(sequence) == expected
#
# def test_concepts_found_are_fully_initialized(self):
# sheerka, context, parser = self.init_parser()
#
# res = parser.parse(context, ParserInput("func(one plus three)"))
# concept = res.body.body.nodes[0].concept
#
# assert res.status
# assert isinstance(concept.get_compiled()["a"], Concept)
#
# # three is not recognized,
# # so it will be transformed into list of ReturnValueConcept that indicate how to recognized it
# assert isinstance(concept.get_compiled()["b"], list)
# for item in concept.get_compiled()["b"]:
# assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)
tests/parsers/test_FunctionParserOld.py
+249
View File
@@ -0,0 +1,249 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import SourceCodeWithConceptNode
from parsers.BaseParser import ErrorSink
from parsers.FunctionParserOld import FunctionParserOld
from parsers.PythonParser import PythonErrorNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, SCN, SCWC, CN, UTN, CNC, RN, FNOld, get_test_obj, \
get_expr_node_from_test_node
cmap = {
"one": Concept("one"),
"two": Concept("two"),
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
"plus": Concept("a plus b").def_var("a").def_var("b"),
}
class TestFunctionParserOld(TestUsingMemoryBasedSheerka):
shared_ontology = None
@classmethod
def setup_class(cls):
init_test_helper = cls().init_test(cache_only=False, ontology="#TestFunctionParserOld#")
sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
for i, concept_name in enumerate(cmap):
cmap[concept_name] = updated[i]
cls.shared_ontology = sheerka.get_ontology(context)
sheerka.pop_ontology(context)
def init_parser(self, my_concepts_map=None, **kwargs):
if my_concepts_map is None:
sheerka, context = self.init_test().unpack()
sheerka.add_ontology(context, self.shared_ontology)
else:
sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
for i, pair in enumerate(my_concepts_map):
my_concepts_map[pair] = updated[i]
parser = FunctionParserOld()
return sheerka, context, parser
def init_parser_with_source(self, source):
sheerka, context, parser = self.init_parser()
error_sink = ErrorSink()
parser_input = ParserInput(source)
parser.reset_parser_input(parser_input, error_sink)
return sheerka, context, parser, parser_input, error_sink
def test_i_can_detect_empty_expression(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(""))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
def test_input_must_be_a_parser_input(self):
sheerka, context, parser = self.init_parser()
parser.parse(context, "not a parser input") is None
def test_i_cannot_parse_when_not_a_function(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput("not a function"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@pytest.mark.parametrize("expression, expected", [
("func()", FNOld("func(", ")", [])),
("concept(one)", FNOld("concept(", ")", ["one"])),
("func(one)", FNOld("func(", ")", ["one"])),
("func(a long two, 'three', ;:$*)", FNOld("func(", ")", ["a long two, ", "'three', ", ";:$*"])),
("func(func1(one), two, func2(func3(), func4(three)))", FNOld("func(", (")", 4), [
(FNOld("func1(", ")", ["one"]), ", "),
"two, ",
(FNOld("func2(", (")", 3), [
(FNOld("func3(", (")", 1), []), ", "),
(FNOld("func4(", (")", 2), ["three"]), None),
]), None)
])),
("func(r:|1:)", FNOld("func(", ")", ["r:|1:"]))
])
def test_i_can_parse_function(self, expression, expected):
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
expected = get_expr_node_from_test_node(expression, expected)
parsed = parser.parse_input(context, parser_input, error_sink)
assert not error_sink.has_error
assert parsed == expected
@pytest.mark.parametrize("text, expected", [
("func()", SCN("func()")),
(" func()", SCN("func()")),
("func(one)", SCWC("func(", ")", CN("one"))),
("func(one, unknown, two)", SCWC("func(", ")", CN("one"), ", ", UTN("unknown"), (", ", 1), CN("two"))),
("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))),
("func(one plus two, three)", SCWC("func(", ")", CNC("plus", a="one", b="two"), ", ", UTN("three"))),
("func(func1(one), two)", SCWC("func(", (")", 1), SCWC("func1(", ")", "one"), ", ", "two"))
])
def test_i_can_parse(self, text, expected):
sheerka, context, parser = self.init_parser()
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
res = parser.parse(context, ParserInput(text))
parser_result = res.body
expression = res.body.body
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
transformed_expression = get_test_obj(expression, resolved_expected)
assert transformed_expression == resolved_expected
assert expression.python_node is not None
assert expression.return_value is not None
def test_i_can_parse_when_multiple_results_when_requested(self):
# the previous output was
# [
# SCWC("func(", ")", "one", ", ", "twenty ", "two"),
# SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))
# ]
# But the first one is now filtered out, as it's not a valid python function call
sheerka, context, parser = self.init_parser()
parser.longest_concepts_only = False
text = "func(one, twenty two)"
expected = [SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))]
resolved_expected = compute_expected_array(cmap, text, expected)
results = parser.parse(context, ParserInput(text))
assert len(results) == 2
res = results[0]
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert len(res.body.body) == 1
assert (res.body.body[0], PythonErrorNode)
res = results[1]
parser_result = res.body
expressions = res.body.body
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
transformed_expressions = get_test_obj(expressions, resolved_expected[0])
assert transformed_expressions == resolved_expected[0]
def test_i_can_parse_when_the_parameter_is_not_a_concept(self):
"""
It's not a concept, but it can be a valid short term memory object
:return:
"""
sheerka, context, parser = self.init_parser()
text = "func(unknown_concept)"
res = parser.parse(context, ParserInput(text))
expected = [SCWC("func(", ")", "unknown_concept")]
resolved_expected = compute_expected_array(cmap, text, expected)
assert res.status
parsed = res.body.body
transformed_parsed = get_test_obj([parsed], resolved_expected)
assert transformed_parsed == resolved_expected
def test_i_can_parse_when_the_concept_is_not_found(self):
"""
We do not check yet if it's a valid concept
If you find a cheap way to do so, simply remove this test
:return:
"""
sheerka, context, parser = self.init_parser()
text = "func(c:|xxx:)"
res = parser.parse(context, ParserInput(text))
assert res.status
def test_i_can_parse_when_rules(self):
sheerka, context, parser = self.init_parser()
text = "func(r:|1:)"
expected = SCWC("func(", ")", RN("1"))
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
res = parser.parse(context, ParserInput(text))
parser_result = res.body
expression = res.body.body
transformed_expression = get_test_obj(expression, resolved_expected)
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert transformed_expression == resolved_expected
assert expression.python_node is not None
assert expression.return_value is not None
def test_i_can_parse_when_the_parameter_is_a_dynamic_concept(self):
sheerka, context, parser = self.init_parser()
text = "func(ones)"
res = parser.parse(context, ParserInput(text))
assert res.status
assert isinstance(res.body.body, SourceCodeWithConceptNode)
assert res.body.body.python_node.source == 'func(__C__ones__1001___PLURAL__C__)'
assert "__C__ones__1001___PLURAL__C__" in res.body.body.python_node.objects
@pytest.mark.parametrize("text, expected_error_type", [
("one", BuiltinConcepts.NOT_FOR_ME), # no function found
("$*!", BuiltinConcepts.NOT_FOR_ME), # no function found
("func(", BuiltinConcepts.ERROR), # function found, but incomplete
("func(one", BuiltinConcepts.ERROR), # function found, but incomplete
("func(one, two, ", BuiltinConcepts.ERROR), # function found, but incomplete
("func(one) and func(two)", BuiltinConcepts.ERROR), # to many function
("one func(one)", BuiltinConcepts.NOT_FOR_ME), # function not found ! (as it is not the first)
("func(a=b, c)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
("func(one two)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
])
def test_i_cannot_parse(self, text, expected_error_type):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(text))
assert not res.status
assert sheerka.isinstance(res.body, expected_error_type)
@pytest.mark.parametrize("sequence, expected", [
(None, None),
([["a"]], [["a"]]),
([["a"], ["b", "c"]], [["a"]]),
([["b", "c"], ["a"]], [["a"]]),
([["b", "c"], ["a"], ["d", "e"], ["f"]], [["a"], ["f"]]),
])
def test_i_can_get_the_longest_concept_sequence(self, sequence, expected):
assert FunctionParserOld.get_longest_concepts(sequence) == expected
def test_concepts_found_are_fully_initialized(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput("func(one plus three)"))
concept = res.body.body.nodes[0].concept
assert res.status
assert isinstance(concept.get_compiled()["a"], Concept)
# three is not recognized,
# so it will be transformed into list of ReturnValueConcept that indicate how to recognized it
assert isinstance(concept.get_compiled()["b"], list)
for item in concept.get_compiled()["b"]:
assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)
tests/parsers/test_ListComprehensionParser.py
+13 -3
View File
@@ -58,10 +58,20 @@ class TestListComprehensionParser(TestUsingMemoryBasedSheerka):
error = res.body.body[0]
assert isinstance(error, UnexpectedTokenParsingError)
def test_i_can_parse_a_simple_expression(self):
@pytest.mark.parametrize("expression, expected_generator", [
("[x for x in ['a', 'b'] if x == 'a']", [(("x", 1), "['a', 'b']", "x == 'a'")]),
("[x for x in ('a', 'b') if x == 'a']", [(("x", 1), "('a', 'b')", "x == 'a'")]),
("[x for x in {'a', 'b'} if x == 'a']", [(("x", 1), "{'a', 'b'}", "x == 'a'")]),
("(x for x in ['a', 'b'] if x == 'a')", [(("x", 1), "['a', 'b']", "x == 'a'")]),
("(x for x in ('a', 'b') if x == 'a')", [(("x", 1), "('a', 'b')", "x == 'a'")]),
("(x for x in {'a', 'b'} if x == 'a')", [(("x", 1), "{'a', 'b'}", "x == 'a'")]),
("{x for x in ['a', 'b'] if x == 'a'}", [(("x", 1), "['a', 'b']", "x == 'a'")]),
("{x for x in ('a', 'b') if x == 'a'}", [(("x", 1), "('a', 'b')", "x == 'a'")]),
("{x for x in {'a', 'b'} if x == 'a'}", [(("x", 1), "{'a', 'b'}", "x == 'a'")]),
])
def test_i_can_parse_a_simple_expression(self, expression, expected_generator):
sheerka, context, parser = self.init_parser()
expression = "[x for x in ['a', 'b'] if x == 'a']"
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
lc_node = res.body.body
@@ -69,7 +79,7 @@ class TestListComprehensionParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected = LC(L_EXPR(None, None, "x", source="x "), [(("x", 1), "['a', 'b']", "x == 'a'")], source=expression)
expected = LC(L_EXPR(None, None, "x", source="x "), expected_generator, source=expression)
to_compare_to = get_expr_node_from_test_node(expression, expected)
assert lc_node == to_compare_to
tests/parsers/test_ListParser.py
+24 -5
View File
@@ -14,9 +14,9 @@ or_token = Token(TokenKind.IDENTIFIER, "or", -1, -1, -1)
class TestListParser(TestUsingMemoryBasedSheerka):
def init_parser(self, sep=None):
def init_parser(self, strict=True, sep=None):
sheerka, context = self.init_concepts()
parser = ListParser(sep)
parser = ListParser(strict, sep)
return sheerka, context, parser
@pytest.mark.parametrize("expression, sep, expected", [
@@ -26,12 +26,13 @@ class TestListParser(TestUsingMemoryBasedSheerka):
("x", None, L_EXPR(None, None, EXPR("x"))),
("[x, foo y, z]", None, L_EXPR("[", "]", EXPR("x"), EXPR("foo y"), EXPR("z"))),
("{x, foo y, z}", None, L_EXPR("{", "}", EXPR("x"), EXPR("foo y"), EXPR("z"))),
("(x; y; z)", semi_colon, L_EXPR("(", ")", EXPR("x"), EXPR("y"), EXPR("z"), sep=semi_colon, source="(x; y; z)")),
("(x; y; z)", semi_colon, L_EXPR("(", ")", "x", "y", "z", sep=semi_colon, source="(x; y; z)")),
("x; y; z", semi_colon, L_EXPR(None, None, EXPR("x"), EXPR("y"), EXPR("z"), sep=semi_colon, source="x; y; z")),
("x or y or z", or_token, L_EXPR(None, None, EXPR("x"), EXPR("y"), EXPR("z"), sep=or_token, source="x or y or z")),
("x or y or z", or_token,
L_EXPR(None, None, EXPR("x"), EXPR("y"), EXPR("z"), sep=or_token, source="x or y or z")),
])
def test_i_can_parse_expression(self, expression, sep, expected):
sheerka, context, parser = self.init_parser(sep)
sheerka, context, parser = self.init_parser(strict=False, sep=sep)
expected = get_expr_node_from_test_node(expression, expected)
res = parser.parse(context, ParserInput(expression))
@@ -42,6 +43,24 @@ class TestListParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert expressions == expected
@pytest.mark.parametrize("expression, expected", [
("()", L_EXPR("(", ")")),
("(a, b)", L_EXPR("(", ")", EXPR("a"), EXPR("b"))),
("a, b", None),
])
def test_parenthesis_are_mandatory_when_strict(self, expression, expected):
sheerka, context, parser = self.init_parser(strict=True)
res = parser.parse(context, ParserInput(expression))
if expected:
expected = get_expr_node_from_test_node(expression, expected)
assert res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT)
assert res.body.body == expected
else:
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@pytest.mark.parametrize("expression, starting", [
("(", TokenKind.LPAR),
("(x, y", TokenKind.LPAR),
tests/parsers/test_LogicalOperatorParser.py
+8 -283
View File
@@ -3,13 +3,12 @@ import pytest
from core.builtin_concepts import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind
from parsers.BaseExpressionParser import TrueifyVisitor, IsAQuestionVisitor, LeftPartNotFoundError, \
ParenthesisMismatchError, compile_disjunctions, NotNode, AndNode, OrNode
from parsers.BaseExpressionParser import AndNode, IsAQuestionVisitor, LeftPartNotFoundError, NotNode, OrNode, \
ParenthesisMismatchError, TrueifyVisitor, compile_disjunctions
from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
from parsers.LogicalOperatorParser import LogicalOperatorParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import EXPR, OR, AND, NOT, \
get_expr_node_from_test_node
from tests.parsers.parsers_utils import AND, EXPR, NOT, OR, get_expr_node_from_test_node
class DoNotCompareStartStopContextManager:
@@ -248,6 +247,11 @@ class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka):
("a or (b or c)", [EXPR("a"),
EXPR("b"),
EXPR("c")]),
("a and b and not c", [AND(EXPR("a"), EXPR("b"), NOT(EXPR("c")))]),
("not a and b", [AND(NOT(EXPR("a")), EXPR("b"))]),
("not a and not b", [AND(NOT(EXPR("a")), NOT(EXPR("b")))]),
("not a or not b", [NOT(EXPR("a")),
NOT(EXPR("b"))]),
])
def test_i_can_compile_disjunction(self, expression, expected):
sheerka, context, parser = self.init_parser()
@@ -258,282 +262,3 @@ class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka):
with DoNotCompareStartStopContextManager():
res = compile_disjunctions(expr_node)
assert res == resolved_expected
# @pytest.mark.parametrize("expression, expected", [
# ("foo", "foo"),
# ("one two", "one two"),
# ("foo is a bar", CMV("is a", x='foo', y='bar')),
# ("one two is a bar", [CNC("is a", "one two is a bar", x="one two", y="bar")]),
# ("foo is an foo bar",
# [CNC("is an", "foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]),
# ])
# def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected):
# concepts_map = {
# "foo": Concept("foo"),
# "bar": Concept("bar"),
# "one two": Concept("one two"),
# "is a": Concept("x is a y").def_var("x").def_var("y"),
# "is an": Concept("x is an y", definition="('foo'|'bar')=x 'is an' 'foo bar'").def_var("x"),
# }
# sheerka, context, *concepts = self.init_test().with_concepts(*concepts_map.values(), create_new=True).unpack()
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# if isinstance(expected, list):
# expected_nodes = compute_expected_array(concepts_map, expression, expected)
# compare_with_test_object(ret.body.body, expected_nodes)
# else:
# expected_concept = resolve_test_concept(concepts_map, expected)
# compare_with_test_object(ret.body.body, expected_concept)
# @pytest.mark.parametrize("expression", [
# "a == 5",
# "foo > 5",
# "func() == 5",
# "not a == 5",
# "not foo > 5",
# "not func() == 5",
# "isinstance(a, int)",
# "func()",
# "not isinstance(a, int)",
# "not func()"
# ])
# def test_i_can_get_compiled_expr_from_simple_python_expressions(self, expression):
# sheerka, context, = self.init_test().unpack()
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# assert ret.status
# python_node = ret.body.body.get_python_node()
# _ast = ast.parse(expression, mode="eval")
# expected_python_node = PythonNode(expression, _ast)
# assert python_node == expected_python_node
#
# @pytest.mark.parametrize("expression", [
# "a and not b",
# "not b and a",
# "__ret and not __ret.status",
# ])
# def test_i_can_compile_negative_conjunctions_when_pure_python(self, expression):
# sheerka, context, *concepts = self.init_concepts("foo")
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
#
# ast_ = ast.parse(expression, "<source>", 'eval')
# expected_python_node = PythonNode(expression, ast_)
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# assert sheerka.objvalue(ret) == expected_python_node
#
# @pytest.mark.parametrize("expression, text_to_compile", [
# ("foo bar == 5", "__C__foo0bar__1001__C__ == 5"),
# ("not foo bar == 5", "not __C__foo0bar__1001__C__ == 5"),
# ])
# def test_i_can_get_compiled_expr_from_python_and_concept(self, expression, text_to_compile):
# sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack()
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# assert ret.status
# python_node = ret.body.body.get_python_node()
# _ast = ast.parse(text_to_compile, mode="eval")
# expected_python_node = PythonNode(text_to_compile, _ast, expression)
# assert python_node == expected_python_node
#
# def test_i_can_get_compiled_expr_from__mix_of_concepts_and_python(self):
# sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
# Concept("animal"),
# Concept("a cat"),
# Concept("dog"),
# Concept("pet"),
# Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
# Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
# create_new=True
# ).unpack()
#
# parser = LogicalOperatorParser()
# expression = "not a cat is a pet and not bird is an animal and not x > 5 and not dog is a pet"
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
#
# to_compile = 'not __C__00var0000is0a000var001__1005__C__'
# to_compile += ' and not __C__00var0000is0an0y__1006__C__'
# to_compile += ' and not x > 5'
# to_compile += ' and not __C__00var0000is0a000var001__1005_1__C__'
# ast_ = ast.parse(to_compile, "<source>", 'eval')
# expected_python_node = PythonNode(to_compile, ast_, expression)
#
# assert len(return_values) == 1
# ret = return_values[0]
# python_node = ret.body.body
# assert python_node == expected_python_node
# compare_with_test_object(python_node.objects, {
# "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
# "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
# "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
# })
#
# def test_i_can_get_compiled_expr_from_mix(self):
# sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
# Concept("animal"),
# Concept("a cat"),
# Concept("dog"),
# Concept("pet"),
# Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
# Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
# create_new=True
# ).unpack()
#
# expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
#
# assert len(return_values) == 1
# ret = return_values[0]
#
# to_compile = '__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1006__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__'
# ast_ = ast.parse(to_compile, "<source>", 'eval')
# expected_python_node = PythonNode(to_compile, ast_, expression)
#
# python_node = ret.body.body
# assert python_node == expected_python_node
# compare_with_test_object(python_node.objects, {
# "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
# "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
# "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
# })
#
# def test_i_can_get_compiled_expr_when_multiple_choices(self):
# sheerka, context, *concepts = self.init_test().with_concepts(
# Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
# Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
# create_new=True
# ).unpack()
#
# parser = LogicalOperatorParser()
# expression = "a is a b"
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
#
# assert len(return_values) == 2
#
# ret = return_values[0]
# compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[0], x="a", y="b"))
#
# ret = return_values[1]
# compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[1], x="a", y="b"))
#
# def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self):
# sheerka, context, *concepts = self.init_test().with_concepts(
# Concept("animal"),
# Concept("a cat"),
# Concept("dog"),
# Concept("pet"),
# Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
# Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
# Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
# create_new=True
# ).unpack()
#
# expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
# return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
#
# assert len(return_values) == 4
# trimmed_source = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
#
# current_ret = return_values[0]
# python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__"
# ast_ = ast.parse(python_source, "<source>", 'eval')
# resolved_expected = PythonNode(python_source, ast_, trimmed_source)
# assert sheerka.objvalue(current_ret) == resolved_expected
#
# current_ret = return_values[1]
# assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
# python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006__C__"
# ast_ = ast.parse(python_source, "<source>", 'eval')
# resolved_expected = PythonNode(python_source, ast_, trimmed_source)
# assert sheerka.objvalue(current_ret) == resolved_expected
#
# current_ret = return_values[2]
# assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
# python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005__C__"
# ast_ = ast.parse(python_source, "<source>", 'eval')
# resolved_expected = PythonNode(python_source, ast_, trimmed_source)
# assert sheerka.objvalue(current_ret) == resolved_expected
#
# current_ret = return_values[3]
# python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006_1__C__"
# ast_ = ast.parse(python_source, "<source>", 'eval')
# resolved_expected = PythonNode(python_source, ast_, trimmed_source)
# assert sheerka.objvalue(current_ret) == resolved_expected
#
# @pytest.mark.skip
# @pytest.mark.parametrize("expression, expected_conditions, test_obj", [
# (
# "__ret",
# ["#__x_00__|__name__|'__ret'"],
# ReturnValueConcept("Test", True, None)
# ),
# (
# "__ret.status == True",
# ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
# ReturnValueConcept("Test", True, None)
# ),
# (
# "__ret.status",
# ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
# ReturnValueConcept("Test", True, None)
# ),
# (
# "__ret and __ret.status",
# ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
# ReturnValueConcept("Test", True, None)
# ),
# ])
# def test_i_can_get_rete_condition_from_python(self, expression, expected_conditions, test_obj):
# sheerka, context, = self.init_test().unpack()
# expected_full_condition = get_rete_conditions(*expected_conditions)
#
# parser = LogicalOperatorParser()
# expr_node = parser.parse(context, ParserInput(expression)).body.body
#
# nodes = expr_node.parts if isinstance(expr_node, AndNode) else [expr_node]
# _, rete_disjunctions = parser.compile_conjunctions(context, nodes, "test")
#
# assert len(rete_disjunctions) == 1
# assert rete_disjunctions == [expected_full_condition]
#
# # check against a Rete network
# network = ReteNetwork()
# rule = Rule("test", expression, None)
# rule.metadata.id = 9999
# rule.metadata.is_compiled = True
# rule.metadata.is_enabled = True
# rule.rete_disjunctions = rete_disjunctions
# network.add_rule(rule)
#
# network.add_obj("__ret", test_obj)
# matches = list(network.matches)
# assert len(matches) > 0
tests/parsers/test_RelationalOperatorParser.py
+8 -8
View File
@@ -3,11 +3,11 @@ import pytest
from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind
from parsers.BaseExpressionParser import ParenthesisMismatchError
from parsers.BaseParser import UnexpectedTokenParsingError
from parsers.RelationalOperatorParser import RelationalOperatorParser
from parsers.BaseExpressionParser import ParenthesisMismatchError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, EQ, NEQ, GT, GTE, LT, LTE, IN, NIN
from tests.parsers.parsers_utils import EQ, EXPR, GT, GTE, IN, LT, LTE, L_EXPR, NEQ, NIN, get_expr_node_from_test_node
class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
@@ -33,15 +33,15 @@ class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
("var_name.attr >= 10", GTE(EXPR("var_name.attr"), EXPR("10"))),
("var_name.attr < 10", LT(EXPR("var_name.attr"), EXPR("10"))),
("var_name.attr <= 10", LTE(EXPR("var_name.attr"), EXPR("10"))),
("var_name.attr in (a, b)", IN(EXPR("var_name.attr"), EXPR("a, b"))),
("var_name.attr not in (a, b)", NIN(EXPR("var_name.attr"), EXPR("a, b"))),
("var_name.attr in (a, b)", IN(EXPR("var_name.attr"), L_EXPR("(", ")", "a", "b"))),
("var_name.attr not in (a, b)", NIN(EXPR("var_name.attr"), L_EXPR("(", ")", "a", "b"))),
("var1.attr1 == var2.attr2", EQ(EXPR("var1.attr1"), EXPR("var2.attr2"))),
("var1.attr1 == (var2.attr2)", EQ(EXPR("var1.attr1"), EXPR("var2.attr2"))),
#("var_name.attr in (a.b, b.c)", IN(EXPR("var_name.attr"), PAREN(EXPR("a.b, b.c"), source="(a.b, b.c)"))),
("var_name.attr in (a.b, b.c)", IN(EXPR("var_name.attr"), L_EXPR("(", ")", "a.b", "b.c"))),
("not a var identifier", EXPR("not a var identifier")),
("func()", EXPR("func()")),
#("func(a, not an identifier, x >5)", EXPR("func(a, not an identifier, x >5)")),
("func(a, not an identifier, x >5)", EXPR("func(a, not an identifier, x >5)")),
("(var_name.attr != var_name2.attr2)", NEQ(EXPR("var_name.attr"), EXPR("var_name2.attr2")))
])
def test_i_can_parse_simple_expressions(self, expression, expected):
@@ -61,11 +61,11 @@ class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
("(", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 0),
(")", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 0),
("something (", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10),
# ("something )", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
("something )", BuiltinConcepts.ERROR, TokenKind.RPAR, 10),
("something == (", BuiltinConcepts.ERROR, TokenKind.LPAR, 13),
("something == )", BuiltinConcepts.ERROR, TokenKind.RPAR, 13),
("something (==", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10),
# ("something )==", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
("something )==", BuiltinConcepts.ERROR, TokenKind.RPAR, 10),
])
def test_i_can_detect_unbalanced_parenthesis(self, expression, expected_error, parenthesis_type, index):
sheerka, context, parser = self.init_parser()
tests/parsers/test_SequenceNodeParser.py
+18 -3
View File
@@ -3,13 +3,16 @@ import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_DEF
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind, Tokenizer
from parsers.BaseExpressionParser import NameExprNode
from parsers.ExactConceptParser import NotSupportedElement
from parsers.SequenceNodeParser import SequenceNodeParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CN, SCN, UTN, compare_with_test_object, compute_expected_array, get_test_obj
class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
def init_parser(self, my_map, create_new=False, singleton=True, use_sheerka=False):
def init_parser(self, my_map, create_new=False, use_sheerka=False):
sheerka, context, *updated_concepts = self.init_test().with_concepts(
*my_map.values(),
create_new=create_new).unpack()
@@ -105,7 +108,7 @@ class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_parse_when_unrecognized(self, text, expected_status, expected):
concepts_map = {
"prefixed": Concept("a prefixed").def_var("a"),
"suffixed": Concept("prefixed a").def_var("a"),
"suffixed": Concept("suffixed a").def_var("a"),
"infix": Concept("a infix b").def_var("a").def_var("b"),
"foo bar": Concept("foo bar"),
"one": Concept("one"),
@@ -332,7 +335,7 @@ class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
"bar": Concept("bar")
}
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, singleton=False)
sheerka, context, parser = self.init_parser(concepts_map, create_new=True)
list_of_res = parser.parse(context, ParserInput(text))
assert len(list_of_res) == len(expected)
@@ -483,3 +486,15 @@ class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
concept_found = lexer_nodes[0].concept
assert concept_found.get_metadata().body == "get_plural_concept_value(self)"
def test_i_cannot_parse_when_expr_token(self):
sheerka, context, parser = self.init_parser({})
tokens = list(Tokenizer("foo bar baz", yield_eof=False))
tokens.append(Token(TokenKind.EXPR, NameExprNode(-1, -1, []), -1, -1, -1))
ret = parser.parse(context, ParserInput(None, tokens))
assert not ret.status
assert sheerka.isinstance(ret.body, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(ret.body.reason, NotSupportedElement)
tests/parsers/test_SyaNodeParser.py
+106 -26
View File
@@ -2,21 +2,22 @@ import pytest
from core.builtin_concepts import ReturnValueConcept
from core.builtin_concepts_ids import BuiltinConcepts
from core.builtin_helpers import get_new_variables_definitions
from core.builtin_helpers import get_new_variables_definitions, longest_only
from core.concept import Concept
from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
from core.sheerka.Sheerka import RECOGNIZED_BY_KEY
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind, Tokenizer, comparable_tokens
from core.utils import get_text_from_tokens
from parsers.BaseExpressionParser import FunctionNode, FunctionParameter, NameExprNode
from parsers.BaseExpressionParser import BinaryNode, FunctionNode, FunctionNodeOld, FunctionParameter, ListNode, \
NameExprNode, VariableNode
from parsers.BaseNodeParser import ConceptNode, SourceCodeNode, UnrecognizedTokensNode
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import FunctionDetected, NoSyaConceptFound, NotEnoughParameters, SyaConceptParser, \
SyaNodeParser, SyaTokensParser, TokensNotFound, TooManyParameters
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CC, CN, CNC, RETVAL, SCN, SCWC, UTN, compute_expected_array, get_test_obj, \
from tests.parsers.parsers_utils import CC, CIO, CN, CNC, RETVAL, SCN, UTN, compute_expected_array, get_test_obj, \
prepare_nodes_comparison
cmap = {
@@ -403,10 +404,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert not concept_parser.has_error()
assert len(concept_parser.expected) == 0
expected = CNC("plus", a=CNC("one"), b=SCWC("func(", ")", CN("twenties", source="twenty two")))
resolved_expected = compute_expected_array(cmap, expression, [expected])[0]
concept_node_as_test_obj = get_test_obj(concept_node, expected)
assert concept_node_as_test_obj == resolved_expected
expected = CNC("plus", a=CNC("one"), b=SCN("func(twenty two)", [CIO("twenties", source="twenty two")]))
actual_as_test, expected_resolved = prepare_nodes_comparison(cmap, expression, concept_node, expected)
assert actual_as_test == expected_resolved
assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
def test_i_can_concept_parse_concepts_composition(self):
@@ -1366,22 +1366,13 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected = [CN(cmap["suffixed"], text, 0, 6)]
expected_return_value = RETVAL("1 + 1")
expected = [CNC(cmap["suffixed"], text, 0, 6, a=[expected_return_value])]
concept_node_as_test_obj = get_test_obj(lexer_nodes, expected)
assert concept_node_as_test_obj == expected
# check the compiled
expected_concept = lexer_nodes[0].concept
assert len(expected_concept.get_compiled()["a"]) == 1
return_value_a = expected_concept.get_compiled()["a"][0]
assert sheerka.isinstance(return_value_a, BuiltinConcepts.RETURN_VALUE)
assert return_value_a.status
assert sheerka.isinstance(return_value_a.body, BuiltinConcepts.PARSER_RESULT)
assert return_value_a.body.source == "1 + 1"
assert isinstance(return_value_a.body.body, PythonNode)
# check metadata
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "1 + 1")]
def test_i_can_parse_when_bnf_concept(self):
@@ -1408,6 +1399,33 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
# check metadata
assert expected_concept.get_metadata().variables == [("a", "twenty one")]
@pytest.mark.skip("Not quite sure that this test is relevant")
def test_i_can_parse_when_bnf_and_python(self):
# twenty one + 1 is not correctly parsed by BNFNode or SequenceNode
# Maybe I should just leave it to ExpressionParser instead
sheerka, context, parser = self.init_parser()
text = "suffixed twenty one + 1"
res = parser.parse(context, ParserInput(text))
# only care about the result that recognizes 'twenty one'
longest = longest_only(context, res)
res = longest.body.body[0]
wrapper = res.body
lexer_nodes = res.body.body
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected_return_value = RETVAL("twenty one + 1", objects={"__o_00__": CIO("twenties", source="twenty one")})
expected = [CNC(cmap["suffixed"], text, 0, 8, a=[expected_return_value])]
concept_node_as_test_obj = get_test_obj(lexer_nodes, expected)
assert concept_node_as_test_obj == expected
# check metadata
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "twenty one + 1")]
def test_i_can_parse_when_function(self):
sheerka, context, parser = self.init_parser()
@@ -1419,7 +1437,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected = [CNC("plus", a=CC("one"), b=[RETVAL("func(twenty two)")], source=text)]
expected_ret_val = RETVAL("func(twenty two)",
"func(__o_00__)",
objects={"__o_00__": CIO("twenties", source="twenty two")})
expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)]
_stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
assert _stack == _expected
@@ -1543,16 +1564,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
_stack, _expected = prepare_nodes_comparison(concepts_map, text, lexer_nodes, expected)
assert _stack == _expected
def test_i_can_parse_when_expr_tokens(self):
def test_i_can_parse_when_function_old_style_expr_tokens(self):
sheerka, context, parser = self.init_parser()
text = "one plus func(twenty two)"
tokens = list(Tokenizer(text, yield_eof=False))
fun_token = tokens[4]
expr = FunctionNode(4, 9, tokens[4:10],
NameExprNode(4, 4, tokens[4:5]),
NameExprNode(9, 9, tokens[9:10]),
[FunctionParameter(NameExprNode(6, 8, tokens[6:9]), None)])
expr = FunctionNodeOld(4, 9, tokens[4:10],
NameExprNode(4, 5, tokens[4:6]),
NameExprNode(9, 9, tokens[9:10]),
[FunctionParameter(NameExprNode(6, 8, tokens[6:9]), None)])
tokens[4:] = [Token(TokenKind.EXPR, expr, fun_token.index, fun_token.line, fun_token.column)]
res = parser.parse(context, ParserInput(None, tokens=tokens))
wrapper = res.body
@@ -1561,7 +1582,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected = [CNC("plus", a=CC("one"), b=[RETVAL("func(twenty two)")], source=text)]
expected_ret_val = RETVAL("func(twenty two)", objects={"__o_00__": CIO("twenties", source="twenty two")})
expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)]
_stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
assert _stack == _expected
@@ -1569,6 +1591,64 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
def test_i_can_parse_when_function_style_expr_tokens(self):
sheerka, context, parser = self.init_parser()
text = "one plus func(twenty two)"
tokens = list(Tokenizer(text, yield_eof=False))
fun_token = tokens[4]
expr = FunctionNode(4, 9, tokens[4:10],
NameExprNode(4, 4, tokens[4:5]),
ListNode(5, 9, tokens[5:10],
NameExprNode(5, 5, tokens[5:6]),
NameExprNode(9, 9, tokens[9:10]),
[NameExprNode(6, 8, tokens[6:9])]))
tokens[4:] = [Token(TokenKind.EXPR, expr, fun_token.index, fun_token.line, fun_token.column)]
res = parser.parse(context, ParserInput(None, tokens=tokens))
wrapper = res.body
lexer_nodes = res.body.body
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected_ret_val = RETVAL("func(twenty two)", objects={"__o_00__": CIO("twenties", source="twenty two")})
expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)]
_stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
assert _stack == _expected
# check the metadata
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
def test_i_can_parse_when_binary_expr_tokens(self):
sheerka, context, parser = self.init_parser()
text = "suffixed twenty one + self"
tokens = list(Tokenizer(text, yield_eof=False))
expr = BinaryNode(2, 8, tokens[2:9], tokens[6],
VariableNode(2, 4, tokens[2:5], "twenty one"),
VariableNode(8, 8, tokens[8:9], "self"))
token_twenty = tokens[2]
tokens[2:] = [Token(TokenKind.EXPR, expr, token_twenty.index, token_twenty.line, token_twenty.column)]
res = parser.parse(context, ParserInput(None, tokens=tokens))
wrapper = res.body
lexer_nodes = res.body.body
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected_ret_val = RETVAL("twenty one + self",
source="__o_00__ + self",
objects={"__o_00__": CIO("twenties", source="twenty one")})
expected = [CNC("suffixed", a=[expected_ret_val], source=text)]
_stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
assert _stack == _expected
# check the metadata
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "twenty one + self")]
@pytest.mark.parametrize("text, expected_result", [
("one plus two foo bar baz", [CNC("plus", a="one", b="two"), UTN(" foo bar baz")]),
("one plus two foo bar", [CNC("plus", a="one", b="two"), UTN(" foo bar")]),
tests/parsers/test_TokenExpressionParser.py
+183
View File
@@ -0,0 +1,183 @@
from core.builtin_concepts_ids import BuiltinConcepts
from core.concept import Concept
from core.global_symbols import OBJECTS_COUNTER_KEY
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind
from parsers.BaseExpressionParser import ExprNode, NameExprNode
from parsers.ExpressionParser import ExpressionParser
from parsers.PythonParser import PythonNode
from parsers.TokenExpressionParser import NoTokenExprFound, TokenExpressionParser
from sheerkapython.python_wrapper import Expando
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class TestTokenExpressionParser(TestUsingMemoryBasedSheerka):
def init_parser(self, my_map=None, create_new=False):
if my_map is None:
my_map = {}
sheerka, context, *updated_concepts = self.init_test().with_concepts(
*my_map.values(),
create_new=create_new).unpack()
parser = TokenExpressionParser()
return sheerka, context, parser
@staticmethod
def get_parser_input(context, expression):
expr_parser = ExpressionParser(auto_compile=False)
parsed_ret = expr_parser.parse(context, ParserInput(expression))
parsed = parsed_ret.body.body
token_expr = Token(TokenKind.EXPR, parsed, 0, 1, 1)
return ParserInput(None, [token_expr])
def test_i_cannot_parse_empty_string(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(""))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
def test_i_cannot_parse_when_no_token_expression(self):
sheerka, context, parser = self.init_parser()
parser_input = ParserInput("not an ExprNode")
res = parser.parse(context, parser_input)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert res.body.body == parser_input
assert isinstance(res.body.reason, NoTokenExprFound)
def test_i_cannot_parse_when_too_many_token_expression(self):
sheerka, context, parser = self.init_parser()
expr = NameExprNode(0, 0, [Token(TokenKind.IDENTIFIER, "hello", 0, 1, 1)])
token_expr = Token(TokenKind.EXPR, expr, 0, 1, 1)
parser_input = ParserInput(None, [token_expr, token_expr])
res = parser.parse(context, parser_input)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert res.body.body == parser_input
assert isinstance(res.body.reason, NoTokenExprFound)
def test_i_can_parse_simple_expression(self):
sheerka, context, parser = self.init_parser()
parser_input = self.get_parser_input(context, "a + b")
res = parser.parse(context, parser_input)
assert len(res) == 1
assert res[0].status
assert res[0].who == "parsers.TokenExpr"
assert sheerka.isinstance(res[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(res[0].body.body, PythonNode)
def test_i_can_parse_when_concept(self):
concepts_map = {"foo": Concept("foo x").def_var("x")}
sheerka, context, parser = self.init_parser(concepts_map, True)
parser_input = self.get_parser_input(context, "foo a")
res = parser.parse(context, parser_input)
assert len(res) == 1
assert res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(res[0].body.body, PythonNode)
python_node = res[0].body.body
assert python_node.source == "call_concept(__o_00__, x=a)"
def test_i_can_parse_when_concept_using_object_counter(self):
concepts_map = {"foo": Concept("foo x").def_var("x")}
sheerka, context, parser = self.init_parser(concepts_map, True)
parser_input = self.get_parser_input(context, "foo a")
objects_counter_entry = Expando(OBJECTS_COUNTER_KEY, {"value": 10})
context.add_to_short_term_memory(OBJECTS_COUNTER_KEY, objects_counter_entry)
res = parser.parse(context, parser_input)
assert len(res) == 1
assert res[0].status
assert sheerka.isinstance(res[0].body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(res[0].body.body, PythonNode)
python_node = res[0].body.body
assert python_node.source == "call_concept(__o_10__, x=a)"
objects_counter_entry = context.get_from_short_term_memory(OBJECTS_COUNTER_KEY)
assert objects_counter_entry.value == 11
def test_i_can_parse_simple_expression_when_is_question_is_set(self):
sheerka, context, parser = self.init_parser()
context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
parser_input = self.get_parser_input(context, "a + b")
res = parser.parse(context, parser_input)
assert res.status
assert res.who == "parsers.TokenExpr"
assert sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(res.body.body, ExprNode)
assert res.body.body.compiled is not None
def test_i_can_parse_when_concept_and_is_question_is_set(self):
concepts_map = {"isa": Concept("x is a y", pre="is_question()").def_var("x").def_var("y")}
sheerka, context, parser = self.init_parser(concepts_map, True)
context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
parser_input = self.get_parser_input(context, "a is a b")
objects_counter_entry = Expando(OBJECTS_COUNTER_KEY, {"value": 10})
context.add_to_short_term_memory(OBJECTS_COUNTER_KEY, objects_counter_entry)
res = parser.parse(context, parser_input)
assert res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT)
assert isinstance(res.body.body, ExprNode)
expr_node = res.body.body
compiled_0 = expr_node.compiled[0]
assert compiled_0.return_value.body.body.source == "evaluate_question(__o_10__, x=a, y=b)"
objects_counter_entry = context.get_from_short_term_memory(OBJECTS_COUNTER_KEY)
assert objects_counter_entry.value == 11
def test_obj_counter_is_not_updated_when_error(self):
concepts_map = {}
sheerka, context, parser = self.init_parser(concepts_map, True)
parser_input = self.get_parser_input(context, "foo a")
objects_counter_entry = Expando(OBJECTS_COUNTER_KEY, {"value": 10})
context.add_to_short_term_memory(OBJECTS_COUNTER_KEY, objects_counter_entry)
res = parser.parse(context, parser_input)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
objects_counter_entry = context.get_from_short_term_memory(OBJECTS_COUNTER_KEY)
assert objects_counter_entry.value == 10
def test_obj_counter_is_not_updated_when_error_and_is_question_is_set(self):
concepts_map = {}
sheerka, context, parser = self.init_parser(concepts_map, True)
context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
parser_input = self.get_parser_input(context, "a is a b")
objects_counter_entry = Expando(OBJECTS_COUNTER_KEY, {"value": 10})
context.add_to_short_term_memory(OBJECTS_COUNTER_KEY, objects_counter_entry)
res = parser.parse(context, parser_input)
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
objects_counter_entry = context.get_from_short_term_memory(OBJECTS_COUNTER_KEY)
assert objects_counter_entry.value == 10
tests/parsers/test_UnrecognizedNodeParser.py
+9 -7
View File
@@ -1,14 +1,14 @@
from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
from core.concept import Concept
from core.tokenizer import Tokenizer, TokenKind
from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, SourceCodeNode
from core.tokenizer import TokenKind, Tokenizer
from parsers.BaseNodeParser import ConceptNode, SourceCodeNode, SourceCodeWithConceptNode, UnrecognizedTokensNode
from parsers.BnfNodeParser import BnfNodeParser
from parsers.SequenceNodeParser import SequenceNodeParser
from parsers.SyaNodeParser import SyaNodeParser
from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, get_node, CC, UTN, CNC, CN, SCWC, \
compare_with_test_object, SCN
from tests.parsers.parsers_utils import CC, CN, CNC, SCN, SCWC, UTN, compare_with_test_object, compute_expected_array, \
get_node, prepare_nodes_comparison
def get_input_nodes_from(my_concepts_map, full_expr, *args):
@@ -281,8 +281,10 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parser_result.source == expression
assert len(actual_nodes) == 1
compare_with_test_object(actual_nodes[0], SCN(expression, 0, 4))
expected = [SCN(expression, start=0, end=4)]
actual_as_test, resolved_expected = prepare_nodes_comparison(concepts_map, expression, actual_nodes, expected)
assert actual_as_test == resolved_expected
def test_i_cannot_parse_unrecognized_python_that_looks_like_concept(self):
sheerka, context, parser = self.init_parser()
tests/parsers/test_parsers_utils.py
+7 -7
View File
@@ -4,7 +4,7 @@ from core.rule import Rule, ACTION_TYPE_EXEC
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import RuleNode
from parsers.BnfNodeParser import BnfNodeParser
from parsers.FunctionParser import FunctionParser
from parsers.FunctionParserOld import FunctionParserOld
from parsers.SyaNodeParser import SyaNodeParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import get_test_obj, CNC, CC, CN, SCN, SCWC, UTN, RN, CB
@@ -133,31 +133,31 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka):
def test_i_can_get_test_obj_when_SCN(self):
sheerka, context = self.init_test().unpack()
parser = FunctionParser()
parser = FunctionParserOld()
scn = parser.parse(context, ParserInput("test()")).body.body
scn_res = get_test_obj(scn, SCN("", start=0, end=1))
assert isinstance(scn_res, SCN)
assert scn_res == SCN("test()", 0, 2)
assert scn_res == SCN("test()", start=0, end=2)
# I can discard start and end
scn_res = get_test_obj(scn, SCN(""))
assert isinstance(scn_res, SCN)
assert scn_res == SCN("test()", None, None)
assert scn_res == SCN("test()", start=None, end=None)
def test_i_can_get_test_obj_when_SCWC(self):
sheerka, context = self.init_test().unpack()
parser = FunctionParser()
parser = FunctionParserOld()
scwc = parser.parse(context, ParserInput("test(param1, test2())")).body.body
scwc_res = get_test_obj(scwc, SCWC(UTN(""), UTN(""), UTN(""), UTN(""), SCN("", 0, 0)))
scwc_res = get_test_obj(scwc, SCWC(UTN(""), UTN(""), UTN(""), UTN(""), SCN("", None, 0, 0)))
assert isinstance(scwc_res, SCWC)
expected = SCWC(UTN("test(", 0, 1),
UTN(")", 8, 8),
UTN("param1", 2, 2),
UTN(", ", 3, 4),
SCN("test2()", 5, 7))
SCN("test2()", None, 5, 7))
expected.start = 0
expected.end = 8
assert scwc_res == expected