Files
Sheerka-Old/tests/parsers/test_ExpressionParser.py
T

491 lines
23 KiB
Python

import ast
import pytest
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.concept import Concept, CMV, DoNotResolve, CC
from core.rule import Rule
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind
from parsers.BaseNodeParser import CNC
from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
from parsers.ExpressionParser import ExpressionParser, LeftPartNotFoundError, ParenthesisMismatchError
from parsers.PythonParser import PythonNode
from parsers.expressions import TrueifyVisitor, IsAQuestionVisitor, AndNode
from sheerkarete.network import ReteNetwork
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import compute_expected_array, resolve_test_concept, EXPR, OR, AND, NOT, \
get_expr_node_from_test_node, get_rete_conditions
class TestExpressionParser(TestUsingMemoryBasedSheerka):
def init_parser(self):
sheerka, context = self.init_concepts()
parser = ExpressionParser()
return sheerka, context, parser
@pytest.mark.parametrize("expression, expected", [
("one complicated expression", EXPR("one complicated expression")),
("function_call(a,b,c)", EXPR("function_call(a,b,c)")),
("one expression or another expression", OR(EXPR("one expression"), EXPR("another expression"))),
("one expression and another expression", AND(EXPR("one expression"), EXPR("another expression"))),
("not one", NOT(EXPR("one"))),
("one and not two", AND(EXPR("one"), NOT(EXPR("two")))),
("not one and two", AND(NOT(EXPR("one")), EXPR("two"))),
("one or not two", OR(EXPR("one"), NOT(EXPR("two")))),
("not one or two", OR(NOT(EXPR("one")), EXPR("two"))),
("one or two or three", OR(EXPR("one"), EXPR("two"), EXPR("three"))),
("one and two and three", AND(EXPR("one"), EXPR("two"), EXPR("three"))),
("one or two and three", OR(EXPR("one"), AND(EXPR("two"), EXPR("three")))),
("one and two or three", OR(AND(EXPR("one"), EXPR("two")), EXPR("three"))),
("one and (two or three)", AND(EXPR("one"), OR(EXPR("two"), EXPR("three")), source="one and (two or three)")),
("not not one", NOT(NOT(EXPR("one")))),
("not (one and two)", NOT(AND(EXPR("one"), EXPR("two")), source="not (one and two)")),
("one 'and' two or three", OR(EXPR("one 'and' two"), EXPR("three"))),
("not ((a or b) and (c or d))", NOT(AND(OR(EXPR("a"), EXPR("b")), OR(EXPR("c"), EXPR("d")),
source="(a or b) and (c or d)"),
source="not ((a or b) and (c or d))")),
("not ((a and b) or (c and d))", NOT(OR(AND(EXPR("a"), EXPR("b")), AND(EXPR("c"), EXPR("d")),
source="(a and b) or (c and d)"),
source="not ((a and b) or (c and d))")),
])
def test_i_can_parse_expression(self, expression, expected):
sheerka, context, parser = self.init_parser()
expected = get_expr_node_from_test_node(expression, expected)
res = parser.parse(context, ParserInput(expression))
wrapper = res.body
expressions = res.body.body
assert res.status
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert expressions == expected
@pytest.mark.parametrize("expression, expected_errors", [
("one or", [UnexpectedEofParsingError("When parsing 'or'")]),
("one and", [UnexpectedEofParsingError("When parsing 'and'")]),
("and one", [LeftPartNotFoundError()]),
("or one", [LeftPartNotFoundError()]),
("or", [LeftPartNotFoundError(), UnexpectedEofParsingError("When parsing 'or'")]),
("and", [LeftPartNotFoundError(), UnexpectedEofParsingError("When parsing 'and'")]),
])
def test_i_can_detect_error(self, expression, expected_errors):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(expression))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert res.body.body == expected_errors
def test_i_can_detect_unexpected_not_error(self):
sheerka, context, parser = self.init_parser()
expression = "a cat is not a human"
res = parser.parse(context, ParserInput(expression))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
def test_i_can_detect_unbalanced_parenthesis(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput("("))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(res.body.reason[0], UnexpectedTokenParsingError)
assert res.body.reason[0].token.type == TokenKind.EOF
assert res.body.reason[0].expected_tokens == [TokenKind.RPAR]
res = parser.parse(context, ParserInput(")"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(res.body.reason[0], UnexpectedTokenParsingError)
assert res.body.reason[0].token.type == TokenKind.RPAR
assert res.body.reason[0].expected_tokens == []
res = parser.parse(context, ParserInput("one and two("))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], ParenthesisMismatchError)
assert res.body.body[0].token.type == TokenKind.LPAR
assert res.body.body[0].token.index == 11
res = parser.parse(context, ParserInput("one ("))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(res.body.reason[0], ParenthesisMismatchError)
assert res.body.reason[0].token.type == TokenKind.LPAR
assert res.body.reason[0].token.index == 4
res = parser.parse(context, ParserInput("one (and"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], ParenthesisMismatchError)
assert res.body.body[0].token.type == TokenKind.LPAR
assert res.body.body[0].token.index == 4
res = parser.parse(context, ParserInput("one and two)"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
assert res.body.body[0].token.type == TokenKind.RPAR
assert res.body.body[0].expected_tokens == []
res = parser.parse(context, ParserInput("one )"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
assert res.body.body[0].token.type == TokenKind.RPAR
assert res.body.body[0].expected_tokens == []
res = parser.parse(context, ParserInput("one ) and"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
assert res.body.body[0].token.type == TokenKind.RPAR
assert res.body.body[0].expected_tokens == []
def test_i_can_detect_empty_expression(self):
sheerka, context, parser = self.init_parser()
res = parser.parse(context, ParserInput(""))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
@pytest.mark.parametrize("expression, to_trueify, to_skip, expected", [
("a", ["b"], ["a"], "a"),
("b", ["b"], ["a"], "True"),
("a and b", ["b"], ["a"], "a and True"),
("b or a", ["b"], ["a"], "True or a"),
("isinstance(b, str)", ["b"], ["a"], "True"),
("isinstance(b, str) or instance(a, str)", ["b"], ["a"], "True or instance(a, str)"),
("a and b or c", ["b", "c"], ["a"], "a and True or True"),
("a + b or a + c", ["b", "c"], ["a"], "a + b or a + c"),
])
def test_i_can_trueify(self, expression, to_trueify, to_skip, expected):
sheerka, context, parser = self.init_parser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
translated_node = TrueifyVisitor(to_trueify, to_skip).visit(expr_node)
assert str(translated_node) == expected
@pytest.mark.parametrize("expression, expected", [
("foo", None),
("", None),
("is_question()", True),
(" is_question() ", True),
("is_question ( ) ", True),
("is _question()", None),
("is_ question()", None),
("is _ question()", None),
("context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", True),
("not is_question()", False),
("not context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", False),
("not foo", None),
("not is_question() and not is_question()", False),
("not is_question() and is_question()", False),
("not is_question() and foo", False),
("is_question() and not is_question()", False),
("is_question() and is_question()", True),
("is_question() and foo", True),
("foo and not is_question()", False),
("foo and is_question()", True),
("foo and bar", None),
("not is_question() or not is_question()", False),
("not is_question() or is_question()", True),
("not is_question() or foo", False),
("is_question() or not is_question()", True),
("is_question() or is_question()", True),
("is_question() or foo", True),
("foo or not is_question()", False),
("foo or is_question()", True),
("foo or bar", None),
])
def test_is_a_question(self, expression, expected):
sheerka, context, parser = self.init_parser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
assert IsAQuestionVisitor().visit(expr_node) == expected
@pytest.mark.parametrize("expression, expected", [
("foo", "foo"),
("one two", "one two"),
("foo is a bar", CMV("is a", x='foo', y='bar')),
("one two is a bar", [CNC("is a", source="one two is a bar", x="one two", y="bar")]),
("foo is an foo bar",
[CNC("is an", source="foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]),
])
def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected):
concepts_map = {
"foo": Concept("foo"),
"bar": Concept("bar"),
"one two": Concept("one two"),
"is a": Concept("x is a y").def_var("x").def_var("y"),
"is an": Concept("x is an y", definition="('foo'|'bar')=x 'is an' 'foo bar'").def_var("x"),
}
sheerka, context, *concepts = self.init_test().with_concepts(*concepts_map.values(), create_new=True).unpack()
parser = ExpressionParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
assert len(return_values) == 1
ret = return_values[0]
if isinstance(expected, list):
expected_nodes = compute_expected_array(concepts_map, expression, expected)
assert ret.body.body == expected_nodes
else:
expected_concept = resolve_test_concept(concepts_map, expected)
assert ret.body.body == expected_concept
@pytest.mark.parametrize("expression", [
"a == 5",
"foo > 5",
"func() == 5",
"not a == 5",
"not foo > 5",
"not func() == 5",
"isinstance(a, int)",
"func()",
"not isinstance(a, int)",
"not func()"
])
def test_i_can_get_compiled_expr_from_simple_python_expressions(self, expression):
sheerka, context, = self.init_test().unpack()
parser = ExpressionParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
assert len(return_values) == 1
ret = return_values[0]
assert ret.status
python_node = ret.body.body.get_python_node()
_ast = ast.parse(expression, mode="eval")
expected_python_node = PythonNode(expression, _ast)
assert python_node == expected_python_node
@pytest.mark.parametrize("expression", [
"a and not b",
"not b and a",
"__ret and not __ret.status",
])
def test_i_can_compile_negative_conjunctions_when_pure_python(self, expression):
sheerka, context, *concepts = self.init_concepts("foo")
parser = ExpressionParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
ast_ = ast.parse(expression, "<source>", 'eval')
expected_python_node = PythonNode(expression, ast_)
assert len(return_values) == 1
ret = return_values[0]
assert sheerka.objvalue(ret) == expected_python_node
@pytest.mark.parametrize("expression, text_to_compile", [
("foo bar == 5", "__C__foo0bar__1001__C__ == 5"),
("not foo bar == 5", "not __C__foo0bar__1001__C__ == 5"),
])
def test_i_can_get_compiled_expr_from_python_and_concept(self, expression, text_to_compile):
sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack()
parser = ExpressionParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
assert len(return_values) == 1
ret = return_values[0]
assert ret.status
python_node = ret.body.body.get_python_node()
_ast = ast.parse(text_to_compile, mode="eval")
expected_python_node = PythonNode(text_to_compile, _ast, expression)
assert python_node == expected_python_node
def test_i_can_get_compiled_expr_from__mix_of_concepts_and_python(self):
sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
Concept("animal"),
Concept("a cat"),
Concept("dog"),
Concept("pet"),
Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
create_new=True
).unpack()
parser = ExpressionParser()
expression = "not a cat is a pet and not bird is an animal and not x > 5 and not dog is a pet"
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
to_compile = 'not __C__00var0000is0a000var001__1005__C__'
to_compile += ' and not __C__00var0000is0an0y__1006__C__'
to_compile += ' and not x > 5'
to_compile += ' and not __C__00var0000is0a000var001__1005_1__C__'
ast_ = ast.parse(to_compile, "<source>", 'eval')
expected_python_node = PythonNode(to_compile, ast_, expression)
assert len(return_values) == 1
ret = return_values[0]
python_node = ret.body.body
assert python_node == expected_python_node
assert python_node.objects == {
"__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
"__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
"__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
}
def test_i_can_get_compiled_expr_from_mix(self):
sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
Concept("animal"),
Concept("a cat"),
Concept("dog"),
Concept("pet"),
Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
create_new=True
).unpack()
expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
parser = ExpressionParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
assert len(return_values) == 1
ret = return_values[0]
to_compile = '__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1006__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__'
ast_ = ast.parse(to_compile, "<source>", 'eval')
expected_python_node = PythonNode(to_compile, ast_, expression)
python_node = ret.body.body
assert python_node == expected_python_node
assert python_node.objects == {
"__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
"__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
"__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
}
def test_i_can_get_compiled_expr_when_multiple_choices(self):
sheerka, context, *concepts = self.init_test().with_concepts(
Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
create_new=True
).unpack()
parser = ExpressionParser()
expression = "a is a b"
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
assert len(return_values) == 2
ret = return_values[0]
assert sheerka.objvalue(ret)[0].concept == CMV(concepts[0], x="a", y="b")
ret = return_values[1]
assert sheerka.objvalue(ret)[0].concept == CMV(concepts[1], x="a", y="b")
def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self):
sheerka, context, *concepts = self.init_test().with_concepts(
Concept("animal"),
Concept("a cat"),
Concept("dog"),
Concept("pet"),
Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
create_new=True
).unpack()
expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
parser = ExpressionParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
assert len(return_values) == 4
trimmed_source = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
current_ret = return_values[0]
python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__"
ast_ = ast.parse(python_source, "<source>", 'eval')
resolved_expected = PythonNode(python_source, ast_, trimmed_source)
assert sheerka.objvalue(current_ret) == resolved_expected
current_ret = return_values[1]
assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006__C__"
ast_ = ast.parse(python_source, "<source>", 'eval')
resolved_expected = PythonNode(python_source, ast_, trimmed_source)
assert sheerka.objvalue(current_ret) == resolved_expected
current_ret = return_values[2]
assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005__C__"
ast_ = ast.parse(python_source, "<source>", 'eval')
resolved_expected = PythonNode(python_source, ast_, trimmed_source)
assert sheerka.objvalue(current_ret) == resolved_expected
current_ret = return_values[3]
python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006_1__C__"
ast_ = ast.parse(python_source, "<source>", 'eval')
resolved_expected = PythonNode(python_source, ast_, trimmed_source)
assert sheerka.objvalue(current_ret) == resolved_expected
@pytest.mark.parametrize("expression, expected_conditions, test_obj", [
(
"__ret",
["#__x_00__|__name__|'__ret'"],
ReturnValueConcept("Test", True, None)
),
(
"__ret.status == True",
["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
ReturnValueConcept("Test", True, None)
),
(
"__ret.status",
["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
ReturnValueConcept("Test", True, None)
),
(
"__ret and __ret.status",
["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
ReturnValueConcept("Test", True, None)
),
])
def test_i_can_get_rete_condition_from_python(self, expression, expected_conditions, test_obj):
sheerka, context, = self.init_test().unpack()
expected_full_condition = get_rete_conditions(*expected_conditions)
parser = ExpressionParser()
expr_node = parser.parse(context, ParserInput(expression)).body.body
nodes = expr_node.parts if isinstance(expr_node, AndNode) else [expr_node]
_, rete_disjunctions = parser.compile_conjunctions(context, nodes, "test")
assert len(rete_disjunctions) == 1
assert rete_disjunctions == [expected_full_condition]
# check against a Rete network
network = ReteNetwork()
rule = Rule("test", expression, None)
rule.metadata.id = 9999
rule.metadata.is_compiled = True
rule.metadata.is_enabled = True
rule.rete_disjunctions = rete_disjunctions
network.add_rule(rule)
network.add_obj("__ret", test_obj)
matches = list(network.matches)
assert len(matches) > 0