Fixed #109 : Mix python and concept. List comprehension
Fixed #110 : SheerkaDebugManager: add list_debug_settings Fixed #111 : SheerkaDebugManager: Implement ListDebugLogger Fixed #112 : SyaNodeParser: rewrite this parser Fixed #113 : Sheerka.: Add enable_parser_caching to disable parsers caching Fixed #114 : SyaNodeParser : Implement fast cache to resolve unrecognized tokens requests Fixed #115 : BnfNodeParser : Implement fast cache to resolve unrecognized tokens requests Fixed #116 : SequenceNodeParser : Implement fast cache to resolve unrecognized tokens requests Fixed #117 : ResolveMultiplePluralAmbiguityEvaluator: Resolve Multiple plural ambiguity
This commit is contained in:
@@ -0,0 +1,422 @@
|
||||
import pytest
|
||||
|
||||
from core.builtin_concepts_ids import BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints
|
||||
from core.sheerka.services.SheerkaExecute import ParserInput
|
||||
from evaluators.PythonEvaluator import PythonEvaluator
|
||||
from parsers.BaseParser import ErrorSink
|
||||
from parsers.ExpressionParser import ExpressionParser
|
||||
from parsers.ListComprehensionParser import ListComprehensionParser
|
||||
from parsers.PythonParser import PythonNode
|
||||
from sheerkapython.ExprToPython import PythonExprVisitor
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
|
||||
|
||||
class TestExprToPython(TestUsingMemoryBasedSheerka):
|
||||
|
||||
@staticmethod
|
||||
def get_expr_node(context, expression, parser=None):
|
||||
parser = parser or ExpressionParser()
|
||||
error_sink = ErrorSink()
|
||||
parser_input = ParserInput(expression)
|
||||
parser.reset_parser_input(parser_input, error_sink)
|
||||
parsed = parser.parse_input(context, parser_input, error_sink)
|
||||
|
||||
assert not error_sink.has_error
|
||||
|
||||
return parsed
|
||||
|
||||
@staticmethod
|
||||
def eval(context, return_value, namespace=None):
|
||||
evaluator = PythonEvaluator()
|
||||
assert evaluator.matches(context, return_value)
|
||||
|
||||
if namespace:
|
||||
for k, v in namespace.items():
|
||||
context.add_to_short_term_memory(k, v)
|
||||
|
||||
res = evaluator.eval(context, return_value)
|
||||
assert res.status
|
||||
return res.body
|
||||
|
||||
@pytest.mark.parametrize("expression, source, objects", [
|
||||
("foo w", "call_concept(__o_00__, x=w)", {"__o_00__": "foo"}),
|
||||
("foo z + 2", "call_concept(__o_00__, x=z + 2)", {"__o_00__": "foo"}),
|
||||
("foo a and bar b",
|
||||
"call_concept(__o_00__, x=a) and call_concept(__o_01__, y=b)",
|
||||
{"__o_00__": "foo", "__o_01__": "bar"}),
|
||||
("foo a or bar b",
|
||||
"call_concept(__o_00__, x=a) or call_concept(__o_01__, y=b)",
|
||||
{"__o_00__": "foo", "__o_01__": "bar"}),
|
||||
("not foo w", "not call_concept(__o_00__, x=w)", {"__o_00__": "foo"}),
|
||||
("foo a >= bar b",
|
||||
"call_concept(__o_00__, x=a) >= call_concept(__o_01__, y=b)",
|
||||
{"__o_00__": "foo", "__o_01__": "bar"}),
|
||||
("function(foo a, bar b)",
|
||||
"function(call_concept(__o_00__, x=a), call_concept(__o_01__, y=b))",
|
||||
{"__o_00__": "foo", "__o_01__": "bar"})
|
||||
|
||||
])
|
||||
def test_i_can_compile_concept_when_is_question_is_false(self, expression, source, objects):
|
||||
sheerka, context, foo, bar = self.init_test().with_concepts(
|
||||
Concept("foo x", body="x").def_var("x"),
|
||||
Concept("bar y", body="y").def_var("y"),
|
||||
create_new=True
|
||||
).unpack()
|
||||
|
||||
concepts = {
|
||||
"foo": foo,
|
||||
"bar": bar
|
||||
}
|
||||
|
||||
node = self.get_expr_node(context, expression)
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == source
|
||||
|
||||
for obj_name, obj_value in objects.items():
|
||||
assert obj_name in python_node.objects
|
||||
|
||||
obj = python_node.objects[obj_name]
|
||||
if isinstance(obj, Concept):
|
||||
assert sheerka.isinstance(obj, concepts[obj_value])
|
||||
assert obj.get_hints().use_copy
|
||||
assert obj.get_hints().is_evaluated
|
||||
else:
|
||||
assert False
|
||||
|
||||
@pytest.mark.parametrize("expression, source, objects", [
|
||||
("foo w", "evaluate_question(__o_00__, x=w)", {"__o_00__": "foo"}),
|
||||
("foo z + 2", "evaluate_question(__o_00__, x=z + 2)", {"__o_00__": "foo"}),
|
||||
])
|
||||
def test_i_can_compile_concept_when_is_question_is_true(self, expression, source, objects):
|
||||
sheerka, context, foo = self.init_test().with_concepts(
|
||||
Concept("foo x", body="x", pre="is_question()").def_var("x"),
|
||||
create_new=True
|
||||
).unpack()
|
||||
|
||||
concepts = {
|
||||
"foo": foo
|
||||
}
|
||||
|
||||
node = self.get_expr_node(context, expression)
|
||||
visitor = PythonExprVisitor(context)
|
||||
ret = visitor.compile(node, EvaluationHints(eval_body=True, eval_question=True))
|
||||
|
||||
assert len(ret) == 1
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == source
|
||||
|
||||
for obj_name, obj_value in objects.items():
|
||||
assert obj_name in python_node.objects
|
||||
|
||||
obj = python_node.objects[obj_name]
|
||||
if isinstance(obj, Concept):
|
||||
assert sheerka.isinstance(obj, concepts[obj_value])
|
||||
assert obj.get_hints().use_copy
|
||||
assert obj.get_hints().is_evaluated
|
||||
else:
|
||||
assert False
|
||||
|
||||
def test_i_can_compile_simple_list_comprehension(self):
|
||||
sheerka, context = self.init_test().unpack()
|
||||
|
||||
expression = "[ x for x in ['a', 'b'] if x == 'a' ]"
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
|
||||
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
|
||||
assert isinstance(ret[0].body.body, PythonNode)
|
||||
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == expression
|
||||
|
||||
assert self.eval(context, ret[0]) == ["a"]
|
||||
|
||||
def test_i_can_compile_simple_list_comprehension_when_no_if(self):
|
||||
sheerka, context = self.init_test().unpack()
|
||||
|
||||
expression = "[ x for x in ['a', 'b'] ]"
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
|
||||
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
|
||||
assert isinstance(ret[0].body.body, PythonNode)
|
||||
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == expression
|
||||
|
||||
assert self.eval(context, ret[0]) == ['a', 'b']
|
||||
|
||||
def test_i_can_compile_list_comprehension_when_element_is_a_concept(self):
|
||||
sheerka, context, foo = self.init_test().with_concepts(
|
||||
Concept("foo x", body="x").def_var("x")
|
||||
).unpack()
|
||||
|
||||
expression = "[ foo w for w in ['a', 'b'] ]"
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
|
||||
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
|
||||
assert isinstance(ret[0].body.body, PythonNode)
|
||||
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == "[ call_concept(__o_00__, x=w) for w in ['a', 'b'] ]"
|
||||
assert "__o_00__" in python_node.objects
|
||||
|
||||
concept = python_node.objects["__o_00__"]
|
||||
assert sheerka.isinstance(concept, foo)
|
||||
assert concept.get_hints().use_copy
|
||||
assert concept.get_hints().is_evaluated
|
||||
|
||||
assert self.eval(context, ret[0]) == ["a", "b"]
|
||||
|
||||
def test_i_can_compile_list_comprehension_when_concept_with_complex_parameter(self):
|
||||
sheerka, context, foo = self.init_test().with_concepts(
|
||||
Concept("foo x", body="x").def_var("x"),
|
||||
create_new=True
|
||||
).unpack()
|
||||
|
||||
expression = "[ foo w + 1 for w in [1, 2] ]"
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
|
||||
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
|
||||
assert isinstance(ret[0].body.body, PythonNode)
|
||||
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == "[ call_concept(__o_00__, x=w + 1) for w in [1, 2] ]"
|
||||
assert "__o_00__" in python_node.objects
|
||||
|
||||
assert self.eval(context, ret[0]) == [2, 3]
|
||||
|
||||
def test_i_can_compile_list_comprehension_when_iter_is_a_concept(self):
|
||||
sheerka, context, red, blue, color, foo = self.init_test().with_concepts(
|
||||
"red",
|
||||
"blue",
|
||||
"color",
|
||||
Concept("foo x", body="x").def_var("x")
|
||||
).unpack()
|
||||
|
||||
global_truth_context = self.get_context(sheerka, global_truth=True)
|
||||
sheerka.set_isa(global_truth_context, red, color)
|
||||
sheerka.set_isa(global_truth_context, blue, color)
|
||||
|
||||
expression = "[ foo x for x in colors ]"
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
|
||||
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
|
||||
assert isinstance(ret[0].body.body, PythonNode)
|
||||
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == "[ call_concept(__o_00__, x=x) for x in call_concept(__o_01__) ]"
|
||||
assert "__o_00__" in python_node.objects
|
||||
assert "__o_01__" in python_node.objects
|
||||
|
||||
concept0 = python_node.objects["__o_00__"]
|
||||
assert sheerka.isinstance(concept0, foo)
|
||||
assert concept0.get_hints().use_copy
|
||||
assert concept0.get_hints().is_evaluated
|
||||
|
||||
concept1 = python_node.objects["__o_01__"]
|
||||
assert sheerka.isinstance(concept1, "colors")
|
||||
assert concept1.get_hints().use_copy
|
||||
assert concept1.get_hints().is_evaluated
|
||||
|
||||
assert set(self.eval(context, ret[0])) == {red, blue}
|
||||
|
||||
def test_i_can_compile_list_comprehension_when_if_expression_is_a_concept(self):
|
||||
sheerka, context, red, blue, color, foo, startswith = self.init_test().with_concepts(
|
||||
"red",
|
||||
"blue",
|
||||
"color",
|
||||
Concept("foo x", body="x").def_var("x"),
|
||||
Concept("x starts with y", body="x.name.startswith(y)", pre="is_question()").def_var("x").def_var("y")
|
||||
).unpack()
|
||||
global_truth_context = self.get_context(sheerka, global_truth=True)
|
||||
sheerka.set_isa(global_truth_context, red, color)
|
||||
sheerka.set_isa(global_truth_context, blue, color)
|
||||
|
||||
expression = "[ foo x for x in colors if x starts with 'b' ]"
|
||||
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
|
||||
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
|
||||
assert isinstance(ret[0].body.body, PythonNode)
|
||||
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == "[ call_concept(__o_00__, x=x) for x in call_concept(__o_01__) if evaluate_question(__o_02__, x=x, y='b') ]"
|
||||
assert "__o_00__" in python_node.objects
|
||||
assert "__o_01__" in python_node.objects
|
||||
assert "__o_02__" in python_node.objects
|
||||
assert visitor.obj_counter == 3
|
||||
|
||||
assert set(self.eval(context, ret[0])) == {blue}
|
||||
|
||||
def test_i_can_compile_list_comprehension_when_multiple_concepts(self):
|
||||
sheerka, context, foo1, foo2, bar1, bar2, colors1, colors2, = self.init_test().with_concepts(
|
||||
Concept("foo x").def_var("x"),
|
||||
Concept("foo y", ).def_var("y"),
|
||||
Concept("bar x", pre="is_question()").def_var("x"),
|
||||
Concept("bar y", pre="is_question()").def_var("y"),
|
||||
Concept("colors", body="[1]"),
|
||||
Concept("colors", body="[2]"),
|
||||
).unpack()
|
||||
|
||||
expression = "[ foo a for a in colors if bar a ]"
|
||||
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 8
|
||||
python_node = ret[0].body.body
|
||||
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
|
||||
assert python_node.source == '[ call_concept(__o_00__, x=a) for a in call_concept(__o_02__) if evaluate_question(__o_04__, x=a) ]'
|
||||
assert object_to_compare == {"__o_00__": "foo x", "__o_02__": "colors", "__o_04__": "bar x"}
|
||||
|
||||
# ...
|
||||
|
||||
python_node = ret[7].body.body
|
||||
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
|
||||
assert python_node.source == '[ call_concept(__o_01__, y=a) for a in call_concept(__o_03__) if evaluate_question(__o_05__, y=a) ]'
|
||||
assert object_to_compare == {"__o_01__": "foo y", "__o_03__": "colors", "__o_05__": "bar y"}
|
||||
|
||||
def test_i_can_compile_list_comprehension_when_missing_concept_parameter(self):
|
||||
sheerka, context, foo = self.init_test().with_concepts(
|
||||
Concept("foo x y", body="x").def_var("x").def_var("y")
|
||||
).unpack()
|
||||
|
||||
expression = "[ foo x k for x in ['a', 'b'] ]"
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
python_node = ret[0].body.body
|
||||
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
|
||||
assert python_node.source == "[ call_concept(__o_00__, x=x, y=k) for x in ['a', 'b'] ]"
|
||||
assert object_to_compare == {"__o_00__": "foo x y"}
|
||||
|
||||
def test_i_can_compile_simple_list_comprehension_when_multiple_for(self):
|
||||
sheerka, context = self.init_test().unpack()
|
||||
|
||||
expression = "[ (x, y) for x in ['a', 'b'] if x == 'a' for y in ['c', 'd'] if y == 'c' ]"
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
|
||||
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
|
||||
assert isinstance(ret[0].body.body, PythonNode)
|
||||
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == expression
|
||||
|
||||
assert self.eval(context, ret[0]) == [("a", "c")]
|
||||
|
||||
def test_i_can_compile_and_when_multiple_results(self):
|
||||
sheerka, context, foo, foo2, bar, bar2 = self.init_test().with_concepts(
|
||||
Concept("foo x", body="x").def_var("x"),
|
||||
Concept("foo y", body="y").def_var("y"),
|
||||
Concept("bar x", body="x").def_var("x"),
|
||||
Concept("bar y", body="y").def_var("y"),
|
||||
create_new=True
|
||||
).unpack()
|
||||
|
||||
node = self.get_expr_node(context, "foo a and bar b")
|
||||
visitor = PythonExprVisitor(context)
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 4
|
||||
python_node = ret[0].body.body
|
||||
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
|
||||
assert python_node.source == 'call_concept(__o_00__, x=a) and call_concept(__o_02__, x=b)'
|
||||
assert object_to_compare == {"__o_00__": "foo x", "__o_02__": "bar x"}
|
||||
|
||||
python_node = ret[1].body.body
|
||||
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
|
||||
assert python_node.source == 'call_concept(__o_00__, x=a) and call_concept(__o_03__, y=b)'
|
||||
assert object_to_compare == {"__o_00__": "foo x", "__o_03__": "bar y"}
|
||||
|
||||
python_node = ret[2].body.body
|
||||
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
|
||||
assert python_node.source == 'call_concept(__o_01__, y=a) and call_concept(__o_02__, x=b)'
|
||||
assert object_to_compare == {"__o_01__": "foo y", "__o_02__": "bar x"}
|
||||
|
||||
python_node = ret[3].body.body
|
||||
object_to_compare = {k: v.name for k, v in python_node.objects.items()}
|
||||
assert python_node.source == 'call_concept(__o_01__, y=a) and call_concept(__o_03__, y=b)'
|
||||
assert object_to_compare == {"__o_01__": "foo y", "__o_03__": "bar y"}
|
||||
|
||||
def test_i_can_compile_when_element_is_missing_its_parenthesis(self):
|
||||
sheerka, context, foo = self.init_test().with_concepts(
|
||||
Concept("foo x", body="x").def_var("x")
|
||||
).unpack()
|
||||
|
||||
expression = "[ w, foo w for w in ['a', 'b'] ]"
|
||||
node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
|
||||
visitor = PythonExprVisitor(context)
|
||||
|
||||
ret = visitor.compile(node)
|
||||
|
||||
assert len(ret) == 1
|
||||
assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
|
||||
assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
|
||||
assert isinstance(ret[0].body.body, PythonNode)
|
||||
|
||||
python_node = ret[0].body.body
|
||||
assert python_node.original_source == expression
|
||||
assert python_node.source == "[ (w, call_concept(__o_00__, x=w)) for w in ['a', 'b'] ]"
|
||||
assert "__o_00__" in python_node.objects
|
||||
|
||||
concept = python_node.objects["__o_00__"]
|
||||
assert sheerka.isinstance(concept, foo)
|
||||
assert concept.get_hints().use_copy
|
||||
assert concept.get_hints().is_evaluated
|
||||
|
||||
assert self.eval(context, ret[0]) == [("a", "a"), ("b", "b")]
|
||||
Reference in New Issue
Block a user