Fixed #131 : Implement ExprToConditions
Fixed #130 : ArithmeticOperatorParser Fixed #129 : python_wrapper : create_namespace Fixed #128 : ExpressionParser: Cannot parse func(x) infixed concept 'xxx'
This commit is contained in:
@@ -3,13 +3,11 @@ import pytest
|
||||
from core.builtin_concepts import BuiltinConcepts
|
||||
from core.concept import Concept
|
||||
from core.sheerka.services.SheerkaExecute import ParserInput
|
||||
from parsers.BaseNodeParser import SourceCodeWithConceptNode
|
||||
from parsers.BaseParser import ErrorSink
|
||||
from parsers.BaseParser import ErrorSink, UnexpectedTokenParsingError
|
||||
from parsers.FunctionParser import FunctionParser
|
||||
from parsers.PythonParser import PythonErrorNode
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
from tests.parsers.parsers_utils import compute_expected_array, SCN, SCWC, CN, UTN, CNC, RN, FN, get_test_obj, \
|
||||
get_expr_node_from_test_node
|
||||
from tests.parsers.parsers_utils import FN, SCN, SEQ, compute_expected_array, \
|
||||
get_expr_node_from_test_node, get_test_obj
|
||||
|
||||
cmap = {
|
||||
"one": Concept("one"),
|
||||
@@ -24,7 +22,7 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
init_test_helper = cls().init_test(cache_only=False, ontology="#TestFunctionParser#")
|
||||
init_test_helper = cls().init_test(cache_only=False, ontology="#TestFunctionParserOld#")
|
||||
sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
|
||||
for i, concept_name in enumerate(cmap):
|
||||
cmap[concept_name] = updated[i]
|
||||
@@ -32,7 +30,7 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
|
||||
cls.shared_ontology = sheerka.get_ontology(context)
|
||||
sheerka.pop_ontology(context)
|
||||
|
||||
def init_parser(self, my_concepts_map=None, **kwargs):
|
||||
def init_parser(self, my_concepts_map=None, strict=True, **kwargs):
|
||||
if my_concepts_map is None:
|
||||
sheerka, context = self.init_test().unpack()
|
||||
sheerka.add_ontology(context, self.shared_ontology)
|
||||
@@ -41,11 +39,12 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
|
||||
for i, pair in enumerate(my_concepts_map):
|
||||
my_concepts_map[pair] = updated[i]
|
||||
|
||||
parser = FunctionParser()
|
||||
expr_parser = kwargs.get("expr_parser", None)
|
||||
parser = FunctionParser(strict, expr_parser=expr_parser)
|
||||
return sheerka, context, parser
|
||||
|
||||
def init_parser_with_source(self, source):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
def init_parser_with_source(self, source, strict=True):
|
||||
sheerka, context, parser = self.init_parser(None, strict)
|
||||
error_sink = ErrorSink()
|
||||
parser_input = ParserInput(source)
|
||||
parser.reset_parser_input(parser_input, error_sink)
|
||||
@@ -70,180 +69,146 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
|
||||
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
||||
|
||||
@pytest.mark.parametrize("expression, expected", [
|
||||
("func()", FN("func(", ")", [])),
|
||||
("concept(one)", FN("concept(", ")", ["one"])),
|
||||
("func(one)", FN("func(", ")", ["one"])),
|
||||
("func(a long two, 'three', ;:$*)", FN("func(", ")", ["a long two, ", "'three', ", ";:$*"])),
|
||||
("func(func1(one), two, func2(func3(), func4(three)))", FN("func(", (")", 4), [
|
||||
(FN("func1(", ")", ["one"]), ", "),
|
||||
"two, ",
|
||||
(FN("func2(", (")", 3), [
|
||||
(FN("func3(", (")", 1), []), ", "),
|
||||
(FN("func4(", (")", 2), ["three"]), None),
|
||||
]), None)
|
||||
])),
|
||||
("func(r:|1:)", FN("func(", ")", ["r:|1:"]))
|
||||
("func()", FN("func")),
|
||||
("concept(one)", FN("concept", "one")),
|
||||
("func(one)", FN("func", "one")),
|
||||
("func(a long two, 'three', ;:$*)", FN("func", "a long two", "'three'", ";:$*")),
|
||||
("func(func1(one), two, func2(func3(), func4(three)))",
|
||||
FN("func", "func1(one)", "two", "func2(func3(), func4(three))")),
|
||||
("func(r:|1:)", FN("func", "r:|1:")),
|
||||
("func(bar fn(x))", FN("func", "bar fn(x)")),
|
||||
("func([one, two])", FN("func", "[one, two]")),
|
||||
("func((one, two))", FN("func", "(one, two)")),
|
||||
("func({one, two})", FN("func", "{one, two}")),
|
||||
])
|
||||
def test_i_can_parse_function(self, expression, expected):
|
||||
def test_i_can_parse_input_function_when_expr_parser_is_none(self, expression, expected):
|
||||
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
|
||||
expected = get_expr_node_from_test_node(expression, expected)
|
||||
|
||||
parsed = parser.parse_input(context, parser_input, error_sink)
|
||||
|
||||
assert not error_sink.has_error
|
||||
assert parsed == expected
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
("func()", SCN("func()")),
|
||||
(" func()", SCN("func()")),
|
||||
("func(one)", SCWC("func(", ")", CN("one"))),
|
||||
("func(one, unknown, two)", SCWC("func(", ")", CN("one"), ", ", UTN("unknown"), (", ", 1), CN("two"))),
|
||||
("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))),
|
||||
("func(one plus two, three)", SCWC("func(", ")", CNC("plus", a="one", b="two"), ", ", UTN("three"))),
|
||||
("func(func1(one), two)", SCWC("func(", (")", 1), SCWC("func1(", ")", "one"), ", ", "two"))
|
||||
parser.strict = False
|
||||
parser.reset_parser_input(parser_input, error_sink)
|
||||
parsed = parser.parse_input(context, parser_input, error_sink)
|
||||
assert not error_sink.has_error
|
||||
assert parsed == expected
|
||||
|
||||
def test_i_can_parse_input_when_prefixed_depending_on_strict_mode(self):
|
||||
expression = "bar fn(x)"
|
||||
expected = SEQ("bar ", FN("fn", "x"))
|
||||
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
|
||||
expected = get_expr_node_from_test_node(expression, expected)
|
||||
|
||||
parsed = parser.parse_input(context, parser_input, error_sink)
|
||||
assert error_sink.has_error
|
||||
assert parsed is None
|
||||
assert isinstance(error_sink.sink[0], UnexpectedTokenParsingError)
|
||||
|
||||
parser.strict = False
|
||||
parser.reset_parser_input(parser_input, error_sink)
|
||||
parsed = parser.parse_input(context, parser_input, error_sink)
|
||||
assert not error_sink.has_error
|
||||
assert parsed == expected
|
||||
|
||||
def test_i_can_parse_input_when_suffixed_depending_on_strict_mode(self):
|
||||
expression = "fn(x) bar"
|
||||
expected = SEQ(FN("fn", "x"), " bar")
|
||||
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
|
||||
expected = get_expr_node_from_test_node(expression, expected)
|
||||
|
||||
parsed = parser.parse_input(context, parser_input, error_sink)
|
||||
assert error_sink.has_error
|
||||
assert parsed is None
|
||||
assert isinstance(error_sink.sink[0], UnexpectedTokenParsingError)
|
||||
|
||||
parser.strict = False
|
||||
parser.reset_parser_input(parser_input, error_sink)
|
||||
parsed = parser.parse_input(context, parser_input, error_sink)
|
||||
assert not error_sink.has_error
|
||||
assert parsed == expected
|
||||
|
||||
def test_i_can_parse_input_when_infixed_and_suffixed(self):
|
||||
expression = "foo fn(x) bar"
|
||||
expected = SEQ("foo ", FN("fn", "x"), " bar")
|
||||
sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
|
||||
expected = get_expr_node_from_test_node(expression, expected)
|
||||
|
||||
parser.strict = False
|
||||
parser.reset_parser_input(parser_input, error_sink)
|
||||
parsed = parser.parse_input(context, parser_input, error_sink)
|
||||
assert not error_sink.has_error
|
||||
assert parsed == expected
|
||||
|
||||
@pytest.mark.parametrize("text, expected, objects", [
|
||||
("func()", SCN("func()"), {}),
|
||||
(" func()", SCN("func()"), {}),
|
||||
("func(one)", SCN("func(one)"), {"__o_00__": "one"}),
|
||||
("func(one, unknown, two)", SCN("func(one, unknown, two)"), {"__o_00__": "one", "__o_01__": "two"}),
|
||||
("func(one, twenty two)", SCN("func(one, twenty two)"), {"__o_00__": "one", "__o_01__": "twenties"}),
|
||||
("func(one plus two, three)", SCN("func(one plus two, three)"), {"__o_00__": "plus"}),
|
||||
("func(func1(one), two)", SCN("func(func1(one), two)"), {"__o_00__": "one", "__o_01__": "two"}),
|
||||
])
|
||||
def test_i_can_parse(self, text, expected):
|
||||
def test_i_can_parse(self, text, expected, objects):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
parser_result = res.body
|
||||
expression = res.body.body
|
||||
node = res.body.body
|
||||
|
||||
assert res.status
|
||||
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
|
||||
transformed_expression = get_test_obj(expression, resolved_expected)
|
||||
transformed_expression = get_test_obj(node, resolved_expected)
|
||||
assert transformed_expression == resolved_expected
|
||||
assert expression.python_node is not None
|
||||
assert expression.return_value is not None
|
||||
assert node.python_node is not None
|
||||
assert node.return_value is not None
|
||||
|
||||
def test_i_can_parse_when_multiple_results_when_requested(self):
|
||||
# the previous output was
|
||||
# [
|
||||
# SCWC("func(", ")", "one", ", ", "twenty ", "two"),
|
||||
# SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))
|
||||
# ]
|
||||
# But the first one is now filtered out, as it's not a valid python function call
|
||||
sheerka, context, parser = self.init_parser()
|
||||
parser.longest_concepts_only = False
|
||||
text = "func(one, twenty two)"
|
||||
expected = [SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))]
|
||||
resolved_expected = compute_expected_array(cmap, text, expected)
|
||||
# compare objects
|
||||
actual_objects = {k: v.id for k, v in node.python_node.objects.items()}
|
||||
expected_objects = {k: cmap[v].id for k, v in objects.items()}
|
||||
assert actual_objects == expected_objects
|
||||
|
||||
results = parser.parse(context, ParserInput(text))
|
||||
|
||||
assert len(results) == 2
|
||||
|
||||
res = results[0]
|
||||
assert not res.status
|
||||
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
|
||||
assert len(res.body.body) == 1
|
||||
assert (res.body.body[0], PythonErrorNode)
|
||||
|
||||
res = results[1]
|
||||
parser_result = res.body
|
||||
expressions = res.body.body
|
||||
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
|
||||
transformed_expressions = get_test_obj(expressions, resolved_expected[0])
|
||||
assert transformed_expressions == resolved_expected[0]
|
||||
|
||||
def test_i_can_parse_when_the_parameter_is_not_a_concept(self):
|
||||
"""
|
||||
It's not a concept, but it can be a valid short term memory object
|
||||
:return:
|
||||
"""
|
||||
sheerka, context, parser = self.init_parser()
|
||||
text = "func(unknown_concept)"
|
||||
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
expected = [SCWC("func(", ")", "unknown_concept")]
|
||||
resolved_expected = compute_expected_array(cmap, text, expected)
|
||||
|
||||
assert res.status
|
||||
parsed = res.body.body
|
||||
transformed_parsed = get_test_obj([parsed], resolved_expected)
|
||||
assert transformed_parsed == resolved_expected
|
||||
|
||||
def test_i_can_parse_when_the_concept_is_not_found(self):
|
||||
"""
|
||||
We do not check yet if it's a valid concept
|
||||
If you find a cheap way to do so, simply remove this test
|
||||
:return:
|
||||
"""
|
||||
sheerka, context, parser = self.init_parser()
|
||||
text = "func(c:|xxx:)"
|
||||
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
|
||||
assert res.status
|
||||
|
||||
def test_i_can_parse_when_rules(self):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
text = "func(r:|1:)"
|
||||
expected = SCWC("func(", ")", RN("1"))
|
||||
resolved_expected = compute_expected_array(cmap, text, [expected])[0]
|
||||
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
parser_result = res.body
|
||||
expression = res.body.body
|
||||
transformed_expression = get_test_obj(expression, resolved_expected)
|
||||
|
||||
assert res.status
|
||||
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
|
||||
assert transformed_expression == resolved_expected
|
||||
assert expression.python_node is not None
|
||||
assert expression.return_value is not None
|
||||
|
||||
def test_i_can_parse_when_the_parameter_is_a_dynamic_concept(self):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
text = "func(ones)"
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
|
||||
assert res.status
|
||||
assert isinstance(res.body.body, SourceCodeWithConceptNode)
|
||||
assert res.body.body.python_node.source == 'func(__C__ones__1001___PLURAL__C__)'
|
||||
assert "__C__ones__1001___PLURAL__C__" in res.body.body.python_node.objects
|
||||
|
||||
@pytest.mark.parametrize("text, expected_error_type", [
|
||||
("one", BuiltinConcepts.NOT_FOR_ME), # no function found
|
||||
("$*!", BuiltinConcepts.NOT_FOR_ME), # no function found
|
||||
("func(", BuiltinConcepts.ERROR), # function found, but incomplete
|
||||
("func(one", BuiltinConcepts.ERROR), # function found, but incomplete
|
||||
("func(one, two, ", BuiltinConcepts.ERROR), # function found, but incomplete
|
||||
("func(one) and func(two)", BuiltinConcepts.ERROR), # to many function
|
||||
("one func(one)", BuiltinConcepts.NOT_FOR_ME), # function not found ! (as it is not the first)
|
||||
("func(a=b, c)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
|
||||
("func(one two)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
|
||||
])
|
||||
def test_i_cannot_parse(self, text, expected_error_type):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
|
||||
assert not res.status
|
||||
assert sheerka.isinstance(res.body, expected_error_type)
|
||||
|
||||
@pytest.mark.parametrize("sequence, expected", [
|
||||
(None, None),
|
||||
([["a"]], [["a"]]),
|
||||
([["a"], ["b", "c"]], [["a"]]),
|
||||
([["b", "c"], ["a"]], [["a"]]),
|
||||
([["b", "c"], ["a"], ["d", "e"], ["f"]], [["a"], ["f"]]),
|
||||
])
|
||||
def test_i_can_get_the_longest_concept_sequence(self, sequence, expected):
|
||||
assert FunctionParser.get_longest_concepts(sequence) == expected
|
||||
|
||||
def test_concepts_found_are_fully_initialized(self):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
res = parser.parse(context, ParserInput("func(one plus three)"))
|
||||
concept = res.body.body.nodes[0].concept
|
||||
|
||||
assert res.status
|
||||
assert isinstance(concept.get_compiled()["a"], Concept)
|
||||
|
||||
# three is not recognized,
|
||||
# so it will be transformed into list of ReturnValueConcept that indicate how to recognized it
|
||||
assert isinstance(concept.get_compiled()["b"], list)
|
||||
for item in concept.get_compiled()["b"]:
|
||||
assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)
|
||||
# @pytest.mark.parametrize("text, expected_error_type", [
|
||||
# ("one", BuiltinConcepts.NOT_FOR_ME), # no function found
|
||||
# ("$*!", BuiltinConcepts.NOT_FOR_ME), # no function found
|
||||
# ("func(", BuiltinConcepts.ERROR), # function found, but incomplete
|
||||
# ("func(one", BuiltinConcepts.ERROR), # function found, but incomplete
|
||||
# ("func(one, two, ", BuiltinConcepts.ERROR), # function found, but incomplete
|
||||
# ("func(one) and func(two)", BuiltinConcepts.ERROR), # to many function
|
||||
# ("one func(one)", BuiltinConcepts.NOT_FOR_ME), # function not found ! (as it is not the first)
|
||||
# ("func(a=b, c)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
|
||||
# ("func(one two)", BuiltinConcepts.ERROR), # function found, but cannot be parsed
|
||||
# ])
|
||||
# def test_i_cannot_parse(self, text, expected_error_type):
|
||||
# sheerka, context, parser = self.init_parser()
|
||||
#
|
||||
# res = parser.parse(context, ParserInput(text))
|
||||
#
|
||||
# assert not res.status
|
||||
# assert sheerka.isinstance(res.body, expected_error_type)
|
||||
#
|
||||
# @pytest.mark.parametrize("sequence, expected", [
|
||||
# (None, None),
|
||||
# ([["a"]], [["a"]]),
|
||||
# ([["a"], ["b", "c"]], [["a"]]),
|
||||
# ([["b", "c"], ["a"]], [["a"]]),
|
||||
# ([["b", "c"], ["a"], ["d", "e"], ["f"]], [["a"], ["f"]]),
|
||||
# ])
|
||||
# def test_i_can_get_the_longest_concept_sequence(self, sequence, expected):
|
||||
# assert FunctionParserOld.get_longest_concepts(sequence) == expected
|
||||
#
|
||||
# def test_concepts_found_are_fully_initialized(self):
|
||||
# sheerka, context, parser = self.init_parser()
|
||||
#
|
||||
# res = parser.parse(context, ParserInput("func(one plus three)"))
|
||||
# concept = res.body.body.nodes[0].concept
|
||||
#
|
||||
# assert res.status
|
||||
# assert isinstance(concept.get_compiled()["a"], Concept)
|
||||
#
|
||||
# # three is not recognized,
|
||||
# # so it will be transformed into list of ReturnValueConcept that indicate how to recognized it
|
||||
# assert isinstance(concept.get_compiled()["b"], list)
|
||||
# for item in concept.get_compiled()["b"]:
|
||||
# assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)
|
||||
|
||||
Reference in New Issue
Block a user