kodjo
89e1f20975
Fixed #130 : ArithmeticOperatorParser Fixed #129 : python_wrapper : create_namespace Fixed #128 : ExpressionParser: Cannot parse func(x) infixed concept 'xxx'
501 lines
20 KiB
Python
501 lines
20 KiB
Python
import pytest
|
|
|
|
from core.builtin_concepts import BuiltinConcepts
|
|
from core.concept import Concept, DEFINITION_TYPE_DEF
|
|
from core.sheerka.services.SheerkaExecute import ParserInput
|
|
from core.tokenizer import Token, TokenKind, Tokenizer
|
|
from parsers.BaseExpressionParser import NameExprNode
|
|
from parsers.ExactConceptParser import NotSupportedElement
|
|
from parsers.SequenceNodeParser import SequenceNodeParser
|
|
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
|
from tests.parsers.parsers_utils import CN, SCN, UTN, compare_with_test_object, compute_expected_array, get_test_obj
|
|
|
|
|
|
class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
|
|
def init_parser(self, my_map, create_new=False, use_sheerka=False):
|
|
sheerka, context, *updated_concepts = self.init_test().with_concepts(
|
|
*my_map.values(),
|
|
create_new=create_new).unpack()
|
|
|
|
if use_sheerka:
|
|
parser = SequenceNodeParser(sheerka=sheerka)
|
|
else:
|
|
parser = SequenceNodeParser()
|
|
parser.init_from_concepts(context, updated_concepts)
|
|
|
|
return sheerka, context, parser
|
|
|
|
def test_i_cannot_parse_empty_string(self):
|
|
sheerka, context, parser = self.init_parser({})
|
|
|
|
res = parser.parse(context, ParserInput(""))
|
|
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
|
|
|
|
@pytest.mark.parametrize("text, expected", [
|
|
("foo", ["foo"]),
|
|
("c:foo:", [CN("foo", source="c:foo:")]),
|
|
("c:|1001:", [CN("foo", source="c:|1001:")]),
|
|
(" foo", ["foo"]),
|
|
("foo ", ["foo"]),
|
|
(" foo ", ["foo"]),
|
|
("foo bar", ["foo", "bar"]),
|
|
("foo bar twenties", ["foo", "bar", "twenties"]),
|
|
("a plus b", [CN("plus", None, 0, 4)]),
|
|
("mult", [CN("mult", "mult", 0, 0)]),
|
|
])
|
|
def test_i_can_parse_simple_sequences(self, text, expected):
|
|
concepts_map = {
|
|
"foo": Concept("foo"),
|
|
"bar": Concept("bar"),
|
|
"plus": Concept("a plus b").def_var("a").def_var("b"),
|
|
"mult": Concept("mult", definition="a mult b", definition_type=DEFINITION_TYPE_DEF).def_var("a").def_var(
|
|
"b"),
|
|
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
|
|
expected_array = compute_expected_array(concepts_map, text, expected)
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
@pytest.mark.parametrize("text, expected", [
|
|
("foo bar", ["foo bar"]),
|
|
("one two three", ["one two three"]),
|
|
("foo bar twenties one two three", ["foo bar", "twenties", "one two three"]),
|
|
])
|
|
def test_i_can_parse_long_names(self, text, expected):
|
|
concepts_map = {
|
|
"foo bar": Concept("foo bar"),
|
|
"one two three": Concept("one two three"),
|
|
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
|
|
expected_array = compute_expected_array(concepts_map, text, expected)
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
@pytest.mark.parametrize("text, expected_status, expected", [
|
|
("foo bar suffixed one", False, ["foo bar", " suffixed ", "one"]),
|
|
("foo bar one prefixed", False, ["foo bar", "one", " prefixed"]),
|
|
("foo bar one infix two", False, ["foo bar", "one", " infix ", "two"]),
|
|
("foo bar 1 + 1", False, ["foo bar", SCN(" 1 + 1")]),
|
|
("foo bar twenty one", False, ["foo bar", " twenty ", "one"]),
|
|
("foo bar x$!#", False, ["foo bar", " x$!#"]),
|
|
|
|
("suffixed one foo bar", False, ["suffixed ", "one", "foo bar"]),
|
|
("one prefixed foo bar", False, ["one", " prefixed ", "foo bar"]),
|
|
("one infix two foo bar", False, ["one", " infix ", "two", "foo bar"]),
|
|
("1 + 1 foo bar", False, [SCN("1 + 1 "), "foo bar"]),
|
|
("twenty one foo bar", False, ["twenty ", "one", "foo bar"]),
|
|
("x$!# foo bar", False, ["x$!# ", "foo bar"]),
|
|
("func(one)", False, ["func(", "one", ")"]),
|
|
])
|
|
def test_i_can_parse_when_unrecognized(self, text, expected_status, expected):
|
|
concepts_map = {
|
|
"prefixed": Concept("a prefixed").def_var("a"),
|
|
"suffixed": Concept("suffixed a").def_var("a"),
|
|
"infix": Concept("a infix b").def_var("a").def_var("b"),
|
|
"foo bar": Concept("foo bar"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status == expected_status
|
|
|
|
expected_array = compute_expected_array(concepts_map, text, expected)
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
@pytest.mark.parametrize("text, expected_status, expected", [
|
|
(" one two ", True, [CN("one", "one", 1, 1), CN("two", "two", 3, 3)]),
|
|
(" one x$!# ", False, [CN("one", "one", 1, 1), UTN(" x$!# ", 2, 7)]),
|
|
(" foo bar x$!# ", False, [CN("foo bar", "foo bar", 1, 3), UTN(" x$!# ", 4, 9)]),
|
|
])
|
|
def test_i_can_parse_when_surrounded_by_spaces(self, text, expected_status, expected):
|
|
concepts_map = {
|
|
"prefixed": Concept("a prefixed").def_var("a"),
|
|
"suffixed": Concept("prefixed a").def_var("a"),
|
|
"infix": Concept("a infix b").def_var("a").def_var("b"),
|
|
"foo bar": Concept("foo bar"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status == expected_status
|
|
|
|
expected_array = compute_expected_array(concepts_map, text, expected)
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
@pytest.mark.parametrize("text, expected", [
|
|
("one two", [["one", "two"], ["one two"]])
|
|
])
|
|
def test_i_can_parse_when_multiple_concepts_start_with_the_same_token(self, text, expected):
|
|
concepts_map = {
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"one two": Concept("one two"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
list_of_res = parser.parse(context, ParserInput(text))
|
|
assert len(list_of_res) == len(expected)
|
|
|
|
for i, res in enumerate(list_of_res):
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
expected_array = compute_expected_array(concepts_map, text, expected[i])
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
def test_i_can_parse_multiple_concepts_when_long_names_and_unrecognized(self):
|
|
concepts_map = {
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"one two": Concept("one two"),
|
|
"one two x$!# one two": Concept("one two x$!# one two"),
|
|
}
|
|
|
|
text = "one two x$!# one two"
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
list_of_res = parser.parse(context, ParserInput(text))
|
|
|
|
expected = [
|
|
(False, ["one", "two", " x$!# ", ("one", 1), ("two", 1)]),
|
|
(False, ["one", "two", " x$!# ", ("one two", 1)]),
|
|
(False, ["one two", " x$!# ", ("one", 1), ("two", 1)]),
|
|
(False, ["one two", " x$!# ", ("one two", 1)]),
|
|
(True, ["one two x$!# one two"]),
|
|
]
|
|
|
|
assert len(list_of_res) == len(expected)
|
|
|
|
for res, expected in zip(list_of_res, expected):
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status == expected[0]
|
|
expected_array = compute_expected_array(concepts_map, text, expected[1])
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
def test_i_can_parse_concepts_with_isa(self):
|
|
concepts_map = {
|
|
"one": Concept("one"),
|
|
"number": Concept("number"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
sheerka.set_isa(context, sheerka.new("one"), sheerka.new("number"))
|
|
|
|
res = parser.parse(context, ParserInput("one"))
|
|
lexer_nodes = res.body.body
|
|
expected_array = compute_expected_array(concepts_map, "one", ["one"])
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
def test_i_can_parse_concepts_with_keyword(self):
|
|
concepts_map = {
|
|
"a special concept": Concept("a special concept"),
|
|
"isa": Concept("isa"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
|
|
res = parser.parse(context, ParserInput("a special concept"))
|
|
assert res.status
|
|
lexer_nodes = res.body.body
|
|
expected_array = compute_expected_array(concepts_map, "a special concept", ["a special concept"])
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
res = parser.parse(context, ParserInput("isa"))
|
|
assert res.status
|
|
lexer_nodes = res.body.body
|
|
expected_array = compute_expected_array(concepts_map, "isa", ["isa"])
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
def test_i_can_parse_concepts_when_sub_tokens(self):
|
|
concepts_map = {
|
|
"foo": Concept("foo"),
|
|
"bar": Concept("bar"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
|
|
text = "not recognized foo bar not not recognized"
|
|
expected = ["foo", "bar"]
|
|
parser_input = ParserInput(text, start=3, end=7)
|
|
res = parser.parse(context, parser_input)
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
|
|
expected_array = compute_expected_array(concepts_map, text, expected)
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
|
|
@pytest.mark.parametrize("concept", [
|
|
Concept("foo x", body="1").def_var("x"),
|
|
Concept("foo"),
|
|
])
|
|
def test_i_can_parse_token_concepts(self, concept):
|
|
concepts_map = {
|
|
"foo": concept,
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
|
|
res = parser.parse(context, ParserInput(f"c:{concept.name}:"))
|
|
|
|
assert res.status
|
|
concept_found = res.body.body[0].concept
|
|
assert concept_found.get_hints().is_evaluated
|
|
assert not concept_found.get_hints().is_instance
|
|
|
|
def test_i_can_parse_when_variable_is_not_a_parameter(self):
|
|
concepts_map = {
|
|
"foo": Concept("foo").def_var("a"), # 'a' is not a parameter (but a property like its color for ex)
|
|
}
|
|
|
|
text, expected = "foo", ["foo"]
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
|
|
expected_array = compute_expected_array(concepts_map, text, expected)
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
compare_with_test_object(lexer_nodes, expected_array)
|
|
assert not lexer_nodes[0].concept.get_hints().is_evaluated
|
|
assert lexer_nodes[0].concept.get_hints().use_copy
|
|
assert lexer_nodes[0].concept.get_hints().is_instance
|
|
|
|
@pytest.mark.parametrize("text", [
|
|
"foo",
|
|
f"foo one",
|
|
"x$!#",
|
|
"twenty one",
|
|
"1 + 1",
|
|
"foo x$!#",
|
|
"1 + 1 twenty one",
|
|
])
|
|
def test_i_cannot_parse_concepts_with_property_or_bnf_or_unrecognized(self, text):
|
|
concepts_map = {
|
|
"foo": Concept("foo a").def_var("a"),
|
|
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
res = parser.parse(context, ParserInput(text))
|
|
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
|
assert res.body.body == text
|
|
|
|
@pytest.mark.parametrize("text, expected", [
|
|
("hello foo bar",
|
|
[
|
|
("a", [CN("hello1", "hello foo "), "bar"]),
|
|
("b", [CN("hello2", "hello foo "), "bar"]),
|
|
]),
|
|
])
|
|
def test_i_can_parse_when_unrecognized_yield_multiple_values(self, text, expected):
|
|
concepts_map = {
|
|
"hello1": Concept("hello a").def_var("a"),
|
|
"hello2": Concept("hello b").def_var("b"),
|
|
"bar": Concept("bar")
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True)
|
|
list_of_res = parser.parse(context, ParserInput(text))
|
|
|
|
assert len(list_of_res) == len(expected)
|
|
|
|
for res, expected in zip(list_of_res, expected):
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
expected_array = compute_expected_array(concepts_map, text, expected[1])
|
|
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
|
|
assert transformed_nodes == expected_array
|
|
assert lexer_nodes[0].concept.get_metadata().variables == [(expected[0], "foo ")]
|
|
|
|
@pytest.mark.parametrize("text, expected", [
|
|
("1 + twenty one", [SCN("1 + twenty "), "one"]),
|
|
("one + twenty one", ["one", SCN(" + twenty "), ("one", 1)]),
|
|
])
|
|
def test_source_code_found_must_be_considered_as_potential_false_positive(self, text, expected):
|
|
concepts_map = {
|
|
"one": Concept("one")
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert not res.status
|
|
|
|
expected_array = compute_expected_array(concepts_map, text, expected)
|
|
assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
transformed_nodes = get_test_obj(lexer_nodes, expected_array)
|
|
assert transformed_nodes == expected_array
|
|
|
|
@pytest.mark.parametrize("text, expected_is_evaluated", [
|
|
("foo", False),
|
|
("bar", False),
|
|
("twenties", True),
|
|
("a plus b", True),
|
|
("mult", True),
|
|
])
|
|
def test_concepts_with_variables_must_not_be_evaluated(self, text, expected_is_evaluated):
|
|
concepts_map = {
|
|
"foo": Concept("foo"),
|
|
"bar": Concept("bar", body="'bar'"),
|
|
"plus": Concept("a plus b").def_var("a").def_var("b"),
|
|
"mult": Concept("mult", definition="a mult b", definition_type=DEFINITION_TYPE_DEF).def_var("a").def_var(
|
|
"b"),
|
|
"twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
|
|
res = parser.parse(context, ParserInput(text))
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
assert lexer_nodes[0].concept.get_hints().is_evaluated == expected_is_evaluated
|
|
|
|
def test_the_parser_always_return_a_new_instance_of_the_concept(self):
|
|
concepts_map = {
|
|
"foo": Concept("foo"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
|
|
res = parser.parse(context, ParserInput("foo"))
|
|
|
|
assert res.status
|
|
assert id(res.body.body[0].concept) != id(sheerka.get_by_name("foo"))
|
|
|
|
def test_i_can_only_parse_when_the_name_is_an_identifier(self):
|
|
# to prove that I can distinguish string from actual concept name
|
|
concepts_map = {
|
|
"foo": Concept("foo"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
|
|
res = parser.parse(context, ParserInput("'foo'"))
|
|
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
|
|
|
def test_i_do_not_parse_rules_instead_of_concepts(self):
|
|
sheerka, context, parser = self.init_parser({})
|
|
res = parser.parse(context, ParserInput("r:|20:"))
|
|
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
|
|
|
@pytest.mark.parametrize("text", [
|
|
"foo",
|
|
"foo bar",
|
|
"long concept",
|
|
"unrecognized foo",
|
|
"bar unrecognized",
|
|
])
|
|
def test_i_correctly_set_up_use_copy(self, text):
|
|
concepts_map = {
|
|
"foo": Concept("foo"),
|
|
"bar": Concept("bar"),
|
|
"long concept": Concept("long concept"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
res = parser.parse(context, ParserInput(text))
|
|
|
|
for node in res.body.body:
|
|
if hasattr(node, "concept"):
|
|
assert node.concept.get_hints().use_copy
|
|
|
|
def test_i_can_parse_plural(self):
|
|
concepts_map = {
|
|
"boy": Concept("boy"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
boy = concepts_map['boy']
|
|
|
|
res = parser.parse(context, ParserInput("boys"))
|
|
assert res.status
|
|
lexer_nodes = res.body.body
|
|
assert len(lexer_nodes) == 1
|
|
|
|
concept_found = lexer_nodes[0].concept
|
|
assert concept_found.id == f"{boy.id}-{BuiltinConcepts.PLURAL}"
|
|
assert concept_found.name == "boys"
|
|
assert concept_found.key == "boys"
|
|
assert concept_found.get_prop(BuiltinConcepts.PLURAL) == boy
|
|
|
|
def test_i_can_set_body_for_plurals_that_are_a_set(self):
|
|
concepts_map = {
|
|
"boy": Concept("boy"),
|
|
"girl": Concept("girl"),
|
|
"human": Concept("human"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
global_truth_concept = self.get_context(sheerka, global_truth=True)
|
|
sheerka.set_isa(global_truth_concept, concepts_map["boy"], concepts_map["human"])
|
|
sheerka.set_isa(global_truth_concept, concepts_map["girl"], concepts_map["human"])
|
|
|
|
res = parser.parse(context, ParserInput("humans"))
|
|
|
|
assert res.status
|
|
lexer_nodes = res.body.body
|
|
assert len(lexer_nodes) == 1
|
|
concept_found = lexer_nodes[0].concept
|
|
|
|
assert concept_found.get_metadata().body == "get_plural_concept_value(self)"
|
|
|
|
def test_i_cannot_parse_when_expr_token(self):
|
|
sheerka, context, parser = self.init_parser({})
|
|
|
|
tokens = list(Tokenizer("foo bar baz", yield_eof=False))
|
|
tokens.append(Token(TokenKind.EXPR, NameExprNode(-1, -1, []), -1, -1, -1))
|
|
|
|
ret = parser.parse(context, ParserInput(None, tokens))
|
|
|
|
assert not ret.status
|
|
assert sheerka.isinstance(ret.body, BuiltinConcepts.NOT_FOR_ME)
|
|
assert isinstance(ret.body.reason, NotSupportedElement)
|