1359 lines
61 KiB
Python
1359 lines
61 KiB
Python
import pytest
|
|
from core.builtin_concepts import BuiltinConcepts
|
|
from core.concept import Concept, CIO, ALL_ATTRIBUTES, CMV
|
|
from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
|
|
from core.sheerka.services.SheerkaExecute import ParserInput
|
|
from core.tokenizer import Tokenizer
|
|
from core.utils import NextIdManager
|
|
from parsers.BaseNodeParser import utnode, cnode, short_cnode, UnrecognizedTokensNode, \
|
|
SCWC, CNC, UTN, SCN, CN
|
|
from parsers.PythonParser import PythonNode
|
|
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
|
|
NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError
|
|
|
|
import tests.parsers.parsers_utils
|
|
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
|
|
|
|
|
def compute_expected_array(concepts_map, expression, expected):
|
|
return tests.parsers.parsers_utils.compute_expected_array(concepts_map, expression, expected, sya=True)
|
|
|
|
|
|
cmap = {
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"three": Concept("three"),
|
|
"four": Concept("four"),
|
|
"plus": Concept("a plus b").def_var("a").def_var("b"),
|
|
"minus": Concept("a minus b").def_var("a").def_var("b"),
|
|
"mult": Concept("a mult b").def_var("a").def_var("b"),
|
|
"prefixed": Concept("a prefixed").def_var("a"),
|
|
"suffixed": Concept("suffixed a").def_var("a"),
|
|
"infix": Concept("a infix b").def_var("a").def_var("b"),
|
|
"?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
|
"if": Concept("if a then b else c end").def_var("a").def_var("b").def_var("c"),
|
|
"square": Concept("square(a)").def_var("a"),
|
|
"foo bar": Concept("foo bar(a)").def_var("a"),
|
|
"long infixed": Concept("a long infixed b").def_var("a").def_var("b"),
|
|
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
|
"is a concept": Concept("c is a concept").def_var("c"),
|
|
}
|
|
|
|
|
|
class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
|
|
shared_ontology = None
|
|
|
|
@classmethod
|
|
def setup_class(cls):
|
|
init_test_helper = cls().init_test(cache_only=False, ontology="#TestSyaNodeParser#")
|
|
sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
|
|
for i, concept_name in enumerate(cmap):
|
|
cmap[concept_name] = updated[i]
|
|
|
|
cmap["plus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
|
|
cmap["mult"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
|
|
cmap["minus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
|
|
TestSyaNodeParser.sheerka.set_is_greater_than(context,
|
|
BuiltinConcepts.PRECEDENCE,
|
|
cmap["mult"],
|
|
cmap["plus"],
|
|
CONCEPT_COMPARISON_CONTEXT)
|
|
TestSyaNodeParser.sheerka.set_is_greater_than(context,
|
|
BuiltinConcepts.PRECEDENCE,
|
|
cmap["mult"],
|
|
cmap["minus"],
|
|
CONCEPT_COMPARISON_CONTEXT)
|
|
|
|
cls.shared_ontology = sheerka.get_ontology(context)
|
|
sheerka.pop_ontology()
|
|
|
|
def init_parser(self,
|
|
my_concepts_map=None,
|
|
sya_def=None,
|
|
post_init_concepts=None,
|
|
**kwargs):
|
|
if my_concepts_map is None:
|
|
sheerka, context = self.init_test().unpack()
|
|
sheerka.add_ontology(context, self.shared_ontology)
|
|
concepts = cmap.values()
|
|
init_from_sheerka = kwargs.get("init_from_sheerka", True)
|
|
else:
|
|
sheerka, context, *concepts = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
|
|
for i, pair in enumerate(my_concepts_map):
|
|
my_concepts_map[pair] = concepts[i]
|
|
init_from_sheerka = kwargs.get("init_from_sheerka", False)
|
|
|
|
if post_init_concepts:
|
|
post_init_concepts(sheerka, context)
|
|
|
|
if sya_def:
|
|
sya_def_to_use = {}
|
|
for k, v in sya_def.items():
|
|
sya_def_to_use[k.id] = v
|
|
else:
|
|
sya_def_to_use = None
|
|
|
|
if init_from_sheerka:
|
|
parser = SyaNodeParser(sheerka=sheerka)
|
|
else:
|
|
parser = SyaNodeParser()
|
|
if my_concepts_map:
|
|
parser.init_from_concepts(context, concepts, sya=sya_def_to_use)
|
|
return sheerka, context, parser
|
|
|
|
@pytest.mark.parametrize("expression, expected_sequences", [
|
|
("one plus two", [["one", "two", "plus"]]),
|
|
("1 + 1 plus two", [["1 + 1", "two", "plus"]]),
|
|
("one + two plus three", [
|
|
["one", " + ", "two", "three", "plus"],
|
|
["one + two", "three", "plus"]]),
|
|
("twenty one plus two", [
|
|
["twenty ", "one", "two", "plus"],
|
|
[short_cnode("twenties", "twenty one"), "two", "plus"]
|
|
]),
|
|
("x$!# plus two", [["x$!#", "two", "plus"]]),
|
|
|
|
("one plus 1 + 1", [["one", "1 + 1", "plus"]]),
|
|
("1 + 1 plus 2 + 2", [["1 + 1", "2 + 2", "plus"]]),
|
|
("one + two plus 1 + 1", [
|
|
["one", " + ", "two", "1 + 1", "plus"],
|
|
["one + two", "1 + 1", "plus"]
|
|
]),
|
|
("twenty one plus 1 + 1", [
|
|
["twenty ", "one", "1 + 1", "plus"],
|
|
[cnode("twenties", 0, 2, "twenty one"), "1 + 1", "plus"]
|
|
]),
|
|
("x$!# plus 1 + 1", [["x$!#", "1 + 1", "plus"]]),
|
|
|
|
("one plus two + three", [
|
|
["one", "two", "plus", " + ", "three"],
|
|
["one", "two + three", "plus"],
|
|
]),
|
|
("1 + 1 plus two + three", [
|
|
["1 + 1", "two", "plus", (" + ", 1), "three"],
|
|
["1 + 1", "two + three", "plus"],
|
|
]),
|
|
("one + two plus two + three", [
|
|
["one", " + ", "two", ("two", 1), "plus", (" + ", 1), "three"],
|
|
["one + two", ("two", 1), "plus", (" + ", 1), "three"],
|
|
["one", " + ", "two", "two + three", "plus"],
|
|
["one + two", "two + three", "plus"],
|
|
]),
|
|
("twenty one plus two + three", [
|
|
["twenty ", "one", "two", "plus", " + ", "three"],
|
|
[cnode("twenties", 0, 2, "twenty one"), "two", "plus", " + ", "three"],
|
|
["twenty ", "one", "two + three", "plus"],
|
|
[cnode("twenties", 0, 2, "twenty one"), "two + three", "plus"],
|
|
]),
|
|
("x$!# plus two + three", [
|
|
["x$!#", "two", "plus", " + ", "three"],
|
|
["x$!#", "two + three", "plus"],
|
|
]),
|
|
|
|
("one plus twenty two", [
|
|
["one", "twenty ", "plus", "two"],
|
|
["one", cnode("twenties", 4, 6, "twenty two"), "plus"],
|
|
]),
|
|
("1 + 1 plus twenty one", [
|
|
["1 + 1", "twenty ", "plus", "one"],
|
|
["1 + 1", cnode("twenties", 8, 10, "twenty one"), "plus"],
|
|
]),
|
|
("one + two plus twenty one", [
|
|
["one", " + ", "two", "twenty ", "plus", ("one", 1)],
|
|
["one + two", "twenty ", "plus", ("one", 1)],
|
|
["one", " + ", "two", cnode("twenties", 8, 10, "twenty one"), "plus"],
|
|
["one + two", cnode("twenties", 8, 10, "twenty one"), "plus"],
|
|
]),
|
|
("twenty one plus twenty two",
|
|
[
|
|
["twenty ", "one", ("twenty ", 1), "plus", "two"],
|
|
[cnode("twenties", 0, 2, "twenty one"), ("twenty ", 1), "plus", "two"],
|
|
["twenty ", "one", cnode("twenties", 6, 8, "twenty two"), "plus"],
|
|
[cnode("twenties", 0, 2, "twenty one"), cnode("twenties", 6, 8, "twenty two"), "plus"],
|
|
]),
|
|
("x$!# plus twenty two", [
|
|
["x$!#", "twenty ", "plus", "two"],
|
|
["x$!#", cnode("twenties", 7, 9, "twenty two"), "plus"]
|
|
]),
|
|
|
|
("one plus z$!#", [["one", "z$!#", "plus"]]),
|
|
("1 + 1 plus z$!#", [["1 + 1", "z$!#", "plus"]]),
|
|
("one + two plus z$!#", [
|
|
["one", " + ", "two", "z$!#", "plus"],
|
|
["one + two", "z$!#", "plus"],
|
|
]),
|
|
("twenty one plus z$!#", [
|
|
["twenty ", "one", "z$!#", "plus"],
|
|
[cnode("twenties", 0, 2, "twenty one"), "z$!#", "plus"],
|
|
]),
|
|
("x$!# plus z$!#", [["x$!#", "z$!#", "plus"]]),
|
|
])
|
|
def test_i_can_post_fix_simple_infix_concepts(self, expression, expected_sequences):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == len(expected_sequences)
|
|
for res_i, expected in zip(res, expected_sequences):
|
|
assert len(res_i.errors) == 0
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
assert res_i.out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected_sequences", [
|
|
("one plus plus plus 1 + 1", [["one", "1 + 1", "plus plus plus"]]),
|
|
("x$!# another long name infix twenty two", [
|
|
["x$!#", "twenty ", "another long name infix", "two"],
|
|
["x$!#", cnode("twenties", 13, 15, "twenty two"), "another long name infix"],
|
|
]),
|
|
])
|
|
def test_i_can_post_fix_infix_concepts_with_long_name(self, expression, expected_sequences):
|
|
concepts_map = {
|
|
"plus plus plus": Concept("a plus plus plus b").def_var("a").def_var("b"),
|
|
"another long name infix": Concept("a another long name infix b").def_var("a").def_var("b"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"three": Concept("three"),
|
|
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == len(expected_sequences)
|
|
for res_i, expected in zip(res, expected_sequences):
|
|
assert len(res_i.errors) == 0
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
assert res_i.out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected_sequences", [
|
|
("one prefixed", [["one", "prefixed"]]),
|
|
("1 + 1 prefixed", [["1 + 1", "prefixed"]]),
|
|
("one + two prefixed", [
|
|
["one", " + ", "two", "prefixed"],
|
|
["one + two", "prefixed"],
|
|
]),
|
|
("twenty one prefixed", [
|
|
["twenty ", "one", "prefixed"],
|
|
[cnode("twenties", 0, 2, "twenty one"), "prefixed"],
|
|
]),
|
|
("x$!# prefixed", [["x$!#", "prefixed"]]),
|
|
])
|
|
def test_i_can_post_fix_simple_prefixed_concepts(self, expression, expected_sequences):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == len(expected_sequences)
|
|
for res_i, expected in zip(res, expected_sequences):
|
|
assert len(res_i.errors) == 0
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
assert res_i.out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected_sequences", [
|
|
("one prefixed prefixed", [["one", "prefixed prefixed"]]),
|
|
("1 + 1 prefixed prefixed", [["1 + 1", "prefixed prefixed"]]),
|
|
("one + two prefixed prefixed", [
|
|
["one", " + ", "two", "prefixed prefixed"],
|
|
["one + two", "prefixed prefixed"],
|
|
]),
|
|
("twenty one prefixed prefixed", [
|
|
["twenty ", "one", "prefixed prefixed"],
|
|
[cnode("twenties", 0, 2, "twenty one"), "prefixed prefixed"],
|
|
]),
|
|
("x$!# prefixed prefixed", [["x$!#", "prefixed prefixed"]]),
|
|
|
|
("one long name prefixed", [["one", "long name prefixed"]]),
|
|
("1 + 1 long name prefixed", [["1 + 1", "long name prefixed"]]),
|
|
("one + two long name prefixed", [
|
|
["one", " + ", "two", "long name prefixed"],
|
|
["one + two", "long name prefixed"],
|
|
]),
|
|
("twenty one long name prefixed", [
|
|
["twenty ", "one", "long name prefixed"],
|
|
[cnode("twenties", 0, 2, "twenty one"), "long name prefixed"],
|
|
]),
|
|
("x$!# long name prefixed", [["x$!#", "long name prefixed"]]),
|
|
])
|
|
def test_i_can_post_fix_prefixed_concepts_with_long_names(self, expression, expected_sequences):
|
|
concepts_map = {
|
|
"prefixed prefixed": Concept("a prefixed prefixed").def_var("a"),
|
|
"long name prefixed": Concept("a long name prefixed").def_var("a"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == len(expected_sequences)
|
|
for res_i, expected in zip(res, expected_sequences):
|
|
assert len(res_i.errors) == 0
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
assert res_i.out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected_sequences", [
|
|
("suffixed one", [["one", "suffixed"]]),
|
|
("suffixed 1 + 1", [["1 + 1", "suffixed"]]),
|
|
("suffixed one + two", [
|
|
["one", "suffixed", " + ", "two"],
|
|
["one + two", "suffixed"],
|
|
]),
|
|
("suffixed twenty one", [
|
|
["twenty ", "suffixed", "one"],
|
|
[cnode("twenties", 2, 4, "twenty one"), "suffixed"],
|
|
]),
|
|
("suffixed x$!#", [["x$!#", "suffixed"]]),
|
|
])
|
|
def test_i_can_post_fix_simple_suffixed_concepts(self, expression, expected_sequences):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == len(expected_sequences)
|
|
for res_i, expected in zip(res, expected_sequences):
|
|
assert len(res_i.errors) == 0
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
assert res_i.out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("suffixed suffixed one", ["one", "suffixed suffixed"]),
|
|
("long name suffixed one", ["one", "long name suffixed"]),
|
|
])
|
|
def test_i_can_post_fix_suffixed_concepts_with_long_names(self, expression, expected):
|
|
concepts_map = {
|
|
"suffixed suffixed": Concept("suffixed suffixed a").def_var("a"),
|
|
"long name suffixed": Concept("long name suffixed a").def_var("a"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, None)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected_sequences", [
|
|
("one ? two : three", [["one", "two", "three", "?"]]),
|
|
("one ? baz qux : two", [["one", "baz qux", "two", "?"]]),
|
|
("1+1 ? one + two : twenty one", [
|
|
["1+1", "one", " + ", "two"], # error is detected so the parsing has stopped
|
|
["1+1", "one + two", "twenty ", "?", ("one", 1)],
|
|
["1+1", "one + two", short_cnode("twenties", "twenty one"), "?"],
|
|
]),
|
|
("x$!# ? y$!# : z$!#", [["x$!#", "y$!#", "z$!#", "?"]]),
|
|
|
|
("if one then two else three end", [["one", "two", "three", "if"]]),
|
|
("if 1+1 then x$!# else twenty one end", [
|
|
["1+1", "x$!#", "twenty ", "one"], # an error is detected
|
|
["1+1", "x$!#", short_cnode("twenties", "twenty one"), "if"],
|
|
]),
|
|
("if x$!# then one + two else z$!# end", [
|
|
["x$!#", "one", " + ", "two"], # error is detected so the parsing has stopped
|
|
["x$!#", "one + two", "z$!#", "if"],
|
|
]),
|
|
])
|
|
def test_i_can_post_fix_ternary_concepts(self, expression, expected_sequences):
|
|
"""
|
|
The purpose of this test is to validate concepts
|
|
that have at least 3 parameters separated by tokens
|
|
Example :
|
|
var_0 token var_1 token var_2
|
|
token var_0 token var_1 token var_2
|
|
token var_0 token var_1 token var_2 token
|
|
var_0 token var_1 token var_2 token
|
|
etc...
|
|
:return:
|
|
"""
|
|
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == len(expected_sequences)
|
|
for res_i, expected in zip(res, expected_sequences):
|
|
# assert len(res_i.errors) == 0 # Do not validate errors
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
assert res_i.out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected_sequences", [
|
|
("one ? ? two : : three", [["one", "two", "three", "? ?"]]),
|
|
("1+1 ? ? one + two : : twenty one", [
|
|
["1+1", "one", " + ", "two"], # error
|
|
["1+1", "one + two", "twenty ", "? ?", ("one", 1)],
|
|
["1+1", "one + two", short_cnode("twenties", "twenty one"), "? ?"],
|
|
]),
|
|
|
|
("if if one then then two else else three end end ", [["one", "two", "three", "if if"]]),
|
|
("if if 1+1 then then x$!# else else twenty one end end ", [
|
|
["1+1", "x$!#", "twenty ", "one"], # error
|
|
["1+1", "x$!#", short_cnode("twenties", "twenty one"), "if if"]]),
|
|
])
|
|
def test_i_can_post_fix_ternary_concept_with_long_names(self, expression, expected_sequences):
|
|
concepts_map = {
|
|
"? ?": Concept("a ? ? b : : c").def_var("a").def_var("b").def_var("c"),
|
|
"if if": Concept("if if a then then b else else c end end").def_var("a").def_var("b").def_var("c"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"three": Concept("three"),
|
|
"twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == len(expected_sequences)
|
|
for res_i, expected in zip(res, expected_sequences):
|
|
# assert len(res_i.errors) == 0 # Do not validate errors
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
assert res_i.out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("foo bar baz", ["baz", "bar", "foo"]),
|
|
("foo bar x$!#", ["x$!#", "bar", "foo"]),
|
|
("foo bar 1 + 1", ["1 + 1", "bar", "foo"]),
|
|
])
|
|
def test_i_can_post_fix_suffixed_unary_composition(self, expression, expected):
|
|
concepts_map = {
|
|
"foo": Concept("foo a").def_var("a"),
|
|
"bar": Concept("bar a").def_var("a"),
|
|
"baz": Concept("baz"),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, None)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("baz bar foo", ["baz", "bar", "foo"]),
|
|
("x$!# bar foo", ["x$!#", "bar", "foo"]),
|
|
("1 + 1 bar foo", ["1 + 1", "bar", "foo"]),
|
|
])
|
|
def test_i_can_post_fix_prefixed_unary_composition(self, expression, expected):
|
|
concepts_map = {
|
|
"foo": Concept("a foo").def_var("a"),
|
|
"bar": Concept("a bar").def_var("a"),
|
|
"baz": Concept("baz"),
|
|
}
|
|
sya_def = {
|
|
concepts_map["foo"]: (5, SyaAssociativity.Left),
|
|
concepts_map["bar"]: (5, SyaAssociativity.Left), # precedence greater than plus
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("one plus two mult three", ["one", "two", "three", "mult", "plus"]),
|
|
("one mult two plus three", ["one", "two", "mult", "three", "plus"]),
|
|
("(one plus two) mult three", ["one", "two", "plus", "three", "mult"]),
|
|
("one mult (two plus three)", ["one", "two", "three", "plus", "mult"]),
|
|
])
|
|
def test_i_can_post_fix_binary_with_precedence(self, expression, expected):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
def test_i_can_post_fix_unary_with_precedence(self):
|
|
concepts_map = {
|
|
"suffixed": Concept("suffixed a").def_var("a"),
|
|
"prefixed": Concept("a prefixed").def_var("a"),
|
|
"a": Concept("a"),
|
|
}
|
|
|
|
sya_def = {
|
|
concepts_map["prefixed"]: (10, SyaAssociativity.Left),
|
|
concepts_map["suffixed"]: (5, SyaAssociativity.Right),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
|
|
|
expression = "suffixed a prefixed"
|
|
expected = ["a", "prefixed", "suffixed"]
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
# change the precedence
|
|
sya_def = {
|
|
concepts_map["prefixed"]: (5, SyaAssociativity.Left),
|
|
concepts_map["suffixed"]: (10, SyaAssociativity.Right),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
|
|
|
expression = "suffixed a prefixed"
|
|
expected = ["a", "suffixed", "prefixed"]
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
def test_i_can_post_fix_right_associated_binary(self):
|
|
concepts_map = {
|
|
"equals": Concept("a equals b").def_var("a").def_var("b"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"three": Concept("three"),
|
|
}
|
|
|
|
sya_def = {
|
|
concepts_map["equals"]: (None, SyaAssociativity.Right),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
|
|
|
expression = "one equals two equals three"
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
expected = ["one", "two", "three", ("equals", 1), "equals"]
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
def test_i_can_post_fix_left_associated_binary(self):
|
|
concepts_map = {
|
|
"plus": Concept("a plus b").def_var("a").def_var("b"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"three": Concept("three"),
|
|
}
|
|
|
|
sya_def = {
|
|
concepts_map["plus"]: (1, SyaAssociativity.Left),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
|
|
|
expression = "one plus two plus three"
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
expected = ["one", "two", "plus", "three", ("plus", 1)]
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("x$!# ? y$!# : z$!# ? two : three", ["x$!#", "y$!#", "z$!#", "two", "three", ("?", 1), "?"]),
|
|
("x$!# ? y$!# : (z$!# ? two : three)", ["x$!#", "y$!#", "z$!#", "two", "three", ("?", 1), "?"]),
|
|
|
|
("one ? x$!# ? y$!# : z$!# : three", ["one", "x$!#", "y$!#", "z$!#", ("?", 1), "three", "?"]),
|
|
("one ? (x$!# ? y$!# : z$!#) : three", ["one", "x$!#", "y$!#", "z$!#", ("?", 1), "three", "?"]),
|
|
|
|
("one ? two : x$!# ? y$!# : z$!#", ["one", "two", "x$!#", "y$!#", "z$!#", ("?", 1), "?"]),
|
|
("one ? two : (x$!# ? y$!# : z$!#)", ["one", "two", "x$!#", "y$!#", "z$!#", ("?", 1), "?"]),
|
|
])
|
|
def test_i_can_post_fix_right_associated_ternary(self, expression, expected):
|
|
concepts_map = {
|
|
"?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"three": Concept("three"),
|
|
}
|
|
sya_def = {
|
|
concepts_map["?"]: (5, SyaAssociativity.Right),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("x$!# ? y$!# : z$!# ? two : three", ["x$!#", "y$!#", "z$!#", "?", "two", "three", ("?", 1)]),
|
|
("(x$!# ? y$!# : z$!#) ? two : three", ["x$!#", "y$!#", "z$!#", "?", "two", "three", ("?", 1)]),
|
|
|
|
# the following one is not possible when Left association
|
|
# ("one ? x$!# ? y$!# : z$!# : three", ["one", "x$!#", "y$!#", "z$!#", ("?", 1), "three", "?"]),
|
|
|
|
("one ? two : x$!# ? y$!# : z$!#", ["one", "two", "x$!#", "?", "y$!#", "z$!#", ("?", 1)]),
|
|
("(one ? two : x$!#) ? y$!# : z$!#", ["one", "two", "x$!#", "?", "y$!#", "z$!#", ("?", 1)]),
|
|
])
|
|
def test_i_can_post_fix_left_associated_ternary(self, expression, expected):
|
|
concepts_map = {
|
|
"?": Concept("a ? b : c").def_var("a").def_var("b").def_var("c"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"three": Concept("three"),
|
|
}
|
|
sya_def = {
|
|
concepts_map["?"]: (5, SyaAssociativity.Left),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
def test_i_can_post_fix_when_multiple_concepts_are_found(self):
|
|
concepts_map = {
|
|
"foo": Concept("foo a").def_var("a"),
|
|
"foo bar": Concept("foo bar a").def_var("a"),
|
|
"baz": Concept("baz"),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map, None)
|
|
|
|
expression = "foo bar baz"
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_sequences = [
|
|
[UTN("bar "), "foo", "baz"],
|
|
["baz", "foo bar"]
|
|
]
|
|
|
|
assert len(res) == len(expected_sequences)
|
|
for res_i, expected in zip(res, expected_sequences):
|
|
assert len(res_i.errors) == 0
|
|
expected_array = compute_expected_array(concepts_map, expression, expected)
|
|
assert res_i.out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
# ("function(one plus three) minus two",
|
|
# [SCWC("function(", ")", CNC("plus", a="one", b="three")), "two", "minus"]),
|
|
("two minus function(one plus three)",
|
|
["two", SCWC("function(", ")", CNC("plus", a="one", b="three")), "minus"]),
|
|
("func1() minus func2()", [SCN("func1()"), SCN("func2()"), "minus"]),
|
|
("func1() comes with func2()", [SCN("func1()"), UTN(" comes with "), SCN("func2()")]),
|
|
|
|
("(one plus two) ", ["one", "two", "plus"]),
|
|
("(one prefixed) ", ["one", "prefixed"]),
|
|
("(suffixed one) ", ["one", "suffixed"]),
|
|
("(one ? two : three)", ["one", "two", "three", "?"]),
|
|
("square(square(one))", ["one", ("square", 1), "square"]),
|
|
("square ( square ( one ) )", ["one", ("square", 1), "square"]),
|
|
|
|
("square(one plus three) minus two", ["one", "three", "plus", "square", "two", "minus"]),
|
|
("square( one plus three ) minus two", ["one", "three", "plus", "square", "two", "minus"]),
|
|
("one minus square( two plus three ) ", ["one", "two", "three", "plus", "square", "minus"]),
|
|
|
|
("((one prefixed) prefixed)", ["one", "prefixed", ("prefixed", 1)]),
|
|
("( ( one prefixed ) prefixed)", ["one", "prefixed", ("prefixed", 1)]),
|
|
("( ( square( one ) prefixed ) prefixed)", ["one", "square", "prefixed", ("prefixed", 1)]),
|
|
|
|
("suffixed (suffixed one)", ["one", ("suffixed", 1), "suffixed"]),
|
|
("suffixed ( suffixed one) ", ["one", ("suffixed", 1), "suffixed"]),
|
|
("suffixed (suffixed square(one))", ["one", "square", ("suffixed", 1), "suffixed"]),
|
|
("suffixed ( suffixed square ( one ) )", ["one", "square", ("suffixed", 1), "suffixed"]),
|
|
|
|
("one plus (two minus three)", ["one", "two", "three", "minus", "plus"]),
|
|
("one plus ( two minus three )", ["one", "two", "three", "minus", "plus"]),
|
|
("(one plus two) minus three", ["one", "two", "plus", "three", "minus"]),
|
|
("(( one plus two ) minus three )", ["one", "two", "plus", "three", "minus"]),
|
|
|
|
("foo bar(one)", ["one", "foo bar"]),
|
|
("foo bar ( one )", ["one", "foo bar"]),
|
|
])
|
|
def test_i_can_pos_fix_when_parenthesis(self, expression, expected):
|
|
sheerka, context, parser = self.init_parser()
|
|
context.add_to_protected_hints(BuiltinConcepts.DEBUG)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected_sequences", [
|
|
# composition
|
|
("function(suffixed one)", [[SCWC("function(", ")", CNC("suffixed", a="one"))]]),
|
|
("function(one prefixed)", [[SCWC("function(", ")", CNC("prefixed", a="one"))]]),
|
|
("function(if one then two else three end)",
|
|
[[SCWC("function(", ")", CNC("if", a="one", b="two", c="three", end=14))]]),
|
|
("function(suffixed twenty two)",
|
|
[[SCWC("function(", ")", CNC("suffixed", a=CIO("twenties", source="twenty two")))]]),
|
|
("function(twenty two prefixed)",
|
|
[[SCWC("function(", ")", CNC("prefixed", a=CIO("twenties", source="twenty two")))]]),
|
|
("function(if one then twenty two else three end)",
|
|
[[SCWC("function(", ")", CNC("if", a="one", b=CIO("twenties", source="twenty two"), c="three", end=16))]]),
|
|
("func1(func2(one two) three)",
|
|
[[SCWC("func1(", (")", 1), SCWC("func2(", ")", "one", "two"), "three")]]),
|
|
|
|
("twenty two(suffixed one)", [
|
|
["twenty ", SCWC("two(", ")", CNC("suffixed", a="one"))],
|
|
[CN("twenties", source="twenty two"), "one", "suffixed"],
|
|
]),
|
|
("twenty two(one prefixed)", [
|
|
["twenty ", SCWC("two(", ")", CNC("prefixed", a="one"))],
|
|
[CN("twenties", source="twenty two"), "one", "prefixed"],
|
|
]),
|
|
("f1(one plus two mult three) plus f2(suffixed x$!# prefixed)", [
|
|
[SCWC("f1(", ")", CN("plus", source="one plus two mult three")),
|
|
SCWC("f2(", (")", 1), CN("suffixed", source="suffixed x$!# prefixed")),
|
|
("plus", 1)]
|
|
]),
|
|
|
|
# plus, suffixed, prefixed, ternary
|
|
("func1(one) plus func2(two)", [[SCWC("func1(", ")", "one"), SCWC("func2(", (")", 1), "two"), "plus"]]),
|
|
("suffixed function(one)", [[SCWC("function(", ")", "one"), "suffixed"]]),
|
|
("function(one) prefixed", [[SCWC("function(", ")", "one"), "prefixed"]]),
|
|
("if f1(one) then f2(two) else f3(three) end", [
|
|
[SCWC("f1(", ")", "one"), SCWC("f2(", (")", 1), "two"), SCWC("f3(", (")", 2), "three"), "if"]]),
|
|
|
|
# Sequence
|
|
("if one then two else three end function(x$!#)", [
|
|
["one", "two", "three", "if", UTN(" ", start=13, end=13), SCWC("function(", ")", "x$!#")]]),
|
|
("one prefixed function(two)", [["one", "prefixed", UTN(" ", start=3, end=3), SCWC("function(", ")", "two")]]),
|
|
("suffixed one function(two)", [["one", "suffixed", UTN(" ", start=3, end=3), SCWC("function(", ")", "two")]]),
|
|
("func(one, two, three)", [[SCWC("func(", ")", "one", ", ", "two", (", ", 1), "three")]]),
|
|
])
|
|
def test_i_can_post_fix_when_parenthesis_and_unknown(self, expression, expected_sequences):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == len(expected_sequences)
|
|
for res_i, expected in zip(res, expected_sequences):
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
assert res_i.out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("(", ("(", 0)),
|
|
("one plus ( 1 + ", ("(", 4)),
|
|
("one( 1 + ", ("(", 1)),
|
|
("one ( 1 + ", ("(", 2)),
|
|
("function(", ("(", 1)),
|
|
("function( 1 + ", ("(", 1)),
|
|
("function ( 1 + ", ("(", 2)),
|
|
("one plus ) 1 + ", (")", 4)),
|
|
("one ) 1 + ", (")", 2)),
|
|
("function ) 1 + ", (")", 2)),
|
|
("one ? ( : two", ("(", 4)),
|
|
("one ? one plus ( : two", ("(", 8)),
|
|
("one ? ) : two", (")", 4)),
|
|
("one ? one plus ) : two", (")", 8)),
|
|
("(one plus ( 1 + )", ("(", 0)),
|
|
])
|
|
def test_i_can_detect_parenthesis_mismatch_error_when_post_fixing(self, expression, expected):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == 1
|
|
assert res[0].errors == [ParenthesisMismatchError(expected)]
|
|
|
|
def test_i_can_detect_parenthesis_mismatch_error_special_case(self):
|
|
sheerka, context, parser = self.init_parser()
|
|
expression = "one ? function( : two"
|
|
expected = [ParenthesisMismatchError(("(", 5)), ParenthesisMismatchError(("(", 5))]
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == 1
|
|
assert res[0].errors == expected
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("one ? one two : three", ("?", ":")),
|
|
])
|
|
def test_i_can_detected_when_too_many_parameters(self, expression, expected):
|
|
sheerka, context, parser = self.init_parser(cmap, None)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == 1
|
|
assert len(res[0].errors) == 1
|
|
error = res[0].errors[0]
|
|
assert isinstance(error, TooManyParametersFoundError)
|
|
assert error.concept == cmap[expected[0]]
|
|
assert error.token.value == expected[1]
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("one infix two x$!#", ["one", "two", "infix", " x$!#"]),
|
|
("x$!# one infix two", ["x$!# ", "one", "two", "infix"]),
|
|
("one prefixed x$!#", ["one", "prefixed", " x$!#"]),
|
|
("x$!# one prefixed", ["x$!# ", "one", "prefixed"]),
|
|
("suffixed one x$!#", ["one", "suffixed", " x$!#"]),
|
|
("x$!# suffixed one", ["x$!# ", "one", "suffixed"]),
|
|
("one ? two : three x$!#", ["one", "two", "three", "?", " x$!#"]),
|
|
("x$!# one ? two : three", ["x$!# ", "one", "two", "three", "?"]),
|
|
|
|
("one infix two three infix four", ["one", "two", "infix", "three", "four", ("infix", 1)]),
|
|
("one infix two three prefixed", ["one", "two", "infix", "three", "prefixed"]),
|
|
("one infix two suffixed three", ["one", "two", "infix", "three", "suffixed"]),
|
|
("one infix two x$!# ? y$!# : z$!#", ["one", "two", "infix", " x$!#", "y$!#", "z$!#", "?"]),
|
|
|
|
("one prefixed two infix three", ["one", "prefixed", "two", "three", "infix"]),
|
|
("one prefixed two prefixed", ["one", "prefixed", "two", ("prefixed", 1)]),
|
|
("one prefixed suffixed two", ["one", "prefixed", "two", "suffixed"]),
|
|
("one prefixed x$!# ? y$!# : z$!#", ["one", "prefixed", " x$!#", "y$!#", "z$!#", "?"]),
|
|
|
|
("(one infix two) (three prefixed)", ["one", "two", "infix", "three", "prefixed"]),
|
|
])
|
|
def test_i_can_post_fix_sequences(self, expression, expected):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression", [
|
|
"one ? two : three",
|
|
"one?two:three",
|
|
"one ?two:three",
|
|
"one? two:three",
|
|
"one ? two :three",
|
|
"one ? two: three",
|
|
])
|
|
def test_whitespaces_may_be_omitted_in_some_circumstances(self, expression):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
expected = ["one", "two", "three", "?"]
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
def test_the_more_concepts_the_more_results(self):
|
|
concepts_map = {
|
|
"plus": Concept("a plus b").def_var("a").def_var("b"),
|
|
"plus plus": Concept("a plus plus").def_var("a"),
|
|
"plus equals": Concept("a plus equals b").def_var("a").def_var("b"),
|
|
}
|
|
|
|
sya_def = {
|
|
concepts_map["plus"]: (1, SyaAssociativity.Right),
|
|
concepts_map["plus plus"]: (1, SyaAssociativity.Right),
|
|
concepts_map["plus equals"]: (1, SyaAssociativity.Right),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
|
|
|
expression = "a plus plus equals b"
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
expected_array = tests.parsers.parsers_utils.compute_debug_array(res)
|
|
assert len(expected_array) == len([
|
|
["T(a)", "C(a plus b)", "C(a plus b)", "T(equals)", "T(b)"],
|
|
["T(a)", "C(a plus b)", "C(a plus plus)", "T(equals)", "T(b)"],
|
|
["T(a)", "C(a plus b)", "C(a plus equals b)", "T(equals)", "T(b)"],
|
|
["T(a)", "C(a plus plus)", "T(plus)", "T(equals)", "T(b)"],
|
|
["T(a)", "C(a plus plus)", "T(plus)", "T(equals)", "T(b)"],
|
|
["T(a)", "C(a plus plus)", "T(plus)", "T(equals)", "T(b)"],
|
|
["T(a)", "C(a plus equals b)", "C(a plus b)", "T(equals)", "T(b)"],
|
|
["T(a)", "C(a plus equals b)", "C(a plus plus)", "T(equals)", "T(b)"],
|
|
["T(a)", "C(a plus equals b)", "C(a plus equals b)", "T(equals)", "T(b)"],
|
|
])
|
|
|
|
def test_i_can_use_string_instead_of_identifier(self):
|
|
concepts_map = {
|
|
"ternary": Concept("a ? ? b '::' c").def_var("a").def_var("b").def_var("c"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"three": Concept("three"),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, None)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput("one ? ? two '::' three"))
|
|
assert len(res) == 1
|
|
assert res[0].out == [
|
|
cnode("one", start=0, end=0, source="one"),
|
|
cnode("two", start=6, end=6, source="two"),
|
|
cnode("three", start=10, end=10, source="three"),
|
|
SyaConceptParserHelper(concepts_map["ternary"], 2),
|
|
]
|
|
|
|
def test_i_cannot_chain_non_associative(self):
|
|
concepts_map = {
|
|
"less than": Concept("a less than b").def_var("a").def_var("b"),
|
|
"one": Concept("one"),
|
|
"two": Concept("two"),
|
|
"three": Concept("three"),
|
|
}
|
|
sya_def = {
|
|
concepts_map["less than"]: (None, SyaAssociativity.No),
|
|
}
|
|
|
|
sheerka, context, parser = self.init_parser(concepts_map, sya_def)
|
|
|
|
res = parser.infix_to_postfix(context, ParserInput("one less than two less than three"))
|
|
assert len(res) == 1
|
|
assert res[0].errors == [NoneAssociativeSequenceError(concepts_map["less than"], 2, 8)]
|
|
|
|
def test_i_can_post_fix_bnf_definition(self):
|
|
"""
|
|
The definition of a BNF concept is considered as an atom concept
|
|
Not quite sure why this test is here
|
|
:return:
|
|
"""
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
expression = "suffixed twenties"
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
expected = [cnode("twenties", 2, 2, "twenties"), "suffixed"]
|
|
expected_array = compute_expected_array(cmap, expression, expected)
|
|
assert len(res) == 1
|
|
assert res[0].out == expected_array
|
|
|
|
@pytest.mark.parametrize("expression, expected_debugs", [
|
|
("one", [[" 0:one => PUSH_UNREC"]]),
|
|
("one plus two", [[' 0:one => PUSH_UNREC',
|
|
' 1:<ws> => PUSH_UNREC',
|
|
' 2:plus ((1005)a plus b, prio=1, assoc=SyaAssociativity.Right) => ??',
|
|
' _: => RECOG [[CN((1001)one)]]',
|
|
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
|
|
' 2:plus ((1005)a plus b, prio=1, assoc=SyaAssociativity.Right) => PUSH',
|
|
' 3:<ws> => EAT',
|
|
' 4:two => PUSH_UNREC',
|
|
' 5:<EOF> => ??',
|
|
' _: => RECOG [[CN((1002)two)]]',
|
|
" _: => POP ConceptNode(concept='(1002)two', source='two', start=4, end=4)",
|
|
' _: => POP SyaConceptParserHelper(concept=(1005)a plus b, start=2, '
|
|
'error=None)']]),
|
|
("suffixed one", [[
|
|
' 0:suffixed ((1009)suffixed a, prio=1, assoc=SyaAssociativity.Right) => PUSH',
|
|
' 1:<ws> => EAT',
|
|
' 2:one => PUSH_UNREC',
|
|
' 3:<EOF> => ??',
|
|
" _: => RECOG [[CN((1001)one)]]",
|
|
" _: => POP ConceptNode(concept='(1001)one', source='one', start=2, end=2)",
|
|
' _: => POP SyaConceptParserHelper(concept=(1009)suffixed a, start=0, error=None)'
|
|
]]),
|
|
("one ? twenty one : three", [[
|
|
' 0:one => PUSH_UNREC',
|
|
' 1:<ws> => PUSH_UNREC',
|
|
' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => ??',
|
|
' _: => RECOG [[CN((1001)one)]]',
|
|
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
|
|
' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => PUSH',
|
|
' 3:<ws> => EAT',
|
|
' 4:twenty => PUSH_UNREC',
|
|
' 5:<ws> => PUSH_UNREC',
|
|
' 6:one => PUSH_UNREC',
|
|
' 7:<ws> => PUSH_UNREC',
|
|
' 8:: => ??',
|
|
" _: => RECOG [[UTN('twenty '), CN((1001)one)], [CN((1016)twenties)]]",
|
|
" _: => POP UnrecognizedTokensNode(source='twenty ', start=4, end=5)",
|
|
" _: => POP ConceptNode(concept='(1001)one', source='one', start=6, end=6)",
|
|
" _: => => ERROR Too many parameters found for '(1011)a ? b : c' before token 'Token(:)'",
|
|
' 8:: => EAT'], [
|
|
' 0:one => PUSH_UNREC',
|
|
' 1:<ws> => PUSH_UNREC',
|
|
' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => ??',
|
|
' _: => RECOG [[CN((1001)one)]]',
|
|
" _: => POP ConceptNode(concept='(1001)one', source='one', start=0, end=0)",
|
|
' 2:? ((1011)a ? b : c, prio=1, assoc=SyaAssociativity.Right) => PUSH',
|
|
' 3:<ws> => EAT',
|
|
' 4:twenty => PUSH_UNREC',
|
|
' 5:<ws> => PUSH_UNREC',
|
|
' 6:one => PUSH_UNREC',
|
|
' 7:<ws> => PUSH_UNREC',
|
|
' 8:: => ??',
|
|
" _: => RECOG [[UTN('twenty '), CN((1001)one)], [CN((1016)twenties)]]",
|
|
" _: => POP ConceptNode(concept='(1016)twenties', source='twenty one', start=4, end=6, #body#='DoNotResolve(value='twenty one')', unit='(1001)one')",
|
|
' 9:<ws> => EAT',
|
|
' 10:three => PUSH_UNREC',
|
|
' 11:<EOF> => ??',
|
|
' _: => RECOG [[CN((1003)three)]]',
|
|
" _: => POP ConceptNode(concept='(1003)three', source='three', start=10, end=10)",
|
|
' _: => POP SyaConceptParserHelper(concept=(1011)a ? b : c, start=2, error=None)']]),
|
|
])
|
|
def test_i_can_debug(self, expression, expected_debugs):
|
|
sheerka, context, parser = self.init_parser()
|
|
sheerka.set_debug(context, True)
|
|
sheerka.debug_var(context, "Sya")
|
|
res = parser.infix_to_postfix(context, ParserInput(expression))
|
|
|
|
assert len(res) == len(expected_debugs)
|
|
for res_i, expected_debug in zip(res, expected_debugs):
|
|
actual_debug = [str(di) for di in res_i.debug]
|
|
assert actual_debug == expected_debug
|
|
|
|
@pytest.mark.parametrize("settings", [
|
|
"Sya.*.*",
|
|
"Sya.*.#0.can_pop"
|
|
])
|
|
def test_i_can_debug_can_pop_using_star(self, settings):
|
|
sheerka, context, parser = self.init_parser()
|
|
sheerka.set_debug(context, True)
|
|
sheerka.debug_var(context, settings)
|
|
res = parser.infix_to_postfix(context, ParserInput("one plus two mult three"))
|
|
|
|
debug = [str(di) for di in res[0].debug]
|
|
assert debug[5] == ' _: => No stack. CAN_POP false.'
|
|
|
|
def test_i_can_parse_when_concept_atom_only(self):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
text = "one plus two mult three"
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == [CN(cmap["plus"], 0, 8, source=text)]
|
|
|
|
# check the compiled
|
|
expected_concept = lexer_nodes[0].concept
|
|
assert expected_concept.get_compiled()["a"] == cmap["one"]
|
|
assert expected_concept.get_compiled()["b"] == CMV(cmap["mult"], a="two", b="three")
|
|
assert expected_concept.get_compiled()["b"].get_compiled()["a"] == cmap["two"]
|
|
assert expected_concept.get_compiled()["b"].get_compiled()["b"] == cmap["three"]
|
|
|
|
# check the metadata
|
|
expected_concept = lexer_nodes[0].concept
|
|
assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "two mult three")]
|
|
assert expected_concept.get_compiled()["b"].get_metadata().variables == [("a", "two"), ("b", "three")]
|
|
|
|
def test_i_can_parse_when_python_code(self):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
text = "suffixed 1 + 1"
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
|
|
|
|
# check the compiled
|
|
expected_concept = lexer_nodes[0].concept
|
|
assert len(expected_concept.get_compiled()["a"]) == 1
|
|
|
|
return_value_a = expected_concept.get_compiled()["a"][0]
|
|
assert sheerka.isinstance(return_value_a, BuiltinConcepts.RETURN_VALUE)
|
|
assert return_value_a.status
|
|
assert sheerka.isinstance(return_value_a.body, BuiltinConcepts.PARSER_RESULT)
|
|
assert return_value_a.body.source == "1 + 1"
|
|
assert isinstance(return_value_a.body.body, PythonNode)
|
|
|
|
# check metadata
|
|
assert expected_concept.get_metadata().variables == [("a", "1 + 1")]
|
|
|
|
def test_i_can_parse_when_bnf_concept(self):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
text = "suffixed twenty one"
|
|
res = parser.parse(context, ParserInput(text))
|
|
assert len(res) == 2
|
|
assert res[1].status
|
|
|
|
wrapper = res[1].body
|
|
lexer_nodes = res[1].body.body
|
|
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == [CN(cmap["suffixed"], 0, 4, source=text)]
|
|
|
|
# check the compiled
|
|
expected_concept = lexer_nodes[0].concept
|
|
assert sheerka.isinstance(expected_concept.get_compiled()["a"], "twenties")
|
|
assert expected_concept.get_compiled()["a"].get_compiled()["unit"] == cmap["one"]
|
|
|
|
# check metadata
|
|
assert expected_concept.get_metadata().variables == [("a", "twenty one")]
|
|
|
|
def test_i_can_parse_sequences(self):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
text = "one plus 1 + 1 suffixed two"
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == [
|
|
CN(cmap["plus"], 0, 9, source="one plus 1 + 1 "),
|
|
CN(cmap["suffixed"], 10, 12, source="suffixed two")]
|
|
|
|
# check the compiled
|
|
concept_plus_a = lexer_nodes[0].concept.get_compiled()["a"]
|
|
concept_plus_b = lexer_nodes[0].concept.get_compiled()["b"]
|
|
concept_suffixed_a = lexer_nodes[1].concept.get_compiled()["a"]
|
|
|
|
assert concept_plus_a == cmap["one"]
|
|
assert len(concept_plus_b) == 1
|
|
assert sheerka.isinstance(concept_plus_b[0], BuiltinConcepts.RETURN_VALUE)
|
|
assert isinstance(concept_plus_b[0].body.body, PythonNode)
|
|
assert concept_suffixed_a == cmap["two"]
|
|
|
|
@pytest.mark.parametrize("text, expected_status, expected_result", [
|
|
("f1(one prefixed) plus f2(suffixed two)", False, [
|
|
CNC("plus",
|
|
a=SCWC("f1(", ")", CNC("prefixed", a="one")),
|
|
b=SCWC("f2(", (")", 1), CNC("suffixed", a="two")))
|
|
]),
|
|
("one is a concept", True, [CNC("is a concept", c="one")]),
|
|
("a is a concept", False, [CNC("is a concept", c=UTN("a"))]),
|
|
])
|
|
def test_i_can_parse_when_one_result(self, text, expected_status, expected_result):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
expected_array = compute_expected_array(cmap, text, expected_result)
|
|
assert res.status == expected_status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == expected_array
|
|
|
|
@pytest.mark.parametrize("text", [
|
|
"function(suffixed one)",
|
|
"function(one plus two mult three)",
|
|
"function(suffixed x$!#)"
|
|
])
|
|
def test_i_cannot_parse_when_function_only(self, text):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.parse(context, ParserInput(text))
|
|
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
|
|
|
@pytest.mark.parametrize("text", [
|
|
"foo bar (one",
|
|
"foo bar one",
|
|
"foo one two",
|
|
"foo x$!# one",
|
|
])
|
|
def test_i_cannot_parse_when_concept_almost_found(self, text):
|
|
"""
|
|
We test that the parsed concept seems like a known one, but it was not.
|
|
The parser has to detected that the predication was incorrect
|
|
:return:
|
|
"""
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.parse(context, ParserInput(text))
|
|
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
|
assert res.body.body == text
|
|
|
|
@pytest.mark.parametrize("text, expected_result", [
|
|
("one plus two foo bar baz", [CNC("plus", a="one", b="two"), UTN(" foo bar baz")]),
|
|
("one plus two foo bar", [CNC("plus", a="one", b="two"), UTN(" foo bar")]),
|
|
("foo bar one plus two", [UTN("foo bar "), CNC("plus", a="one", b="two")]),
|
|
("one plus two a long other b", [CNC("plus", a="one", b="two"), UTN(" a long other b")]),
|
|
("one plus two a long infixed", [CNC("plus", a="one", b="two"), UTN(" a long infixed")]),
|
|
("one plus two a long", [CNC("plus", a="one", b="two"), UTN(" a long")]),
|
|
("one ? a long infixed : two", [CNC("?", a="one", b=UTN("a long infixed"), c="two")]),
|
|
("one ? a long infix : two", [CNC("?", a="one", b=UTN("a long infix"), c="two")]),
|
|
])
|
|
def test_i_can_almost_parse_when_one_part_is_recognized_but_not_the_rest(self, text, expected_result):
|
|
"""
|
|
We test that the parsed concept seems like a known one, but it was not.
|
|
The parser has to detected that the predication was incorrect
|
|
:return:
|
|
"""
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.parse(context, ParserInput(text))
|
|
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
expected_array = compute_expected_array(cmap, text, expected_result)
|
|
assert not res.status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == expected_array
|
|
|
|
@pytest.mark.parametrize("text, expected_result", [
|
|
("a plus b", [CN("plus", source="a plus b")]),
|
|
("suffixed a plus b", [CN("suffixed", source="suffixed a plus b")]),
|
|
])
|
|
def test_i_can_almost_parse_concept_definition(self, text, expected_result):
|
|
"""
|
|
In these examples, 'a' and 'b' are not defined.
|
|
So the status of the return value cannot be True
|
|
:param text:
|
|
:param expected_result:
|
|
:return:
|
|
"""
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.parse(context, ParserInput(text))
|
|
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
expected_array = compute_expected_array(cmap, text, expected_result)
|
|
assert not res.status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == expected_array
|
|
|
|
@pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [
|
|
("x$!# prefixed", "prefixed", ["a"]),
|
|
("suffixed x$!#", "suffixed", ["a"]),
|
|
("one infix x$!#", "infix", ["b"]),
|
|
("x$!# infix one", "infix", ["a"]),
|
|
("x$!# infix z$!#", "infix", ["a", "b"]),
|
|
])
|
|
def test_i_cannot_parse_when_unrecognized(self, text, expected_concept, expected_unrecognized):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
expected_end = len(list(Tokenizer(text))) - 2
|
|
|
|
assert not res.status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == [CN(cmap[expected_concept], 0, expected_end, source=text)]
|
|
|
|
concept_found = lexer_nodes[0].concept
|
|
for unrecognized in expected_unrecognized:
|
|
assert isinstance(concept_found.get_compiled()[unrecognized], UnrecognizedTokensNode)
|
|
|
|
@pytest.mark.parametrize("text, expected", [
|
|
("x$!# suffixed one", [utnode(0, 4, "x$!# "), cnode("suffixed __var__0", 5, 7, "suffixed one")]),
|
|
("one prefixed x$!#", [cnode("__var__0 prefixed", 0, 2, "one prefixed"), utnode(3, 7, " x$!#")]),
|
|
])
|
|
def test_i_cannot_parse_when_part_of_the_sequence_is_not_recognized(self, text, expected):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert not res.status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == expected
|
|
|
|
def test_i_cannot_parse_function_using_short_name(self):
|
|
concepts_map = {
|
|
"infixed": self.from_def_concept("infixed", "a infixed b", ["a", "b"]),
|
|
"suffixed": self.from_def_concept("suffixed", "suffixed a", ["a"]),
|
|
"prefixed": self.from_def_concept("prefixed", "a prefixed", ["a"]),
|
|
}
|
|
sheerka, context, parser = self.init_parser(concepts_map)
|
|
|
|
res = parser.parse(context, ParserInput("desc(infixed)"))
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
|
|
|
res = parser.parse(context, ParserInput("desc(suffixed)"))
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
|
|
|
res = parser.parse(context, ParserInput("desc(prefixed)"))
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
|
|
|
@pytest.mark.parametrize("text", [
|
|
"one",
|
|
"1 + 1",
|
|
"x$!#",
|
|
"twenty one"
|
|
"",
|
|
"function(not an sya concept)",
|
|
])
|
|
def test_i_cannot_parse_when_no_concept_is_recognized(self, text):
|
|
"""
|
|
it's actually no concept with property
|
|
Atoms concepts, source code or BNF concepts alone are discarded by the lexer
|
|
:return:
|
|
"""
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
res = parser.parse(context, ParserInput(text))
|
|
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
|
|
assert res.body.body == text
|
|
|
|
def test_i_cannot_parse_empty_string(self):
|
|
sheerka, context, parser = self.init_parser({}, None)
|
|
|
|
res = parser.parse(context, ParserInput(""))
|
|
|
|
assert not res.status
|
|
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
|
|
|
|
@pytest.mark.parametrize("expression, expected", [
|
|
("function(", ([], "function(")),
|
|
("before the function(", (["before the "], "function(")),
|
|
("one two function(", (["one", "two", UTN(" ", 3, 3)], "function(")),
|
|
("one(", ([], "one(")),
|
|
("one before the function(", (["one", " before the "], "function(")),
|
|
])
|
|
def test_i_can_get_functions_names_from_unrecognized(self, expression, expected):
|
|
sheerka, context, parser = self.init_parser()
|
|
infix_to_postfix = InFixToPostFix(context, NextIdManager())
|
|
|
|
tokens = list(Tokenizer(expression, yield_eof=False))
|
|
for pos, token in enumerate(tokens[:-1]):
|
|
infix_to_postfix.eat_unrecognized(token, pos)
|
|
|
|
resolved_to_out = compute_expected_array(cmap, expression, expected[0])
|
|
resolved_function_name = compute_expected_array(cmap, expression, [expected[1]])
|
|
actual = infix_to_postfix.get_functions_names_from_unrecognized(tokens[-1], len(tokens) - 1)
|
|
|
|
assert len(actual) == 1
|
|
|
|
assert actual[0].to_out == resolved_to_out
|
|
actual[0].function.fix_source()
|
|
assert actual[0].function == resolved_function_name[0]
|
|
|
|
@pytest.mark.parametrize("expression, expected_list", [
|
|
("twenty two function(", [(["twenty ", "two", UTN(" ", 3, 3)], "function("),
|
|
([CN("twenties", source="twenty two"), UTN(" ", 3, 3)], "function(")]),
|
|
("twenty two(", [(["twenty "], "two("),
|
|
([CN("twenties", source="twenty two")], None)]),
|
|
])
|
|
def test_i_can_get_functions_names_from_unrecognized_when_multiple_results(self, expression, expected_list):
|
|
sheerka, context, parser = self.init_parser()
|
|
infix_to_postfix = InFixToPostFix(context, NextIdManager())
|
|
|
|
tokens = list(Tokenizer(expression, yield_eof=False))
|
|
for pos, token in enumerate(tokens[:-1]):
|
|
infix_to_postfix.eat_unrecognized(token, pos)
|
|
|
|
actual_list = infix_to_postfix.get_functions_names_from_unrecognized(tokens[-1], len(tokens) - 1)
|
|
|
|
assert len(actual_list) == len(expected_list)
|
|
|
|
for actual, expected in zip(actual_list, expected_list):
|
|
resolved_to_out = compute_expected_array(cmap, expression, expected[0])
|
|
|
|
assert actual.to_out == resolved_to_out
|
|
if actual.function:
|
|
actual.function.fix_source()
|
|
resolved_function_name = compute_expected_array(cmap, expression, [expected[1]])
|
|
assert actual.function == resolved_function_name[0]
|
|
else:
|
|
assert actual.function is None
|
|
|
|
def test_i_can_parse_when_multiple_ontologies(self):
|
|
sheerka, context, parser = self.init_parser()
|
|
|
|
text = "suffixed 1 + 1"
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
|
|
|
|
# add an ontology layer and make sure will still can parse
|
|
sheerka.push_ontology(context, "new ontology")
|
|
parser = SyaNodeParser(sheerka=sheerka)
|
|
|
|
res = parser.parse(context, ParserInput(text))
|
|
wrapper = res.body
|
|
lexer_nodes = res.body.body
|
|
|
|
assert res.status
|
|
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
|
|
assert lexer_nodes == [CN(cmap["suffixed"], 0, 6, source=text)]
|