341 lines
13 KiB
Python
341 lines
13 KiB
Python
from dataclasses import dataclass
|
|
|
|
import core.utils
|
|
import pytest
|
|
from core.concept import Concept
|
|
from core.tokenizer import Token, TokenKind, Tokenizer, Keywords
|
|
|
|
|
|
@dataclass
|
|
class Obj:
|
|
prop1: str
|
|
prop2: str
|
|
|
|
def __hash__(self):
|
|
return hash((self.prop1, self.prop1))
|
|
|
|
|
|
@dataclass
|
|
class Obj2:
|
|
prop1: object
|
|
prop2: object
|
|
|
|
|
|
def get_tokens(lst):
|
|
res = []
|
|
for e in lst:
|
|
if e == " ":
|
|
res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0))
|
|
elif e == "\n":
|
|
res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0))
|
|
elif e == "<EOF>":
|
|
res.append(Token(TokenKind.EOF, "\n", 0, 0, 0))
|
|
else:
|
|
res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0))
|
|
|
|
return res
|
|
|
|
|
|
@pytest.mark.parametrize("lst, as_string", [
|
|
(None, "",),
|
|
([], ""),
|
|
(["hello", "world"], "hello world"),
|
|
# (["hello world", "my friend"], '"hello world" "my friend"')
|
|
])
|
|
def test_i_can_create_string_from_a_list(lst, as_string):
|
|
assert core.utils.sysarg_to_string(lst) == as_string
|
|
|
|
|
|
def test_i_can_get_classes():
|
|
classes = list(core.utils.get_classes("core.builtin_concepts"))
|
|
error_concept = core.utils.get_class("core.builtin_concepts.ErrorConcept")
|
|
return_value_concept = core.utils.get_class("core.builtin_concepts.ReturnValueConcept")
|
|
|
|
assert len(classes) > 2
|
|
assert error_concept in classes
|
|
assert return_value_concept in classes
|
|
|
|
|
|
def test_i_can_get_base_classes():
|
|
classes = list(core.utils.get_classes_from_package("parsers"))
|
|
|
|
# example of classes that should be in the result
|
|
base_parser = core.utils.get_class("parsers.BaseParser.BaseParser")
|
|
def_concept_parser = core.utils.get_class("parsers.DefConceptParser.DefConceptParser")
|
|
exact_concept_parser = core.utils.get_class("parsers.ExactConceptParser.ExactConceptParser")
|
|
python_parser = core.utils.get_class("parsers.PythonParser.PythonParser")
|
|
node = core.utils.get_class("parsers.BaseParser.Node")
|
|
def_concept_node = core.utils.get_class("parsers.DefConceptParser.DefConceptNode")
|
|
python_node = core.utils.get_class("parsers.PythonParser.PythonNode")
|
|
|
|
assert base_parser in classes
|
|
assert def_concept_parser in classes
|
|
assert exact_concept_parser in classes
|
|
assert python_parser in classes
|
|
assert node in classes
|
|
assert def_concept_node in classes
|
|
assert python_node in classes
|
|
|
|
|
|
def test_i_can_get_sub_classes():
|
|
sub_classes = core.utils.get_sub_classes("parsers", "parsers.BaseParser.BaseParser")
|
|
|
|
# example of classes that should be (or not) in the result
|
|
base_parser = core.utils.get_class("parsers.BaseParser.BaseParser")
|
|
def_concept_parser = core.utils.get_class("parsers.DefConceptParser.DefConceptParser")
|
|
exact_concept_parser = core.utils.get_class("parsers.ExactConceptParser.ExactConceptParser")
|
|
python_parser = core.utils.get_class("parsers.PythonParser.PythonParser")
|
|
bnf_node_parser = core.utils.get_class("parsers.BnfNodeParser.BnfNodeParser")
|
|
|
|
assert base_parser not in sub_classes
|
|
assert def_concept_parser in sub_classes
|
|
assert exact_concept_parser in sub_classes
|
|
assert python_parser in sub_classes
|
|
assert bnf_node_parser in sub_classes
|
|
|
|
|
|
@pytest.mark.parametrize("a,b, expected", [
|
|
([], [], []),
|
|
([], ['a'], ['a']),
|
|
([[]], ['a'], [['a']]),
|
|
(['a'], [], ['a']),
|
|
([['a']], [], [['a']]),
|
|
|
|
([['a']], ['b'], [['a', 'b']]),
|
|
([['a'], ['b']], ['c'], [['a', 'c'], ['b', 'c']]),
|
|
([['a1', 'a2'], ['b1', 'b2', 'b3']], ['c'], [['a1', 'a2', 'c'], ['b1', 'b2', 'b3', 'c']]),
|
|
([[]], ['a', 'b'], [['a'], ['b']]),
|
|
([['a'], ['b']], ['c', 'd', 'e'], [['a', 'c'], ['b', 'c'], ['a', 'd'], ['b', 'd'], ['a', 'e'], ['b', 'e']]),
|
|
])
|
|
def test_i_can_product(a, b, expected):
|
|
res = core.utils.product(a, b)
|
|
assert res == expected
|
|
|
|
|
|
@pytest.mark.parametrize("input_as_list, expected_as_list", [
|
|
([" "], []),
|
|
([" ", "one"], ["one"]),
|
|
(["one", " "], ["one"]),
|
|
([" ", "one", " "], ["one"]),
|
|
|
|
(["\n", "one"], ["one"]),
|
|
(["one", "\n"], ["one"]),
|
|
(["\n", "one", "\n"], ["one"]),
|
|
|
|
([" ", "\n", "one"], ["one"]),
|
|
(["one", " ", "\n"], ["one"]),
|
|
([" ", "\n", "one", " ", "\n"], ["one"]),
|
|
|
|
(["\n", " ", "one"], ["one"]),
|
|
(["one", "\n", " "], ["one"]),
|
|
(["\n", " ", "one", "\n", " "], ["one"]),
|
|
|
|
([" ", "\n", " ", "one"], ["one"]),
|
|
(["one", " ", "\n", " "], ["one"]),
|
|
([" ", "\n", " ", "one", " ", "\n", " "], ["one"]),
|
|
|
|
(["\n", " ", "\n", "one"], ["one"]),
|
|
(["one", "\n", " ", "\n"], ["one"]),
|
|
(["\n", " ", "\n", "one", "\n", " ", "\n"], ["one"]),
|
|
|
|
])
|
|
def test_i_can_strip(input_as_list, expected_as_list):
|
|
actual = core.utils.strip_tokens(get_tokens(input_as_list)) # KSI 20201007 Why not use Tokenizer ?!! For perf ?
|
|
expected = get_tokens(expected_as_list)
|
|
assert actual == expected
|
|
|
|
|
|
@pytest.mark.parametrize("text, value, expected", [
|
|
("xxx=yyy", "=", 1),
|
|
("xxx", "=", -1),
|
|
("xxx = yyy", "=", 2),
|
|
("xxx = yyy", " = ", -1),
|
|
])
|
|
def test_i_can_index(text, value, expected):
|
|
assert core.utils.index_tokens(Tokenizer(text), value) == expected
|
|
|
|
|
|
def test_i_can_manage_non_in_index_tokens():
|
|
assert core.utils.index_tokens(None, "=") == -1
|
|
|
|
|
|
def test_by_default_eof_is_not_stripped():
|
|
actual = core.utils.strip_tokens(get_tokens(["one", "two", " ", "\n", "<EOF>"]))
|
|
expected = get_tokens(["one", "two", " ", "\n", "<EOF>"])
|
|
assert actual == expected
|
|
|
|
|
|
def test_i_can_strip_eof():
|
|
actual = core.utils.strip_tokens(get_tokens(["one", "two", " ", "\n", "<EOF>"]), True)
|
|
expected = get_tokens(["one", "two"])
|
|
assert actual == expected
|
|
|
|
|
|
def test_i_can_escape():
|
|
actual = core.utils.escape_char("hello 'world' my friend", "'")
|
|
assert actual == "hello \\'world\\' my friend"
|
|
|
|
|
|
@pytest.mark.parametrize("text, expected_key, expected_id", [
|
|
(None, None, None),
|
|
(10, None, None),
|
|
("", None, None),
|
|
("xxx", None, None),
|
|
("c:", None, None),
|
|
("c:key", None, None),
|
|
("c:key:", "key", None),
|
|
("c:key|id", None, None),
|
|
("c:key|id:", "key", "id"),
|
|
("c:|id:", None, "id"),
|
|
("c:key|:", "key", None),
|
|
("c:key|id:x", None, None),
|
|
])
|
|
def test_i_can_unstr_concept(text, expected_key, expected_id):
|
|
k, i = core.utils.unstr_concept(text)
|
|
assert k == expected_key
|
|
assert i == expected_id
|
|
|
|
|
|
@pytest.mark.parametrize("text, expected_key, expected_id", [
|
|
("r:key:", "key", None),
|
|
("r:key|id:", "key", "id"),
|
|
])
|
|
def test_i_can_unstr_concept_rules(text, expected_key, expected_id):
|
|
k, i = core.utils.unstr_concept(text, prefix="r:")
|
|
assert k == expected_key
|
|
assert i == expected_id
|
|
|
|
|
|
def test_i_can_str_concept():
|
|
assert core.utils.str_concept(("key", "id")) == "c:key|id:"
|
|
assert core.utils.str_concept((None, "id")) == "c:|id:"
|
|
assert core.utils.str_concept(("key", None)) == "c:key:"
|
|
assert core.utils.str_concept((None, None)) == ""
|
|
assert core.utils.str_concept(("key", "id"), drop_name=True) == "c:|id:"
|
|
|
|
concept = Concept("foo").init_key()
|
|
assert core.utils.str_concept(concept) == "c:foo:"
|
|
|
|
concept.get_metadata().id = "1001"
|
|
assert core.utils.str_concept(concept) == "c:foo|1001:"
|
|
assert core.utils.str_concept(concept, drop_name=True) == "c:|1001:"
|
|
|
|
assert core.utils.str_concept(("key", "id"), prefix='r:') == "r:key|id:"
|
|
|
|
|
|
@pytest.mark.parametrize("text, expected", [
|
|
(None, None),
|
|
(10, None),
|
|
("", None),
|
|
("xxx", None),
|
|
("xxx.", None),
|
|
("xxx.yyy", None),
|
|
("core.tokenizer.Keywords.CONCEPT", Keywords.CONCEPT),
|
|
])
|
|
def test_i_can_decode_enum(text, expected):
|
|
actual = core.utils.decode_enum(text)
|
|
assert actual == expected
|
|
|
|
|
|
def test_encode_concept_key_id():
|
|
assert core.utils.encode_concept(("key", "id")) == "__C__KEY_key__ID_id__C__"
|
|
assert core.utils.encode_concept((None, "id")) == "__C__KEY_00None00__ID_id__C__"
|
|
assert core.utils.encode_concept(("key", None)) == "__C__KEY_key__ID_00None00__C__"
|
|
assert core.utils.encode_concept(("k + y", "id")) == "__C__KEY_k000y__ID_id__C__"
|
|
|
|
concept = Concept("foo").init_key()
|
|
assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_00None00__C__"
|
|
|
|
concept.get_metadata().id = "1001"
|
|
assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_1001__C__"
|
|
|
|
assert core.utils.encode_concept(("key", "id"), "R") == "__R__KEY_key__ID_id__R__"
|
|
assert core.utils.encode_concept((None, "id"), "R") == "__R__KEY_00None00__ID_id__R__"
|
|
assert core.utils.encode_concept(("key", None), "R") == "__R__KEY_key__ID_00None00__R__"
|
|
assert core.utils.encode_concept(("k + y", "id"), "R") == "__R__KEY_k000y__ID_id__R__"
|
|
|
|
|
|
def test_decode_concept_key_id():
|
|
assert core.utils.decode_concept("__C__KEY_key__ID_id__C__") == ("key", "id")
|
|
assert core.utils.decode_concept("__C__KEY_00None00__ID_id__C__") == (None, "id")
|
|
assert core.utils.decode_concept("__C__KEY_key__ID_00None00__C__") == ("key", None)
|
|
|
|
assert core.utils.decode_concept("__R__KEY_key__ID_id__R__", "R") == ("key", "id")
|
|
assert core.utils.decode_concept("__R__KEY_00None00__ID_id__R__", "R") == (None, "id")
|
|
assert core.utils.decode_concept("__R__KEY_key__ID_00None00__R__", "R") == ("key", None)
|
|
|
|
|
|
@pytest.mark.parametrize("a,b,expected", [
|
|
([], [], []),
|
|
([{"a": "a", "b": "b"}], [], [{"a": "a", "b": "b"}]),
|
|
([], [{"a": "a", "b": "b"}], [{"a": "a", "b": "b"}]),
|
|
([{"a": "a", "b": "b"}], [{"d": "d1"}, {"d": "d2"}], [{"a": "a", "b": "b", "d": "d1"},
|
|
{"a": "a", "b": "b", "d": "d2"}]),
|
|
([{"d": "d1"}, {"d": "d2"}], [{"a": "a", "b": "b"}], [{"a": "a", "b": "b", "d": "d1"},
|
|
{"a": "a", "b": "b", "d": "d2"}]),
|
|
([{"a": "a", "b": "b"}], [{"d": "d", "e": "e"}], [{"a": "a", "b": "b", "d": "d", "e": "e"}]),
|
|
([{"a": "a"}, {"b": "b"}], [{"d": "d"}, {"e": "e"}], [{"a": "a", "d": "d"},
|
|
{"a": "a", "e": "e"},
|
|
{"b": "b", "d": "d"},
|
|
{"b": "b", "e": "e"}])
|
|
])
|
|
def test_dict_product(a, b, expected):
|
|
assert core.utils.dict_product(a, b) == expected
|
|
|
|
|
|
def test_i_can_make_unique():
|
|
assert core.utils.make_unique(["a", "a", "b", "c", "c"]) == ["a", "b", "c"]
|
|
assert core.utils.make_unique([Obj("a", "b"), Obj("a", "c"), Obj("a", "b")]) == [Obj("a", "b"), Obj("a", "c")]
|
|
assert core.utils.make_unique([Obj("a", "b"), Obj("a", "c")], lambda o: o.prop1) == [Obj("a", "b")]
|
|
|
|
|
|
@pytest.mark.parametrize("expression, bag, expected", [
|
|
("", {}, None),
|
|
(None, {}, None),
|
|
("a", {"a": 1}, 1),
|
|
("a.x", {"a.x": 1}, 1),
|
|
("a.prop1", {"a": Obj("prop1", "prop2")}, "prop1"),
|
|
("a.prop1.prop2.prop1", {"a": Obj2(Obj2("prop11", Obj2("prop121", "prop122")), "2")}, "prop121"),
|
|
("a.prop1.prop2.prop1", {"a": Obj2(Obj2("prop11", None), "2")}, None),
|
|
("a[1]", {"a": ['lst-first', 'lst-second']}, 'lst-second'),
|
|
("a['bar']", {"a": {'foo': 'dict-first', 'bar': 'dict-second'}}, 'dict-second'),
|
|
("a.prop1[0]", {"a": Obj2(['lst-first', 'lst-second'], None)}, 'lst-first'),
|
|
("a.prop2['bar']", {"a": Obj2(None, {'foo': 'dict-first', 'bar': 'dict-second'})}, 'dict-second'),
|
|
("a.bar", {"a": {'foo': 'dict-first', 'bar': 'dict-second'}}, 'dict-second'),
|
|
("a.prop2.bar", {"a": Obj2(None, {'foo': 'dict-first', 'bar': 'dict-second'})}, 'dict-second'),
|
|
])
|
|
def test_i_can_evaluate_expression(expression, bag, expected):
|
|
assert core.utils.evaluate_expression(expression, bag) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("expression, bag, expected_error, prop_name", [
|
|
("a", {}, NameError, "a"),
|
|
("a.prop3", {"a": Obj("prop1", "prop2")}, NameError, "prop3"),
|
|
])
|
|
def test_i_cannot_evaluate_expression(expression, bag, expected_error, prop_name):
|
|
with pytest.raises(expected_error) as e:
|
|
core.utils.evaluate_expression(expression, bag)
|
|
|
|
assert e.value.args == (prop_name,)
|
|
|
|
|
|
@pytest.mark.parametrize("text, expected_text", [
|
|
("hello world", "hello world"),
|
|
("'hello' 'world'", "'hello' 'world'"),
|
|
("def concept a from", "def concept a from"),
|
|
("()[]{}1=1.5+-/*><&é", "()[]{}1=1.5+-/*><&é"),
|
|
("execute(c:concept_name:)", "execute(c:concept_name:)")
|
|
|
|
])
|
|
def test_i_can_get_text_from_tokens(text, expected_text):
|
|
tokens = list(Tokenizer(text))
|
|
assert core.utils.get_text_from_tokens(tokens) == expected_text
|
|
|
|
|
|
@pytest.mark.parametrize("text, custom, expected_text", [
|
|
("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)")
|
|
])
|
|
def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
|
|
tokens = list(Tokenizer(text))
|
|
assert core.utils.get_text_from_tokens(tokens, custom) == expected_text
|