import pytest from core.builtin_concepts import ReturnValueConcept from core.builtin_concepts_ids import BuiltinConcepts from core.builtin_helpers import get_new_variables_definitions, longest_only from core.concept import Concept from core.global_symbols import CONCEPT_COMPARISON_CONTEXT from core.sheerka.Sheerka import RECOGNIZED_BY_KEY from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Token, TokenKind, Tokenizer, comparable_tokens from core.utils import get_text_from_tokens from parsers.BaseExpressionParser import BinaryNode, FunctionNode, ListNode, NameExprNode, VariableNode from parsers.BaseNodeParser import ConceptNode, SourceCodeNode, UnrecognizedTokensNode from parsers.PythonParser import PythonNode from parsers.SyaNodeParser import FunctionDetected, NoSyaConceptFound, NotEnoughParameters, SyaConceptParser, \ SyaNodeParser, SyaTokensParser, TokensNotFound, TooManyParameters from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import CC, CIO, CN, CNC, RETVAL, SCN, UTN, compute_expected_array, get_test_obj, \ prepare_nodes_comparison cmap = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"), "plus": Concept("a plus b").def_var("a").def_var("b"), "mult": Concept("a mult b").def_var("a").def_var("b"), "prefixed": Concept("a prefixed").def_var("a"), "suffixed": Concept("suffixed a").def_var("a"), "infixed": Concept("a infixed b").def_var("a").def_var("b"), "begin": Concept("begin x long end").def_var("x"), } class TestSyaNodeParser(TestUsingMemoryBasedSheerka): shared_ontology = None @classmethod def setup_class(cls): init_test_helper = cls().init_test(cache_only=False, ontology="#TestSyaNodeParser#") sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack() for i, concept_name in enumerate(cmap): cmap[concept_name] = updated[i] cmap["plus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") cmap["mult"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right") TestSyaNodeParser.sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, cmap["mult"], cmap["plus"], CONCEPT_COMPARISON_CONTEXT) cls.shared_ontology = sheerka.get_ontology(context) sheerka.pop_ontology(context) def initialize_test(self, concepts_map=None): if concepts_map is None: sheerka, context = self.init_test().unpack() sheerka.add_ontology(context, self.shared_ontology) else: sheerka, context, *updated = super().init_test().with_concepts(*concepts_map.values(), create_new=True).unpack() for i, concept_name in enumerate(concepts_map): concepts_map[concept_name] = updated[i] return sheerka, context def init_parser(self, concepts_map=None): sheerka, context = self.initialize_test(concepts_map) parser = SyaNodeParser() return sheerka, context, parser def get_real_node(self, concepts_map, expression, source): expr_tokens = list(Tokenizer(expression)) def _to_real_obj(_test_node): if isinstance(_test_node, (CN, CNC)): concept = _test_node.concept if _test_node.concept else concepts_map[_test_node.concept_key] if hasattr(_test_node, "compiled"): for compiled_name, compiled_value in _test_node.compiled.items(): concept.get_compiled()[compiled_name] = _to_real_obj(compiled_value) concept.get_metadata().variables = get_new_variables_definitions(concept) tokens = expr_tokens[_test_node.start:_test_node.start + 1] return ConceptNode(concept, _test_node.start, _test_node.end, tokens=tokens, source=_test_node.source or get_text_from_tokens(tokens)) if isinstance(_test_node, UTN): tokens = expr_tokens[_test_node.start:_test_node.start + 1] return UnrecognizedTokensNode(_test_node.start, _test_node.end, tokens=tokens) if isinstance(_test_node, SCN): tokens = expr_tokens[_test_node.start:_test_node.start + 1] return SourceCodeNode(_test_node.start, _test_node.end, tokens=tokens, ) raise NotImplementedError(_test_node) test_nodes = compute_expected_array(concepts_map, expression, [source]) real_nodes = [_to_real_obj(n) for n in test_nodes] return real_nodes[0] @staticmethod def resolve_expected(actual, concepts_map, expression, array_of_expected): resolved_expected = compute_expected_array(concepts_map, expression, array_of_expected) return get_test_obj(actual, resolved_expected[0]) @pytest.mark.parametrize("concept_key, expected_list", [ ["a long token name", [("a long token name", 0)]], ["__var__0 __var__1 __var__2", [("", 3)]], ["__var__0 __var__1 prefixed", [(" prefixed", 2)]], ["suffixed __var__0 __var__1", [("suffixed ", 0), ["", 2]]], ["__var__0 __var__1 infixed __var__0 __var__1", [(" infixed ", 2), ["", 2]]], ["if __var__0 __var__1 then __var__2 end", [("if ", 0), (" then ", 2), (" end", 1)]] ]) def test_i_can_initialize_expected_parameters(self, concept_key, expected_list): resolved_expected_list = [] for expected in expected_list: source, nb = expected tokens = list(Tokenizer(source, yield_eof=False)) resolved_expected_list.append((tokens, nb)) with comparable_tokens(): assert SyaConceptParser.compute_expected_parameters(concept_key) == resolved_expected_list def test_i_can_concept_parse_simple_infixed_concept(self): sheerka, context = self.initialize_test() expression = "one plus two" param1 = self.get_real_node(cmap, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("plus", a=CNC("one"), b=CNC("two"), source=expression) resolved_expected = compute_expected_array(cmap, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two")] def test_i_can_concept_parse_simple_prefixed_concept(self): sheerka, context = self.initialize_test() expression = "one prefixed" param1 = self.get_real_node(cmap, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, cmap["prefixed"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("prefixed", a=CNC("one"), source=expression) resolved_expected = compute_expected_array(cmap, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one")] def test_i_can_concept_parse_simple_suffixed_concept(self): sheerka, context = self.initialize_test() expression = "suffixed one" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, cmap["suffixed"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("suffixed", a=CNC("one"), source=expression) resolved_expected = compute_expected_array(cmap, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one")] def test_i_can_concept_parse_simple_ternary_concept(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "if": Concept("if a then b else c end").def_var("a").def_var("b").def_var("c") } sheerka, context = self.initialize_test(concepts_map) expression = "if one then two else three end" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, concepts_map["if"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("if", a=CNC("one"), b=CNC("two"), c=CNC("three"), source=expression) resolved_expected = compute_expected_array(concepts_map, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two"), ("c", "three")] def test_i_can_concept_parse_concepts_with_long_names(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "plus": Concept("a plus plus b").def_var("a").def_var("b") } sheerka, context = self.initialize_test(concepts_map) expression = "one plus plus two" param1 = self.get_real_node(concepts_map, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, concepts_map["plus"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("plus", a=CNC("one"), b=CNC("two")) resolved_expected = compute_expected_array(concepts_map, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two")] def test_i_can_concept_parse_infix_when_multiple_variables_are_expected(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "four": Concept("four"), "infixed": Concept("a b infixed c d").def_var("a").def_var("b").def_var("c").def_var("d") } sheerka, context = self.initialize_test(concepts_map) expression = "one two infixed three four" param1 = self.get_real_node(concepts_map, expression, "one") param2 = self.get_real_node(concepts_map, expression, "two") parser_input = ParserInput(expression).reset() parser_input.seek(4) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1, param2] concept_parser = SyaConceptParser(tokens_parser, concepts_map["infixed"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("infixed", a=CNC("one"), b=CNC("two"), c=CNC("three"), d=CNC("four"), source=expression) resolved_expected = compute_expected_array(concepts_map, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two"), ("c", "three"), ("d", "four")] def test_i_can_concept_parse_ternary_when_multiple_variables_are_expected(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "if": Concept("if a b then c end").def_var("a").def_var("b").def_var("c") } sheerka, context = self.initialize_test(concepts_map) expression = "if one two then three end" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, concepts_map["if"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("if", a=CNC("one"), b=CNC("two"), c=CNC("three"), source=expression) resolved_expected = compute_expected_array(concepts_map, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two"), ("c", "three")] def test_i_can_concept_parse_unrecognized(self): concepts_map = { "plus": Concept("a plus b").def_var("a").def_var("b") } sheerka, context = self.initialize_test(concepts_map) expression = "one plus two" param1 = self.get_real_node(concepts_map, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, concepts_map["plus"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("plus", a="one", b="two", source=expression) resolved_expected = compute_expected_array(concepts_map, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two")] def test_i_can_concept_parse_source_code(self): sheerka, context = self.initialize_test() expression = "1 + 1 plus 2 + 2" param1 = SourceCodeNode(0, 5, source="1 + 1 ") parser_input = ParserInput(expression).reset() parser_input.seek(6) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("plus", a=SCN(source="1 + 1 "), b=SCN(source="2 + 2")) resolved_expected = compute_expected_array(cmap, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "1 + 1 "), ("b", "2 + 2")] def test_i_can_concept_parse_function(self): sheerka, context = self.initialize_test() expression = "one plus func(twenty two)" param1 = self.get_real_node(cmap, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("plus", a=CNC("one"), b=SCN("func(twenty two)", [CIO("twenties", source="twenty two")])) actual_as_test, expected_resolved = prepare_nodes_comparison(cmap, expression, concept_node, expected) assert actual_as_test == expected_resolved assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")] def test_i_can_concept_parse_concepts_composition(self): sheerka, context = self.initialize_test() expression = "one plus two mult three" param1 = self.get_real_node(cmap, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("plus", a=CNC("one"), b=CNC("mult", a=CN("two"), b=CN("three")), source=expression) _stack, _expected = prepare_nodes_comparison(cmap, expression, concept_node, expected) assert _stack == _expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two mult three")] def test_i_can_concept_parse_embedded_concepts(self): sheerka, context = self.initialize_test() expression = "begin one plus two long end" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, cmap["begin"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("begin", x=CNC("plus", a=CNC("one"), b=CNC("two")), source=expression) _stack, _expected = prepare_nodes_comparison(cmap, expression, concept_node, expected) assert _stack == _expected assert concept_node.concept.get_metadata().variables == [("x", "one plus two")] def test_i_can_concept_parse_concepts_composition_when_unrecognized_yield_multiple_results(self): sheerka, context = self.initialize_test() expression = "one plus twenty two mult three" param1 = self.get_real_node(cmap, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 # the first part is found as the concept_node expected = CNC("plus", a=CNC("one"), b=UTN("twenty ")) _stack, _expected = prepare_nodes_comparison(cmap, expression, concept_node, expected) assert _stack == _expected # the second part can be found in parameters expected = [CNC("mult", a=CNC("two"), b=CNC("three"))] _stack, _expected = prepare_nodes_comparison(cmap, expression, concept_parser.parameters, expected) assert _stack == _expected # there is a fork assert len(sya_node_parser.forks) == 1 forked_parser = sya_node_parser.forks[0].state_context forked_parser.parse() assert not forked_parser.has_error() expected = CNC("plus", a=CNC("one"), b=CNC("mult", a=CN("twenties", source="twenty two"), b=CN("three"))) _stack, _expected = prepare_nodes_comparison(cmap, expression, forked_parser.concept_node, expected) assert _stack == _expected def test_i_can_concept_parse_when_too_many_suffix_parameters(self): sheerka, context = self.initialize_test() expression = "one plus two three" param1 = self.get_real_node(cmap, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() assert not concept_parser.has_error() expected = CNC("plus", a=CNC("one"), b=CNC("two"), source="one plus two") resolved_expected = compute_expected_array(cmap, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_parser.concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert not concept_parser.has_error() assert concept_parser.concept_node.start == 0 assert concept_parser.concept_node.end == 4 def test_i_can_concept_parse_when_too_many_prefix_parameters(self): sheerka, context = self.initialize_test() expression = "one two plus three" param1 = self.get_real_node(cmap, expression, "one") param2 = self.get_real_node(cmap, expression, "two") parser_input = ParserInput(expression).reset() parser_input.seek(4) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1, param2] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() assert not concept_parser.has_error() expected = CNC("plus", a=CNC("two"), b=CNC("three"), source="two plus three") resolved_expected = compute_expected_array(cmap, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_parser.concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert not concept_parser.has_error() assert concept_parser.concept_node.start == 2 assert concept_parser.concept_node.end == 6 def test_i_can_concept_rollback_when_the_incorrect_concept_is_parsed(self): concepts_map = { "one": Concept("one"), "twenty one": Concept("twenty one"), "twenty two": Concept("twenty two x").def_var("x"), "plus": Concept("a plus b").def_var("a").def_var("b"), } sheerka, context = self.initialize_test(concepts_map) expression = "one plus twenty one" param1 = self.get_real_node(concepts_map, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, concepts_map["plus"], tokens_parser.stack) concept_parser.parse() assert not concept_parser.has_error() concept_node = concept_parser.concept_node expected = CNC("plus", a=CNC("one"), b=CNC("twenty one"), source=expression) resolved_expected = compute_expected_array(concepts_map, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "twenty one")] def test_i_can_concept_parse_when_unrecognized_tokens_yields_multiple_results_ternary_concept(self): sheerka, context = self.initialize_test() expression = "begin twenty two long end" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, cmap["begin"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert concept_parser.errors == [TooManyParameters(cmap["begin"], " long end", 5, 1)] assert len(sya_node_parser.forks) == 1 forked_parser = sya_node_parser.forks[0].state_context forked_parser.parse() assert not forked_parser.has_error() expected = CNC("begin", x=CN("twenties", source="twenty two"), source=expression) _stack, _expected = prepare_nodes_comparison(cmap, expression, forked_parser.concept_node, expected) assert _stack == _expected def test_i_can_concept_parse_when_multiple_concepts_found(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "plus": Concept("a plus b").def_var("a").def_var("b"), "minus1": Concept("a minus b").def_var("a").def_var("b"), "minus2": Concept("x minus y").def_var("x").def_var("y"), } sheerka, context = self.initialize_test(concepts_map) expression = "one plus two minus three" param1 = self.get_real_node(concepts_map, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, concepts_map["plus"], tokens_parser.stack) concept_parser.parse() assert not concept_parser.has_error() expected = CNC("plus", a=CN("one"), b=CNC("minus1", a=CN("two"), b=CN("three"), source="two minus three")) _stack, _expected = prepare_nodes_comparison(concepts_map, expression, concept_parser.concept_node, expected) assert _stack == _expected assert len(sya_node_parser.forks) == 1 forked_parser = sya_node_parser.forks[0].state_context forked_parser.parse() assert not forked_parser.has_error() expected = CNC("plus", a=CN("one"), b=CNC("minus2", x=CN("two"), y=CN("three"), source="two minus three")) _stack, _expected = prepare_nodes_comparison(concepts_map, expression, forked_parser.concept_node, expected) assert _stack == _expected def test_i_can_concept_detect_the_end_of_a_concept(self): sheerka, context = self.initialize_test() expression = "begin one long end two" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, cmap["begin"], tokens_parser.stack) concept_parser.parse() concept_node = concept_parser.concept_node assert not concept_parser.has_error() assert len(concept_parser.expected) == 0 expected = CNC("begin", x=CNC("one"), source="begin one long end") _concept_node, _expected = prepare_nodes_comparison(cmap, expression, concept_node, expected) assert _concept_node == _expected def test_i_can_concept_early_detect_when_an_incorrect_concept_is_parsed(self): concepts_map = { "foo": Concept("foo"), "car": Concept("beautiful but expensive car"), "isa": Concept("x is a command").def_var("x"), } sheerka, context = self.initialize_test(concepts_map) expression = "foo is a beautiful but expensive car" param1 = self.get_real_node(concepts_map, expression, "foo") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, concepts_map["isa"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert isinstance(concept_parser.errors[0], TokensNotFound) assert concept_parser.checkpoint == 3 # I stopped before the end of the input def test_i_can_tokens_parse_unrecognized(self): """ When there is no sya concept, there is no need to manage unrecognized """ sheerka, context = self.initialize_test() expression = "one two three" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() assert len(tokens_parser.stack) == 1 expected = [UTN("one two three")] _stack, _expected = prepare_nodes_comparison(cmap, expression, tokens_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_parse_simple_infixed_concept(self): sheerka, context = self.initialize_test() expression = "one plus two" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() assert len(tokens_parser.stack) == 1 concept_node = tokens_parser.stack[0] expected = CNC("plus", a=CNC("one"), b=CNC("two"), source=expression) resolved_expected = compute_expected_array(cmap, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two")] def test_i_can_tokens_parse_concepts_composition(self): sheerka, context = self.initialize_test() expression = "one plus two mult three" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() assert len(tokens_parser.stack) == 1 concept_node = tokens_parser.stack[0] expected = CNC("plus", a=CNC("one"), b=CNC("mult", a=CN("two"), b=CN("three")), source=expression) resolved_expected = compute_expected_array(cmap, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two mult three")] def test_i_can_tokens_parse_concepts_composition_when_pop(self): sheerka, context = self.initialize_test() expression = "one mult two plus three" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() assert len(tokens_parser.stack) == 1 concept_node = tokens_parser.stack[0] expected = CNC("plus", a=CNC("mult", a=CN("one"), b=CN("two")), b=CNC("three"), source=expression) resolved_expected = compute_expected_array(cmap, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one mult two"), ("b", "three")] @pytest.mark.parametrize("expression, expected", [ ("suffixed foo bar", ["suffixed x", "bar"]), ("foo suffixed bar", ["foo", "suffixed x"]), ("foo prefixed bar", ["x prefixed", "bar"]), ("foo bar prefixed", ["foo", "x prefixed"]), ("foo infixed bar baz", ["x infixed y", "baz"]), ("foo bar infixed bar", ["foo", "x infixed y"]), ("foo infixed bar baz infixed qux", ["x infixed y", "x infixed y"]), ]) def test_i_can_tokens_parse_sequences(self, expression, expected): concepts_map = { "foo": Concept("foo"), "bar": Concept("bar"), "baz": Concept("baz"), "qux": Concept("qux"), "suffixed": Concept("suffixed x").def_var("x"), "prefixed": Concept("x prefixed").def_var("x"), "infixed": Concept("x infixed y").def_var("x").def_var("y"), } sheerka, context = self.initialize_test(concepts_map) parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() assert len(tokens_parser.stack) == 2 to_compare_to = [concept_node.concept.name for concept_node in tokens_parser.stack] assert to_compare_to == expected assert len(sya_node_parser.forks) == 0 def test_i_can_tokens_parse_sequences_sya_concept_and_concept(self): concepts_map = { "foo": Concept("foo"), "two": Concept("two"), "one x": Concept("one x").def_var("x"), } sheerka, context = self.initialize_test(concepts_map) expression = "two one foo" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() assert len(tokens_parser.stack) == 2 assert isinstance(tokens_parser.stack[0], ConceptNode) assert sheerka.isinstance(tokens_parser.stack[0].concept, concepts_map["two"].key) assert isinstance(tokens_parser.stack[1], ConceptNode) assert sheerka.isinstance(tokens_parser.stack[1].concept, concepts_map["one x"].key) def test_i_can_tokens_parse_when_trailing_unrecognized_tokens_yields_multiple_results(self): sheerka, context = self.initialize_test() expression = "one plus twenty two" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() expected = [CNC("plus", a=CNC("one"), b=UTN("twenty ")), "two"] _stack, _expected = prepare_nodes_comparison(cmap, expression, tokens_parser.stack, expected) assert _stack == _expected assert len(sya_node_parser.forks) == 1 forked_parser = sya_node_parser.forks[0] forked_parser.parse() assert not forked_parser.has_error() expected = [CNC("plus", a=CNC("one"), b=CN("twenties", source="twenty two"))] _stack, _expected = prepare_nodes_comparison(cmap, expression, forked_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_parse_when_leading_unrecognized_tokens_yields_multiple_results(self): sheerka, context = self.initialize_test() expression = "twenty one plus two" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() expected = [UTN("twenty "), CNC("plus", a=CNC("one"), b=CNC("two"))] _stack, _expected = prepare_nodes_comparison(cmap, expression, tokens_parser.stack, expected) assert _stack == _expected assert len(sya_node_parser.forks) == 1 forked_parser = sya_node_parser.forks[0] forked_parser.parse() assert not forked_parser.has_error() expected = [CNC("plus", a=CN("twenties", source="twenty one"), b=CNC("two"))] _stack, _expected = prepare_nodes_comparison(cmap, expression, forked_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_parse_when_unrecognized_tokens_yields_multiple_results_ternary_concept(self): sheerka, context = self.initialize_test() expression = "begin twenty two long end" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() concept_parser = SyaTokensParser(context, sya_node_parser, parser_input) concept_parser.parse() assert not concept_parser.has_error() expected = [UTN("begin twenty two long end")] _stack, _expected = prepare_nodes_comparison(cmap, expression, concept_parser.stack, expected) assert _stack == _expected assert len(sya_node_parser.forks) == 1 forked_parser = sya_node_parser.forks[0] forked_parser.parse() assert not forked_parser.has_error() expected = [CNC("begin", x=CN("twenties", source="twenty two"), source=expression)] _stack, _expected = prepare_nodes_comparison(cmap, expression, forked_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_rollback_when_the_incorrect_concept_is_parsed(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "one and only": Concept("one and only x").def_var("x"), "plus": Concept("a plus b").def_var("a").def_var("b"), } sheerka, context = self.initialize_test(concepts_map) expression = "one plus two" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() assert len(tokens_parser.stack) == 1 concept_node = tokens_parser.stack[0] expected = CNC("plus", a=CNC("one"), b=CNC("two"), source=expression) resolved_expected = compute_expected_array(concepts_map, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two")] def test_i_can_tokens_parse_when_tokens_parser_yields_multiple_results(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "plus": Concept("a plus b").def_var("a").def_var("b"), "mult1": Concept("a mult b").def_var("a").def_var("b"), "mult2": Concept("x mult y").def_var("x").def_var("y"), } sheerka, context = self.initialize_test(concepts_map) sheerka.set_precedence(context, concepts_map["mult1"], concepts_map["plus"]) sheerka.set_precedence(context, concepts_map["mult2"], concepts_map["plus"]) expression = "one mult two plus three" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() expected = [CNC("plus", a=CNC("mult1", a=CN("one"), b=CN("two"), source="one mult two"), b=CN("three"))] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, tokens_parser.stack, expected) assert _stack == _expected assert len(sya_node_parser.forks) == 1 forked_parser = sya_node_parser.forks[0] assert type(forked_parser) == SyaTokensParser forked_parser.parse() assert not forked_parser.has_error() expected = [CNC("plus", a=CNC("mult2", x=CN("one"), y=CN("two"), source="one mult two"), b=CN("three"))] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, forked_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_parse_when_concept_parsers_yield_multiple_results(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "plus": Concept("a plus b").def_var("a").def_var("b"), "minus1": Concept("a minus b").def_var("a").def_var("b"), "minus2": Concept("x minus y").def_var("x").def_var("y"), } sheerka, context = self.initialize_test(concepts_map) expression = "one plus two minus three" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() expected = [CNC("plus", a=CN("one"), b=CNC("minus1", a=CN("two"), b=CN("three"), source="two minus three"))] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, tokens_parser.stack, expected) assert _stack == _expected assert len(sya_node_parser.forks) == 1 forked_parser = sya_node_parser.forks[0] assert type(forked_parser) == SyaTokensParser forked_parser.parse() assert not forked_parser.has_error() expected = [CNC("plus", a=CN("one"), b=CNC("minus2", x=CN("two"), y=CN("three"), source="two minus three"))] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, forked_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_parse_when_concept_parsers_pop_multiple_results(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "plus1": Concept("a plus b").def_var("a").def_var("b"), "plus2": Concept("x plus y").def_var("x").def_var("y"), "mult": Concept("a mult b").def_var("a").def_var("b"), } sheerka, context = self.initialize_test(concepts_map) sheerka.set_precedence(context, concepts_map["mult"], concepts_map["plus1"]) sheerka.set_precedence(context, concepts_map["mult"], concepts_map["plus2"]) expression = "one mult two plus three" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() expected = [CNC("plus1", a=CNC("mult", a=CN("one"), b=CN("two")), b=CN("three"), source=expression)] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, tokens_parser.stack, expected) assert _stack == _expected assert len(sya_node_parser.forks) == 1 forked_parser = sya_node_parser.forks[0] assert type(forked_parser) == SyaTokensParser forked_parser.parse() assert not forked_parser.has_error() expected = [CNC("plus2", x=CNC("mult", a=CN("one"), b=CN("two")), y=CN("three"), source=expression)] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, forked_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_parse_when_concept_parsers_mix_multiple_results(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "plus1": Concept("a plus b").def_var("a").def_var("b"), "plus2": Concept("x plus y").def_var("x").def_var("y"), "plus3": Concept("u plus v").def_var("u").def_var("v"), "plus4": Concept("i plus j").def_var("i").def_var("j"), "mult": Concept("a mult b").def_var("a").def_var("b"), } sheerka, context = self.initialize_test(concepts_map) sheerka.set_precedence(context, concepts_map["mult"], concepts_map["plus1"]) sheerka.set_precedence(context, concepts_map["mult"], concepts_map["plus2"]) sheerka.set_precedence(context, concepts_map["plus3"], concepts_map["mult"]) sheerka.set_precedence(context, concepts_map["plus4"], concepts_map["mult"]) expression = "one mult two plus three" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() expected = [CNC("plus1", a=CNC("mult", a=CN("one"), b=CN("two")), b=CN("three"), source=expression)] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, tokens_parser.stack, expected) assert _stack == _expected assert len(sya_node_parser.forks) == 3 forked_parser = sya_node_parser.forks[0] forked_parser.parse() assert not forked_parser.has_error() expected = [CNC("plus2", x=CNC("mult", a=CN("one"), b=CN("two")), y=CN("three"), source=expression)] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, forked_parser.stack, expected) assert _stack == _expected forked_parser = sya_node_parser.forks[1] forked_parser.parse() assert not forked_parser.has_error() expected = [CNC("mult", a=CN("one"), b=CNC("plus3", u=CN("two"), v=CN("three"), source="two plus three"))] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, forked_parser.stack, expected) assert _stack == _expected forked_parser = sya_node_parser.forks[2] forked_parser.parse() assert not forked_parser.has_error() expected = [CNC("mult", a=CN("one"), b=CNC("plus4", i=CN("two"), j=CN("three"), source="two plus three"))] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, forked_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_rollback_incorrect_concept_when_prefix_parameters(self): sheerka, context = self.initialize_test() expression = "desc(infixed)" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() expected = [UTN("desc(infixed)")] _stack, _expected = prepare_nodes_comparison(cmap, expression, tokens_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_parse_when_multiple_levels_of_pop(self): concepts_map = { "what": Concept("what is the x of y").def_var("x").def_var("y"), "?": Concept("q ?").def_var("q"), "the": Concept("the x").def_var("x"), "color": Concept("color"), "short": Concept("short"), } sheerka, context = self.initialize_test(concepts_map) sheerka.set_precedence(context, concepts_map["what"], concepts_map["?"]) sheerka.set_precedence(context, concepts_map["the"], concepts_map["?"]) expression = "what is the color of the short ?" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() assert len(tokens_parser.stack) == 1 expected = [CNC("?", q=CNC("what", x=CNC("color"), y=CNC("the", x=CNC("short"), source="the short")), source=expression)] _stack, _expected = prepare_nodes_comparison(concepts_map, expression, tokens_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_manage_when_leading_parameter_is_a_whitespace(self): sheerka, context = self.initialize_test() expression = " plus two" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() expected = [UTN(" plus two")] _stack, _expected = prepare_nodes_comparison(cmap, expression, tokens_parser.stack, expected) assert _stack == _expected def test_i_can_tokens_manage_when_trailing_parameter_is_a_whitespace(self): sheerka, context = self.initialize_test() expression = "one plus " parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() expected = ["one", UTN("plus ")] _stack, _expected = prepare_nodes_comparison(cmap, expression, tokens_parser.stack, expected) assert _stack == _expected def test_i_always_look_for_the_longest_match(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "one x": Concept("one x").def_var("x"), "plus": Concept("a plus b").def_var("a").def_var("b"), } sheerka, context = self.initialize_test(concepts_map) expression = "one plus two" parser_input = ParserInput(expression).reset() sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.parse() assert not tokens_parser.has_error() assert len(tokens_parser.stack) == 2 assert len(sya_node_parser.forks) == 1 forked_tokens_parser = sya_node_parser.forks[0] forked_tokens_parser.parse() concept_node = forked_tokens_parser.stack[0] expected = CNC("plus", a=CNC("one"), b=CNC("two"), source=expression) resolved_expected = compute_expected_array(concepts_map, expression, [expected])[0] concept_node_as_test_obj = get_test_obj(concept_node, resolved_expected) assert concept_node_as_test_obj == resolved_expected assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "two")] def test_i_cannot_concept_parse_concept_when_not_enough_prefix_parameters(self): sheerka, context = self.initialize_test() expression = "one plus two" parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert concept_parser.errors == [NotEnoughParameters(cmap["plus"], " plus ", 2, 1)] def test_i_cannot_concept_parse_concept_when_leading_parameters_is_a_white_space(self): sheerka, context = self.initialize_test() expression = " plus two" parser_input = ParserInput(expression).reset() parser_input.seek(1) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [UTN(" ", start=0, end=0)] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert concept_parser.errors == [NotEnoughParameters(cmap["plus"], " plus ", 1, 1)] def test_i_cannot_concept_parse_concept_when_trailing_parameters_is_a_white_space(self): sheerka, context = self.initialize_test() expression = "one plus " param1 = self.get_real_node(cmap, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert concept_parser.errors == [NotEnoughParameters(cmap["plus"], "", 4, 1)] def test_i_cannot_concept_parse_when_not_enough_suffix_parameters(self): concepts_map = { "one": Concept("one"), "plus": Concept("a plus b").def_var("a").def_var("b") } sheerka, context = self.initialize_test(concepts_map) expression = "one plus " param1 = self.get_real_node(concepts_map, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, concepts_map["plus"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert concept_parser.errors == [NotEnoughParameters(concepts_map["plus"], '', 4, 1)] def test_i_cannot_concept_parse_when_not_enough_infix_parameters(self): concepts_map = { "one": Concept("one"), "begin": Concept("begin a b end").def_var("a").def_var("b") } sheerka, context = self.initialize_test(concepts_map) expression = "begin one end" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, concepts_map["begin"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert concept_parser.errors == [NotEnoughParameters(concepts_map["begin"], ' end', 3, 2)] def test_i_cannot_concept_parse_when_too_many_infix_parameters(self): sheerka, context = self.initialize_test() expression = "begin one two long end" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, cmap["begin"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert concept_parser.errors == [TooManyParameters(cmap["begin"], ' long end', 5, 1)] def test_i_cannot_concept_parse_when_missing_parts(self): sheerka, context = self.initialize_test() expression = "begin twenty" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, cmap["begin"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert concept_parser.errors == [TokensNotFound(cmap['begin'], " long end")] def test_i_can_detect_when_parsing_the_wrong_concept(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "if": Concept("if a then b").def_var("a").def_var("b") } sheerka, context = self.initialize_test(concepts_map) expression = "if one else two" parser_input = ParserInput(expression).reset() parser_input.seek(0) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [] concept_parser = SyaConceptParser(tokens_parser, concepts_map["if"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() assert concept_parser.errors == [TokensNotFound(concepts_map["if"], " then ")] def test_i_can_detect_invalid_long_name_parsing(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "plus": Concept("a plus plus b").def_var("a").def_var("b") } sheerka, context = self.initialize_test(concepts_map) expression = "one plus two" param1 = self.get_real_node(concepts_map, expression, "one") parser_input = ParserInput(expression).reset() parser_input.seek(2) sya_node_parser = SyaNodeParser() tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input) tokens_parser.stack = [param1] concept_parser = SyaConceptParser(tokens_parser, concepts_map["plus"], tokens_parser.stack) concept_parser.parse() assert concept_parser.has_error() with comparable_tokens(): assert concept_parser.errors == [TokensNotFound(concepts_map["plus"], " plus ")] def test_i_can_parse_simple_concepts_composition(self): sheerka, context, parser = self.init_parser() text = "one plus two mult three" res = parser.parse(context, ParserInput(text)) wrapper = res.body lexer_nodes = res.body.body assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected = [CNC("plus", a=CC("one"), b=CC("mult", a=CC("two"), b=CC("three")), source=text)] _stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected) assert _stack == _expected # check the metadata expected_concept = lexer_nodes[0].concept assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "two mult three")] assert expected_concept.get_compiled()["b"].get_metadata().variables == [("a", "two"), ("b", "three")] def test_i_can_parse_when_python_code(self): sheerka, context, parser = self.init_parser() text = "suffixed 1 + 1" res = parser.parse(context, ParserInput(text)) wrapper = res.body lexer_nodes = res.body.body assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected_return_value = RETVAL("1 + 1") expected = [CNC(cmap["suffixed"], text, 0, 6, a=[expected_return_value])] concept_node_as_test_obj = get_test_obj(lexer_nodes, expected) assert concept_node_as_test_obj == expected # check metadata expected_concept = lexer_nodes[0].concept assert expected_concept.get_metadata().variables == [("a", "1 + 1")] def test_i_can_parse_when_bnf_concept(self): sheerka, context, parser = self.init_parser() text = "suffixed twenty one" res = parser.parse(context, ParserInput(text)) assert len(res) == 2 assert res[1].status wrapper = res[1].body lexer_nodes = res[1].body.body assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected = [CN(cmap["suffixed"], text, 0, 4)] concept_node_as_test_obj = get_test_obj(lexer_nodes, expected) assert concept_node_as_test_obj == expected # check the compiled expected_concept = lexer_nodes[0].concept assert sheerka.isinstance(expected_concept.get_compiled()["a"], "twenties") assert expected_concept.get_compiled()["a"].get_compiled()["unit"] == cmap["one"] # check metadata assert expected_concept.get_metadata().variables == [("a", "twenty one")] @pytest.mark.skip("Not quite sure that this test is relevant") def test_i_can_parse_when_bnf_and_python(self): # twenty one + 1 is not correctly parsed by BNFNode or SequenceNode # Maybe I should just leave it to ExpressionParser instead sheerka, context, parser = self.init_parser() text = "suffixed twenty one + 1" res = parser.parse(context, ParserInput(text)) # only care about the result that recognizes 'twenty one' longest = longest_only(context, res) res = longest.body.body[0] wrapper = res.body lexer_nodes = res.body.body assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected_return_value = RETVAL("twenty one + 1", objects={"__o_00__": CIO("twenties", source="twenty one")}) expected = [CNC(cmap["suffixed"], text, 0, 8, a=[expected_return_value])] concept_node_as_test_obj = get_test_obj(lexer_nodes, expected) assert concept_node_as_test_obj == expected # check metadata expected_concept = lexer_nodes[0].concept assert expected_concept.get_metadata().variables == [("a", "twenty one + 1")] def test_i_can_parse_when_function(self): sheerka, context, parser = self.init_parser() text = "one plus func(twenty two)" res = parser.parse(context, ParserInput(text)) wrapper = res.body lexer_nodes = res.body.body assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected_ret_val = RETVAL("func(twenty two)", "func(__o_00__)", objects={"__o_00__": CIO("twenties", source="twenty two")}) expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)] _stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected) assert _stack == _expected # check the metadata expected_concept = lexer_nodes[0].concept assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")] def test_i_can_parse_sequences(self): sheerka, context, parser = self.init_parser() text = "one plus 1 + 1 suffixed two" res = parser.parse(context, ParserInput(text)) wrapper = res.body lexer_nodes = res.body.body assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected = [ CN(cmap["plus"], "one plus 1 + 1 ", 0, 9), CN(cmap["suffixed"], "suffixed two", 10, 12)] _stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected) assert _stack == _expected # check the compiled concept_plus_a = lexer_nodes[0].concept.get_compiled()["a"] concept_plus_b = lexer_nodes[0].concept.get_compiled()["b"] concept_suffixed_a = lexer_nodes[1].concept.get_compiled()["a"] assert concept_plus_a == cmap["one"] assert len(concept_plus_b) == 1 assert sheerka.isinstance(concept_plus_b[0], BuiltinConcepts.RETURN_VALUE) assert isinstance(concept_plus_b[0].body.body, PythonNode) assert concept_suffixed_a == cmap["two"] def test_i_can_parse_when_mix_bnf_and_python_code(self): sheerka, context, parser = self.init_parser() text = "twenty two plus two mult 3" res = parser.parse(context, ParserInput(text)) assert len(res) == 2 assert res[1].status lexer_nodes = res[1].body.body assert len(lexer_nodes) == 1 concept_found = lexer_nodes[0].concept assert sheerka.isinstance(concept_found.get_compiled()["a"], "twenties") assert sheerka.isinstance(concept_found.get_compiled()["b"], cmap["mult"]) concept_mult = concept_found.get_compiled()["b"] assert sheerka.isinstance(concept_mult.get_compiled()["a"], "two") assert isinstance(concept_mult.get_compiled()["b"], list) assert isinstance(concept_mult.get_compiled()["b"][0], ReturnValueConcept) assert "Python" in concept_mult.get_compiled()["b"][0].who @pytest.mark.parametrize("text, expected", [ ("suffixed3 a b c", [("x", "a"), ("y", "b"), ("z", "c")]), ("a b c prefixed3", [("x", "a"), ("y", "b"), ("z", "c")]), ("start3 a b c stop", [("x", "a"), ("y", "b"), ("z", "c")]), ]) def test_i_can_parse_when_multiple_parameters_are_expected(self, text, expected): concepts_map = { "a": Concept("a"), "b": Concept("b"), "c": Concept("c"), "suffixed3": Concept("suffixed3 x y z").def_var("x").def_var("y").def_var("z"), "prefixed3": Concept("x y z prefixed3").def_var("x").def_var("y").def_var("z"), "start3": Concept("start3 x y z stop").def_var("x").def_var("y").def_var("z"), } sheerka, context, parser = self.init_parser(concepts_map) res = parser.parse(context, ParserInput(text)) lexer_nodes = res.body.body assert res.status assert lexer_nodes[0].concept.get_metadata().variables == expected def test_i_can_parse_when_all_variables_are_not_parameters(self): my_map = { "foo": Concept("foo a").def_var("a", "'default_a'").def_var("b", "'default_b'"), # 'b' is not a parameter "bar": Concept("bar b").def_var("a", "'default_a'").def_var("b", "'default_b'"), # 'a' is not a parameter "baz": Concept("baz") } sheerka, context, parser = self.init_parser(my_map) text = "foo baz" res = parser.parse(context, ParserInput(text)) assert res.status assert len(res.body.body) == 1 expected_concept = res.body.body[0].concept assert sheerka.isinstance(expected_concept.get_compiled()["a"], "baz") assert "b" not in expected_concept.get_compiled() assert expected_concept.get_metadata().variables == [("a", "baz"), ("b", "'default_b'")] text = "bar baz" res = parser.parse(context, ParserInput(text)) assert res.status assert len(res.body.body) == 1 expected_concept = res.body.body[0].concept assert sheerka.isinstance(expected_concept.get_compiled()["b"], "baz") assert "a" not in expected_concept.get_compiled() assert expected_concept.get_metadata().variables == [("a", "'default_a'"), ("b", "baz")] def test_i_can_parse_when_defining_parenthesis(self): concepts_map = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "plus": Concept("a plus b").def_var("a").def_var("b"), "mult": Concept("a mult b").def_var("a").def_var("b"), "(": Concept("( x )").def_var("x") } sheerka, context, parser = self.init_parser(concepts_map) text = "( one plus two ) mult three" res = parser.parse(context, ParserInput(text)) lexer_nodes = res.body.body expected = [CNC("mult", a=CC("(", x=CC("plus"), source="( one plus two )"), b="three")] _stack, _expected = prepare_nodes_comparison(concepts_map, text, lexer_nodes, expected) assert _stack == _expected def test_i_can_parse_when_function_style_expr_tokens(self): sheerka, context, parser = self.init_parser() text = "one plus func(twenty two)" tokens = list(Tokenizer(text, yield_eof=False)) fun_token = tokens[4] expr = FunctionNode(4, 9, tokens[4:10], NameExprNode(4, 4, tokens[4:5]), ListNode(5, 9, tokens[5:10], NameExprNode(5, 5, tokens[5:6]), NameExprNode(9, 9, tokens[9:10]), [NameExprNode(6, 8, tokens[6:9])])) tokens[4:] = [Token(TokenKind.EXPR, expr, fun_token.index, fun_token.line, fun_token.column)] res = parser.parse(context, ParserInput(None, tokens=tokens)) wrapper = res.body lexer_nodes = res.body.body assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected_ret_val = RETVAL("func(twenty two)", objects={"__o_00__": CIO("twenties", source="twenty two")}) expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)] _stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected) assert _stack == _expected # check the metadata expected_concept = lexer_nodes[0].concept assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")] def test_i_can_parse_when_binary_expr_tokens(self): sheerka, context, parser = self.init_parser() text = "suffixed twenty one + self" tokens = list(Tokenizer(text, yield_eof=False)) expr = BinaryNode(2, 8, tokens[2:9], tokens[6], VariableNode(2, 4, tokens[2:5], "twenty one"), VariableNode(8, 8, tokens[8:9], "self")) token_twenty = tokens[2] tokens[2:] = [Token(TokenKind.EXPR, expr, token_twenty.index, token_twenty.line, token_twenty.column)] res = parser.parse(context, ParserInput(None, tokens=tokens)) wrapper = res.body lexer_nodes = res.body.body assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected_ret_val = RETVAL("twenty one + self", source="__o_00__ + self", objects={"__o_00__": CIO("twenties", source="twenty one")}) expected = [CNC("suffixed", a=[expected_ret_val], source=text)] _stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected) assert _stack == _expected # check the metadata expected_concept = lexer_nodes[0].concept assert expected_concept.get_metadata().variables == [("a", "twenty one + self")] @pytest.mark.parametrize("text, expected_result", [ ("one plus two foo bar baz", [CNC("plus", a="one", b="two"), UTN(" foo bar baz")]), ("one plus two foo bar", [CNC("plus", a="one", b="two"), UTN(" foo bar")]), ("foo bar one plus two", [UTN("foo bar "), CNC("plus", a="one", b="two")]), ("one plus two a long other b", [CNC("plus", a="one", b="two"), UTN(" a long other b")]), ("one plus two a long infixed", [CNC("plus", a="one", b="two"), UTN(" a long infixed")]), ("one plus two a long", [CNC("plus", a="one", b="two"), UTN(" a long")]), ]) def test_i_can_almost_parse_when_one_part_is_recognized_but_not_the_rest(self, text, expected_result): """ Return value status must be False when there are unrecognized tokens :return: """ sheerka, context, parser = self.init_parser() res = parser.parse(context, ParserInput(text)) wrapper = res.body lexer_nodes = res.body.body expected_array = compute_expected_array(cmap, text, expected_result) assert not res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) transformed_nodes = get_test_obj(lexer_nodes, expected_array) assert transformed_nodes == expected_array @pytest.mark.parametrize("text, expected_result", [ ("a plus b", [CN("plus", "a plus b")]), ("suffixed a plus b", [CN("suffixed", "suffixed a plus b")]), ]) def test_i_can_almost_parse_concept_definition(self, text, expected_result): """ In these examples, 'a' and 'b' are not defined. But the concepts are recognized :param text: :param expected_result: :return: """ sheerka, context, parser = self.init_parser() res = parser.parse(context, ParserInput(text)) wrapper = res.body lexer_nodes = res.body.body expected_array = compute_expected_array(cmap, text, expected_result) assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) transformed_nodes = get_test_obj(lexer_nodes, expected_array) assert transformed_nodes == expected_array @pytest.mark.parametrize("text, expected_error", [ ("x$!# prefixed", "Cannot parse 'x$!# '"), ("suffixed x$!#", "Cannot parse 'x$!#'"), ("one infixed x$!#", "Cannot parse 'x$!#'"), ("x$!# infixed one", "Cannot parse 'x$!# '"), ("x$!# infixed z$!#", ["Cannot parse 'x$!# '", "Cannot parse 'z$!#'"]), ("suffixed alpha beta", "Cannot parse 'alpha beta'"), ("alpha beta prefixed", "Cannot parse 'alpha beta '"), ("one plus alpha beta", "Cannot parse 'alpha beta'"), ]) def test_i_cannot_parse_when_unrecognized(self, text, expected_error): sheerka, context, parser = self.init_parser() res = parser.parse(context, ParserInput(text)) wrapper = res.body assert not res.status assert sheerka.isinstance(wrapper, BuiltinConcepts.ERROR) assert wrapper.body == expected_error def test_i_cannot_parse_empty_string(self): sheerka, context, parser = self.init_parser() res = parser.parse(context, ParserInput("")) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY) def test_i_cannot_parse_concept_when_variable_is_not_a_parameter(self): my_map = { "foo": Concept("foo").def_var("a") # 'a' is not a parameter (but a property like its color for ex) } sheerka, context, parser = self.init_parser(my_map) text = "foo" res = parser.parse(context, ParserInput(text)) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) assert res.body.body == text @pytest.mark.parametrize("text, expected_error", [ ("one", NoSyaConceptFound()), ("1 + 1", NoSyaConceptFound()), ("x$!#", NoSyaConceptFound()), ("twenty one", NoSyaConceptFound()), ("function(not an sya concept)", FunctionDetected()), ]) def test_i_cannot_parse_when_no_concept_is_recognized(self, text, expected_error): """ it's actually no concept with property Atoms concepts, source code or BNF concepts alone are discarded by the lexer :return: """ sheerka, context, parser = self.init_parser() res = parser.parse(context, ParserInput(text)) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) assert res.body.body == text assert res.body.reason == [expected_error] def test_i_cannot_parse_function_using_short_name(self): concepts_map = { "infixed": self.from_def_concept("infixed", "a infixed b", ["a", "b"]), "suffixed": self.from_def_concept("suffixed", "suffixed a", ["a"]), "prefixed": self.from_def_concept("prefixed", "a prefixed", ["a"]), } sheerka, context, parser = self.init_parser(concepts_map) res = parser.parse(context, ParserInput("desc(suffixed)")) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) res = parser.parse(context, ParserInput("desc(prefixed)")) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) res = parser.parse(context, ParserInput("desc(infixed)")) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) @pytest.mark.parametrize("text", [ "function(suffixed one)", "function(one plus two mult three)", "function(suffixed x$!#)" ]) def test_i_cannot_parse_when_function_only(self, text): sheerka, context, parser = self.init_parser() res = parser.parse(context, ParserInput(text)) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) def test_i_can_parse_when_multiple_ontologies(self): sheerka, context, parser = self.init_parser() text = "suffixed 1 + 1" res = parser.parse(context, ParserInput(text)) wrapper = res.body lexer_nodes = res.body.body assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected = [CN(cmap["suffixed"], text, 0, 6)] concept_node_as_test_obj = get_test_obj(lexer_nodes, expected) assert concept_node_as_test_obj == expected # add an ontology layer and make sure will still can parse sheerka.push_ontology(context, "new ontology") parser = SyaNodeParser(sheerka=sheerka) res = parser.parse(context, ParserInput(text)) wrapper = res.body lexer_nodes = res.body.body assert res.status assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected = [CN(cmap["suffixed"], text, 0, 6)] concept_node_as_test_obj = get_test_obj(lexer_nodes, expected) assert concept_node_as_test_obj == expected def test_i_correctly_set_up_concept_hints(self): my_map = { "shirt": Concept("shirt"), "a x": Concept("a x", ret="x").def_var("x"), "red x": Concept("red x", ret="x").def_var("x"), } sheerka, context, parser = self.init_parser(my_map) res = parser.parse(context, ParserInput("a red shirt")) concept_found = res.body.body[0].concept assert concept_found.get_hints().use_copy assert concept_found.get_hints().need_validation assert concept_found.get_hints().recognized_by == RECOGNIZED_BY_KEY concept_found_x = concept_found.get_compiled()["x"] assert concept_found_x.get_hints().use_copy assert concept_found_x.get_hints().need_validation assert concept_found_x.get_hints().recognized_by == RECOGNIZED_BY_KEY concept_found_x_x = concept_found_x.get_compiled()["x"] assert concept_found_x_x.get_hints().use_copy def test_i_can_rollback_concept_during_finalize(self): concepts_map = { "[": Concept("[z for x in y]").def_var("z").def_var("x").def_var("y"), "foo": Concept("foo"), "bar": Concept("bar"), "baz": Concept("baz"), } sheerka, context, parser = self.init_parser(concepts_map) text = "[ foo for foo in [bar, baz] ]" res = parser.parse(context, ParserInput(text)) nodes = res.body.body assert res.status assert len(nodes) == 1 node = nodes[0] assert sheerka.isinstance(node.concept, concepts_map["["]) assert node.concept.get_compiled()["z"] == concepts_map["foo"] assert node.concept.get_compiled()["x"] == concepts_map["foo"] assert node.concept.get_compiled()["y"][0].who == "parsers.Python" assert node.concept.get_compiled()["y"][0].body.source == '[bar, baz]' class TestFileBaseSyaNodeParser(TestUsingFileBasedSheerka): def test_i_can_parse_after_restart(self): sheerka, context, one, two, plus = self.init_test().with_concepts("one", "two", Concept("a plus b").def_var("a").def_var("b"), create_new=True).unpack() sheerka.om.commit(context) parser = SyaNodeParser() # sanity check # Remove this commented section to make sure that the nominal case still works # res = parser.parse(context, ParserInput("one plus two")) # assert res.status # assert sheerka.isinstance(res.body.body[0].concept, plus.key) sheerka = self.new_sheerka_instance(False) context = self.get_context(sheerka) res = parser.parse(context, ParserInput("one plus two")) assert res.status assert sheerka.isinstance(res.body.body[0].concept, plus.key) # adds an ontology layer and make the test again sheerka.push_ontology(context, "new ontology") sheerka = self.new_sheerka_instance(False) context = self.get_context(sheerka) res = parser.parse(context, ParserInput("one plus two")) assert res.status assert sheerka.isinstance(res.body.body[0].concept, plus.key)