from core.builtin_concepts import ParserResultConcept, BuiltinConcepts from core.concept import Concept, CC from core.tokenizer import Tokenizer, TokenKind from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, scnode, cnode, \ utnode, SyaAssociativity, CN, CNC, UTN from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import compute_expected_array, get_node def get_input_nodes_from(my_concepts_map, full_expr, *args): def _get_real_node(n): if isinstance(n, CC): concept = n.concept or Concept.update_from(my_concepts_map[n.concept_key]) for k, v in n.compiled.items(): concept.compiled[k] = _get_real_node(v) return concept if isinstance(n, (utnode, UTN)): return UnrecognizedTokensNode(n.start, n.end, full_expr_as_tokens[n.start: n.end + 1]) if isinstance(n, (CNC, CN, cnode)): concept = n.concept if hasattr(n, "concept") and n.concept else \ Concept().update_from(my_concepts_map[n.concept_key]) tokens = full_expr_as_tokens[n.start: n.end + 1] if hasattr(node, "compiled"): for k, v in n.compiled.items(): concept.compiled[k] = _get_real_node(v) return ConceptNode(concept, n.start, n.end, tokens) raise NotImplementedError() res = [] full_expr_as_tokens = list(Tokenizer(full_expr)) tokens_for_get_node = [token.value for token in full_expr_as_tokens if token.type != TokenKind.EOF] for arg in args: node = get_node(my_concepts_map, tokens_for_get_node, arg) res.append(_get_real_node(node)) return res concepts_map = { "5params": Concept("5params").def_prop("a").def_prop("b").def_prop("c").def_prop("d").def_prop("e"), "plus": Concept("a plus b", body="a + b").def_prop("a").def_prop("b"), "mult": Concept("a mult b", body="a * b").def_prop("a").def_prop("b"), "one": Concept("one", body="1"), "two": Concept("two", body="2"), "three": Concept("three", body="3"), "twenties": Concept("twenties", definition="'twenty' (one|two)=unit", body="20 + unit").def_prop("unit"), "hello_atom": Concept("hello one"), "hello_sya": Concept("hello a").def_prop("a"), "greetings_a": Concept("greetings a").def_prop("a"), "greetings_b": Concept("greetings b").def_prop("b"), } class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): sheerka = None @classmethod def setup_class(cls): t = TestUnrecognizedNodeParser() TestUnrecognizedNodeParser.sheerka, context, _ = t.init_parser(concepts_map, create_new=True) TestUnrecognizedNodeParser.sheerka.set_sya_def(context, [ (concepts_map["mult"].id, 20, SyaAssociativity.Right), (concepts_map["plus"].id, 10, SyaAssociativity.Right), ]) def init_parser(self, my_concepts_map=None, **kwargs): if my_concepts_map: sheerka, context, *updated_concepts = self.init_concepts(*my_concepts_map.values(), **kwargs) for i, pair in enumerate(my_concepts_map): my_concepts_map[pair] = updated_concepts[i] else: sheerka = TestUnrecognizedNodeParser.sheerka context = self.get_context(sheerka) parser = UnrecognizedNodeParser() return sheerka, context, parser def test_i_can_validate_a_valid_concept_node(self): sheerka, context, parser = self.init_parser() node = get_input_nodes_from(concepts_map, "one", "one")[0] res = UnrecognizedNodeParser().validate_concept_node(context, node) assert res.status assert res.body.concept == concepts_map["one"] def test_i_can_validate_concept_unrecognized_tokens(self): sheerka, context, parser = self.init_parser() node = get_input_nodes_from( concepts_map, "5params one two three twenty one 1 + 2 one plus two mult three", CNC("5params", a=" one ", b=" two three ", c=" twenty one ", d=utnode(12, 18, " 1 + 2 "), e=" one plus two mult three"))[0] res = UnrecognizedNodeParser().validate_concept_node(context, node) assert res.status concept = res.body.concept assert concept == concepts_map["5params"] assert len(concept.compiled["a"]) == 1 assert sheerka.isinstance(concept.compiled["a"][0], BuiltinConcepts.RETURN_VALUE) assert concept.compiled["a"][0].status assert concept.compiled["a"][0].who == "parsers.AtomNode" assert concept.compiled["a"][0].body.body == [cnode("one", 1, 1, "one")] assert len(concept.compiled["b"]) == 1 assert sheerka.isinstance(concept.compiled["b"][0], BuiltinConcepts.RETURN_VALUE) assert concept.compiled["b"][0].status assert concept.compiled["b"][0].who == "parsers.AtomNode" assert concept.compiled["b"][0].body.body == [cnode("two", 1, 1, "two"), cnode("three", 3, 3, "three")] assert len(concept.compiled["c"]) == 1 assert sheerka.isinstance(concept.compiled["c"][0], BuiltinConcepts.RETURN_VALUE) assert concept.compiled["c"][0].status assert concept.compiled["c"][0].who == "parsers.BnfNode" expected_nodes = compute_expected_array( concepts_map, " twenty one ", [CNC("twenties", source="twenty one", unit="one", one="one")]) assert concept.compiled["c"][0].body.body == expected_nodes assert len(concept.compiled["d"]) == 1 assert sheerka.isinstance(concept.compiled["d"][0], BuiltinConcepts.RETURN_VALUE) assert concept.compiled["d"][0].status assert concept.compiled["d"][0].who == "parsers.Python" assert concept.compiled["d"][0].body.source == "1 + 2" assert len(concept.compiled["e"]) == 1 assert sheerka.isinstance(concept.compiled["e"][0], BuiltinConcepts.RETURN_VALUE) assert concept.compiled["e"][0].status assert concept.compiled["e"][0].who == "parsers.SyaNode" expected_nodes = compute_expected_array( concepts_map, " one plus two mult three ", [CNC("plus", a="one", b=CC("mult", a="two", b="three"))]) assert concept.compiled["e"][0].body.body == expected_nodes # # sanity check, I can evaluate the concept # evaluated = sheerka.evaluate_concept(self.get_context(sheerka, eval_body=True), concept) # assert evaluated.key == concept.key # assert evaluated.get_prop("a") == def test_i_can_validate_with_recursion(self): sheerka, context, parser = self.init_parser() node = get_input_nodes_from( concepts_map, "1 plus 2 mult twenty two", CNC("plus", a="1 ", b=CC("mult", a=" 2 ", b=" twenty two")))[0] res = UnrecognizedNodeParser().validate_concept_node(context, node) assert res.status assert res.body.concept == concepts_map["plus"] assert len(res.body.concept.compiled["a"]) == 1 assert res.body.concept.compiled["a"][0].status assert res.body.concept.compiled["a"][0].who == "parsers.Python" assert res.body.concept.compiled["a"][0].body.source == "1" assert res.body.concept.compiled["b"] == concepts_map["mult"] assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["a"][0], BuiltinConcepts.RETURN_VALUE) assert res.body.concept.compiled["b"].compiled["a"][0].status assert res.body.concept.compiled["b"].compiled["a"][0].who == "parsers.Python" assert res.body.concept.compiled["b"].compiled["a"][0].body.source == "2" assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["b"][0], BuiltinConcepts.RETURN_VALUE) assert res.body.concept.compiled["b"].compiled["b"][0].status assert res.body.concept.compiled["b"].compiled["b"][0].who == "parsers.BnfNode" expected_nodes = compute_expected_array( concepts_map, " twenty two", [CNC("twenties", source="twenty two", unit="two", two="two")]) assert res.body.concept.compiled["b"].compiled["b"][0].body.body == expected_nodes # def test_i_can_validate_and_evaluate_a_concept_node_with_python(self): # sheerka, context, parser = self.init_parser() # # node = get_input_nodes_from( # concepts_map, # "one plus 1 + 1", # CNC("plus", # a=UTN("one "), # b=UTN("1 + 1")))[0] # # res = UnrecognizedNodeParser().validate_concept_node(context, node) # # assert res.status # assert res.body.concept == concepts_map["plus"] # assert res.body.concept.compiled["a"] == concepts_map["one"] # assert len(res.body.concept.compiled["b"]) == 1 # assert sheerka.isinstance(res.body.concept.compiled["b"][0], BuiltinConcepts.RETURN_VALUE) # assert res.body.concept.compiled["b"][0].status # assert res.body.concept.compiled["b"][0].who == "parsers.Python" # assert res.body.concept.compiled["b"][0].body.source == "1 + 1" # # # # evaluate # # context = self.get_context(sheerka, eval_body=True) # # evaluated = sheerka.evaluate_concept(context, res.body.concept) # # assert evaluated.body == 3 # def test_i_can_validate_and_evaluate_concept_when_bnf_concept(self): # sheerka, context, parser = self.init_parser() # node = get_concept_node(concepts_map, "one plus twenty one", "plus", "one", "twenty one") # # res = UnrecognizedNodeParser().validate_concept_node(context, node) # # assert res.status # assert res.body.concept == concepts_map["plus"] # assert res.body.concept.compiled["a"] == concepts_map["one"] # assert len(res.body.concept.compiled["b"]) == 1 # assert res.body.concept.compiled["b"][0].status # assert res.body.concept.compiled["b"][0].who == "parsers.BnfNode" # # # evaluate # context = self.get_context(sheerka, eval_body=True) # evaluated = sheerka.evaluate_concept(context, res.body.concept) # assert evaluated.body == 22 def test_i_can_parse_and_evaluate_unrecognized_python_node(self): sheerka, context, parser = self.init_parser() expression = "1 + 1" nodes = get_input_nodes_from(concepts_map, expression, UTN(expression)) parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes) res = parser.parse(context, parser_input) parser_result = res.body actual_nodes = res.body.body assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert len(actual_nodes) == 1 assert actual_nodes[0] == scnode(0, 4, expression) def test_i_can_parse_unrecognized_bnf_concept_node(self): sheerka, context, parser = self.init_parser() expression = "twenty one" nodes = get_input_nodes_from(concepts_map, expression, UTN(expression)) parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes) res = parser.parse(context, parser_input) parser_result = res.body actual_nodes = res.body.body assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert len(actual_nodes) == 1 expected_array = compute_expected_array( concepts_map, expression, [CNC("twenties", source=expression, unit="one", one="one")]) assert actual_nodes == expected_array def test_i_can_parse_unrecognized_sya_concept_node(self): sheerka, context, parser = self.init_parser() expression = "one plus two mult three" nodes = get_input_nodes_from(concepts_map, expression, UTN(expression)) parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes) res = parser.parse(context, parser_input) parser_result = res.body actual_nodes = res.body.body assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert len(actual_nodes) == 1 expected_array = compute_expected_array( concepts_map, expression, [CNC("plus", a="one", b=CC("mult", source="two mult three", a="two", b="three"))]) assert actual_nodes == expected_array def test_i_can_parse_sequences(self): sheerka, context, parser = self.init_parser() expression = "one plus two three" sequence = get_input_nodes_from(concepts_map, expression, CNC("plus", a="one", b="two"), utnode(5, 6, " three")) parser_input = ParserResultConcept("parsers.xxx", source="one plus two three", value=sequence) res = parser.parse(context, parser_input) actual_nodes = res.body.body assert res.status expected_array = compute_expected_array( concepts_map, expression, [ CNC("plus", a="one", b="two"), CN("three", start=6, end=6)]) assert actual_nodes == expected_array def test_i_can_parse_when_multiple_atom_and_sya(self): sheerka, context, parser = self.init_parser() expression = "two hello one three" nodes = get_input_nodes_from(concepts_map, expression, "two", UTN("hello one"), "three") parser_input = ParserResultConcept("parsers.xxx", source="one plus two hello one", value=nodes) res = parser.parse(context, parser_input) assert len(res) == 2 assert res[0].status assert res[1].status actual_nodes0 = res[0].body.body expected_0 = compute_expected_array(concepts_map, expression, [ CN("two", 0, 0), CN("hello_atom", source="hello one", start=2, end=4), CN("three", 6, 6)]) assert actual_nodes0 == expected_0 actual_nodes1 = res[1].body.body expected_1 = compute_expected_array(concepts_map, expression, [ CN("two", 0, 0), CNC("hello_sya", source="hello one", start=2, end=4, a="one"), CN("three", 6, 6)]) assert actual_nodes1 == expected_1 def test_i_can_parse_when_multiple_sya_concepts(self): sheerka, context, parser = self.init_parser() expression = "greetings two" nodes = get_input_nodes_from(concepts_map, expression, UTN("greetings two")) parser_input = ParserResultConcept("parsers.xxx", source="greetings two", value=nodes) res = parser.parse(context, parser_input) assert len(res) == 2 assert res[0].status assert res[1].status actual_nodes0 = res[0].body.body expected_0 = compute_expected_array(concepts_map, expression, [ CNC("greetings_a", source="greetings two", start=0, end=2, a="two")]) assert actual_nodes0 == expected_0 actual_nodes1 = res[1].body.body expected_1 = compute_expected_array(concepts_map, expression, [ CNC("greetings_b", source="greetings two", start=0, end=2, b="two")]) assert actual_nodes1 == expected_1 def test_i_cannot_parse_when_i_cannot_validate(self): sheerka, context, parser = self.init_parser(concepts_map, create_new=True) expression = "one plus unknown tokens" nodes = get_input_nodes_from(concepts_map, expression, CNC("plus", a="one ", b=" unknown tokens")) parser_input = ParserResultConcept("parsers.xxx", source="six", value=nodes) res = parser.parse(context, parser_input) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR) def test_i_cannot_parse_when_unrecognized(self): sheerka, context, parser = self.init_parser(concepts_map, create_new=True) expression = "unknown tokens" nodes = get_input_nodes_from(concepts_map, expression, UTN(expression)) parser_input = ParserResultConcept("parsers.xxx", source="six", value=nodes) res = parser.parse(context, parser_input) actual_nodes = res.body.body assert not res.status assert actual_nodes == nodes