import pytest import tests.parsers.parsers_utils from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF from core.global_symbols import NotInit from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from core.sheerka.services.SheerkaExecute import ParserInput from parsers.BaseNodeParser import CNC, UTN, CN from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \ Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser from tests.BaseTest import BaseTest from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka cmap = { "one": Concept("one"), "two": Concept("two"), "three": Concept("three"), "four": Concept("four"), "thirty": Concept("thirty", body="30"), "forty": Concept("forty", body="40"), "fifty": Concept("fifty", body="50"), "number": Concept("number"), "foo": Concept("foo"), "bar": Concept("bar"), "baz": Concept("baz"), "one hundred": Concept("one hundred", body="100"), "one_hundred": Concept("'one hundred'", body="100"), "hundreds": Concept("hundreds", definition="number=n1 'hundred' 'and' number=n2", where="n1 < 10 and n2 < 100", body="n1 * 100 + n2").def_var("n1").def_var("n2"), "bnf baz": Concept("bnf baz", definition="'baz'"), # this one should be chosen "plus": Concept("plus", definition="one 'plus' two").def_var("a").def_var("b"), 'foo then bar': Concept("foo then bar", definition="foo bar").def_var("foo").def_var("bar"), 'foo or bar': Concept("foo or bar", definition="foo | bar").def_var("foo").def_var("bar"), 'one or more foo': Concept("one or more foo", definition="foo+").def_var("foo"), "t1": Concept("t1", definition="'twenty' (one|two)=unit").def_var("unit").def_var("one").def_var("two"), "three_four": Concept("three_four", definition="three | four").def_var("three").def_var("four"), "t2": Concept("t2", definition="'twenty' three_four=unit").def_var("unit").def_var("three").def_var("four"), # testing keywords "def_only": Concept("def"), "def number": Concept("def number", definition="def (one|two)=number"), # sequence of keywords using bnf definition # "def_concept_bnf": Concept("def_concept_bnf", definition="'def' 'concept'"), # "def concept_bnf number": Concept("def number", definition="def_concept_bnf (one|two)=number"), # sequence of keywords using def definition # "def_concept_def": Concept("def_concept_def", definition="def concept", definition_type=DEFINITION_TYPE_DEF), # "def concept_def number": Concept("def number", definition="def_concept_def (one|two|three)=number"), "filter": Concept("filter", definition="'--filter' (one | two)") } def u(parsing_expression, start, end, children=None): """ u stands for underlying :param parsing_expression: :param start: :param end: :param children: :return: """ if isinstance(parsing_expression, str): parsing_expression = StrMatch(parsing_expression) if isinstance(parsing_expression, StrMatch): return TerminalNode(parsing_expression, start, end, parsing_expression.to_match) return NonTerminalNode(parsing_expression, start, end, [], children) def compute_expected_array(my_concepts_map, expression, expected, exclude_body=False): return tests.parsers.parsers_utils.compute_expected_array( my_concepts_map, expression, expected, init_empty_body=True, exclude_body=exclude_body) class TestBnfNodeParser(TestUsingMemoryBasedSheerka): shared_ontology = None @classmethod def setup_class(cls): init_test_helper = cls().init_test(cache_only=False, ontology="#TestBnfNodeParser#") sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack() for i, concept_name in enumerate(cmap): cmap[concept_name] = updated[i] # end of initialisation sheerka = TestBnfNodeParser.sheerka sheerka.set_isa(context, cmap["one"], cmap["number"]) sheerka.set_isa(context, cmap["two"], cmap["number"]) sheerka.set_isa(context, cmap["three"], cmap["number"]) sheerka.set_isa(context, cmap["four"], cmap["number"]) sheerka.set_isa(context, cmap["thirty"], cmap["number"]) sheerka.set_isa(context, cmap["forty"], cmap["number"]) sheerka.set_isa(context, cmap["fifty"], cmap["number"]) sheerka.set_isa(context, cmap["one hundred"], cmap["number"]) sheerka.set_isa(context, cmap["hundreds"], cmap["number"]) # Pay attention. 'twenties (t1 and t2) are not set as number thirties = cls.update_bnf(context, Concept("thirties", definition="thirty number", where="number < 10", body="thirty + number").def_var("thirty").def_var("number")) cmap["thirties"] = sheerka.create_new_concept(context, thirties).body.body sheerka.set_isa(context, sheerka.new("thirties"), sheerka.new("number")) forties = cls.update_bnf(context, Concept("forties", definition="forty number", where="number < 10", body="forty + number").def_var("forty").def_var("number")) cmap["forties"] = sheerka.create_new_concept(context, forties).body.body sheerka.set_isa(context, sheerka.new("forties"), sheerka.new("number")) fifties = cls.update_bnf(context, Concept("fifties", definition="fifty number", where="number < 10", body="fifty + number").def_var("fifty").def_var("number")) cmap["fifties"] = sheerka.create_new_concept(context, fifties).body.body sheerka.set_isa(context, sheerka.new("fifties"), sheerka.new("number")) thousands = cls.update_bnf(context, Concept("thousands", definition="number 'thousand'", where="number < 999", body="number * 1000").def_var("number")) cmap["thousands"] = sheerka.create_new_concept(context, thousands).body.body sheerka.set_isa(context, sheerka.new("thousands"), sheerka.new("number")) cls.shared_ontology = sheerka.get_ontology(context) sheerka.pop_ontology(context) @staticmethod def update_bnf(context, concept): bnf_parser = BnfDefinitionParser() res = bnf_parser.parse(context, concept.get_metadata().definition) if res.status: concept.set_bnf(res.value.value) concept.get_metadata().definition_type = DEFINITION_TYPE_BNF else: raise Exception(res) return concept def init_parser(self, my_concepts_map=None, init_from_sheerka=False, **kwargs): if my_concepts_map is None: sheerka, context = self.init_test().unpack() sheerka.add_ontology(context, self.shared_ontology) else: sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack() for i, pair in enumerate(my_concepts_map): my_concepts_map[pair] = updated[i] parser = BnfNodeParser(sheerka=sheerka) if init_from_sheerka else BnfNodeParser() return sheerka, context, parser def validate_get_concepts_sequences(self, my_map, text, expected, multiple_result=False, post_init_concepts=None): sheerka, context, *updated = self.init_test().with_concepts(*my_map.values(), create_new=False).unpack() sequences = self.exec_get_concepts_sequences(context, my_map, text, expected, multiple_result, post_init_concepts, *updated) return sequences @staticmethod def exec_get_concepts_sequences(context, my_map, text, expected, multiple_result=False, post_init_concepts=None, *concepts): sheerka = context.sheerka if not multiple_result: expected_array = [compute_expected_array(my_map, text, expected)] else: expected_array = [compute_expected_array(my_map, text, e) for e in expected] if post_init_concepts: post_init_concepts(sheerka, context) parser = BnfNodeParser() parser.init_from_concepts(context, concepts) parser.reset_parser(context, ParserInput(text)) bnf_parsers_helpers = parser.get_concepts_sequences(context) assert len(bnf_parsers_helpers) == len(expected_array) for parser_helper, expected_sequence in zip(bnf_parsers_helpers, expected_array): assert parser_helper.sequence == expected_sequence if len(bnf_parsers_helpers) == 1: return bnf_parsers_helpers[0].sequence else: return [pe.sequence for pe in bnf_parsers_helpers] def test_i_cannot_parse_empty_strings(self): sheerka, context, parser = self.init_parser({}, singleton=True) res = parser.parse(context, ParserInput("")) assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) assert res.body.reason == BuiltinConcepts.IS_EMPTY @pytest.mark.parametrize("expr, text", [ (StrMatch("foo"), "foo"), (StrMatch("'foo'"), "'foo'"), (StrMatch("1"), "1"), (StrMatch("3.14"), "3.14"), (StrMatch("+"), "+"), ]) def test_i_can_match_simple_bnf(self, expr, text): my_map = { text: self.bnf_concept("foo", expr) } sequence = self.validate_get_concepts_sequences(my_map, text, [text]) assert sequence[0].underlying == u(expr, 0, 0) def test_i_can_match_multiple_concepts_in_one_input(self): my_map = { "one": self.bnf_concept("one"), "two": self.bnf_concept("two"), } text = "one two one" expected = ["one", "two", ("one", 1)] self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one two three", [CNC("foo", source="one two three")]), ("one two", []), ("one two four", []), ]) def test_i_can_match_sequence(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence("one", "two", "three")), } self.validate_get_concepts_sequences(my_map, text, expected) def test_i_always_choose_the_longest_match(self): my_map = { "foo": self.bnf_concept("foo", Sequence("one", "two", "three")), "bar": self.bnf_concept("bar", Sequence("one", "two")), } text = "one two three" expected = [CNC("foo", source=text)] self.validate_get_concepts_sequences(my_map, text, expected) def test_i_can_use_skip_whitespace_when_mixing_sequence_and_strmatch(self): # to match '--filter' in one word my_map = { "filter": self.bnf_concept("filter", Sequence(StrMatch("-", skip_whitespace=False), StrMatch("-", skip_whitespace=False), "filter")), } sheerka, context, *updated = self.init_concepts(*my_map.values(), create_new=False, singleton=True) parser = BnfNodeParser() parser.init_from_concepts(context, updated) text = "--filter" expected = [CN("filter", source="--filter")] expected_array = compute_expected_array(my_map, text, expected) parser.reset_parser(context, ParserInput(text)) bnf_parsers_helpers = parser.get_concepts_sequences(context) assert bnf_parsers_helpers[0].sequence == expected_array assert not bnf_parsers_helpers[0].has_unrecognized # but I cannot parse text = "- - filter" parser.reset_parser(context, ParserInput(text)) bnf_parsers_helpers = parser.get_concepts_sequences(context) assert bnf_parsers_helpers[0].has_unrecognized def test_i_can_match_multiple_sequences(self): my_map = { "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), StrMatch("two"), StrMatch("three"))), "bar": self.bnf_concept("bar", Sequence(StrMatch("one"), StrMatch("two"))), } text = "one two three one two" expected = [ CNC("foo", source="one two three"), CNC("bar", source="one two", start=6, end=8)] self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one", [CNC("foo", source="one")]), ("two", [CNC("foo", source="two")]), ("three", []), ]) def test_i_can_match_ordered_choice(self, text, expected): my_map = { "foo": self.bnf_concept("foo", OrderedChoice(StrMatch("one"), StrMatch("two"))) } self.validate_get_concepts_sequences(my_map, text, expected) def test_i_do_not_match_ordered_choice_with_empty_alternative(self): my_map = { "foo": self.bnf_concept("foo", OrderedChoice(StrMatch("one"), StrMatch(""))) } text = "" expected = [] self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("concept_three, expected", [ (Concept("three"), []), (BaseTest.bnf_concept("three", StrMatch("three")), [UTN('twenty '), "three"]) ]) def test_i_can_manage_sequence_with_wrong_order_choice(self, concept_three, expected): my_map = { "foo": self.bnf_concept("foo", Sequence( StrMatch("twenty"), OrderedChoice(StrMatch("one"), StrMatch("two")))), "three": concept_three} text = "twenty three" self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("ok thirty one", [CNC("foo", source="ok thirty one")]), ("ok twenty one", [CNC("foo", source="ok twenty one")]), ("ok one", []), ]) def test_i_can_mix_sequence_and_ordered(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence( StrMatch("ok"), OrderedChoice(StrMatch("twenty"), StrMatch("thirty")), StrMatch("one")) )} self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ # ("twenty one", [CNC("foo", source="twenty one")]), # ("twenty three", []), # three does not exist ("twenty four", []), # four exists but should not be seen ]) def test_i_can_mix_sequence_and_ordered_2(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence( StrMatch("twenty"), OrderedChoice(StrMatch("one"), StrMatch("two")))), "four": Concept("four")} self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("twenty thirty", [CNC("foo", source="twenty thirty")]), ("one", [CNC("foo", source="one")]), ]) def test_i_can_mix_ordered_choices_and_sequences(self, text, expected): my_map = { "foo": self.bnf_concept("foo", OrderedChoice( Sequence(StrMatch("twenty"), StrMatch("thirty")), StrMatch("one")))} self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one", [CNC("foo", source="one")]), ("one two", [CNC("foo", source="one two")]), ("three", []), ]) def test_i_can_parse_unordered_choice(self, text, expected): my_map = { "foo": self.bnf_concept("foo", UnOrderedChoice( StrMatch("one"), Sequence(StrMatch("one"), StrMatch("two")))), } self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one", [CNC("foo", source="one")]), ("", []), ("two", []), ]) def test_i_can_parse_optional(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Optional(StrMatch("one"))) } self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("twenty one", [CNC("foo", source="twenty one")]), ("one", [CNC("foo", source="one")]), ]) def test_i_can_parse_sequence_starting_with_optional(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence( Optional(StrMatch("twenty")), StrMatch("one"))) } self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one two three", [CNC("foo", source="one two three")]), ("one two", [CNC("foo", source="one two")]), ]) def test_i_can_parse_sequence_ending_with_optional(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence( StrMatch("one"), StrMatch("two"), Optional(StrMatch("three")))) } self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one two three", [CNC("foo", source="one two three")]), ("one three", [CNC("foo", source="one three")]), ]) def test_i_can_parse_sequence_with_optional_in_between(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence( StrMatch("one"), Optional(StrMatch("two")), StrMatch("three"))) } self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("", []), ("two", []), ("one", [CNC("foo", source="one")]), ("one one", [CNC("foo", source="one one")]), ]) def test_i_can_parse_zero_or_more(self, text, expected): my_map = { "foo": self.bnf_concept("foo", ZeroOrMore(StrMatch("one"))) } self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("two", [CNC("foo", source="two")]), ("one two", [CNC("foo", source="one two")]), ("one one two", [CNC("foo", source="one one two")]), ]) def test_i_can_parse_sequence_and_zero_or_more(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence( ZeroOrMore(StrMatch("one")), StrMatch("two") )) } self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one, one , one", [CNC("foo", source="one, one , one")]), ]) def test_i_can_parse_zero_or_more_with_separator(self, text, expected): my_map = { "foo": self.bnf_concept("foo", ZeroOrMore(StrMatch("one"), sep=",")) } self.validate_get_concepts_sequences(my_map, text, expected) def test_that_zero_or_more_is_greedy(self): my_map = { "foo": self.bnf_concept("foo", ZeroOrMore(StrMatch("one"))), "bar": self.bnf_concept("foo", StrMatch("one")) } text = "one one one" expected = [CNC("foo", source=text)] self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("", []), ("two", []), ("one", [CNC("foo", source="one")]), ("one one one", [CNC("foo", source="one one one")]), ]) def test_i_can_parse_one_or_more(self, text, expected): my_map = { "foo": self.bnf_concept("foo", OneOrMore(StrMatch("one"))), } self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("two", []), ("one two", [CNC("foo", source="one two")]), ("one one two", [CNC("foo", source="one one two")]), ]) def test_i_can_parse_sequence_one_and_or_more(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence( OneOrMore(StrMatch("one")), StrMatch("two") )) } self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one, one , one", [CNC("foo", source="one, one , one")]), ]) def test_i_can_parse_one_or_more_with_separator(self, text, expected): my_map = { "foo": self.bnf_concept("foo", OneOrMore(StrMatch("one"), sep=",")) } self.validate_get_concepts_sequences(my_map, text, expected) def test_that_one_or_more_is_greedy(self): my_map = { "foo": self.bnf_concept("foo", OneOrMore(StrMatch("one"))), "bar": self.bnf_concept("foo", StrMatch("one")) } text = "one one one" expected = [CNC("foo", source=text)] self.validate_get_concepts_sequences(my_map, text, expected) @pytest.mark.parametrize("text, expected", [ ("one two", [ [CNC("foo", source="one two")], [CNC("bar", source="one two")]]), ("one two one two", [ [CNC("bar", source="one two"), CNC("bar", source="one two")], [CNC("foo", source="one two"), CNC("bar", source="one two")], [CNC("bar", source="one two"), CNC("foo", source="one two")], [CNC("foo", source="one two"), CNC("foo", source="one two")]]), ]) def test_i_can_have_multiple_results(self, text, expected): my_map = { "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), StrMatch("two"))), "bar": self.bnf_concept("bar", Sequence( StrMatch("one"), OrderedChoice(StrMatch("two"), StrMatch("three")))), } text = "one two" expected = [[CNC("foo", source=text)], [CNC("bar", source=text)]] self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) def test_i_can_refer_to_other_concepts(self): my_map = { "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), StrMatch("two"))), "bar": self.bnf_concept("bar", ConceptExpression("foo")) } text = "one two" expected = [ [CNC("foo", source=text)], [CN("bar", source=text)] # Do not check the compiled part ] sequences = self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) # explicit validations of the compiled concept_foo = sequences[0][0].concept assert concept_foo.body == NotInit assert concept_foo.get_compiled() == {ConceptParts.BODY: DoNotResolve("one two")} concept_bar = sequences[1][0].concept assert concept_bar.body == NotInit assert concept_bar.get_compiled() == { ConceptParts.BODY: concept_foo, "foo": concept_foo } assert id(concept_bar.get_compiled()[ConceptParts.BODY]) == id(concept_bar.get_compiled()["foo"]) def test_i_can_refer_to_other_concepts_with_body(self): my_map = { "foo": self.bnf_concept(Concept("foo", body="'foo'"), Sequence(StrMatch("one"), StrMatch("two"))), "bar": self.bnf_concept("bar", ConceptExpression("foo")) } text = "one two" expected = [ [CNC("foo", source=text)], [CN("bar", source=text)] # Do not check the compiled part ] sequences = self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) # explicit validations of the compiled concept_foo = sequences[0][0].concept assert concept_foo.body == NotInit assert len(concept_foo.get_compiled()) == 0 # because there is a body defined in the metadata concept_bar = sequences[1][0].concept assert concept_bar.body == NotInit assert concept_bar.get_compiled() == { ConceptParts.BODY: concept_foo, "foo": concept_foo } def test_i_can_manage_concepts_reference_when_multiple_levels(self): my_map = { "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), StrMatch("two"))), "bar": self.bnf_concept("bar", ConceptExpression("foo")), "baz": self.bnf_concept("baz", ConceptExpression("bar")), } text = "one two" expected = [ [CNC("foo", source=text)], [CN("bar", source=text)], # Do not check the compiled part [CN("baz", source=text)], # Do not check the compiled part ] sequences = self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) # explicit validations of the compiled concept_foo = sequences[0][0].concept assert concept_foo.body == NotInit assert concept_foo.get_compiled() == {ConceptParts.BODY: DoNotResolve("one two")} concept_bar = sequences[1][0].concept assert concept_bar.body == NotInit assert concept_bar.get_compiled() == { ConceptParts.BODY: concept_foo, "foo": concept_foo } concept_baz = sequences[2][0].concept assert concept_baz.body == NotInit assert concept_baz.get_compiled() == { ConceptParts.BODY: concept_bar, "bar": concept_bar } def test_i_can_mix_reference_to_other_concepts(self): my_map = { "foo": self.bnf_concept("foo", OrderedChoice(StrMatch("twenty"), StrMatch("thirty"))), "bar": self.bnf_concept("bar", Sequence( ConceptExpression("foo"), OrderedChoice(StrMatch("one"), StrMatch("two")))), "three": Concept("three") } text = "twenty two" expected = [CN("bar", source="twenty two")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_bar = sequences[0].concept assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("twenty two"), "foo": my_map["foo"], } assert concept_bar.get_compiled()["foo"].get_compiled() == {ConceptParts.BODY: DoNotResolve("twenty")} text = "thirty one" expected = [CN("bar", source="thirty one")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_bar = sequences[0].concept assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("thirty one"), "foo": my_map["foo"], } assert concept_bar.get_compiled()["foo"].get_compiled() == {ConceptParts.BODY: DoNotResolve("thirty")} text = "thirty three" expected = [[CN("foo", source="thirty"), CN("three")], []] self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) def test_i_can_mix_reference_to_other_concepts_2(self): # this time, we use concept expression my_map = { "twenty": self.bnf_concept("twenty", StrMatch("twenty")), "number": self.bnf_concept("number", OrderedChoice(StrMatch("one"), StrMatch("two"))), "twenties": self.bnf_concept("twenties", Sequence(ConceptExpression("twenty"), ConceptExpression("number"))), "three": Concept("three") } text = "twenty two" expected = [CNC("twenties", source="twenty two", twenty=CC("twenty", body=DoNotResolve("twenty")), number=CC("number", source="two", body=DoNotResolve("two")) )] self.validate_get_concepts_sequences(my_map, text, expected) text = "twenty three" expected = [[CN("twenty"), CN("three")], []] self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True) def test_i_can_mix_reference_to_other_concepts_when_body(self): my_map = { "foo": self.bnf_concept(Concept("foo", body="'foo'"), OrderedChoice(StrMatch("twenty"), StrMatch("thirty"))), "bar": self.bnf_concept("bar", Sequence( ConceptExpression("foo"), OrderedChoice(StrMatch("one"), StrMatch("two")))), } sheerka, context, *concepts = self.init_test().with_concepts(*my_map.values(), create_new=False).unpack() text = "twenty two" expected = [CN("bar", source="twenty two")] sequences = self.exec_get_concepts_sequences(context, my_map, text, expected, False, None, *concepts) concept_bar = sequences[0].concept assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("twenty two"), "foo": sheerka.new("foo"), } assert concept_bar.get_compiled()["foo"].get_compiled() == {} # as foo as a body text = "thirty one" expected = [CN("bar", source="thirty one")] sequences = self.exec_get_concepts_sequences(context, my_map, text, expected, False, None, *concepts) concept_bar = sequences[0].concept assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("thirty one"), "foo": sheerka.new("foo"), } assert concept_bar.get_compiled()["foo"].get_compiled() == {} def test_i_can_mix_zero_and_more_and_reference_to_other_concepts(self): my_map = { "foo": self.bnf_concept("foo", OrderedChoice(StrMatch("one"), StrMatch("two"), StrMatch("three"))), "bar": self.bnf_concept("bar", ZeroOrMore(ConceptExpression("foo"))), } text = "one two three" expected = [CN("bar", source="one two three")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_bar = sequences[0].concept assert concept_bar.get_compiled() == { ConceptParts.BODY: DoNotResolve("one two three"), "foo": [my_map["foo"], my_map["foo"], my_map["foo"]] } assert concept_bar.get_compiled()["foo"][0].get_compiled() == {ConceptParts.BODY: DoNotResolve("one")} assert concept_bar.get_compiled()["foo"][1].get_compiled() == {ConceptParts.BODY: DoNotResolve("two")} assert concept_bar.get_compiled()["foo"][2].get_compiled() == {ConceptParts.BODY: DoNotResolve("three")} def test_i_can_parse_concept_reference_that_is_not_in_grammar(self): my_map = { "one": Concept("one"), "two": Concept("two"), "foo": self.bnf_concept("foo", Sequence( StrMatch("twenty"), OrderedChoice( ConceptExpression("one"), ConceptExpression("two"), rule_name="unit"))), } text = "twenty one" expected = [CN("foo", source="twenty one")] sequences = self.validate_get_concepts_sequences(my_map, text, expected) concept_foo = sequences[0].concept assert concept_foo.get_compiled() == { ConceptParts.BODY: DoNotResolve("twenty one"), "unit": my_map["one"], } def test_i_can_refer_to_group_concepts(self): my_map = { "one": Concept("one"), "two": Concept("two"), "number": Concept("number"), "foo": self.bnf_concept("foo", Sequence("twenty", ConceptExpression("number"))) } def pic(s, c): s.add_concept_to_set(c, my_map["one"], my_map["number"]) s.add_concept_to_set(c, my_map["two"], my_map["number"]) text = "twenty two" expected = [CN("foo", source="twenty two")] sequences = self.validate_get_concepts_sequences(my_map, text, expected, post_init_concepts=pic) # explicit validations of the compiled concept_foo = sequences[0].concept assert concept_foo.body == NotInit assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]), ConceptParts.BODY: DoNotResolve(value='twenty two')} text = "twenty one" expected = [CN("foo", source="twenty one")] sequences = self.validate_get_concepts_sequences(my_map, text, expected, post_init_concepts=pic) # explicit validations of the compiled concept_foo = sequences[0].concept assert concept_foo.body == NotInit assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]), ConceptParts.BODY: DoNotResolve(value='twenty one')} @pytest.mark.parametrize("bar_expr, expected", [ (ConceptExpression("foo"), {}), (OrderedChoice(ConceptExpression("foo"), StrMatch("one")), {'one': ['1002']}), (Sequence(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")), {'one': ['1001', '1002']}), ]) def test_i_can_detect_infinite_recursion(self, bar_expr, expected): my_map = { "foo": self.bnf_concept("foo", ConceptExpression("bar")), "bar": self.bnf_concept("bar", bar_expr), } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.context = context parser.sheerka = sheerka # every obvious cyclic recursion are removed from concept_by_first_keyword dict parser.init_from_concepts(context, my_map.values()) assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == expected # get_parsing_expression() also returns CHICKEN_AND_EGG parsing_expression = parser.get_parsing_expression(context, my_map["foo"]) assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) parsing_expression = parser.get_parsing_expression(context, my_map["bar"]) assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG) def test_i_can_detect_longer_infinite_recursion(self): my_map = { "foo": self.bnf_concept("foo", ConceptExpression("bar")), "bar": self.bnf_concept("bar", ConceptExpression("baz")), "baz": self.bnf_concept("baz", ConceptExpression("qux")), "qux": self.bnf_concept("qux", ConceptExpression("foo")), } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.context = context parser.sheerka = sheerka # every obvious cyclic recursion are removed from concept_by_first_keyword dict parser.init_from_concepts(context, my_map.values()) assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {} parsing_expression = parser.get_parsing_expression(context, my_map["foo"]) assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert parser.concepts_grammars.get(my_map["foo"].id).body == ["1001", "1002", "1003", "1004", "1001"] assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG) def test_i_can_detect_partial_infinite_recursion(self): my_map = { "foo": self.bnf_concept("foo", ConceptExpression("bar")), "bar": self.bnf_concept("bar", ConceptExpression("baz")), "baz": self.bnf_concept("baz", ConceptExpression("qux")), "qux": self.bnf_concept("qux", ConceptExpression("baz")), } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.context = context parser.sheerka = sheerka # every obvious cyclic recursion are removed from concept_by_first_keyword dict parser.init_from_concepts(context, my_map.values()) assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {} parsing_expression = parser.get_parsing_expression(context, my_map["foo"]) assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert parser.concepts_grammars.get(my_map["foo"].id).body == ["1001", "1002", "1003", "1004", "1003"] assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG) assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG) @pytest.mark.parametrize("expr, expected", [ (OrderedChoice(StrMatch("bar"), ConceptExpression("foo")), False), (OrderedChoice(ConceptExpression("foo"), StrMatch("bar")), True), (OrderedChoice(Sequence(StrMatch("bar"), ConceptExpression("foo")), StrMatch("baz")), False), (OrderedChoice(Sequence(ConceptExpression("foo"), StrMatch("bar")), StrMatch("baz")), True) ]) def test_i_can_detect_ordered_choice_infinite_recursion(self, expr, expected): my_map = { "foo": self.bnf_concept("foo", expr), } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.init_from_concepts(context, my_map.values()) parser.context = context parser.sheerka = sheerka res = parser.get_parsing_expression(context, my_map["foo"]) assert sheerka.isinstance(res, BuiltinConcepts.CHICKEN_AND_EGG) == expected def test_i_can_get_parsing_expression_when_ending_by_concept_isa(self): my_map = { "one": Concept("one"), "twenty": Concept("twenty"), "number": Concept("number"), "twenties": self.bnf_concept("twenties", Sequence(ConceptExpression("twenty"), ConceptExpression("number"))) } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.context = context parser.sheerka = sheerka sheerka.set_isa(context, sheerka.new("one"), my_map["number"]) sheerka.set_isa(context, sheerka.new("twenty"), my_map["number"]) parser.concepts_grammars.clear() # make sure parsing expression is created from scratch parsing_expression = parser.get_parsing_expression(context, my_map["twenties"]) assert parsing_expression == Sequence( ConceptExpression(my_map["twenty"], rule_name="twenty"), ConceptExpression(my_map["number"], rule_name="number")) assert len(parsing_expression.nodes) == len(parsing_expression.elements) twenty_nodes = parsing_expression.nodes[0].nodes assert twenty_nodes == [StrMatch("twenty")] number_nodes = parsing_expression.nodes[1].nodes assert len(number_nodes) == 1 assert isinstance(number_nodes[0], UnOrderedChoice) assert len(number_nodes[0].nodes) == len(number_nodes[0].elements) assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes def test_i_can_get_parsing_expression_when_starting_by_isa_concept(self): my_map = { "one": Concept("one"), "two": Concept("two"), "number": Concept("number"), "hundreds": self.bnf_concept("hundreds", Sequence(ConceptExpression("number"), StrMatch("hundred"))) } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.context = context parser.sheerka = sheerka sheerka.set_isa(context, sheerka.new("one"), my_map["number"]) sheerka.set_isa(context, sheerka.new("two"), my_map["number"]) sheerka.set_isa(context, sheerka.new("hundreds"), my_map["number"]) parser.concepts_grammars.clear() # make sure parsing expression is created from scratch parsing_expression = parser.get_parsing_expression(context, my_map["hundreds"]) assert parsing_expression == Sequence( ConceptExpression(my_map["number"], rule_name="number"), StrMatch("hundred")) assert len(parsing_expression.nodes) == len(parsing_expression.elements) number_nodes = parsing_expression.nodes[0].nodes assert len(number_nodes) == 1 assert isinstance(number_nodes[0], UnOrderedChoice) assert len(number_nodes[0].nodes) == len(number_nodes[0].elements) assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes assert ConceptExpression(my_map["two"], rule_name="two") in number_nodes[0].nodes def test_i_can_get_parsing_expression_when_concept_is_part_of_a_group(self): my_map = { "one": Concept("one"), "twenty": Concept("twenty"), "number": Concept("number"), "twenties": self.bnf_concept("twenties", Sequence(ConceptExpression("twenty"), ConceptExpression("number"))) } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.context = context parser.sheerka = sheerka sheerka.set_isa(context, sheerka.new("one"), my_map["number"]) sheerka.set_isa(context, sheerka.new("twenty"), my_map["number"]) sheerka.set_isa(context, sheerka.new("twenties"), my_map["number"]) # <- twenties is also a number parser.concepts_grammars.clear() # make sure parsing expression is created from scratch parsing_expression = parser.get_parsing_expression(context, my_map["twenties"]) assert parsing_expression == Sequence( ConceptExpression(my_map["twenty"], rule_name="twenty"), ConceptExpression(my_map["number"], rule_name="number")) assert len(parsing_expression.nodes) == len(parsing_expression.elements) twenty_nodes = parsing_expression.nodes[0].nodes assert twenty_nodes == [StrMatch("twenty")] number_nodes = parsing_expression.nodes[1].nodes assert len(number_nodes) == 1 assert isinstance(number_nodes[0], UnOrderedChoice) assert len(number_nodes[0].nodes) == len(number_nodes[0].elements) assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes def test_i_can_get_parsing_expression_when_sequence_of_concepts(self): my_map = { "one": Concept("one"), "two_ones": self.bnf_concept("two_ones", Sequence(ConceptExpression("one"), ConceptExpression("one"))) } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.context = context parser.sheerka = sheerka parsing_expression = parser.get_parsing_expression(context, my_map["two_ones"]) assert parsing_expression == Sequence( ConceptExpression(my_map["one"], rule_name="one"), ConceptExpression(my_map["one"], rule_name="one")) @pytest.mark.parametrize("expr, text, expected", [ # (ZeroOrMore(StrMatch("one"), sep=","), "one,", [CNC("foo", source="one"), UTN(",")]), # (StrMatch("one"), "one two", [CNC("foo", source="one"), UTN(" two")]), (StrMatch("one"), "two one", [UTN("two "), CNC("foo", source="one")]), ]) def test_i_can_recognize_unknown_concepts(self, expr, text, expected): my_map = { "foo": self.bnf_concept("foo", expr) } self.validate_get_concepts_sequences(my_map, text, expected) def test_i_can_recognize_unknown_when_they_look_like_known(self): my_map = { "one two": self.bnf_concept("one two", Sequence("one", "two")), "three": self.bnf_concept("three") } text = "one three" expected = [UTN("one "), CNC("three", source="three")] self.validate_get_concepts_sequences(my_map, text, expected) def test_i_can_remove_duplicates(self): my_map = { "one two": self.bnf_concept("one two", Sequence("one", "two")), "one four": self.bnf_concept("one four", Sequence("one", "four")), "three": self.bnf_concept("three") } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.init_from_concepts(context, my_map.values()) parser.reset_parser(context, ParserInput("one three")) sequences = parser.get_concepts_sequences(context) sequence = parser.get_valid(sequences) assert len(sequence) == 1 @pytest.mark.parametrize("parser_input, expected_status, expected", [ ("baz", True, [CNC("bnf baz", source="baz")]), # the bnf one is chosen ("foo bar", True, [CNC("foo then bar", source="foo bar", foo="foo", bar="bar")]), ("bar", True, [CNC("foo or bar", source="bar", bar="bar", body="bar")]), ("one plus two", True, [CNC("plus", source="one plus two", one="one", two="two")]), ("twenty one", True, [CNC("t1", source="twenty one", unit="one")]), ]) def test_i_can_parse_simple_expressions(self, parser_input, expected_status, expected): sheerka, context, parser = self.init_parser(init_from_sheerka=True) res = parser.parse(context, ParserInput(parser_input)) expected_array = compute_expected_array(cmap, parser_input, expected) parser_result = res.value concepts_nodes = res.value.value assert res.status == expected_status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_parse_when_multiple_times_the_same_variable(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) text = "foo foo foo" expected_array = compute_expected_array(cmap, text, [CNC("one or more foo", source=text)]) expected_array[0].compiled["foo"] = [cmap["foo"], cmap["foo"], cmap["foo"]] res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_test_when_expression_references_other_expressions(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) text = "twenty four" expected = CNC("t2", source=text, unit=CC("three_four", source="four", four=CC("four", body=DoNotResolve("four")), body=CC("four", body=DoNotResolve("four")))) expected_array = compute_expected_array(cmap, text, [expected]) res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_parse_bnf_concept_mixed_with_isa_concepts(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) # thirties is defined in the global variable cmap as # thirties = cls.update_bnf(context, Concept("thirties", # definition="thirty number", # where="number < 10", # body="thirty + number").def_var("thirty").def_var("number")) text = "thirty one" expected = CNC("thirties", source=text, number=CC("number", source="one", one=CC("one", body=DoNotResolve("one")), body=CC("one", body=DoNotResolve("one"))), thirty="thirty") expected_array = compute_expected_array(cmap, text, [expected]) res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_parse_bnf_concept_mixed_with_isa_concepts_2(self): # this time, three is a number, and also part of three_four, even if it is not relevant in t3 sheerka, context, parser = self.init_parser(init_from_sheerka=True) text = "thirty three" expected = CNC("thirties", source=text, number=CC("number", source="three", three=CC("three", body=DoNotResolve("three")), body=CC("three", body=DoNotResolve("three"))), thirty="thirty") expected_array = compute_expected_array(cmap, text, [expected]) res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_parse_when_starting_by_isa_concept(self): """ Test of simple number + 'thousand' :return: """ sheerka, context, parser = self.init_parser(init_from_sheerka=True) sheerka.clear_bnf_definition() # to simulate restart text = "one thousand" one = CC("one", body=DoNotResolve("one")) expected = CNC("thousands", source=text, number=CC("number", source="one", one=one, body=one)) expected_array = compute_expected_array(cmap, text, [expected]) res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_parse_fifty_one_thousand(self): """ Test of complex number + 'thousand' (complex because the number is a BNF concept) :return: """ sheerka, context, parser = self.init_parser(init_from_sheerka=True) sheerka.clear_bnf_definition() # to simulate restart text = "fifty one thousand" one = CC("one", body=DoNotResolve("one")) fifty_one = CC("fifties", source="fifty one", fifty="fifty", number=CC("number", source="one", body=one, one=one)) one_thousand = CC("thousands", source="one thousand", number=CC("number", source="one", body=one, one=one)) expected_thousand = CNC("thousands", source=text, number=CC("number", source="fifty one", fifties=fifty_one, body=fifty_one)) expected_fifties = CNC("fifties", source=text, fifty="fifty", number=CC("number", source="one thousand", thousands=one_thousand, body=one_thousand)) expected_thousands = compute_expected_array(cmap, text, [expected_thousand]) expected_fifties = compute_expected_array(cmap, text, [expected_fifties]) res = parser.parse(context, ParserInput(text)) assert res[0].status assert res[0].value.value == expected_thousands assert res[1].status assert res[1].value.value == expected_fifties def test_i_can_parse_one_hundred_thousand(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) sheerka.clear_bnf_definition() # to simulate restart text = "one hundred thousand" res = parser.parse(context, ParserInput(text)) parser_result = res.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) def test_i_can_parse_hundreds_like_expression(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) sheerka.clear_bnf_definition() # to simulate restart text = "three hundred and thirty two" three = CC("three", body=DoNotResolve("three")) two = CC("two", body=DoNotResolve("two")) thirty_two = CC("thirties", source="thirty two", thirty="thirty", number=CC("number", source="two", body=two, two=two)) expected = CNC("hundreds", source=text, n1=CC("number", source="three", body=three, three=three), n2=CC("number", source="thirty two", body=thirty_two, thirties=thirty_two)) expected_array = compute_expected_array(cmap, text, [expected]) res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_parse_bnf_concept_mixed_with_isa_after_restart(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) sheerka.clear_bnf_definition() # to simulate restart for c in cmap.values(): sheerka.get_by_id(c.id).set_bnf(None) text = "thirty three" expected = CNC("thirties", source=text, number=CC("number", source="three", three=CC("three", body=DoNotResolve("three")), body=CC("three", body=DoNotResolve("three"))), thirty="thirty") expected_array = compute_expected_array(cmap, text, [expected]) res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array text = "forty one" expected = CNC("forties", source=text, number=CC("number", source="one", one=CC("one", body=DoNotResolve("one")), body=CC("one", body=DoNotResolve("one"))), forty="forty") expected_array = compute_expected_array(cmap, text, [expected]) res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_parse_when_keyword(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) parser_input = "def one" expected = [CNC("def number", source="def one", number="one")] res = parser.parse(context, ParserInput(parser_input)) expected_array = compute_expected_array(cmap, parser_input, expected) expected_array[0].compiled["def"] = cmap["def_only"] parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_parse_filter(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) expression = "--filter one" expected = [CN("filter", source="--filter one")] res = parser.parse(context, ParserInput(expression)) expected_array = compute_expected_array(cmap, expression, expected) parser_result = res.value concepts_nodes = res.value.value assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == expected_array def test_i_can_parse_descent_grammar(self): my_map = { "factor": Concept("factor", definition="1 | 2 | 3"), "term": Concept("term", definition="factor ('*' factor)*"), "expr": Concept("expr", definition="term ('+' term)*"), } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.init_from_concepts(context, my_map.values()) text = "1 + 2 * 3" res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value factor = my_map["factor"] term = my_map["term"] expr = my_map["expr"] assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == [CNC(expr, term=[CC(term, body=CC(factor, body=DoNotResolve("1")), factor=CC(factor, body=DoNotResolve("1"))), CC(term, body=DoNotResolve("2 * 3"), factor=[ CC(factor, body=DoNotResolve("2")), CC(factor, body=DoNotResolve("3")), ])], body=DoNotResolve("1 + 2 * 3"))] def test_i_can_parse_recursive_descent_grammar(self): my_map = { "factor": Concept("factor", definition="1 | 2 | 3"), "term": self.bnf_concept("term", OrderedChoice( Sequence(ConceptExpression("factor"), StrMatch("*"), ConceptExpression("term")), ConceptExpression("factor"))), "expr": self.bnf_concept("expr", OrderedChoice( Sequence(ConceptExpression("term"), StrMatch("+"), ConceptExpression("expr")), ConceptExpression("term"))), } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.init_from_concepts(context, my_map.values()) text = "1 + 2 * 3" res = parser.parse(context, ParserInput(text)) parser_result = res.value concepts_nodes = res.value.value factor = my_map["factor"] term = my_map["term"] expr = my_map["expr"] # concepts_nodes = res.value.value is too complicated to be validated assert res.status assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert concepts_nodes == [CNC(expr, term=CC(term, body=CC(factor, body=DoNotResolve("1")), factor=CC(factor, body=DoNotResolve("1"))), expr=CC(expr, body=CC(term, body=DoNotResolve("2 * 3"), factor=CC(factor, body=DoNotResolve("2")), term=CC(term, body=CC(factor, body=DoNotResolve("3")), factor=CC(factor, body=DoNotResolve("3")))), term=CC(term, body=DoNotResolve("2 * 3"), factor=CC(factor, body=DoNotResolve("2")), term=CC(term, body=CC(factor, body=DoNotResolve("3")), factor=CC(factor, body=DoNotResolve("3"))))), body=DoNotResolve("1 + 2 * 3"))] def test_i_can_parse_simple_recursive_grammar(self): my_map = { "foo": self.bnf_concept("foo", Sequence(StrMatch("foo"), OrderedChoice(StrMatch("bar"), ConceptExpression("foo")))), } sheerka, context, parser = self.init_parser(my_map, singleton=True) parser.init_from_concepts(context, my_map.values()) assert parser.parse(context, ParserInput("foo bar")).status assert parser.parse(context, ParserInput("foo foo foo bar")).status assert not parser.parse(context, ParserInput("foo baz")).status @pytest.mark.parametrize("name, expected", [ (None, []), ("", []), ("foo", StrMatch("foo")), ("foo bar", Sequence(StrMatch("foo"), StrMatch("bar"))), ("'foo bar baz' qux", Sequence(StrMatch("foo", skip_whitespace=False), StrMatch(" ", skip_whitespace=False), StrMatch("bar", skip_whitespace=False), StrMatch(" ", skip_whitespace=False), StrMatch("baz"), StrMatch("qux"))), ]) def test_i_can_get_expression_from_concept_name(self, name, expected): assert BnfNodeParser.get_expression_from_concept_name(name) == expected def test_i_can_parse_when_multiple_layers(self): sheerka, context, parser = self.init_parser(init_from_sheerka=True) # sanity text = "thirty one" res = parser.parse(context, ParserInput(text)) assert res.status assert res.value.value == compute_expected_array(cmap, text, [CN("thirties", source=text)]) # add a layer, I still can parse the text sheerka.push_ontology(context, "new layer") parser = BnfNodeParser(sheerka=sheerka) res = parser.parse(context, ParserInput(text)) assert res.status assert res.value.value == compute_expected_array(cmap, text, [CN("thirties", source=text)]) # @pytest.mark.parametrize("parser_input, expected", [ # ("one", [ # (True, [CNC("bnf_one", source="one", one="one", body="one")]), # (True, [CNC("one_or_two", source="one", one="one", body="one")]), # ]), # ("two plus two", [ # (False, [CN("bnf_one"), UTN(" plus "), CN("one_or_two")]), # (False, [CN("one_or_two"), UTN(" plus "), CN("one_or_two")]), # ]) # ]) # def test_i_can_parse_when_multiple_results(self, parser_input, expected): # sheerka, context, parser = self.init_parser(init_from_sheerka=True) # # res = parser.parse(context, parser_input) # assert len(res) == len(expected) # # for res_i, expected_i in zip(res, expected): # assert res_i.status == expected_i[0] # expected_array = compute_expected_array(cmap, parser_input, expected_i[1]) # assert res_i.value.value == expected_array