kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed BnfNodeParser to allow expressions like 'number hundred' when number is a group

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-06-27 18:56:04 +02:00
parent d4468da8a3
commit 2c5840752a
14 changed files with 593 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/core/test_SheerkaEvaluateConcept.py
+29
View File
@@ -494,3 +494,32 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
evaluated = sheerka.evaluate_concept(self.get_context(sheerka, True), one)
assert evaluated.key == one.key
assert evaluated.body == 1
def test_i_can_evaluate_thousand_concept(self):
sheerka, context, thousand, number, forties, forty, one = self.init_concepts(
Concept("thousand", body="number * 1000").def_var("number"),
Concept("number"),
Concept("forties", body="forty + number").def_var("forty").def_var("number"),
Concept("forty", body="40"),
Concept("one", body="1"),
eval_body=True
)
one = sheerka.new("one")
number2 = sheerka.new("number")
number2.compiled["one"] = one
number2.compiled[ConceptParts.BODY] = one
forties = sheerka.new("forties")
forties.compiled["forty"] = sheerka.new("forty")
forties.compiled["number"] = number2
number1 = sheerka.new("number")
number1.compiled["forties"] = forties
number1.compiled[ConceptParts.BODY] = forties
forty_one_thousand = sheerka.new("thousand")
forty_one_thousand.compiled["number"] = number1
evaluated = sheerka.evaluate_concept(context, forty_one_thousand)
assert evaluated.body == 41000
tests/core/test_utils.py
+17
View File
@@ -1,9 +1,20 @@
from dataclasses import dataclass
import core.utils
import pytest
from core.concept import ConceptParts, Concept
from core.tokenizer import Token, TokenKind
@dataclass
class Obj:
prop1: str
prop2: str
def __hash__(self):
return hash((self.prop1, self.prop1))
def get_tokens(lst):
res = []
for e in lst:
@@ -229,3 +240,9 @@ def test_decode_concept_key_id():
])
def test_dict_product(a, b, expected):
assert core.utils.dict_product(a, b) == expected
def test_i_can_make_unique():
assert core.utils.make_unique(["a", "a", "b", "c", "c"]) == ["a", "b", "c"]
assert core.utils.make_unique([Obj("a", "b"), Obj("a", "c"), Obj("a", "b")]) == [Obj("a", "b"), Obj("a", "c")]
assert core.utils.make_unique([Obj("a", "b"), Obj("a", "c")], lambda o: o.prop1) == [Obj("a", "b")]
tests/evaluators/test_ConceptEvaluator.py
+1
View File
@@ -118,3 +118,4 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
assert not context.sheerka.is_success(error_concept) # it's indeed an error
assert result.status
assert result.value == error_concept
tests/parsers/test_BaseNodeParser.py
+72 -40
View File
@@ -137,28 +137,38 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
}
def test_i_can_resolve_when_concepts_are_sets(self):
sheerka, context, one, two, three, number, foo = self.init_concepts(
sheerka, context, number, *concepts = self.init_concepts(
"number",
"one",
"two",
"three",
"number",
Concept("foo", definition="number three"),
"twenty",
"hundred",
Concept("twenties", definition="twenty number"),
Concept("hundreds", definition="number hundred"),
create_new=True # mandatory because set_isa() needs it
)
sheerka.set_isa(context, sheerka.new("one"), number)
sheerka.set_isa(context, sheerka.new("two"), number)
sheerka.set_isa(context, sheerka.new("twenty"), number)
sheerka.set_isa(context, sheerka.new("thirty"), number)
sheerka.set_isa(context, sheerka.new("hundred"), number)
sheerka.set_isa(context, sheerka.new("twenties"), number)
sheerka.set_isa(context, sheerka.new("hundreds"), number)
cbfk = BaseNodeParser.get_concepts_by_first_token(context, [one, two, three, number, foo]).body
sheerka.concepts_grammars.clear() # reset all the grammar to simulate Sheerka restart
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, cbfk)
# cbft : concept_by_first_token (I usually don't use abbreviation)
cbft = BaseNodeParser.get_concepts_by_first_token(context, [number] + concepts).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, cbft)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"one": ["1001", "1005"],
"two": ["1002", "1005"],
"three": ["1003"],
"number": ["1004"],
'number': ['1001'],
'one': ['1002', '1007'],
'two': ['1003', '1007'],
'twenty': ['1004', '1006', '1007'],
'hundred': ['1005', '1007'],
}
def test_concepts_are_defined_once(self):
@@ -196,36 +206,58 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
"one": ["1001", "1002"],
}
# def tests_i_can_detect_direct_recursion(self):
# sheerka = self.get_sheerka()
# good = self.get_concept(sheerka, "good")
# foo = self.get_concept(sheerka, "foo", ConceptExpression("bar"))
# bar = self.get_concept(sheerka, "bar", ConceptExpression("foo"))
#
# concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, foo, bar]).body
#
# resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
# assert resolved_ret_val.status
# assert resolved_ret_val.body == {
# "good": ["1001"],
# BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1003"]
# }
#
# def test_i_can_detect_indirect_infinite_recursion(self):
# sheerka = self.get_sheerka()
# good = self.get_concept(sheerka, "good")
# one = self.get_concept(sheerka, "one", ConceptExpression("two"))
# two = self.get_concept(sheerka, "two", ConceptExpression("three"))
# three = self.get_concept(sheerka, "three", ConceptExpression("two"))
#
# concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two, three]).body
#
# resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
# assert resolved_ret_val.status
# assert resolved_ret_val.body == {
# "good": ["1001"],
# BuiltinConcepts.CHICKEN_AND_EGG: ["1002", "1004", "1003"]
# }
def tests_i_can_detect_direct_recursion(self):
sheerka, context, good, foo, bar = self.init_concepts(
"good",
self.bnf_concept("foo", ConceptExpression("bar")),
self.bnf_concept("bar", ConceptExpression("foo")),
)
concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(context, [good, foo, bar]).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001"],
}
assert sheerka.chicken_and_eggs.get(foo.id) == {foo.id, bar.id}
assert sheerka.chicken_and_eggs.get(bar.id) == {foo.id, bar.id}
def test_i_can_detect_indirect_infinite_recursion(self):
sheerka, context, good, one, two, three = self.init_concepts(
"good",
self.bnf_concept("one", ConceptExpression("two")),
self.bnf_concept("two", ConceptExpression("three")),
self.bnf_concept("three", ConceptExpression("two")),
)
concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(context, [good, one, two, three]).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001"],
}
assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id}
def test_i_can_detect_the_longest_infinite_recursion_chain(self):
sheerka, context, good, one, two, three = self.init_concepts(
"good",
self.bnf_concept("two", ConceptExpression("three")),
self.bnf_concept("three", ConceptExpression("two")),
self.bnf_concept("one", ConceptExpression("three")),
)
concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(context, [good, one, two, three]).body
resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
assert resolved_ret_val.status
assert resolved_ret_val.body == {
"good": ["1001"],
}
assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id}
assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id}
#
# def test_i_can_detect_infinite_recursion_from_ordered_choice(self):
# sheerka = self.get_sheerka()
tests/parsers/test_BnfNodeParser.py
+177 -100
View File
@@ -4,7 +4,7 @@ from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYP
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import CNC, UTN, CN
from parsers.BnfNodeParser import BnfNodeParser, StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
Optional, ZeroOrMore, OneOrMore, ConceptExpression
Optional, ZeroOrMore, OneOrMore, ConceptExpression, LongestChoice
from parsers.BnfParser import BnfParser
import tests.parsers.parsers_utils
@@ -16,13 +16,15 @@ cmap = {
"two": Concept("two"),
"three": Concept("three"),
"four": Concept("four"),
"thirty": Concept("thirty", body=30),
"forty": Concept("forty", body=40),
"fifty": Concept("fifty", body=50),
"thirty": Concept("thirty", body="30"),
"forty": Concept("forty", body="40"),
"fifty": Concept("fifty", body="50"),
"number": Concept("number"),
"foo": Concept("foo"),
"bar": Concept("bar"),
"baz": Concept("baz"),
"one hundred": Concept("one hundred", body="100"),
"one_hundred": Concept("'one hundred'", body="100"),
"bnf baz": Concept("bnf baz", definition="'baz'"), # this one should be chosen
@@ -80,17 +82,6 @@ def compute_expected_array(my_concepts_map, expression, expected, exclude_body=F
class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
sheerka = None
@staticmethod
def update_bnf(context, concept):
bnf_parser = BnfParser()
res = bnf_parser.parse(context, concept.metadata.definition)
if res.status:
concept.bnf = res.value.value
concept.metadata.definition_type = DEFINITION_TYPE_BNF
else:
raise Exception(res)
return concept
@classmethod
def setup_class(cls):
t = cls()
@@ -109,6 +100,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
sheerka.set_isa(context, sheerka.new("thirty"), sheerka.new("number"))
sheerka.set_isa(context, sheerka.new("forty"), sheerka.new("number"))
sheerka.set_isa(context, sheerka.new("fifty"), sheerka.new("number"))
sheerka.set_isa(context, sheerka.new("one hundred"), sheerka.new("number"))
thirties = cls.update_bnf(context, Concept("thirties",
definition="thirty number",
@@ -131,6 +123,24 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
cmap["fifties"] = sheerka.create_new_concept(context, fifties).body.body
sheerka.set_isa(context, sheerka.new("fifties"), sheerka.new("number"))
thousands = cls.update_bnf(context, Concept("thousands",
definition="number 'thousand'",
where="number < 999",
body="number * 1000").def_var("number"))
cmap["thousands"] = sheerka.create_new_concept(context, thousands).body.body
sheerka.set_isa(context, sheerka.new("thousands"), sheerka.new("number"))
@staticmethod
def update_bnf(context, concept):
bnf_parser = BnfParser()
res = bnf_parser.parse(context, concept.metadata.definition)
if res.status:
concept.bnf = res.value.value
concept.metadata.definition_type = DEFINITION_TYPE_BNF
else:
raise Exception(res)
return concept
def init_parser(self, my_concepts_map=None, init_from_sheerka=False, **kwargs):
if my_concepts_map is not None:
sheerka, context, *updated = self.init_concepts(*my_concepts_map.values(), **kwargs)
@@ -351,6 +361,21 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one", [CNC("foo", source="one")]),
("one two", [CNC("foo", source="one two")]),
("three", []),
])
def test_i_can_parse_longest_choice(self, text, expected):
my_map = {
"foo": self.bnf_concept("foo", LongestChoice(
StrMatch("one"),
Sequence(StrMatch("one"), StrMatch("two")))),
}
self.validate_get_concepts_sequences(my_map, text, expected)
@pytest.mark.parametrize("text, expected", [
("one", [CNC("foo", source="one")]),
("", []),
@@ -724,7 +749,10 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
"foo": self.bnf_concept("foo",
Sequence(
StrMatch("twenty"),
OrderedChoice(ConceptExpression("one"), ConceptExpression("two")))),
OrderedChoice(
ConceptExpression("one"),
ConceptExpression("two"),
rule_name="unit"))),
}
text = "twenty one"
@@ -733,7 +761,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
concept_foo = sequences[0].concept
assert concept_foo.compiled == {
ConceptParts.BODY: DoNotResolve("twenty one"),
"one": my_map["one"],
"unit": my_map["one"],
}
def test_i_can_refer_to_group_concepts(self):
@@ -755,8 +783,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
# explicit validations of the compiled
concept_foo = sequences[0].concept
assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED
assert concept_foo.compiled == {'number': my_map["number"],
'two': my_map["two"],
assert concept_foo.compiled == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]),
ConceptParts.BODY: DoNotResolve(value='twenty two')}
text = "twenty one"
@@ -766,8 +793,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
# explicit validations of the compiled
concept_foo = sequences[0].concept
assert concept_foo.body == BuiltinConcepts.NOT_INITIALIZED
assert concept_foo.compiled == {'number': my_map["number"],
'one': my_map["one"],
assert concept_foo.compiled == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]),
ConceptParts.BODY: DoNotResolve(value='twenty one')}
@pytest.mark.parametrize("bar_expr, expected", [
@@ -842,8 +868,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
res = parser.get_parsing_expression(context, my_map["foo"])
assert sheerka.isinstance(res, BuiltinConcepts.CHICKEN_AND_EGG) == expected
def test_i_can_get_parsing_expression_when_concept_isa(self):
def test_i_can_get_parsing_expression_when_ending_by_concept_isa(self):
my_map = {
"one": Concept("one"),
"twenty": Concept("twenty"),
@@ -869,42 +894,43 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
number_nodes = parsing_expression.nodes[1].nodes
assert len(number_nodes) == 1
assert isinstance(number_nodes[0], OrderedChoice)
assert isinstance(number_nodes[0], LongestChoice)
assert len(number_nodes[0].nodes) == len(number_nodes[0].elements)
assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes
assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes
assert my_map["number"].id not in parser.concepts_grammars
#
# def test_i_cannot_get_parsing_expression_when_concept_is_part_of_a_group(self):
# """
# In this test, twenties isa number
# # So 'number' in Sequence(thirty, number) will spawn 'twenties' which, because there is no other indication,
# # will create an infinite loop
# :return:
# """
# my_map = {
# "one": Concept("one"),
# "twenty": Concept("twenty"),
# "number": Concept("number"),
# "twenties": self.bnf_concept("twenties", Sequence(ConceptExpression("twenty"), ConceptExpression("number")))
# }
# sheerka, context, parser = self.init_parser(my_map, singleton=True)
# parser.context = context
# parser.sheerka = sheerka
# sheerka.set_isa(context, sheerka.new("one"), my_map["number"])
# sheerka.set_isa(context, sheerka.new("twenty"), my_map["number"])
# sheerka.set_isa(context, sheerka.new("twenties"), my_map["number"]) # <- twenties is also a number
#
# parser.concepts_grammars.clear() # make sure parsing expression is created from scratch
#
# parsing_expression = parser.get_parsing_expression(context, my_map["twenties"])
# assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
# assert parsing_expression.body == {my_map["twenties"].id, my_map["number"].id}
#
# assert isinstance(parser.concepts_grammars.get(my_map["one"].id), ParsingExpression)
# assert isinstance(parser.concepts_grammars.get(my_map["twenty"].id), ParsingExpression)
def test_i_can_get_parsing_expression_when_starting_by_isa_concept(self):
my_map = {
"one": Concept("one"),
"two": Concept("two"),
"number": Concept("number"),
"hundreds": self.bnf_concept("hundreds", Sequence(ConceptExpression("number"), StrMatch("hundred")))
}
sheerka, context, parser = self.init_parser(my_map, singleton=True)
parser.context = context
parser.sheerka = sheerka
sheerka.set_isa(context, sheerka.new("one"), my_map["number"])
sheerka.set_isa(context, sheerka.new("two"), my_map["number"])
sheerka.set_isa(context, sheerka.new("hundreds"), my_map["number"])
parser.concepts_grammars.clear() # make sure parsing expression is created from scratch
parsing_expression = parser.get_parsing_expression(context, my_map["hundreds"])
assert parsing_expression == Sequence(
ConceptExpression(my_map["number"], rule_name="number"),
StrMatch("hundred"))
assert len(parsing_expression.nodes) == len(parsing_expression.elements)
number_nodes = parsing_expression.nodes[0].nodes
assert len(number_nodes) == 1
assert isinstance(number_nodes[0], LongestChoice)
assert len(number_nodes[0].nodes) == len(number_nodes[0].elements)
assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes
assert ConceptExpression(my_map["two"], rule_name="two") in number_nodes[0].nodes
def test_i_can_get_parsing_expression_when_concept_is_part_of_a_group(self):
my_map = {
@@ -933,7 +959,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
number_nodes = parsing_expression.nodes[1].nodes
assert len(number_nodes) == 1
assert isinstance(number_nodes[0], OrderedChoice)
assert isinstance(number_nodes[0], LongestChoice)
assert len(number_nodes[0].nodes) == len(number_nodes[0].elements)
assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes
assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes
@@ -994,7 +1020,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
("foo bar", True, [CNC("foo then bar", source="foo bar", foo="foo", bar="bar")]),
("bar", True, [CNC("foo or bar", source="bar", bar="bar", body="bar")]),
("one plus two", True, [CNC("plus", source="one plus two", one="one", two="two")]),
("twenty one", True, [CNC("t1", source="twenty one", unit="one", one="one")]),
("twenty one", True, [CNC("t1", source="twenty one", unit="one")]),
])
def test_i_can_parse_simple_expressions(self, parser_input, expected_status, expected):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
@@ -1008,7 +1034,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
def test_i_can_when_multiple_times_the_same_variable(self):
def test_i_can_parse_when_multiple_times_the_same_variable(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
text = "foo foo foo"
@@ -1032,8 +1058,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
unit=CC("three_four",
source="four",
four=CC("four", body=DoNotResolve("four")),
body=CC("four", body=DoNotResolve("four"))),
four="four")
body=CC("four", body=DoNotResolve("four"))))
expected_array = compute_expected_array(cmap, text, [expected])
res = parser.parse(context, ParserInput(text))
@@ -1044,40 +1069,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
# def test_i_cannot_parse_bnf_concept_mixed_with_isa_concepts(self):
# sheerka, context, parser = self.init_parser(init_from_sheerka=True)
#
# # thirties = cls.update_bnf(context, Concept("thirties",
# # definition="thirty number",
# # where="number < 10",
# # body="thirty + number").def_var("thirty").def_var("number"))
# # with thirties isa number
# # So number in 'thirty number' will spawn 'thirties' which, because there is no other indication, will
# # create an infinite loop
#
# text = "thirty one"
# expected = CNC("thirties",
# source=text,
# number=CC("number",
# source="one",
# one=CC("one", body=DoNotResolve("one")),
# body=CC("one", body=DoNotResolve("one"))),
# one=CC("one", body=DoNotResolve("one")),
# thirty="thirty")
# expected_array = compute_expected_array(cmap, text, [expected])
#
# res = parser.parse(context, ParserInput(text))
# not_for_me = res.value
# reason = res.value.body
#
# assert not res.status
# assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME)
# assert sheerka.isinstance(reason, BuiltinConcepts.CHICKEN_AND_EGG)
# assert reason.body == {cmap["thirties"].id, cmap["number"].id}
def test_i_can_parse_bnf_concept_mixed_with_isa_concepts(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
# thirties is defined in the global variable cmap as
# thirties = cls.update_bnf(context, Concept("thirties",
# definition="thirty number",
# where="number < 10",
@@ -1090,7 +1084,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
source="one",
one=CC("one", body=DoNotResolve("one")),
body=CC("one", body=DoNotResolve("one"))),
one=CC("one", body=DoNotResolve("one")),
thirty="thirty")
expected_array = compute_expected_array(cmap, text, [expected])
@@ -1113,7 +1106,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
source="three",
three=CC("three", body=DoNotResolve("three")),
body=CC("three", body=DoNotResolve("three"))),
three=CC("three", body=DoNotResolve("three")),
thirty="thirty")
expected_array = compute_expected_array(cmap, text, [expected])
@@ -1125,6 +1117,56 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
def test_i_can_parse_bnf_concept_mixed_with_isa_concepts_when_concept_starts_with_isa(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
one = CC("one", body=DoNotResolve("one"))
text = "one thousand"
expected = CNC("thousands",
source=text,
number=CC("number",
source="one",
one=one,
body=one))
expected_array = compute_expected_array(cmap, text, [expected])
res = parser.parse(context, ParserInput(text))
parser_result = res.value
concepts_nodes = res.value.value
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
sheerka.concepts_grammars.clear() # to simulate restart
text = "fifty one thousand"
fifty_one = CC("fifties",
source="fifty one",
fifty="fifty",
number=CC("number", source="one", body=one, one=one))
expected = CNC("thousands",
source=text,
number=CC("number",
source="fifty one",
fifties=fifty_one,
body=fifty_one))
expected_array = compute_expected_array(cmap, text, [expected])
res = parser.parse(context, ParserInput(text))
parser_result = res.value
concepts_nodes = res.value.value
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
text = "one hundred thousand"
res = parser.parse(context, ParserInput(text))
parser_result = res.value
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
def test_i_can_parse_bnf_concept_mixed_with_isa_after_restart(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
sheerka.concepts_grammars.clear() # simulate restart
@@ -1138,7 +1180,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
source="three",
three=CC("three", body=DoNotResolve("three")),
body=CC("three", body=DoNotResolve("three"))),
three=CC("three", body=DoNotResolve("three")),
thirty="thirty")
expected_array = compute_expected_array(cmap, text, [expected])
@@ -1157,7 +1198,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
source="one",
one=CC("one", body=DoNotResolve("one")),
body=CC("one", body=DoNotResolve("one"))),
one=CC("one", body=DoNotResolve("one")),
forty="forty")
expected_array = compute_expected_array(cmap, text, [expected])
@@ -1173,7 +1213,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
parser_input = "def one"
expected = [CNC("def number", source="def one", number="one", one="one")]
expected = [CNC("def number", source="def one", number="one")]
res = parser.parse(context, ParserInput(parser_input))
expected_array = compute_expected_array(cmap, parser_input, expected)
@@ -1190,7 +1230,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
expression = "--filter one"
expected = [CNC("filter", source="--filter one", one="one")]
expected = [CN("filter", source="--filter one")]
res = parser.parse(context, ParserInput(expression))
expected_array = compute_expected_array(cmap, expression, expected)
@@ -1233,10 +1273,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
CC(factor, body=DoNotResolve("2")),
CC(factor, body=DoNotResolve("3")),
])],
factor=[
CC(factor, body=DoNotResolve("1")),
CC(factor, body=DoNotResolve("2")),
CC(factor, body=DoNotResolve("3"))],
body=DoNotResolve("1 + 2 * 3"))]
def test_i_can_parse_recursive_descent_grammar(self):
@@ -1255,8 +1291,34 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
text = "1 + 2 * 3"
res = parser.parse(context, ParserInput(text))
parser_result = res.value
concepts_nodes = res.value.value
factor = my_map["factor"]
term = my_map["term"]
expr = my_map["expr"]
# concepts_nodes = res.value.value is too complicated to be validated
assert res.status
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == [CNC(expr,
term=CC(term,
body=CC(factor, body=DoNotResolve("1")),
factor=CC(factor, body=DoNotResolve("1"))),
expr=CC(expr,
body=CC(term,
body=DoNotResolve("2 * 3"),
factor=CC(factor, body=DoNotResolve("2")),
term=CC(term,
body=CC(factor, body=DoNotResolve("3")),
factor=CC(factor, body=DoNotResolve("3")))),
term=CC(term,
body=DoNotResolve("2 * 3"),
factor=CC(factor, body=DoNotResolve("2")),
term=CC(term,
body=CC(factor, body=DoNotResolve("3")),
factor=CC(factor, body=DoNotResolve("3"))))),
body=DoNotResolve("1 + 2 * 3"))]
def test_i_can_parse_simple_recursive_grammar(self):
my_map = {
@@ -1271,6 +1333,21 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert parser.parse(context, ParserInput("foo foo foo bar")).status
assert not parser.parse(context, ParserInput("foo baz")).status
@pytest.mark.parametrize("name, expected", [
(None, []),
("", []),
("foo", StrMatch("foo")),
("foo bar", Sequence(StrMatch("foo"), StrMatch("bar"))),
("'foo bar baz' qux", Sequence(StrMatch("foo", skip_whitespace=False),
StrMatch(" ", skip_whitespace=False),
StrMatch("bar", skip_whitespace=False),
StrMatch(" ", skip_whitespace=False),
StrMatch("baz"),
StrMatch("qux"))),
])
def test_i_can_get_expression_from_concept_name(self, name, expected):
assert BnfNodeParser.get_expression_from_concept_name(name) == expected
# @pytest.mark.parametrize("parser_input, expected", [
# ("one", [
# (True, [CNC("bnf_one", source="one", one="one", body="one")]),
tests/parsers/test_UnrecognizedNodeParser.py
+3 -3
View File
@@ -128,7 +128,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
expected_nodes = compute_expected_array(
concepts_map,
" twenty one ",
[CNC("twenties", source="twenty one", unit="one", one="one")])
[CNC("twenties", source="twenty one", unit="one")])
assert concept.compiled["c"][0].body.body == expected_nodes
assert len(concept.compiled["d"]) == 1
@@ -185,7 +185,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
expected_nodes = compute_expected_array(
concepts_map,
" twenty two",
[CNC("twenties", source="twenty two", unit="two", two="two")])
[CNC("twenties", source="twenty two", unit="two")])
assert res.body.concept.compiled["b"].compiled["b"][0].body.body == expected_nodes
# def test_i_can_validate_and_evaluate_a_concept_node_with_python(self):
@@ -281,7 +281,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
assert len(actual_nodes) == 1
expected_array = compute_expected_array(
concepts_map,
expression, [CNC("twenties", source=expression, unit="one", one="one")])
expression, [CNC("twenties", source=expression, unit="one")])
assert actual_nodes == expected_array
def test_i_can_parse_unrecognized_sya_concept_node(self):