kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bug when evaluating numbers several times

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-11-27 11:01:02 +01:00
parent cac732bd93
commit 4f899280c4
16 changed files with 887 additions and 491 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parsers/test_BnfNodeParser.py
+35 -20
View File
@@ -3,9 +3,9 @@ from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF, NotInit
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import CNC, UTN, CN
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser
from parsers.BnfDefinitionParser import BnfDefinitionParser
import tests.parsers.parsers_utils
from tests.BaseTest import BaseTest
@@ -172,7 +172,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
parser.init_from_concepts(context, updated)
parser.reset_parser(context, ParserInput(text))
bnf_parsers_helpers = parser.get_concepts_sequences()
bnf_parsers_helpers = parser.get_concepts_sequences(context)
assert len(bnf_parsers_helpers) == len(expected_array)
for parser_helper, expected_sequence in zip(bnf_parsers_helpers, expected_array):
@@ -263,14 +263,14 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
expected_array = compute_expected_array(my_map, text, expected)
parser.reset_parser(context, ParserInput(text))
bnf_parsers_helpers = parser.get_concepts_sequences()
bnf_parsers_helpers = parser.get_concepts_sequences(context)
assert bnf_parsers_helpers[0].sequence == expected_array
assert not bnf_parsers_helpers[0].has_unrecognized
# but I cannot parse
text = "- - filter"
parser.reset_parser(context, ParserInput(text))
bnf_parsers_helpers = parser.get_concepts_sequences()
bnf_parsers_helpers = parser.get_concepts_sequences(context)
assert bnf_parsers_helpers[0].has_unrecognized
def test_i_can_match_multiple_sequences(self):
@@ -788,7 +788,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
concept_foo = sequences[0].concept
assert concept_foo.body == NotInit
assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["two"], two=my_map["two"]),
ConceptParts.BODY: DoNotResolve(value='twenty two')}
ConceptParts.BODY: DoNotResolve(value='twenty two')}
text = "twenty one"
expected = [CN("foo", source="twenty one")]
@@ -798,13 +798,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
concept_foo = sequences[0].concept
assert concept_foo.body == NotInit
assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]),
ConceptParts.BODY: DoNotResolve(value='twenty one')}
ConceptParts.BODY: DoNotResolve(value='twenty one')}
@pytest.mark.parametrize("bar_expr, expected", [
(ConceptExpression("foo"), {}),
(OrderedChoice(ConceptExpression("foo"), StrMatch("one")), {'one': ['1002']}),
(Sequence(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")), {'one': ['1001', '1002']}),
# (UnOrderedChoice(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")), {'one': ['1001', '1002']})
])
def test_i_can_detect_infinite_recursion(self, bar_expr, expected):
my_map = {
@@ -850,6 +849,33 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert parser.concepts_grammars.get(my_map["foo"].id).body == ["1001", "1002", "1003", "1004", "1001"]
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG)
def test_i_can_detect_partial_infinite_recursion(self):
my_map = {
"foo": self.bnf_concept("foo", ConceptExpression("bar")),
"bar": self.bnf_concept("bar", ConceptExpression("baz")),
"baz": self.bnf_concept("baz", ConceptExpression("qux")),
"qux": self.bnf_concept("qux", ConceptExpression("baz")),
}
sheerka, context, parser = self.init_parser(my_map, singleton=True)
parser.context = context
parser.sheerka = sheerka
# every obvious cyclic recursion are removed from concept_by_first_keyword dict
parser.init_from_concepts(context, my_map.values())
assert parser.concepts_by_first_keyword == {}
parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert parser.concepts_grammars.get(my_map["foo"].id).body == ["1001", "1002", "1003", "1004", "1003"]
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG)
assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG)
@@ -904,8 +930,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes
assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes
assert my_map["number"].id not in parser.concepts_grammars
def test_i_can_get_parsing_expression_when_starting_by_isa_concept(self):
my_map = {
"one": Concept("one"),
@@ -1015,7 +1039,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
parser.init_from_concepts(context, my_map.values())
parser.reset_parser(context, ParserInput("one three"))
sequences = parser.get_concepts_sequences()
sequences = parser.get_concepts_sequences(context)
sequence = parser.get_valid(sequences)
assert len(sequence) == 1
@@ -1122,7 +1146,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert concepts_nodes == expected_array
def test_i_can_parse_one_thousand(self):
def test_i_can_parse_when_starting_by_isa_concept(self):
"""
Test of simple number + 'thousand'
:return:
@@ -1403,15 +1427,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert parser.parse(context, ParserInput("foo foo foo bar")).status
assert not parser.parse(context, ParserInput("foo baz")).status
def test_i_only_get_the_requested_parsing_expression(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
parser.context = context
parser.sheerka = sheerka
sheerka.concepts_grammars.clear() # to simulate restart
parser.get_parsing_expression(context, sheerka.resolve("thirties"))
assert len(parser.concepts_grammars) == 9 # requested concept + concepts that do not contains UnorderedChoice
@pytest.mark.parametrize("name, expected", [
(None, []),
("", []),
tests/parsers/test_BnfParser.py
+3 -4
View File
@@ -2,13 +2,13 @@ import pytest
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_BNF
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
from core.tokenizer import Tokenizer, TokenKind, LexerError
from parsers.BaseNodeParser import cnode
from parsers.BaseParser import UnexpectedTokenErrorNode
from parsers.BnfDefinitionParser import BnfDefinitionParser, UnexpectedEndOfFileError
from parsers.BnfNodeParser import BnfNodeParser
from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \
OneOrMore, ConceptExpression
from parsers.BnfNodeParser import BnfNodeParser
from parsers.BnfDefinitionParser import BnfDefinitionParser, UnexpectedEndOfFileError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -236,4 +236,3 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
assert res.status
pexpression = res.value.value
assert pexpression == Sequence(StrMatch('twenty'), ConceptExpression(number, "n1"))
assert pexpression.elements[1].recurse_id == "1004#1003(n1)"