kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed some bugs

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-08-27 18:54:28 +02:00
parent 351c16f946
commit 37cd3ed757
27 changed files with 685 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parsers/parsers_utils.py
+7 -4
View File
@@ -87,10 +87,13 @@ def get_node(
return sub_expr
if isinstance(sub_expr, SCWC):
first = get_node(concepts_map, expression_as_tokens, sub_expr.first, sya=sya)
last = get_node(concepts_map, expression_as_tokens, sub_expr.last, sya=sya)
content = [get_node(concepts_map, expression_as_tokens, c, sya=sya) for c in sub_expr.content]
return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
sub_expr.first = get_node(concepts_map, expression_as_tokens, sub_expr.first, sya=sya)
sub_expr.last = get_node(concepts_map, expression_as_tokens, sub_expr.last, sya=sya)
sub_expr.content = [get_node(concepts_map, expression_as_tokens, c, sya=sya) for c in sub_expr.content]
sub_expr.fix_pos(sub_expr.first)
sub_expr.fix_pos(sub_expr.last)
return sub_expr
#return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
if isinstance(sub_expr, SCN):
node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya)
tests/parsers/test_BnfNodeParser.py
+15 -3
View File
@@ -3,8 +3,8 @@ from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.BaseNodeParser import CNC, UTN, CN
from parsers.BnfNodeParser import BnfNodeParser, StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice
from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser
from parsers.BnfParser import BnfParser
import tests.parsers.parsers_utils
@@ -969,7 +969,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes
assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes
def test_i_can_get_parsing_expression_when_sequence_of_concept(self):
def test_i_can_get_parsing_expression_when_sequence_of_concepts(self):
my_map = {
"one": Concept("one"),
"two_ones": self.bnf_concept("two_ones", Sequence(ConceptExpression("one"), ConceptExpression("one")))
@@ -1203,6 +1203,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
def test_i_can_parse_hundreds_like_expression(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
sheerka.concepts_grammars.clear()
text = "three hundred and thirty two"
three = CC("three", body=DoNotResolve("three"))
@@ -1226,7 +1227,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
thirties=thirty_two))
expected_array = compute_expected_array(cmap, text, [expected])
res = parser.parse(context, ParserInput(text))
parser_result = res.value
concepts_nodes = res.value.value
@@ -1400,6 +1403,15 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
assert parser.parse(context, ParserInput("foo foo foo bar")).status
assert not parser.parse(context, ParserInput("foo baz")).status
def test_i_only_get_the_requested_parsing_expression(self):
sheerka, context, parser = self.init_parser(init_from_sheerka=True)
parser.context = context
parser.sheerka = sheerka
sheerka.concepts_grammars.clear() # to simulate restart
parser.get_parsing_expression(context, sheerka.resolve("thirties"))
assert len(parser.concepts_grammars) == 9 # requested concept + concepts that do not contains UnorderedChoice
@pytest.mark.parametrize("name, expected", [
(None, []),
("", []),
tests/parsers/test_BnfParser.py
+4 -3
View File
@@ -5,8 +5,9 @@ from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
from parsers.BaseNodeParser import cnode
from parsers.BaseParser import UnexpectedTokenErrorNode
from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
BnfNodeParser, ConceptExpression
from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \
OneOrMore, ConceptExpression
from parsers.BnfNodeParser import BnfNodeParser
from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -235,4 +236,4 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
assert res.status
pexpression = res.value.value
assert pexpression == Sequence(StrMatch('twenty'), ConceptExpression(number, "n1"))
assert pexpression.elements[1].recurse_id == "1003#n1#1004"
assert pexpression.elements[1].recurse_id == "1004#1003(n1)"
tests/parsers/test_SyaNodeParser.py
+64 -17
View File
@@ -5,7 +5,7 @@ from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonMana
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer
from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \
SCWC, CNC, UTN
SCWC, CNC, UTN, SourceCodeWithConceptNode
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
NoneAssociativeSequenceErrorNode, TooManyParametersFound
@@ -56,10 +56,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
cmap["minus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context,
BuiltinConcepts.PRECEDENCE,
cmap["mult"], cmap["plus"])
cmap["mult"],
cmap["plus"])
TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context,
BuiltinConcepts.PRECEDENCE,
cmap["mult"], cmap["minus"])
cmap["mult"],
cmap["minus"])
# TestSyaNodeParser.sheerka.force_sya_def(context, [
# (cmap["plus"].id, 5, SyaAssociativity.Right),
@@ -716,6 +718,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
["one", "two", "three", "if", SCWC(" function(", ")", "x$!#")]]),
("one prefixed function(two)", [["one", "prefixed", SCWC(" function(", ")", "two")]]),
("suffixed one function(two)", [["one", "suffixed", SCWC(" function(", ")", "two")]]),
(
"func1(suffixed one func2(two))",
[[SCWC("func1(", (")", 1), "one", "suffixed", SCWC(" func2(", ")", "two"))]]),
])
def test_i_can_post_fix_when_parenthesis_and_unknown(self, expression, expected_sequences):
sheerka, context, parser = self.init_parser()
@@ -728,19 +733,19 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res_i.out == expected_array
@pytest.mark.parametrize("expression, expected", [
# ("(", ("(", 0)),
# ("one plus ( 1 + ", ("(", 4)),
# ("one( 1 + ", ("(", 1)),
# ("one ( 1 + ", ("(", 2)),
# ("function( 1 + ", ("(", 1)),
# ("function ( 1 + ", ("(", 2)),
# ("one plus ) 1 + ", (")", 4)),
# ("one ) 1 + ", (")", 2)),
# ("function ) 1 + ", (")", 2)),
# ("one ? ( : two", ("(", 4)),
# ("one ? one plus ( : two", ("(", 8)),
# ("one ? ) : two", (")", 4)),
# ("one ? one plus ) : two", (")", 8)),
("(", ("(", 0)),
("one plus ( 1 + ", ("(", 4)),
("one( 1 + ", ("(", 1)),
("one ( 1 + ", ("(", 2)),
("function( 1 + ", ("(", 1)),
("function ( 1 + ", ("(", 2)),
("one plus ) 1 + ", (")", 4)),
("one ) 1 + ", (")", 2)),
("function ) 1 + ", (")", 2)),
("one ? ( : two", ("(", 4)),
("one ? one plus ( : two", ("(", 8)),
("one ? ) : two", (")", 4)),
("one ? one plus ) : two", (")", 8)),
("(one plus ( 1 + )", ("(", 0)),
])
def test_i_can_detect_parenthesis_mismatch_error_when_post_fixing(self, expression, expected):
@@ -797,6 +802,29 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert len(res) == 1
assert res[0].out == expected_array
def test_i_cannot_post_fix_using_concept_short_name(self):
concepts_map = {
"infixed": self.from_def_concept("infixed", "a infixed b", ["a", "b"]),
"suffixed": self.from_def_concept("suffixed", "suffixed a", ["a"]),
"prefixed": self.from_def_concept("prefixed", "a prefixed", ["a"]),
}
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.infix_to_postfix(context, ParserInput("desc(infixed)"))
assert len(res) == 1
assert isinstance(res[0].out[0], SourceCodeWithConceptNode)
assert res[0].out[0].nodes[0].error == 'Not enough prefix parameters'
res = parser.infix_to_postfix(context, ParserInput("desc(suffixed)"))
assert len(res) == 1
assert isinstance(res[0].out[0], SourceCodeWithConceptNode)
assert res[0].out[0].nodes[0].error == 'Not enough suffix parameters'
res = parser.infix_to_postfix(context, ParserInput("desc(prefixed)"))
assert len(res) == 1
assert isinstance(res[0].out[0], SourceCodeWithConceptNode)
assert res[0].out[0].nodes[0].error == 'Not enough prefix parameters'
@pytest.mark.parametrize("expression", [
"one ? two : three",
"one?two:three",
@@ -1117,6 +1145,26 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
assert lexer_nodes == expected
def test_i_cannot_parse_function_using_short_name(self):
concepts_map = {
"infixed": self.from_def_concept("infixed", "a infixed b", ["a", "b"]),
"suffixed": self.from_def_concept("suffixed", "suffixed a", ["a"]),
"prefixed": self.from_def_concept("prefixed", "a prefixed", ["a"]),
}
sheerka, context, parser = self.init_parser(concepts_map)
res = parser.parse(context, ParserInput("desc(infixed)"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
res = parser.parse(context, ParserInput("desc(suffixed)"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
res = parser.parse(context, ParserInput("desc(prefixed)"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@pytest.mark.parametrize("text", [
"one",
"1 + 1",
@@ -1146,4 +1194,3 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
tests/parsers/test_UnrecognizedNodeParser.py
+46 -2
View File
@@ -2,7 +2,7 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
from core.concept import Concept, CC
from core.tokenizer import Tokenizer, TokenKind
from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, scnode, cnode, \
utnode, SyaAssociativity, CN, CNC, UTN
utnode, SyaAssociativity, CN, CNC, UTN, SourceCodeWithConceptNode, SCWC, SourceCodeNode
from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -24,11 +24,20 @@ def get_input_nodes_from(my_concepts_map, full_expr, *args):
concept = n.concept if hasattr(n, "concept") and n.concept else \
Concept().update_from(my_concepts_map[n.concept_key])
tokens = full_expr_as_tokens[n.start: n.end + 1]
if hasattr(node, "compiled"):
if hasattr(n, "compiled"):
for k, v in n.compiled.items():
concept.compiled[k] = _get_real_node(v)
return ConceptNode(concept, n.start, n.end, tokens)
if isinstance(n, SCWC):
n.first = _get_real_node(n.first)
n.last = _get_real_node(n.first)
n.content = tuple(_get_real_node(nn) for nn in n.content)
return SourceCodeWithConceptNode(n.first, n.last, list(n.content))
if isinstance(n, (UnrecognizedTokensNode, ConceptNode, SourceCodeNode, SourceCodeWithConceptNode)):
return n
raise NotImplementedError()
res = []
@@ -307,6 +316,41 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
exclude_body=True)
assert actual_nodes == expected_array
def test_i_can_parse_unrecognized_source_code_with_concept_node(self):
sheerka, context, parser = self.init_parser()
expression = "desc(a plus b)"
source_code_concepts = SCWC("desc(", ")", CNC("plus", a=UTN("a"), b=UTN("b")))
nodes = get_input_nodes_from(concepts_map, expression, source_code_concepts)
parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
res = parser.parse(context, parser_input)
parser_result = res.body
actual_nodes = res.body.body
assert not res.status # status is False to let PythonWithConceptParser validate the code
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert len(actual_nodes) == 1
assert actual_nodes[0].nodes[0].concept.metadata.is_evaluated # 'a plus b' is recognized as concept definition
def test_i_can_parse_unrecognized_source_code_with_concept_node_when_var_in_short_term_memory(self):
sheerka, context, parser = self.init_parser()
expression = "desc(a plus b)"
source_code_concepts = SCWC("desc(", ")", CNC("plus", a=UTN("a"), b=UTN("b")))
nodes = get_input_nodes_from(concepts_map, expression, source_code_concepts)
parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes)
context.add_to_short_term_memory("a", 1)
res = parser.parse(context, parser_input)
parser_result = res.body
actual_nodes = res.body.body
assert not res.status # status is False to let PythonWithConceptParser validate the code
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert len(actual_nodes) == 1
assert not actual_nodes[0].nodes[0].concept.metadata.is_evaluated # 'a plus b' need to be evaluated
def test_i_can_parse_sequences(self):
sheerka, context, parser = self.init_parser()