kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #131 : Implement ExprToConditions

Browse Source 
Fixed #130 : ArithmeticOperatorParser
Fixed #129 : python_wrapper : create_namespace
Fixed #128 : ExpressionParser: Cannot parse func(x) infixed concept 'xxx'
This commit is contained in:
Kodjo Sossouvi
2021-10-13 16:06:57 +02:00
parent a61a1c0d2b
commit 89e1f20975
76 changed files with 5867 additions and 3206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parsers/test_SyaNodeParser.py
+106 -26
View File
@@ -2,21 +2,22 @@ import pytest
from core.builtin_concepts import ReturnValueConcept
from core.builtin_concepts_ids import BuiltinConcepts
from core.builtin_helpers import get_new_variables_definitions
from core.builtin_helpers import get_new_variables_definitions, longest_only
from core.concept import Concept
from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
from core.sheerka.Sheerka import RECOGNIZED_BY_KEY
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Token, TokenKind, Tokenizer, comparable_tokens
from core.utils import get_text_from_tokens
from parsers.BaseExpressionParser import FunctionNode, FunctionParameter, NameExprNode
from parsers.BaseExpressionParser import BinaryNode, FunctionNode, FunctionNodeOld, FunctionParameter, ListNode, \
NameExprNode, VariableNode
from parsers.BaseNodeParser import ConceptNode, SourceCodeNode, UnrecognizedTokensNode
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import FunctionDetected, NoSyaConceptFound, NotEnoughParameters, SyaConceptParser, \
SyaNodeParser, SyaTokensParser, TokensNotFound, TooManyParameters
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import CC, CN, CNC, RETVAL, SCN, SCWC, UTN, compute_expected_array, get_test_obj, \
from tests.parsers.parsers_utils import CC, CIO, CN, CNC, RETVAL, SCN, UTN, compute_expected_array, get_test_obj, \
prepare_nodes_comparison
cmap = {
@@ -403,10 +404,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert not concept_parser.has_error()
assert len(concept_parser.expected) == 0
expected = CNC("plus", a=CNC("one"), b=SCWC("func(", ")", CN("twenties", source="twenty two")))
resolved_expected = compute_expected_array(cmap, expression, [expected])[0]
concept_node_as_test_obj = get_test_obj(concept_node, expected)
assert concept_node_as_test_obj == resolved_expected
expected = CNC("plus", a=CNC("one"), b=SCN("func(twenty two)", [CIO("twenties", source="twenty two")]))
actual_as_test, expected_resolved = prepare_nodes_comparison(cmap, expression, concept_node, expected)
assert actual_as_test == expected_resolved
assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
def test_i_can_concept_parse_concepts_composition(self):
@@ -1366,22 +1366,13 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected = [CN(cmap["suffixed"], text, 0, 6)]
expected_return_value = RETVAL("1 + 1")
expected = [CNC(cmap["suffixed"], text, 0, 6, a=[expected_return_value])]
concept_node_as_test_obj = get_test_obj(lexer_nodes, expected)
assert concept_node_as_test_obj == expected
# check the compiled
expected_concept = lexer_nodes[0].concept
assert len(expected_concept.get_compiled()["a"]) == 1
return_value_a = expected_concept.get_compiled()["a"][0]
assert sheerka.isinstance(return_value_a, BuiltinConcepts.RETURN_VALUE)
assert return_value_a.status
assert sheerka.isinstance(return_value_a.body, BuiltinConcepts.PARSER_RESULT)
assert return_value_a.body.source == "1 + 1"
assert isinstance(return_value_a.body.body, PythonNode)
# check metadata
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "1 + 1")]
def test_i_can_parse_when_bnf_concept(self):
@@ -1408,6 +1399,33 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
# check metadata
assert expected_concept.get_metadata().variables == [("a", "twenty one")]
@pytest.mark.skip("Not quite sure that this test is relevant")
def test_i_can_parse_when_bnf_and_python(self):
# twenty one + 1 is not correctly parsed by BNFNode or SequenceNode
# Maybe I should just leave it to ExpressionParser instead
sheerka, context, parser = self.init_parser()
text = "suffixed twenty one + 1"
res = parser.parse(context, ParserInput(text))
# only care about the result that recognizes 'twenty one'
longest = longest_only(context, res)
res = longest.body.body[0]
wrapper = res.body
lexer_nodes = res.body.body
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected_return_value = RETVAL("twenty one + 1", objects={"__o_00__": CIO("twenties", source="twenty one")})
expected = [CNC(cmap["suffixed"], text, 0, 8, a=[expected_return_value])]
concept_node_as_test_obj = get_test_obj(lexer_nodes, expected)
assert concept_node_as_test_obj == expected
# check metadata
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "twenty one + 1")]
def test_i_can_parse_when_function(self):
sheerka, context, parser = self.init_parser()
@@ -1419,7 +1437,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected = [CNC("plus", a=CC("one"), b=[RETVAL("func(twenty two)")], source=text)]
expected_ret_val = RETVAL("func(twenty two)",
"func(__o_00__)",
objects={"__o_00__": CIO("twenties", source="twenty two")})
expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)]
_stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
assert _stack == _expected
@@ -1543,16 +1564,16 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
_stack, _expected = prepare_nodes_comparison(concepts_map, text, lexer_nodes, expected)
assert _stack == _expected
def test_i_can_parse_when_expr_tokens(self):
def test_i_can_parse_when_function_old_style_expr_tokens(self):
sheerka, context, parser = self.init_parser()
text = "one plus func(twenty two)"
tokens = list(Tokenizer(text, yield_eof=False))
fun_token = tokens[4]
expr = FunctionNode(4, 9, tokens[4:10],
NameExprNode(4, 4, tokens[4:5]),
NameExprNode(9, 9, tokens[9:10]),
[FunctionParameter(NameExprNode(6, 8, tokens[6:9]), None)])
expr = FunctionNodeOld(4, 9, tokens[4:10],
NameExprNode(4, 5, tokens[4:6]),
NameExprNode(9, 9, tokens[9:10]),
[FunctionParameter(NameExprNode(6, 8, tokens[6:9]), None)])
tokens[4:] = [Token(TokenKind.EXPR, expr, fun_token.index, fun_token.line, fun_token.column)]
res = parser.parse(context, ParserInput(None, tokens=tokens))
wrapper = res.body
@@ -1561,7 +1582,8 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected = [CNC("plus", a=CC("one"), b=[RETVAL("func(twenty two)")], source=text)]
expected_ret_val = RETVAL("func(twenty two)", objects={"__o_00__": CIO("twenties", source="twenty two")})
expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)]
_stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
assert _stack == _expected
@@ -1569,6 +1591,64 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
def test_i_can_parse_when_function_style_expr_tokens(self):
sheerka, context, parser = self.init_parser()
text = "one plus func(twenty two)"
tokens = list(Tokenizer(text, yield_eof=False))
fun_token = tokens[4]
expr = FunctionNode(4, 9, tokens[4:10],
NameExprNode(4, 4, tokens[4:5]),
ListNode(5, 9, tokens[5:10],
NameExprNode(5, 5, tokens[5:6]),
NameExprNode(9, 9, tokens[9:10]),
[NameExprNode(6, 8, tokens[6:9])]))
tokens[4:] = [Token(TokenKind.EXPR, expr, fun_token.index, fun_token.line, fun_token.column)]
res = parser.parse(context, ParserInput(None, tokens=tokens))
wrapper = res.body
lexer_nodes = res.body.body
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected_ret_val = RETVAL("func(twenty two)", objects={"__o_00__": CIO("twenties", source="twenty two")})
expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)]
_stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
assert _stack == _expected
# check the metadata
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
def test_i_can_parse_when_binary_expr_tokens(self):
sheerka, context, parser = self.init_parser()
text = "suffixed twenty one + self"
tokens = list(Tokenizer(text, yield_eof=False))
expr = BinaryNode(2, 8, tokens[2:9], tokens[6],
VariableNode(2, 4, tokens[2:5], "twenty one"),
VariableNode(8, 8, tokens[8:9], "self"))
token_twenty = tokens[2]
tokens[2:] = [Token(TokenKind.EXPR, expr, token_twenty.index, token_twenty.line, token_twenty.column)]
res = parser.parse(context, ParserInput(None, tokens=tokens))
wrapper = res.body
lexer_nodes = res.body.body
assert res.status
assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
expected_ret_val = RETVAL("twenty one + self",
source="__o_00__ + self",
objects={"__o_00__": CIO("twenties", source="twenty one")})
expected = [CNC("suffixed", a=[expected_ret_val], source=text)]
_stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
assert _stack == _expected
# check the metadata
expected_concept = lexer_nodes[0].concept
assert expected_concept.get_metadata().variables == [("a", "twenty one + self")]
@pytest.mark.parametrize("text, expected_result", [
("one plus two foo bar baz", [CNC("plus", a="one", b="two"), UTN(" foo bar baz")]),
("one plus two foo bar", [CNC("plus", a="one", b="two"), UTN(" foo bar")]),