Fixed #127 : SyaNodeParser : Allow tokens parsing

Fixed #128 : parser_utils.get_node() : Refactor
2021-09-12 11:26:14 +02:00
parent 945807b375
commit a61a1c0d2b
12 changed files with 327 additions and 290 deletions
@@ -7,17 +7,16 @@ from core.concept import Concept
 from core.global_symbols import CONCEPT_COMPARISON_CONTEXT
 from core.sheerka.Sheerka import RECOGNIZED_BY_KEY
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Tokenizer, comparable_tokens
+from core.tokenizer import Token, TokenKind, Tokenizer, comparable_tokens
 from core.utils import get_text_from_tokens
+from parsers.BaseExpressionParser import FunctionNode, FunctionParameter, NameExprNode
 from parsers.BaseNodeParser import ConceptNode, SourceCodeNode, UnrecognizedTokensNode
 from parsers.PythonParser import PythonNode
 from parsers.SyaNodeParser import FunctionDetected, NoSyaConceptFound, NotEnoughParameters, SyaConceptParser, \
-    SyaNodeParser, \
-    SyaTokensParser, \
-    TokensNotFound, TooManyParameters
+    SyaNodeParser, SyaTokensParser, TokensNotFound, TooManyParameters
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import CC, CN, CNC, SCN, UTN, compute_expected_array, get_test_obj, \
+from tests.parsers.parsers_utils import CC, CN, CNC, RETVAL, SCN, SCWC, UTN, compute_expected_array, get_test_obj, \
    prepare_nodes_comparison

 cmap = {
@@ -34,7 +33,7 @@ cmap = {
 }


-class TestSyaNodeParser2(TestUsingMemoryBasedSheerka):
+class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
    shared_ontology = None

    @classmethod
@@ -46,11 +45,11 @@ class TestSyaNodeParser2(TestUsingMemoryBasedSheerka):

        cmap["plus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
        cmap["mult"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
-        TestSyaNodeParser2.sheerka.set_is_greater_than(context,
-                                                       BuiltinConcepts.PRECEDENCE,
-                                                       cmap["mult"],
-                                                       cmap["plus"],
-                                                       CONCEPT_COMPARISON_CONTEXT)
+        TestSyaNodeParser.sheerka.set_is_greater_than(context,
+                                                      BuiltinConcepts.PRECEDENCE,
+                                                      cmap["mult"],
+                                                      cmap["plus"],
+                                                      CONCEPT_COMPARISON_CONTEXT)

        cls.shared_ontology = sheerka.get_ontology(context)
        sheerka.pop_ontology(context)
@@ -386,6 +385,30 @@ class TestSyaNodeParser2(TestUsingMemoryBasedSheerka):
        assert concept_node_as_test_obj == resolved_expected
        assert concept_node.concept.get_metadata().variables == [("a", "1 + 1 "), ("b", "2 + 2")]

+    def test_i_can_concept_parse_function(self):
+        sheerka, context = self.initialize_test()
+
+        expression = "one plus func(twenty two)"
+        param1 = self.get_real_node(cmap, expression, "one")
+        parser_input = ParserInput(expression).reset()
+        parser_input.seek(2)
+        sya_node_parser = SyaNodeParser()
+        tokens_parser = SyaTokensParser(context, sya_node_parser, parser_input)
+        tokens_parser.stack = [param1]
+
+        concept_parser = SyaConceptParser(tokens_parser, cmap["plus"], tokens_parser.stack)
+        concept_parser.parse()
+        concept_node = concept_parser.concept_node
+
+        assert not concept_parser.has_error()
+        assert len(concept_parser.expected) == 0
+
+        expected = CNC("plus", a=CNC("one"), b=SCWC("func(", ")", CN("twenties", source="twenty two")))
+        resolved_expected = compute_expected_array(cmap, expression, [expected])[0]
+        concept_node_as_test_obj = get_test_obj(concept_node, expected)
+        assert concept_node_as_test_obj == resolved_expected
+        assert concept_node.concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
+
    def test_i_can_concept_parse_concepts_composition(self):
        sheerka, context = self.initialize_test()

@@ -1385,6 +1408,25 @@ class TestSyaNodeParser2(TestUsingMemoryBasedSheerka):
        # check metadata
        assert expected_concept.get_metadata().variables == [("a", "twenty one")]

+    def test_i_can_parse_when_function(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "one plus func(twenty two)"
+        res = parser.parse(context, ParserInput(text))
+        wrapper = res.body
+        lexer_nodes = res.body.body
+
+        assert res.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+
+        expected = [CNC("plus", a=CC("one"), b=[RETVAL("func(twenty two)")], source=text)]
+        _stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
+        assert _stack == _expected
+
+        # check the metadata
+        expected_concept = lexer_nodes[0].concept
+        assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
+
    def test_i_can_parse_sequences(self):
        sheerka, context, parser = self.init_parser()

@@ -1501,6 +1543,32 @@ class TestSyaNodeParser2(TestUsingMemoryBasedSheerka):
        _stack, _expected = prepare_nodes_comparison(concepts_map, text, lexer_nodes, expected)
        assert _stack == _expected

+    def test_i_can_parse_when_expr_tokens(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "one plus func(twenty two)"
+        tokens = list(Tokenizer(text, yield_eof=False))
+        fun_token = tokens[4]
+        expr = FunctionNode(4, 9, tokens[4:10],
+                            NameExprNode(4, 4, tokens[4:5]),
+                            NameExprNode(9, 9, tokens[9:10]),
+                            [FunctionParameter(NameExprNode(6, 8, tokens[6:9]), None)])
+        tokens[4:] = [Token(TokenKind.EXPR, expr, fun_token.index, fun_token.line, fun_token.column)]
+        res = parser.parse(context, ParserInput(None, tokens=tokens))
+        wrapper = res.body
+        lexer_nodes = res.body.body
+
+        assert res.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+
+        expected = [CNC("plus", a=CC("one"), b=[RETVAL("func(twenty two)")], source=text)]
+        _stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
+        assert _stack == _expected
+
+        # check the metadata
+        expected_concept = lexer_nodes[0].concept
+        assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
+
    @pytest.mark.parametrize("text, expected_result", [
        ("one plus two foo bar baz", [CNC("plus", a="one", b="two"), UTN(" foo bar baz")]),
        ("one plus two foo bar", [CNC("plus", a="one", b="two"), UTN(" foo bar")]),