Working on #48 : Refactored FunctionParser, introducting ExpressionParser

tests/parsers/parsers_utils.py

@@ -14,7 +14,7 @@ from parsers.FunctionParser import FunctionNode
 from parsers.PythonParser import PythonNode
 from parsers.SyaNodeParser import SyaConceptParserHelper
 from parsers.expressions import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, ComparisonType, \
-    ParenthesisNode
+    FunctionParameter
 from sheerkarete.common import V
 from sheerkarete.conditions import Condition, AndConditions
 
@@ -972,8 +972,13 @@ def get_expr_node_from_test_node(full_text, test_node):
         return start, end
 
     def get_pos_from_source(source):
+        if isinstance(source, tuple):
+            source, to_skip = source[0], source[1]
+        else:
+            to_skip = 0
+
         source_as_node = list(Tokenizer(source, yield_eof=False))
-        start = tokens_index(full_text_as_tokens, source_as_node)
+        start = tokens_index(full_text_as_tokens, source_as_node, skip=to_skip)
         end = start + len(source_as_node) - 1
         return start, end
 
@@ -1017,11 +1022,31 @@ def get_expr_node_from_test_node(full_text, test_node):
             return ComparisonNode(start, end, full_text_as_tokens[start: end + 1],
                                   node_type, left_node, right_node)
 
-        if isinstance(node, PAREN):
-            value_as_tokens = list(Tokenizer(node.source, yield_eof=False))
-            start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
-            end = start + len(value_as_tokens) - 1
-            return ParenthesisNode(start, end, value_as_tokens, get_expr_node(node.node))
+        if isinstance(node, FN):
+            start, end = get_pos_from_source(node.first)
+            first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+            start, end = get_pos_from_source(node.last)
+            last = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+            parameters = []
+            for param_value, sep in node.parameters:
+                if isinstance(param_value, str):
+                    start, end = get_pos_from_source(param_value)
+                    param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+                else:
+                    param_as_expr_node = get_expr_node(param_value)
+
+                if sep:
+                    sep_tokens = Tokenizer(sep, yield_eof=False)
+                    start = param_as_expr_node.end + 1
+                    end = start + len(list(sep_tokens)) - 1
+                    sep_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+                else:
+                    sep_as_expr_node = None
+
+                parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
+
+            start, end = first.start, last.end
+            return FunctionNode(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
 
     return get_expr_node(test_node)
 

tests/parsers/test_ExpressionParser.py

@@ -0,0 +1,51 @@
+import pytest
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import Tokenizer
+from parsers.ExpressionParser import ExpressionParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR
+
+
+class TestExpressionParser(TestUsingMemoryBasedSheerka):
+    def init_parser(self):
+        sheerka, context = self.init_concepts()
+        parser = ExpressionParser()
+        return sheerka, context, parser
+
+    def input_parser_with_source(self, source):
+        sheerka, context, parser = self.init_parser()
+        parser.reset_parser(context, ParserInput(source))
+        return sheerka, context, parser
+
+    def test_i_can_detect_empty_expression(self):
+        sheerka, context, parser = self.init_parser()
+        res = parser.parse(context, ParserInput(""))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
+
+    @pytest.mark.parametrize("expression, expected", [
+        ("something that i do not recognize", EXPR("something that i do not recognize")),
+    ])
+    def test_i_can_parse_input(self, expression, expected):
+        sheerka, context, parser = self.input_parser_with_source(expression)
+        expected = get_expr_node_from_test_node(expression, expected)
+
+        parsed = parser.parse_input()
+
+        assert not parser.has_error
+        assert parsed == expected
+
+    def test_i_can_parse_sub_tokens(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "do not care var.attr do not care either"
+        parser_input = ParserInput("text", Tokenizer(expression, yield_eof=False), start=6, end=8)
+
+        parser.reset_parser(context, parser_input)
+        parsed = parser.parse_input()
+
+        assert not parser.has_error
+        assert parsed == get_expr_node_from_test_node(expression, [VAR("var.attr")])

tests/parsers/test_FunctionParser.py

@@ -6,7 +6,8 @@ from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.FunctionParser import FunctionParser
 from parsers.PythonParser import PythonErrorNode
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import compute_expected_array, SCN, SCWC, CN, UTN, CNC, RN, FN, get_test_obj
+from tests.parsers.parsers_utils import compute_expected_array, SCN, SCWC, CN, UTN, CNC, RN, FN, get_test_obj, \
+    get_expr_node_from_test_node
 
 cmap = {
     "one": Concept("one"),
@@ -64,12 +65,12 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
         ("concept(one)", FN("concept(", ")", ["one"])),
         ("func(one)", FN("func(", ")", ["one"])),
         ("func(a long two, 'three', ;:$*)", FN("func(", ")", ["a long two, ", "'three', ", ";:$*"])),
-        ("func(func1(one), two, func2(func3(), func4(three)))", FN("func(", ")", [
+        ("func(func1(one), two, func2(func3(), func4(three)))", FN("func(", (")", 4), [
             (FN("func1(", ")", ["one"]), ", "),
             "two, ",
-            (FN("func2(", ")", [
-                (FN("func3(", ")", []), ", "),
-                (FN("func4(", ")", ["three"]), None),
+            (FN("func2(", (")", 3), [
+                (FN("func3(", (")", 1), []), ", "),
+                (FN("func4(", (")", 2), ["three"]), None),
             ]), None)
         ])),
     ])
@@ -80,8 +81,8 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
         parser.parser_input.next_token()
 
         res = parser.parse_function()
-        transformed_res = get_test_obj(res, expected)
-        assert transformed_res == expected
+        expected = get_expr_node_from_test_node(expression, expected)
+        assert res == expected
 
     def test_i_can_parse_function_when_rule(self):
         sheerka, context, parser = self.init_parser()

tests/parsers/test_RelationalOperatorParser.py

@@ -3,6 +3,7 @@ import pytest
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import TokenKind, Tokenizer
+from parsers.BaseParser import UnexpectedTokenParsingError
 from parsers.RelationalOperatorParser import RelationalOperatorParser
 from parsers.expressions import ParenthesisMismatchError
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -94,9 +95,7 @@ class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
-        assert isinstance(res.body.body[0], UnexpectedTokenError)
-
-
+        assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
 
     def test_i_can_parse_tokens_rather_than_parser_input(self):
         sheerka, context, parser = self.init_parser()