Working on #48 : Working

2021-03-11 16:13:55 +01:00
parent 30c99b2d67
commit e303b32eb9
8 changed files with 186 additions and 345 deletions
@@ -1,11 +1,12 @@
 from dataclasses import dataclass
 from typing import List, Tuple, Union

+from core.builtin_concepts_ids import BuiltinConcepts
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Token, TokenKind, Tokenizer, LexerError
 from core.utils import tokens_are_matching
 from parsers.BaseNodeParser import UnrecognizedTokensNode
-from parsers.BaseParser import Node, ParsingError, BaseParser
+from parsers.BaseParser import Node, ParsingError, BaseParser, ErrorSink, UnexpectedTokenParsingError


 class ComparisonType:
@@ -331,9 +332,6 @@ class FunctionNode(ExprNode):

 class BaseExpressionParser(BaseParser):

-    def parse_input(self, context, parser_input, error_sink):
-        raise NotImplementedError
-
    def reset_parser_input(self, parser_input: ParserInput, error_sink):
        try:
            error_sink.clear()
@@ -345,6 +343,129 @@ class BaseExpressionParser(BaseParser):
        parser_input.next_token()
        return True

+    def parse(self, context, parser_input: ParserInput):
+        """
+        :param context:
+        :param parser_input:
+        :return:
+        """
+
+        if not isinstance(parser_input, ParserInput):
+            return None
+
+        context.log(f"Parsing '{parser_input}' with {self.NAME}Parser", self.name)
+        sheerka = context.sheerka
+
+        if parser_input.is_empty():
+            return context.sheerka.ret(self.name,
+                                       False,
+                                       sheerka.new(BuiltinConcepts.IS_EMPTY))
+
+        error_sink = ErrorSink()
+        if not self.reset_parser_input(parser_input, error_sink):
+            return context.sheerka.ret(
+                self.name,
+                False,
+                context.sheerka.new(BuiltinConcepts.ERROR, body=error_sink.sink))
+
+        node = self.parse_input(context, parser_input, error_sink)
+
+        token = parser_input.token
+        if token and token.type != TokenKind.EOF:
+            if token.type == TokenKind.RPAR:
+                error_sink.add_error(ParenthesisMismatchError(token))
+            else:
+                error_sink.add_error(UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.EOF]))
+
+        if isinstance(node, ParenthesisNode):
+            node = node.node
+
+        value = self.get_return_value_body(context.sheerka,
+                                           parser_input.as_text(),
+                                           node,
+                                           node,
+                                           error_sink.sink)
+
+        ret = context.sheerka.ret(self.name,
+                                  not error_sink.has_error,
+                                  value)
+
+        return ret
+
+    def parse_input(self, context, parser_input, error_sink):
+        raise NotImplementedError
+
+    def inner_parse_names(self, context, parser_input, error_sink, stop_condition):
+
+        # def stop():
+        #     return token.type == TokenKind.EOF or \
+        #            paren_count == 0 and token.type == TokenKind.RPAR or \
+        #            token.type == TokenKind.IDENTIFIER and token.value in ("and", "or") or \
+        #            token.value == "not" and parser_input.the_token_after(True).value != "in"
+
+        def stop():
+            return token.type == TokenKind.EOF or \
+                   paren_count == 0 and token.type == TokenKind.RPAR or \
+                   stop_condition(token, parser_input)
+
+        token = parser_input.token
+        if token.type == TokenKind.EOF:
+            return None
+
+        if token.type == TokenKind.LPAR:
+            last_paren = token
+            start = parser_input.pos
+            parser_input.next_token()
+            expr = self.parse_input(context, parser_input, error_sink)
+            token = parser_input.token
+            if token.type != TokenKind.RPAR:
+                error_sink.add_error(ParenthesisMismatchError(last_paren))
+                return expr
+            end = parser_input.pos
+            parser_input.next_token()
+            return ParenthesisNode(start, end, None, expr)
+
+        paren_count = 0
+        last_paren = None
+        start = parser_input.pos
+        end = parser_input.pos
+        last_is_whitespace = False
+        while not stop():
+            last_is_whitespace = token.type == TokenKind.WHITESPACE
+            end += 1
+            if token.type == TokenKind.LPAR:
+                last_paren = token
+                paren_count += 1
+            if token.type == TokenKind.RPAR:
+                paren_count -= 1
+            parser_input.next_token(False)
+            token = parser_input.token
+
+        if last_is_whitespace:
+            end -= 1
+
+        if start == end:
+            if token.type != TokenKind.RPAR:
+                error_sink.add_error(LeftPartNotFoundError())
+            return None
+
+        if paren_count != 0:
+            error_sink.add_error(ParenthesisMismatchError(last_paren))
+            return None
+
+        if self.expr_parser:
+            new_parsing_input = ParserInput(
+                None,
+                tokens=parser_input.tokens,
+                length=parser_input.length,
+                start=start,
+                end=end - 1,
+                yield_oef=False).reset()
+            new_parsing_input.next_token()
+            return self.expr_parser.parse_input(context, new_parsing_input, error_sink)
+        else:
+            return NameExprNode(start, end - 1, parser_input.tokens[start:end])
+

 class ExpressionVisitor:
    """
@@ -1,16 +1,15 @@
 from itertools import product

-from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import only_successful, get_inner_body, get_lexer_nodes_using_positions
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.sheerka_service import FailedToCompileError
 from core.tokenizer import TokenKind, Tokenizer, Keywords
 from core.utils import get_text_from_tokens
+from parsers.BaseExpressionParser import ParenthesisNode, OrNode, AndNode, NotNode, ExprNode, VariableNode, \
+    ComparisonNode, BaseExpressionParser
 from parsers.BaseNodeParser import UnrecognizedTokensNode
-from parsers.BaseParser import UnexpectedTokenParsingError, UnexpectedEofParsingError, ErrorSink
+from parsers.BaseParser import UnexpectedEofParsingError, ErrorSink
 from parsers.PythonWithConceptsParser import PythonWithConceptsParser
-from parsers.BaseExpressionParser import ParenthesisNode, OrNode, AndNode, NotNode, LeftPartNotFoundError, \
-    ParenthesisMismatchError, NameExprNode, ExprNode, VariableNode, ComparisonNode, BaseExpressionParser
 from sheerkarete.common import V
 from sheerkarete.conditions import Condition, AndConditions

@@ -87,52 +86,6 @@ class LogicalOperatorParser(BaseExpressionParser):
            if isinstance(node, ParenthesisNode):
                nodes[i] = node.node

-    def parse(self, context, parser_input: ParserInput):
-        """
-        :param context:
-        :param parser_input:
-        :return:
-        """
-
-        if not isinstance(parser_input, ParserInput):
-            return None
-
-        context.log(f"Parsing '{parser_input}' with {self.NAME}Parser", self.name)
-        sheerka = context.sheerka
-
-        if parser_input.is_empty():
-            return context.sheerka.ret(self.name,
-                                       False,
-                                       sheerka.new(BuiltinConcepts.IS_EMPTY))
-
-        error_sink = ErrorSink()
-        if not self.reset_parser_input(parser_input, error_sink):
-            return context.sheerka.ret(
-                self.name,
-                False,
-                context.sheerka.new(BuiltinConcepts.ERROR, body=error_sink.sink))
-
-        tree = self.parse_input(context, parser_input, error_sink)
-
-        token = parser_input.token
-        if token and token.type != TokenKind.EOF:
-            error_sink.add_error(UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, []))
-
-        if isinstance(tree, ParenthesisNode):
-            tree = tree.node
-
-        value = self.get_return_value_body(context.sheerka,
-                                           parser_input.as_text(),
-                                           tree,
-                                           tree,
-                                           error_sink.sink)
-
-        ret = context.sheerka.ret(self.name,
-                                  not error_sink.has_error,
-                                  value)
-
-        return ret
-
    def parse_input(self, context, parser_input, error_sink):
        return self.parse_or(context, parser_input, error_sink)

@@ -197,71 +150,13 @@ class LogicalOperatorParser(BaseExpressionParser):
        else:
            return self.parse_names(context, parser_input, error_sink)

+    @staticmethod
+    def stop_condition(token, parser_input):
+        return token.type == TokenKind.IDENTIFIER and token.value in ("and", "or") or \
+               token.value == "not" and parser_input.the_token_after(True).value != "in"
+
    def parse_names(self, context, parser_input, error_sink):
-
-        def stop():
-            return token.type == TokenKind.EOF or \
-                   paren_count == 0 and token.type == TokenKind.RPAR or \
-                   token.type == TokenKind.IDENTIFIER and token.value in ("and", "or") or \
-                   token.value == "not" and parser_input.the_token_after(True).value != "in"
-
-        token = parser_input.token
-        if token.type == TokenKind.EOF:
-            return None
-
-        if token.type == TokenKind.LPAR:
-            start = parser_input.pos
-            parser_input.next_token()
-            expr = self.parse_or(context, parser_input, error_sink)
-            token = parser_input.token
-            if token.type != TokenKind.RPAR:
-                error_sink.add_error(
-                    UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.RPAR]))
-                return expr
-            end = parser_input.pos
-            parser_input.next_token()
-            return ParenthesisNode(start, end, None, expr)
-
-        paren_count = 0
-        last_paren = None
-        start = parser_input.pos
-        end = parser_input.pos
-        last_is_whitespace = False
-        while not stop():
-            last_is_whitespace = token.type == TokenKind.WHITESPACE
-            end += 1
-            if token.type == TokenKind.LPAR:
-                last_paren = token
-                paren_count += 1
-            if token.type == TokenKind.RPAR:
-                paren_count -= 1
-            parser_input.next_token(False)
-            token = parser_input.token
-
-        if last_is_whitespace:
-            end -= 1
-
-        if start == end:
-            if token.type != TokenKind.RPAR:
-                error_sink.add_error(LeftPartNotFoundError())
-            return None
-
-        if paren_count != 0:
-            error_sink.add_error(ParenthesisMismatchError(last_paren))
-            return None
-
-        if self.expr_parser:
-            new_parsing_input = ParserInput(
-                None,
-                tokens=parser_input.tokens,
-                length=parser_input.length,
-                start=start,
-                end=end - 1,
-                yield_oef=False).reset()
-            new_parsing_input.next_token()
-            return self.expr_parser.parse_input(context, new_parsing_input, error_sink)
-        else:
-            return NameExprNode(start, end - 1, parser_input.tokens[start:end])
+        return self.inner_parse_names(context, parser_input, error_sink, self.stop_condition)

    def compile_conjunctions(self, context, conjunctions, who):
        """
@@ -1,11 +1,7 @@
-from core.builtin_concepts_ids import BuiltinConcepts
-from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import TokenKind
-from core.utils import get_text_from_tokens
-from parsers.BaseParser import UnexpectedTokenParsingError, ErrorSink
-from parsers.BaseExpressionParser import ComparisonNode, ParenthesisMismatchError, NameExprNode, ComparisonType, \
-    VariableNode, \
-    ParenthesisNode, LeftPartNotFoundError, BaseExpressionParser
+from parsers.BaseExpressionParser import ComparisonNode, ComparisonType, \
+    ParenthesisNode, BaseExpressionParser
+from parsers.BaseParser import UnexpectedTokenParsingError


 class RelationalOperatorParser(BaseExpressionParser):
@@ -20,55 +16,6 @@ class RelationalOperatorParser(BaseExpressionParser):
        super().__init__(self.NAME, 60, False, yield_eof=True)
        self.expr_parser = kwargs.get("expr_parser", None)

-    def parse(self, context, parser_input: ParserInput):
-        """
-        :param context:
-        :param parser_input:
-        :return:
-        """
-
-        if not isinstance(parser_input, ParserInput):
-            return None
-
-        context.log(f"Parsing '{parser_input}' with {self.NAME}Parser", self.name)
-        sheerka = context.sheerka
-
-        if parser_input.is_empty():
-            return context.sheerka.ret(self.name,
-                                       False,
-                                       sheerka.new(BuiltinConcepts.IS_EMPTY))
-
-        error_sink = ErrorSink()
-        if not self.reset_parser_input(parser_input, error_sink):
-            return context.sheerka.ret(
-                self.name,
-                False,
-                context.sheerka.new(BuiltinConcepts.ERROR, body=error_sink.sink))
-
-        node = self.parse_input(context, parser_input, error_sink)
-
-        token = parser_input.token
-        if token and token.type != TokenKind.EOF:
-            if token.type == TokenKind.RPAR:
-                error_sink.add_error(ParenthesisMismatchError(token))
-            else:
-                error_sink.add_error(UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.EOF]))
-
-        if isinstance(node, ParenthesisNode):
-            node = node.node
-
-        value = self.get_return_value_body(context.sheerka,
-                                           parser_input.as_text(),
-                                           node,
-                                           node,
-                                           error_sink.sink)
-
-        ret = context.sheerka.ret(self.name,
-                                  not error_sink.has_error,
-                                  value)
-
-        return ret
-
    def parse_input(self, context, parser_input, error_sink):
        return self.parse_compare(context, parser_input, error_sink)

@@ -93,77 +40,11 @@ class RelationalOperatorParser(BaseExpressionParser):
        end = right.end if right else parser_input.pos
        return ComparisonNode(start, end, parser_input.tokens[start: end + 1], comp, left, right)

+    def stop_condition(self, token, parser_input):
+        return self.eat_comparison(parser_input, False)
+
    def parse_names(self, context, parser_input, error_sink):
-
-        def stop():
-            return token.type == TokenKind.EOF or \
-                   paren_count == 0 and token.type == TokenKind.RPAR or \
-                   self.eat_comparison(parser_input, False)
-
-        token = parser_input.token
-        if token.type == TokenKind.EOF:
-            return None
-
-        if token.type == TokenKind.LPAR:
-            last_paren = token
-            start = parser_input.pos
-            parser_input.next_token()
-            expr = self.parse_compare(context, parser_input, error_sink)
-            token = parser_input.token
-            if token.type != TokenKind.RPAR:
-                error_sink.add_error(ParenthesisMismatchError(last_paren))
-                return expr
-            end = parser_input.pos
-            parser_input.next_token()
-            return ParenthesisNode(start, end, None, expr)
-
-        paren_count = 0
-        last_left_paren = None
-        last_right_paren = None
-        start = parser_input.pos
-        end = parser_input.pos
-        last_is_whitespace = False
-        while not stop():
-            last_is_whitespace = token.type == TokenKind.WHITESPACE
-
-            end += 1
-            if token.type == TokenKind.LPAR:
-                last_left_paren = token
-                paren_count += 1
-            if token.type == TokenKind.RPAR:
-                last_right_paren = token
-                paren_count -= 1
-            parser_input.next_token(False)
-            token = parser_input.token
-
-        if last_is_whitespace:
-            end -= 1
-
-        if start == end:
-            if token.type != TokenKind.RPAR:
-                error_sink.add_error(LeftPartNotFoundError())
-            return None
-
-        if paren_count > 0:
-            error_sink.add_error(ParenthesisMismatchError(last_left_paren))
-            return None
-
-        if paren_count < 0:
-            error_sink.add_error(ParenthesisMismatchError(last_right_paren))
-            return None
-
-        if self.expr_parser:
-            new_parsing_input = ParserInput(
-                None,
-                tokens=parser_input.tokens,
-                length=parser_input.length,
-                start=start,
-                end=end - 1,
-                yield_oef=False).reset()
-            new_parsing_input.next_token()
-            return self.expr_parser.parse_input(context, new_parsing_input, error_sink)
-        else:
-            return self.try_to_recognize(NameExprNode(start, end - 1, parser_input.tokens[start:end]))
+        return self.inner_parse_names(context, parser_input, error_sink, self.stop_condition)

    @staticmethod
    def eat_comparison(parser_input, eat=True):
@@ -214,29 +95,3 @@ class RelationalOperatorParser(BaseExpressionParser):
            return ComparisonType.NOT_EQUAlS

        return None
-
-    @staticmethod
-    def try_to_recognize(expr: NameExprNode):
-        not_a_variable = False
-        expect_dot = False
-        for t in expr.tokens:
-            if expect_dot and t.type != TokenKind.DOT:
-                not_a_variable = True
-            if t.type == TokenKind.DOT:
-                break  # Only interested in the root part
-            elif t.type == TokenKind.WHITESPACE:
-                expect_dot = True
-            elif t.type == TokenKind.LPAR:
-                pass  # try to recognize function
-            elif not str(t.value).isidentifier():
-                not_a_variable = True
-
-        if not_a_variable:
-            return expr
-
-        full_name = get_text_from_tokens(expr.tokens)
-        split = full_name.split(".")
-        if len(split) == 1:
-            return VariableNode(expr.start, expr.end, expr.tokens, split[0])
-        else:
-            return VariableNode(expr.start, expr.end, expr.tokens, split[0], *split[1:])