Refactored Parsers. Introduced BaseCustomGrammarParser. Renamed DefaultParser into DefConceptParser

src/parsers/BaseParser.py

@@ -1,7 +1,7 @@
 import logging
 from dataclasses import dataclass
+from typing import Union
 
-import core.utils
 from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
 from core.concept import Concept
 from core.sheerka.ExecutionContext import ExecutionContext
@@ -57,7 +57,7 @@ class ErrorNode(Node):
 @dataclass()
 class UnexpectedTokenErrorNode(ErrorNode):
     message: str
-    token: Token
+    token: Union[Token, str]
     expected_tokens: list
 
     def __eq__(self, other):
@@ -70,31 +70,25 @@ class UnexpectedTokenErrorNode(ErrorNode):
         if self.message != other.message:
             return False
 
-        if self.token.type != other.token.type or self.token.value != other.token.value:
+        to_compare = self.token.repr_value if isinstance(other.token, str) else self.token
+        if to_compare != other.token:
             return False
 
-        if len(self.expected_tokens) != len(other.expected_tokens):
-            return False
-
-        for i, t in enumerate(self.expected_tokens):
-            if t != other.expected_tokens[i]:
-                return False
-
-        return True
+        return self.expected_tokens == other.expected_tokens
 
     def __hash__(self):
         return hash((self.message, self.token, self.expected_tokens))
 
 
 @dataclass()
-class UnexpectedEof(ErrorNode):
+class UnexpectedEofNode(ErrorNode):
     message: str
 
 
 class BaseParser:
     PREFIX = "parsers."
 
-    def __init__(self, name, priority: int, enabled=True):
+    def __init__(self, name, priority: int, enabled=True, yield_eof=False):
         self.log = get_logger("parsers." + self.__class__.__name__)
         self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__)
         self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__)
@@ -107,6 +101,7 @@ class BaseParser:
         self.context: ExecutionContext = None
         self.sheerka = None
         self.parser_input: ParserInput = None
+        self.yield_eof = yield_eof
 
     def __eq__(self, other):
         if not isinstance(other, self.__class__):
@@ -126,10 +121,9 @@ class BaseParser:
         self.error_sink.clear()
 
         try:
-            self.parser_input.reset(False)
-            self.parser_input.next_token()
+            self.parser_input.reset(self.yield_eof)
         except LexerError as e:
-            self.add_error(self.sheerka.new(BuiltinConcepts.ERROR, body=e), False)
+            self.add_error(e, False)
             return False
         return True
 
@@ -165,12 +159,12 @@ class BaseParser:
             value = context.return_value_to_str(r)
             context.log(f"  Recognized '{value}'", self.name)
 
-    def get_return_value_body(self, sheerka, source, tree, try_parse):
+    def get_return_value_body(self, sheerka, source, parsed, try_parse):
         """
         All parsers must return their result in a standard way
         :param sheerka:
         :param source:
-        :param tree:
+        :param parsed:
         :param try_parse:
         :return:
         """
@@ -178,17 +172,19 @@ class BaseParser:
             return self.error_sink[0]
 
         if self.has_error:
-            return sheerka.new(
-                BuiltinConcepts.ERROR,
-                body=self.error_sink
-            )
+            if parsed is None:
+                return sheerka.new(BuiltinConcepts.NOT_FOR_ME,
+                                   body=source,
+                                   reason=self.error_sink)
+            else:
+                return sheerka.new(BuiltinConcepts.ERROR,
+                                   body=self.error_sink)
 
-        return sheerka.new(
-            BuiltinConcepts.PARSER_RESULT,
-            parser=self,
-            source=source,
-            body=tree,
-            try_parsed=try_parse)
+        return sheerka.new(BuiltinConcepts.PARSER_RESULT,
+                           parser=self,
+                           source=source,
+                           body=parsed,
+                           try_parsed=try_parse)
 
     @staticmethod
     def get_input_as_lexer_nodes(parser_input, expected_parser=None):
@@ -242,7 +238,7 @@ class BaseParser:
             tokens = [tokens]
 
         switcher = {
-         #   TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value),
+            #   TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value),
         }
 
         if custom_switcher: