Refactored Parsers. Introduced BaseCustomGrammarParser. Renamed DefaultParser into DefConceptParser

src/core/sheerka/services/SheerkaExecute.py

@@ -15,11 +15,29 @@ class ParserInput:
     def __init__(self, text, tokens=None, start=None, end=None, yield_oef=True):
         self.text = text
         self.tokens = tokens or None
-        self.length = len(tokens) if tokens else None
+        if self.tokens:
+            # make sure tokens ends with EOF token
+            # and do not modify the original token list
+            if len(self.tokens) == 0:
+                self.tokens = [Token(TokenKind.EOF, "", 0, 1, 1)]
+
+            elif (last_token := self.tokens[-1]).type != TokenKind.EOF:
+                self.tokens = self.tokens + [Token(TokenKind.EOF,
+                                                   "",
+                                                   last_token.index + 1,
+                                                   last_token.line,
+                                                   last_token.column + 1)]
+
+        self.length = None  # to be computed in reset()
         self.yield_oef = yield_oef
 
         self.start = start or 0
-        self.end = end + 1 if end else None
+        if end:
+            self.original_end = end + 1
+            self.end = self.original_end
+        else:
+            self.original_end = self.end = None
+
         self.sub_text = None
         self.sub_tokens = None
 
@@ -32,15 +50,19 @@ class ParserInput:
         from_tokens = "from_tokens" if self.from_tokens else ""
         return f"ParserInput({from_tokens}'{self.text}')"
 
-    def reset(self, yield_oef=True):
+    def reset(self, yield_oef=None):
+        if yield_oef is None:
+            yield_oef = self.yield_oef
+
+        # make sure tokens is correctly initialized
         if self.tokens is None:
-            self.tokens = list(Tokenizer(self.text))
-            self.length = len(self.tokens)
+            self.tokens = list(Tokenizer(self.text, yield_eof=True))
 
-        if self.end is None:
-            self.end = self.length
+        if self.original_end is None:
+            self.end = len(self.tokens) if yield_oef else len(self.tokens) - 1
+        else:
+            self.end = self.original_end if self.original_end <= len(self.tokens) else self.tokens
 
-        self.yield_oef = yield_oef
         self.pos = self.start - 1
         self.token = None
         return self
@@ -70,13 +92,10 @@ class ParserInput:
         self.pos += 1
 
         if self.pos >= self.end:
-            if self.yield_oef:
-                self.token = Token(TokenKind.EOF, "", -1, -1, -1)
             return False
 
         self.token = self.tokens[self.pos]
-
-        if self.token.type == TokenKind.EOF and not self.yield_oef:
+        if self.token.type == TokenKind.EOF:
             return False
 
         if skip_whitespace: