Hardened DefaultParser

src/core/sheerka/services/SheerkaExecute.py

@@ -88,6 +88,19 @@ class ParserInput:
 
         return self.pos < self.end
 
+    def the_token_after(self, skip_whitespace=True):
+        my_pos = self.pos + 1
+        if my_pos >= self.end:
+            return Token(TokenKind.EOF, "", -1, -1, -1)
+
+        if skip_whitespace:
+            while self.tokens[my_pos].type in (TokenKind.WHITESPACE, TokenKind.NEWLINE):
+                my_pos += 1
+                if my_pos == self.end:
+                    return Token(TokenKind.EOF, "", -1, -1, -1)
+
+        return self.tokens[my_pos]
+
     def seek(self, pos):
         """
         Move the token offset to position pos

src/core/tokenizer.py

@@ -68,9 +68,9 @@ class Token:
         if self.type == TokenKind.IDENTIFIER:
             value = str(self.value)
         elif self.type == TokenKind.WHITESPACE:
-            value = "<ws>"
+            value = "<tab>" if self.value[0] == "\t" else "<ws>"
         elif self.type == TokenKind.NEWLINE:
-            value = r"\n"
+            value = "<nl>"
         elif self.type == TokenKind.EOF:
             value = "<EOF>"
         else: