Hardened DefaultParser

This commit is contained in:
2020-09-22 17:39:42 +02:00
parent 310c9ae839
commit 9b965105e9
5 changed files with 220 additions and 40 deletions
@@ -88,6 +88,19 @@ class ParserInput:
return self.pos < self.end
def the_token_after(self, skip_whitespace=True):
my_pos = self.pos + 1
if my_pos >= self.end:
return Token(TokenKind.EOF, "", -1, -1, -1)
if skip_whitespace:
while self.tokens[my_pos].type in (TokenKind.WHITESPACE, TokenKind.NEWLINE):
my_pos += 1
if my_pos == self.end:
return Token(TokenKind.EOF, "", -1, -1, -1)
return self.tokens[my_pos]
def seek(self, pos):
"""
Move the token offset to position pos