First steps of ConceptLexer. Need to update DefaultParser before continuing

2019-11-29 17:26:50 +01:00
parent 5d37addc7d
commit 5e539a4b28
21 changed files with 1409 additions and 55 deletions
@@ -54,12 +54,14 @@ class Token:
    column: int

    def __repr__(self):
-        if type == TokenKind.IDENTIFIER:
-            value = "ident:" + str(self.value)
-        elif type == TokenKind.WHITESPACE:
-            value = " "
-        elif type == TokenKind.NEWLINE:
+        if self.type == TokenKind.IDENTIFIER:
+            value = str(self.value)
+        elif self.type == TokenKind.WHITESPACE:
+            value = "<ws>"
+        elif self.type == TokenKind.NEWLINE:
            value = r"\n"
+        elif self.type == TokenKind.EOF:
+            value = "<EOF>"
        else:
            value = self.value