Added first version of console autocompletion

src/core/tokenizer.py

@@ -49,6 +49,7 @@ class TokenKind(Enum):
     WORD = "word"
     EQUALSEQUALS = "=="
     VAR_DEF = "__var__"
+    REGEX = "r'xxx' or r\"xxx\" or r:xxx: or r|xxx| or r/xxx/"
 
 
 @dataclass()
@@ -322,6 +323,12 @@ class Tokenizer:
                 yield Token(TokenKind.CONCEPT, (name, id), self.i, self.line, self.column)
                 self.i += length + 2
                 self.column += length + 2
+            elif c == "r" and self.i + 1 < self.text_len and self.text[self.i + 1] in "'\":|/":
+                string, newlines, column_index = self.eat_string(self.i + 1, self.line, self.column)
+                yield Token(TokenKind.REGEX, string, self.i, self.line, self.column)  # quotes are kept
+                self.i += len(string) + 1
+                self.column = column_index  # 1 if newlines > 0 else self.column + len(string)
+                self.line += newlines
             elif self.parse_word and (c.isalpha() or c.isdigit()):
                 word = self.eat_word(self.i)
                 yield Token(TokenKind.WORD, word, self.i, self.line, self.column)
@@ -340,10 +347,10 @@ class Tokenizer:
                 self.i += len(number)
                 self.column += len(number)
             elif c == "'" or c == '"':
-                string, newlines = self.eat_string(self.i, self.line, self.column)
+                string, newlines, column_index = self.eat_string(self.i, self.line, self.column)
                 yield Token(TokenKind.STRING, string, self.i, self.line, self.column)  # quotes are kept
                 self.i += len(string)
-                self.column = 1 if newlines > 0 else self.column + len(string)
+                self.column = column_index  # 1 if newlines > 0 else self.column + len(string)
                 self.line += newlines
             elif c == "_":
                 yield Token(TokenKind.UNDERSCORE, "_", self.i, self.line, self.column)
@@ -445,21 +452,20 @@ class Tokenizer:
         quote = self.text[start_index]
         result = self.text[start_index]
         lines_count = 0
+        column_index = start_column + 1
 
         i = start_index + 1
         escape = False
-        newline = None
+        #newline = None
         while i < self.text_len:
             c = self.text[i]
             result += c
             i += 1
+            column_index += 1
 
-            if newline:
+            if c == "\n":
                 lines_count += 1
-                newline = c if c == newline else None
-            else:
-                if c == "\r" or c == "\n":
-                    newline = c
+                column_index = 1
 
             if c == "\\":
                 escape = True
@@ -468,15 +474,16 @@ class Tokenizer:
             else:
                 escape = False
 
-        # add trailing new line if needed
-        if newline:
-            lines_count += 1
+        # # add trailing new line if needed
+        # if newline:
+        #     lines_count += 1
+        #     column_index = 1
 
         if result[-1] != quote:
             raise LexerError("Missing Trailing quote", result, i, start_line + lines_count,
                              1 if lines_count > 0 else start_column + len(result))
 
-        return result, lines_count
+        return result, lines_count, column_index
 
     def eat_word(self, start):
         result = self.text[start]