Fixed SyaNodeParser false positive recognition issue

src/core/tokenizer.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 
 
@@ -48,6 +48,7 @@ class TokenKind(Enum):
     DEGREE = "degree"  # °
     WORD = "word"
     EQUALSEQUALS = "=="
+    VAR_DEF = "__var__"
 
 
 @dataclass()
@@ -58,6 +59,8 @@ class Token:
     line: int
     column: int
 
+    _str_value: str = field(default=None, repr=False, compare=False, hash=None)
+
     def __repr__(self):
         if self.type == TokenKind.IDENTIFIER:
             value = str(self.value)
@@ -72,6 +75,23 @@ class Token:
 
         return f"Token({value})"
 
+    @property
+    def str_value(self):
+        if self._str_value:
+            return self._str_value
+
+        if self.type == TokenKind.STRING:
+            self._str_value = self.value[1:-1]
+        elif self.type == TokenKind.KEYWORD:
+            self._str_value = self.value.value
+        else:
+            self._str_value = str(self.value)
+        return self._str_value
+
+    @staticmethod
+    def is_whitespace(token):
+        return token and token.type == TokenKind.WHITESPACE
+
 
 @dataclass()
 class LexerError(Exception):
@@ -101,12 +121,13 @@ class Tokenizer:
 
     KEYWORDS = set(x.value for x in Keywords)
 
-    def __init__(self, text, parse_word=False):
+    def __init__(self, text, yield_eof=True, parse_word=False):
         self.text = text
         self.text_len = len(text)
         self.column = 1
         self.line = 1
         self.i = 0
+        self.yield_eof = yield_eof
         self.parse_word = parse_word
 
     def __iter__(self):
@@ -134,6 +155,7 @@ class Tokenizer:
                     self.i += 1
                     self.column += 1
             elif c == "_":
+                from core.concept import VARIABLE_PREFIX
                 if self.i + 1 < self.text_len and self.text[self.i + 1].isalpha():
                     identifier = self.eat_identifier(self.i)
                     token_type = TokenKind.KEYWORD if identifier in self.KEYWORDS else TokenKind.IDENTIFIER
@@ -141,6 +163,13 @@ class Tokenizer:
                     yield Token(token_type, value, self.i, self.line, self.column)
                     self.i += len(identifier)
                     self.column += len(identifier)
+                elif self.i + 7 < self.text_len and \
+                    self.text[self.i: self.i + 7] == VARIABLE_PREFIX and \
+                    self.text[self.i + 7].isdigit():
+                    number = self.eat_number(self.i + 7)
+                    yield Token(TokenKind.VAR_DEF, VARIABLE_PREFIX + number, self.i, self.line, self.column)
+                    self.i += 7 + len(number)
+                    self.column += 7 + len(number)
                 else:
                     yield Token(TokenKind.UNDERSCORE, "_", self.i, self.line, self.column)
                     self.i += 1
@@ -308,7 +337,8 @@ class Tokenizer:
             else:
                 raise LexerError(f"Unknown token '{c}'", self.text, self.i, self.line, self.column)
 
-        yield Token(TokenKind.EOF, "", self.i, self.line, self.column)
+        if self.yield_eof:
+            yield Token(TokenKind.EOF, "", self.i, self.line, self.column)
 
     def eat_concept(self, start, line, column):
         key, id, buffer = None, None, ""