Implemented FunctionParser

src/core/tokenizer.py

@@ -62,6 +62,7 @@ class Token:
 
     _strip_quote: str = field(default=None, repr=False, compare=False, hash=None)
     _str_value: str = field(default=None, repr=False, compare=False, hash=None)
+    _repr_value: str = field(default=None, repr=False, compare=False, hash=None)
 
     def __repr__(self):
         if self.type == TokenKind.IDENTIFIER:
@@ -82,7 +83,7 @@ class Token:
         if self._strip_quote:
             return self._strip_quote
 
-        self._strip_quote = self._to_str(True)
+        self._strip_quote = self.value[1:-1] if self.type == TokenKind.STRING else self.value
         return self._strip_quote
 
     @property
@@ -90,18 +91,36 @@ class Token:
         if self._str_value:
             return self._str_value
 
-        self._str_value = self._to_str(False)
+        self._str_value = self.to_str(False)
         return self._str_value
 
+    @property
+    def repr_value(self):
+        if self._repr_value:
+            return self._repr_value
+
+        if self.type == TokenKind.EOF:
+            self._repr_value = "<EOF>"
+        elif self.type == TokenKind.WHITESPACE:
+            self._repr_value = "<ws>"
+        elif self.type == TokenKind.NEWLINE:
+            self._repr_value = "<nl>"
+        else:
+            self._repr_value = self.str_value
+        return self._repr_value
+
     @staticmethod
     def is_whitespace(token):
         return token and token.type == TokenKind.WHITESPACE
 
-    def _to_str(self, strip_quote):
+    def to_str(self, strip_quote):
         if strip_quote and self.type == TokenKind.STRING:
             return self.value[1:-1]
         elif self.type == TokenKind.KEYWORD:
             return self.value.value
+        elif self.type == TokenKind.CONCEPT:
+            from core.utils import str_concept
+            return str_concept(self.value)
         else:
             return str(self.value)
 
@@ -136,8 +155,6 @@ class Tokenizer:
     Class that can iterate on the tokens
     """
 
-    KEYWORDS = set(x.value for x in Keywords)
-
     def __init__(self, text, yield_eof=True, parse_word=False):
         self.text = text
         self.text_len = len(text)
@@ -175,9 +192,7 @@ class Tokenizer:
                 from core.concept import VARIABLE_PREFIX
                 if self.i + 1 < self.text_len and self.text[self.i + 1].isalpha():
                     identifier = self.eat_identifier(self.i)
-                    token_type = TokenKind.KEYWORD if identifier in self.KEYWORDS else TokenKind.IDENTIFIER
-                    value = Keywords(identifier) if identifier in self.KEYWORDS else identifier
-                    yield Token(token_type, value, self.i, self.line, self.column)
+                    yield Token(TokenKind.IDENTIFIER, identifier, self.i, self.line, self.column)
                     self.i += len(identifier)
                     self.column += len(identifier)
                 elif self.i + 7 < self.text_len and \
@@ -335,11 +350,9 @@ class Tokenizer:
                 yield Token(TokenKind.WORD, word, self.i, self.line, self.column)
                 self.i += len(word)
                 self.column += len(word)
-            elif c.isalpha() or c == "_":
+            elif c.isalpha():
                 identifier = self.eat_identifier(self.i)
-                token_type = TokenKind.KEYWORD if identifier in self.KEYWORDS else TokenKind.IDENTIFIER
-                value = Keywords(identifier) if identifier in self.KEYWORDS else identifier
-                yield Token(token_type, value, self.i, self.line, self.column)
+                yield Token(TokenKind.IDENTIFIER, identifier, self.i, self.line, self.column)
                 self.i += len(identifier)
                 self.column += len(identifier)
             elif c.isdigit():
@@ -457,7 +470,7 @@ class Tokenizer:
 
         i = start_index + 1
         escape = False
-        #newline = None
+        # newline = None
         while i < self.text_len:
             c = self.text[i]
             result += c