Tokenizer exceptions are not catched

parsers/PythonParser.py

@@ -1,5 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
-from core.tokenizer import Tokenizer
+from core.tokenizer import Tokenizer, LexerError
 from parsers.BaseParser import BaseParser, Node, ErrorNode
 from dataclasses import dataclass
 import ast
@@ -63,27 +63,33 @@ class PythonParser(BaseParser):
         self.source = kwargs.get("source", "<undef>")
 
     def parse(self, context, text):
-        if isinstance(text, str) and "c:" in text:
-            source = self.get_text_from_tokens(list(Tokenizer(text)))
-        elif isinstance(text, str):
-            source = text
-        else:
-            source = self.get_text_from_tokens(text)
-        source = source.strip()
-
-        text = text if isinstance(text, str) else source
-
         sheerka = context.sheerka
+        tree = None
 
-        #  first, try to parse an expression
-        res, tree, error = self.try_parse_expression(source)
-        if not res:
-            # then try to parse a statement
-            res, tree, error = self.try_parse_statement(source)
+        try:
+            if isinstance(text, str) and "c:" in text:
+                source = self.get_text_from_tokens(list(Tokenizer(text)))
+            elif isinstance(text, str):
+                source = text
+            else:
+                source = self.get_text_from_tokens(text)
+            source = source.strip()
+
+            text = text if isinstance(text, str) else source
+
+            #  first, try to parse an expression
+            res, tree, error = self.try_parse_expression(source)
             if not res:
-                self.has_error = True
-                error_node = PythonErrorNode(text, error)
-                self.error_sink.append(error_node)
+                # then try to parse a statement
+                res, tree, error = self.try_parse_statement(source)
+                if not res:
+                    self.has_error = True
+                    error_node = PythonErrorNode(text, error)
+                    self.error_sink.append(error_node)
+
+        except LexerError as e:
+            self.has_error = True
+            self.error_sink.append(e)
 
         ret = sheerka.ret(
             self.name,