Tokenizer exceptions are not catched

tests/test_PythonParser.py

@@ -4,7 +4,7 @@ import pytest
 
 from core.builtin_concepts import ParserResultConcept
 from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer
+from core.tokenizer import Tokenizer, LexerError
 from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
 from sdp.sheerkaDataProvider import Event
 
@@ -44,7 +44,12 @@ def test_i_can_parse_from_tokens(text, expected):
     assert res.value.value == expected
 
 
-def test_i_can_detect_error():
+@pytest.mark.parametrize("text", [
+    "1+",
+    "'name",
+    "foo = 'name"
+])
+def test_i_can_detect_error(text):
     text = "1+"
 
     parser = PythonParser()
@@ -57,6 +62,21 @@ def test_i_can_detect_error():
     assert isinstance(res.value.value[0].exception, SyntaxError)
 
 
+@pytest.mark.parametrize("text, error_msg, error_text", [
+    ("c::", "Concept name not found", ""),
+    ("c:: + 1", "Concept name not found", ""),
+])
+def test_i_can_detect_lexer_errors(text, error_msg, error_text):
+    parser = PythonParser()
+    res = parser.parse(get_context(), text)
+
+    assert not res.status
+    assert isinstance(res.body, ParserResultConcept)
+    assert isinstance(res.body.body[0], LexerError)
+    assert res.body.body[0].message == error_msg
+    assert res.body.body[0].text == error_text
+
+
 def test_i_can_parse_a_concept():
     text = "c:concept_name: + 1"