Tokenizer exceptions are not catched

tests/test_AddConceptEvaluator.py

@@ -180,3 +180,5 @@ def test_i_can_get_props_from_definition():
     ret_val = get_concept_definition("mult (('+'|'-') add)?", parsing_expression)
 
     assert AddConceptEvaluator.get_props(get_context(), ret_val, []) == ["add", "mult"]
+
+

tests/test_BnfParser.py

@@ -2,7 +2,7 @@ import pytest
 
 from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer, TokenKind
+from core.tokenizer import Tokenizer, TokenKind, LexerError
 from parsers.BaseParser import UnexpectedTokenErrorNode
 from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
 from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
@@ -80,6 +80,7 @@ def test_i_can_parse_regex(expression, expected):
     ("1|", UnexpectedEndOfFileError()),
     ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.RPAR])),
     ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.IDENTIFIER])),
+    ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
 ])
 def test_i_can_detect_errors(expression, error):
     parser = BnfParser()

tests/test_DefaultParser.py

@@ -5,12 +5,11 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnVa
 from core.sheerka import Sheerka, ExecutionContext
 from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptMatch
 from parsers.PythonParser import PythonParser, PythonNode
-from core.tokenizer import Keywords, Tokenizer
+from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode
 from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode
 from parsers.BnfParser import BnfParser
 
-
 from sdp.sheerkaDataProvider import Event
 
 
@@ -321,3 +320,22 @@ def test_i_cannot_parse_invalid_entries(text):
     assert not res.status
     assert isinstance(res.body, ParserResultConcept)
     assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
+
+
+@pytest.mark.parametrize("text, error_msg, error_text", [
+    ("'name", "Missing Trailing quote", "'name"),
+    ("foo isa 'name", "Missing Trailing quote", "'name"),
+    ("def concept 'name", "Missing Trailing quote", "'name"),
+    ("def concept name as 'body", "Missing Trailing quote", "'body"),
+    ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"),
+    ("def concept c::", "Concept name not found", ""),
+])
+def test_i_cannot_parse_when_tokenizer_fails(text, error_msg, error_text):
+    parser = DefaultParser()
+    res = parser.parse(get_context(), text)
+
+    assert not res.status
+    assert isinstance(res.body, ParserResultConcept)
+    assert isinstance(res.body.body[0], LexerError)
+    assert res.body.body[0].message == error_msg
+    assert res.body.body[0].text == error_text

tests/test_PythonParser.py

@@ -4,7 +4,7 @@ import pytest
 
 from core.builtin_concepts import ParserResultConcept
 from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer
+from core.tokenizer import Tokenizer, LexerError
 from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
 from sdp.sheerkaDataProvider import Event
 
@@ -44,7 +44,12 @@ def test_i_can_parse_from_tokens(text, expected):
     assert res.value.value == expected
 
 
-def test_i_can_detect_error():
+@pytest.mark.parametrize("text", [
+    "1+",
+    "'name",
+    "foo = 'name"
+])
+def test_i_can_detect_error(text):
     text = "1+"
 
     parser = PythonParser()
@@ -57,6 +62,21 @@ def test_i_can_detect_error():
     assert isinstance(res.value.value[0].exception, SyntaxError)
 
 
+@pytest.mark.parametrize("text, error_msg, error_text", [
+    ("c::", "Concept name not found", ""),
+    ("c:: + 1", "Concept name not found", ""),
+])
+def test_i_can_detect_lexer_errors(text, error_msg, error_text):
+    parser = PythonParser()
+    res = parser.parse(get_context(), text)
+
+    assert not res.status
+    assert isinstance(res.body, ParserResultConcept)
+    assert isinstance(res.body.body[0], LexerError)
+    assert res.body.body[0].message == error_msg
+    assert res.body.body[0].text == error_text
+
+
 def test_i_can_parse_a_concept():
     text = "c:concept_name: + 1"
 

tests/test_sheerka_non_reg.py

@@ -416,3 +416,23 @@ def test_eval_does_not_break_valid_result():
     assert len(res) == 1
     assert res[0].status
     assert res[0].body == 3
+
+
+@pytest.mark.parametrize("text", [
+    "'hello",
+    '"foo" + "string',
+    "c::",
+    "c:foo\nbar:",
+    "c:foo",
+    "def concept 'name",
+    "def concept name from bnf 'name"
+])
+def test_i_can_manage_tokenizer_error(text):
+    sheerka = get_sheerka()
+    sheerka.add_in_cache(Concept("foo"))
+
+    res = sheerka.evaluate_user_input(text)
+
+    assert len(res) > 1
+    for r in [r for r in res if r.who.startswith("parsers.")]:
+        assert not r.status

tests/test_tokenizer.py

@@ -66,11 +66,11 @@ def test_i_can_tokenize_identifiers(text, expected):
     ('"string', "Missing Trailing quote", '"string', 7, 1, 8),
     ('"a" + "string', "Missing Trailing quote", '"string', 13, 1, 14),
     ('"a"\n\n"string', "Missing Trailing quote", '"string', 12, 3, 8),
-    ("c::", "Context name not found", "", 2, 1, 3),
+    ("c::", "Concept name not found", "", 2, 1, 3),
     ("c:foo\nbar:", "New line is forbidden in concept name", "foo", 5, 1, 6),
     ("c:foo", "Missing ending colon", "foo", 5, 1, 6)
 ])
-def test_i_can_detect_unfinished_strings(text, message, error_text, index, line, column):
+def test_i_can_detect_tokenizer_errors(text, message, error_text, index, line, column):
     with pytest.raises(LexerError) as e:
         list(Tokenizer(text))
     assert e.value.message == message