Fixed SyaNodeParser false positive recognition issue
This commit is contained in:
@@ -4,7 +4,7 @@ from core.tokenizer import Tokenizer, Token, TokenKind, LexerError, Keywords
|
||||
|
||||
def test_i_can_tokenize():
|
||||
source = "+*-/{}[]() ,;:.?\n\n\r\r\r\nidentifier_0\t \t10.15 10 'string\n' \"another string\"=|&<>c:name:"
|
||||
source += "$£€!_identifier°~_^\\`==#"
|
||||
source += "$£€!_identifier°~_^\\`==#__var__10"
|
||||
tokens = list(Tokenizer(source))
|
||||
assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
|
||||
assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
|
||||
@@ -54,8 +54,9 @@ def test_i_can_tokenize():
|
||||
assert tokens[45] == Token(TokenKind.BACK_QUOTE, '`', 108, 6, 50)
|
||||
assert tokens[46] == Token(TokenKind.EQUALSEQUALS, '==', 109, 6, 51)
|
||||
assert tokens[47] == Token(TokenKind.HASH, '#', 111, 6, 53)
|
||||
assert tokens[48] == Token(TokenKind.VAR_DEF, '__var__10', 112, 6, 54)
|
||||
|
||||
assert tokens[48] == Token(TokenKind.EOF, '', 112, 6, 54)
|
||||
assert tokens[49] == Token(TokenKind.EOF, '', 121, 6, 63)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
@@ -88,6 +89,19 @@ def test_i_can_parse_word(text):
|
||||
assert tokens[1].index == len(text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"__var__0",
|
||||
"__var__1",
|
||||
"__var__10",
|
||||
"__var__999",
|
||||
])
|
||||
def test_i_can_parse_var_def(text):
|
||||
tokens = list(Tokenizer(text))
|
||||
assert len(tokens) == 2
|
||||
assert tokens[0].type == TokenKind.VAR_DEF
|
||||
assert tokens[0].value == text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, message, error_text, index, line, column", [
|
||||
("'string", "Missing Trailing quote", "'string", 7, 1, 8),
|
||||
('"string', "Missing Trailing quote", '"string', 7, 1, 8),
|
||||
|
||||
Reference in New Issue
Block a user