Refactored to use a single implementation for concept evaluation

tests/test_tokenizer.py

@@ -3,7 +3,7 @@ from core.tokenizer import Tokenizer, Token, TokenKind, LexerError, Keywords
 
 
 def test_i_can_tokenize():
-    source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&"
+    source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&<>"
     tokens = list(Tokenizer(source))
     assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
     assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
@@ -37,6 +37,8 @@ def test_i_can_tokenize():
     assert tokens[29] == Token(TokenKind.EQUALS, '=', 76, 6, 18)
     assert tokens[30] == Token(TokenKind.VBAR, '|', 77, 6, 19)
     assert tokens[31] == Token(TokenKind.AMPER, '&', 78, 6, 20)
+    assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 21)
+    assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 22)
 
 
 @pytest.mark.parametrize("text, expected", [