Added simple form of concept composition

tests/test_tokenizer.py

@@ -4,6 +4,7 @@ from core.tokenizer import Tokenizer, Token, TokenKind, LexerError, Keywords
 
 def test_i_can_tokenize():
     source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&<>c:name:"
+    source += "$£€!_identifier°~_^\\`#"
     tokens = list(Tokenizer(source))
     assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
     assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
@@ -40,8 +41,20 @@ def test_i_can_tokenize():
     assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 21)
     assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 22)
     assert tokens[34] == Token(TokenKind.CONCEPT, 'name', 81, 6, 23)
+    assert tokens[35] == Token(TokenKind.DOLLAR, '$', 88, 6, 30)
+    assert tokens[36] == Token(TokenKind.STERLING, '£', 89, 6, 31)
+    assert tokens[37] == Token(TokenKind.EURO, '€', 90, 6, 32)
+    assert tokens[38] == Token(TokenKind.EMARK, '!', 91, 6, 33)
+    assert tokens[39] == Token(TokenKind.IDENTIFIER, '_identifier', 92, 6, 34)
+    assert tokens[40] == Token(TokenKind.DEGREE, '°', 103, 6, 45)
+    assert tokens[41] == Token(TokenKind.TILDE, '~', 104, 6, 46)
+    assert tokens[42] == Token(TokenKind.UNDERSCORE, '_', 105, 6, 47)
+    assert tokens[43] == Token(TokenKind.CARAT, '^', 106, 6, 48)
+    assert tokens[44] == Token(TokenKind.BACK_SLASH, '\\', 107, 6, 49)
+    assert tokens[45] == Token(TokenKind.BACK_QUOTE, '`', 108, 6, 50)
+    assert tokens[46] == Token(TokenKind.HASH, '#', 109, 6, 51)
 
-    assert tokens[35] == Token(TokenKind.EOF, '', 88, 6, 30)
+    assert tokens[47] == Token(TokenKind.EOF, '', 110, 6, 52)
 
 
 @pytest.mark.parametrize("text, expected", [