Added basic implentation for where

tests/core/test_tokenizer.py

@@ -40,7 +40,7 @@ def test_i_can_tokenize():
     assert tokens[31] == Token(TokenKind.AMPER, '&', 78, 6, 20)
     assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 21)
     assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 22)
-    assert tokens[34] == Token(TokenKind.CONCEPT, 'name', 81, 6, 23)
+    assert tokens[34] == Token(TokenKind.CONCEPT, ('name', None), 81, 6, 23)
     assert tokens[35] == Token(TokenKind.DOLLAR, '$', 88, 6, 30)
     assert tokens[36] == Token(TokenKind.STERLING, '£', 89, 6, 31)
     assert tokens[37] == Token(TokenKind.EURO, '€', 90, 6, 32)
@@ -79,8 +79,8 @@ def test_i_can_tokenize_identifiers(text, expected):
     ('"string', "Missing Trailing quote", '"string', 7, 1, 8),
     ('"a" + "string', "Missing Trailing quote", '"string', 13, 1, 14),
     ('"a"\n\n"string', "Missing Trailing quote", '"string', 12, 3, 8),
-    ("c::", "Concept name not found", "", 2, 1, 3),
-    ("c:foo\nbar:", "New line is forbidden in concept name", "foo", 5, 1, 6),
+    ("c::", "Concept identifiers not found", "", 2, 1, 3),
+    ("c:foo\nbar:", "New line in concept name", "foo", 5, 1, 6),
     ("c:foo", "Missing ending colon", "foo", 5, 1, 6)
 ])
 def test_i_can_detect_tokenizer_errors(text, message, error_text, index, line, column):
@@ -139,3 +139,17 @@ def test_i_can_recognize_keywords(text, expected):
     tokens = list(Tokenizer(text))
     assert tokens[0].type == TokenKind.KEYWORD
     assert tokens[0].value == expected
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("c:key:", ("key", None)),
+    ("c:key|id:", ("key", "id")),
+    ("c:key|:", ("key", None)),
+    ("c:|id:", (None, "id")),
+    ("c:125:", ("125", None)),
+])
+def test_i_can_parse_concept_token(text, expected):
+    tokens = list(Tokenizer(text))
+
+    assert tokens[0].type == TokenKind.CONCEPT
+    assert tokens[0].value == expected