Refactored Parsers. Introduced BaseCustomGrammarParser. Renamed DefaultParser into DefConceptParser

tests/core/test_ParserInput.py

@@ -1,6 +1,6 @@
 import pytest
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Tokenizer
+from core.tokenizer import Tokenizer, TokenKind
 
 
 @pytest.mark.parametrize("text, start, end, expected", [
@@ -14,38 +14,86 @@ def test_i_can_use_parser_input(text, start, end, expected):
     assert parser_input.as_text() == expected
 
 
-def test_i_can_get_the_next_token():
-    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'").reset()
+def test_i_can_get_the_next_token_when_yield_eof_is_activated():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=True).reset()
+    res = []
+    parser_input.next_token()
+    while True:
+        res.append(f"{parser_input.token.repr_value}")
+        if parser_input.token.type == TokenKind.EOF:
+            break
+        parser_input.next_token()
+
+    expected = ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", '<EOF>']
+
+    assert res == expected
+
+
+def test_i_can_get_the_next_token_when_yield_eof_is_deactivated():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=False).reset()
     res = []
     while parser_input.next_token():
-        res.append(f"{parser_input.token.str_value}")
+        res.append(f"{parser_input.token.repr_value}")
 
-    assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", '']
+    expected = ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'"]
 
+    assert res == expected
+
+
+def test_i_can_get_the_next_token_when_start_and_end_are_provided():
     parser_input = ParserInput("def concept a concept name from bnf 'xyz' as 'xyz'", start=4, end=9).reset()
     res = []
     while parser_input.next_token(skip_whitespace=False):
-        res.append(f"{parser_input.token.str_value}")
+        res.append(f"{parser_input.token.repr_value}")
 
-    assert res == ['a', ' ', 'concept', ' ', 'name', ' ']
+    assert res == ['a', '<ws>', 'concept', '<ws>', 'name', '<ws>']
 
 
-def test_i_can_get_the_next_token_when_initialised_with_tokens():
-    tokens = list(Tokenizer(" def concept a as 'xyz' "))
-    parser_input = ParserInput(" def concept a as 'xyz' ", tokens).reset()
+def test_i_can_get_next_token_when_yield_eof_is_false():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=False).reset()
     res = []
     while parser_input.next_token():
-        res.append(f"{parser_input.token.str_value}")
+        res.append(f"{parser_input.token.repr_value}")
 
-    assert res == ['def', 'concept', 'a', 'as', "'xyz'", '']
+    assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'"]
 
-    tokens = list(Tokenizer(" def concept a as 'xyz' ", yield_eof=False))
-    parser_input = ParserInput(" def concept a as 'xyz' ", tokens).reset()
+
+def test_i_can_override_yield_oef_within_reset():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=False).reset(yield_oef=True)
+    res = []
+    parser_input.next_token()
+    while True:
+        res.append(f"{parser_input.token.repr_value}")
+        if parser_input.token.type == TokenKind.EOF:
+            break
+        parser_input.next_token()
+
+    assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", "<EOF>"]
+    assert not parser_input.yield_oef
+
+
+@pytest.mark.parametrize("list_has_eof, parser_has_eof, reset_has_eof", [
+    (True, True, True),
+    (True, False, True),
+    (False, True, True),
+    (False, False, True),
+    (True, True, False),
+    (True, False, False),
+    (False, True, False),
+    (False, False, False),
+])
+def test_i_can_get_the_next_token_when_initialised_with_tokens(list_has_eof, parser_has_eof, reset_has_eof):
+    tokens = list(Tokenizer(" def concept a as 'xyz' ", yield_eof=list_has_eof))
+    parser_input = ParserInput(" def concept a as 'xyz' ", tokens, yield_oef=parser_has_eof).reset()
+    parser_input.reset(reset_has_eof)
     res = []
     while parser_input.next_token():
-        res.append(f"{parser_input.token.str_value}")
+        res.append(f"{parser_input.token.repr_value}")
 
-    assert res == ['def', 'concept', 'a', 'as', "'xyz'"]
+    expected = ['def', 'concept', 'a', 'as', "'xyz'"]
+    if reset_has_eof:
+        expected.append("<EOF>")
+    assert res == expected
 
 
 def test_i_can_parse_twice():