Refactored Parsers. Introduced BaseCustomGrammarParser. Renamed DefaultParser into DefConceptParser

tests/core/test_ParserInput.py

@@ -1,6 +1,6 @@
 import pytest
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Tokenizer
+from core.tokenizer import Tokenizer, TokenKind
 
 
 @pytest.mark.parametrize("text, start, end, expected", [
@@ -14,38 +14,86 @@ def test_i_can_use_parser_input(text, start, end, expected):
     assert parser_input.as_text() == expected
 
 
-def test_i_can_get_the_next_token():
-    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'").reset()
+def test_i_can_get_the_next_token_when_yield_eof_is_activated():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=True).reset()
+    res = []
+    parser_input.next_token()
+    while True:
+        res.append(f"{parser_input.token.repr_value}")
+        if parser_input.token.type == TokenKind.EOF:
+            break
+        parser_input.next_token()
+
+    expected = ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", '<EOF>']
+
+    assert res == expected
+
+
+def test_i_can_get_the_next_token_when_yield_eof_is_deactivated():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=False).reset()
     res = []
     while parser_input.next_token():
-        res.append(f"{parser_input.token.str_value}")
+        res.append(f"{parser_input.token.repr_value}")
 
-    assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", '']
+    expected = ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'"]
 
+    assert res == expected
+
+
+def test_i_can_get_the_next_token_when_start_and_end_are_provided():
     parser_input = ParserInput("def concept a concept name from bnf 'xyz' as 'xyz'", start=4, end=9).reset()
     res = []
     while parser_input.next_token(skip_whitespace=False):
-        res.append(f"{parser_input.token.str_value}")
+        res.append(f"{parser_input.token.repr_value}")
 
-    assert res == ['a', ' ', 'concept', ' ', 'name', ' ']
+    assert res == ['a', '<ws>', 'concept', '<ws>', 'name', '<ws>']
 
 
-def test_i_can_get_the_next_token_when_initialised_with_tokens():
-    tokens = list(Tokenizer(" def concept a as 'xyz' "))
-    parser_input = ParserInput(" def concept a as 'xyz' ", tokens).reset()
+def test_i_can_get_next_token_when_yield_eof_is_false():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=False).reset()
     res = []
     while parser_input.next_token():
-        res.append(f"{parser_input.token.str_value}")
+        res.append(f"{parser_input.token.repr_value}")
 
-    assert res == ['def', 'concept', 'a', 'as', "'xyz'", '']
+    assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'"]
 
-    tokens = list(Tokenizer(" def concept a as 'xyz' ", yield_eof=False))
-    parser_input = ParserInput(" def concept a as 'xyz' ", tokens).reset()
+
+def test_i_can_override_yield_oef_within_reset():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=False).reset(yield_oef=True)
+    res = []
+    parser_input.next_token()
+    while True:
+        res.append(f"{parser_input.token.repr_value}")
+        if parser_input.token.type == TokenKind.EOF:
+            break
+        parser_input.next_token()
+
+    assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", "<EOF>"]
+    assert not parser_input.yield_oef
+
+
+@pytest.mark.parametrize("list_has_eof, parser_has_eof, reset_has_eof", [
+    (True, True, True),
+    (True, False, True),
+    (False, True, True),
+    (False, False, True),
+    (True, True, False),
+    (True, False, False),
+    (False, True, False),
+    (False, False, False),
+])
+def test_i_can_get_the_next_token_when_initialised_with_tokens(list_has_eof, parser_has_eof, reset_has_eof):
+    tokens = list(Tokenizer(" def concept a as 'xyz' ", yield_eof=list_has_eof))
+    parser_input = ParserInput(" def concept a as 'xyz' ", tokens, yield_oef=parser_has_eof).reset()
+    parser_input.reset(reset_has_eof)
     res = []
     while parser_input.next_token():
-        res.append(f"{parser_input.token.str_value}")
+        res.append(f"{parser_input.token.repr_value}")
 
-    assert res == ['def', 'concept', 'a', 'as', "'xyz'"]
+    expected = ['def', 'concept', 'a', 'as', "'xyz'"]
+    if reset_has_eof:
+        expected.append("<EOF>")
+    assert res == expected
 
 
 def test_i_can_parse_twice():

tests/core/test_sheerka.py

@@ -28,7 +28,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         sheerka = self.get_sheerka()
 
         # test existence of some parser (not all)
-        assert "parsers.DefaultParser.DefaultParser" in sheerka.parsers
+        assert "parsers.DefConceptParser.DefConceptParser" in sheerka.parsers
         assert "parsers.BnfNodeParser.BnfNodeParser" in sheerka.parsers
         assert "parsers.SyaNodeParser.SyaNodeParser" in sheerka.parsers
         assert "parsers.AtomNodeParser.AtomNodeParser" in sheerka.parsers

tests/core/test_utils.py

@@ -55,15 +55,15 @@ def test_i_can_get_base_classes():
 
     # example of classes that should be in the result
     base_parser = core.utils.get_class("parsers.BaseParser.BaseParser")
-    default_parser = core.utils.get_class("parsers.DefaultParser.DefaultParser")
+    def_concept_parser = core.utils.get_class("parsers.DefConceptParser.DefConceptParser")
     exact_concept_parser = core.utils.get_class("parsers.ExactConceptParser.ExactConceptParser")
     python_parser = core.utils.get_class("parsers.PythonParser.PythonParser")
     node = core.utils.get_class("parsers.BaseParser.Node")
-    def_concept_node = core.utils.get_class("parsers.DefaultParser.DefConceptNode")
+    def_concept_node = core.utils.get_class("parsers.DefConceptParser.DefConceptNode")
     python_node = core.utils.get_class("parsers.PythonParser.PythonNode")
 
     assert base_parser in classes
-    assert default_parser in classes
+    assert def_concept_parser in classes
     assert exact_concept_parser in classes
     assert python_parser in classes
     assert node in classes
@@ -76,13 +76,13 @@ def test_i_can_get_sub_classes():
 
     # example of classes that should be (or not) in the result
     base_parser = core.utils.get_class("parsers.BaseParser.BaseParser")
-    default_parser = core.utils.get_class("parsers.DefaultParser.DefaultParser")
+    def_concept_parser = core.utils.get_class("parsers.DefConceptParser.DefConceptParser")
     exact_concept_parser = core.utils.get_class("parsers.ExactConceptParser.ExactConceptParser")
     python_parser = core.utils.get_class("parsers.PythonParser.PythonParser")
     bnf_node_parser = core.utils.get_class("parsers.BnfNodeParser.BnfNodeParser")
 
     assert base_parser not in sub_classes
-    assert default_parser in sub_classes
+    assert def_concept_parser in sub_classes
     assert exact_concept_parser in sub_classes
     assert python_parser in sub_classes
     assert bnf_node_parser in sub_classes