Introduced ParserInput

tests/core/test_ParserInput.py

@@ -0,0 +1,79 @@
+import pytest
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import Tokenizer
+
+
+@pytest.mark.parametrize("text, start, end, expected", [
+    ("def concept a", None, None, "def concept a"),
+    ("&é#(-è_çà)='string'", None, None, "&é#(-è_çà)='string'"),
+    ("def concept a", 2, None, "concept a"),
+    ("def concept a", 0, 2, "def concept"),
+])
+def test_i_can_use_parser_input(text, start, end, expected):
+    parser_input = ParserInput(text, start=start, end=end).reset()
+    assert parser_input.as_text() == expected
+
+
+def test_i_can_get_the_next_token():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'").reset()
+    res = []
+    while parser_input.next_token():
+        res.append(f"{parser_input.token.str_value}")
+
+    assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", '']
+
+    parser_input = ParserInput("def concept a concept name from bnf 'xyz' as 'xyz'", start=4, end=9).reset()
+    res = []
+    while parser_input.next_token(skip_whitespace=False):
+        res.append(f"{parser_input.token.str_value}")
+
+    assert res == ['a', ' ', 'concept', ' ', 'name', ' ']
+
+
+def test_i_can_get_the_next_token_when_initialised_with_tokens():
+    tokens = list(Tokenizer(" def concept a as 'xyz' "))
+    parser_input = ParserInput(" def concept a as 'xyz' ", tokens).reset()
+    res = []
+    while parser_input.next_token():
+        res.append(f"{parser_input.token.str_value}")
+
+    assert res == ['def', 'concept', 'a', 'as', "'xyz'", '']
+
+    tokens = list(Tokenizer(" def concept a as 'xyz' ", yield_eof=False))
+    parser_input = ParserInput(" def concept a as 'xyz' ", tokens).reset()
+    res = []
+    while parser_input.next_token():
+        res.append(f"{parser_input.token.str_value}")
+
+    assert res == ['def', 'concept', 'a', 'as', "'xyz'"]
+
+
+def test_i_can_parse_twice():
+    text = """
+    def concept a + b
+    where isinstance(a, int) and isinstance(b, int)
+    pre isinstance(a, int) and isinstance(b, int)
+    post isinstance(res, int)
+    as:
+        def func(x,y):
+            return x+y
+        func(a,b)
+                """
+
+    p1 = ParserInput(text).reset()
+    while p1.next_token():
+        pass
+
+    p1.reset()
+    p2 = ParserInput(text).reset()
+
+    while p1.next_token():
+        p2.next_token()
+        assert p1.token == p2.token
+
+    p1.reset()
+    p2 = ParserInput(text).reset()
+
+    while p2.next_token():
+        p1.next_token()
+        assert p1.token == p2.token

tests/core/test_sheerka_call_parsers.py

@@ -1,4 +1,5 @@
 from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept
+from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.BaseParser import BaseParser
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -22,7 +23,7 @@ class BaseTestParser(BaseParser):
 
     @staticmethod
     def _get_source(text_):
-        return text_ if isinstance(text_, str) else text_.body
+        return text_.as_text() if isinstance(text_, ParserInput) else text_.body
 
     def _out(self, name, priority, status, source):
         debug = f"name={name}"
@@ -33,7 +34,7 @@ class BaseTestParser(BaseParser):
 
     def parse(self, context, text):
         self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        value = self._get_name(self.name) + ":" + self._get_source(text)
         parser_result = ParserResultConcept(parser=self, value=value)
         return ReturnValueConcept(self, self.status, self.parser_result or parser_result)
 
@@ -54,8 +55,8 @@ class Enabled80MultipleFalseParser(BaseTestParser):
 
     def parse(self, context, text):
         self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
-        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        value1 = self._get_name(self.name) + ":" + self._get_source(text) + "_1"
+        value2 = self._get_name(self.name) + ":" + self._get_source(text) + "_2"
         return [
             ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)),
             ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)),
@@ -68,8 +69,8 @@ class Enabled80MultipleTrueParser(BaseTestParser):
 
     def parse(self, context, text):
         self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
-        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        value1 = self._get_name(self.name) + ":" + self._get_source(text) + "_1"
+        value2 = self._get_name(self.name) + ":" + self._get_source(text) + "_2"
         return [
             ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)),
             ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)),
@@ -90,7 +91,7 @@ class Enabled50TrueParser(BaseTestParser):
         status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world"
         self._out(self._get_name(self.name), self.priority, status, source)
 
-        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        value = self._get_name(self.name) + ":" + self._get_source(text)
         return_value = ParserResultConcept(parser=self, value=value)
         return ReturnValueConcept(self, status, return_value)