Introduced ParserInput

tests/BaseTest.py

@@ -24,10 +24,10 @@ class BaseTest:
     def get_default_concept(self):
         concept = Concept(
             name="a + b",
-            where="isinstance(a, int) and isinstance(b, int)",
-            pre="isinstance(a, int) and isinstance(b, int)",
-            post="isinstance(res, int)",
-            body="def func(x,y):\n    return x+y\nfunc(a,b)",
+            where="isinstance(a, int) and isinstance(b, int)\n",
+            pre="isinstance(a, int) and isinstance(b, int)\n",
+            post="isinstance(res, int)\n",
+            body="def func(x,y):\n    return x+y\nfunc(a,b)\n",
             desc="specific description")
         concept.def_var("a", "value1")
         concept.def_var("b", "value2")

tests/core/test_ParserInput.py

@@ -0,0 +1,79 @@
+import pytest
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import Tokenizer
+
+
+@pytest.mark.parametrize("text, start, end, expected", [
+    ("def concept a", None, None, "def concept a"),
+    ("&é#(-è_çà)='string'", None, None, "&é#(-è_çà)='string'"),
+    ("def concept a", 2, None, "concept a"),
+    ("def concept a", 0, 2, "def concept"),
+])
+def test_i_can_use_parser_input(text, start, end, expected):
+    parser_input = ParserInput(text, start=start, end=end).reset()
+    assert parser_input.as_text() == expected
+
+
+def test_i_can_get_the_next_token():
+    parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'").reset()
+    res = []
+    while parser_input.next_token():
+        res.append(f"{parser_input.token.str_value}")
+
+    assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", '']
+
+    parser_input = ParserInput("def concept a concept name from bnf 'xyz' as 'xyz'", start=4, end=9).reset()
+    res = []
+    while parser_input.next_token(skip_whitespace=False):
+        res.append(f"{parser_input.token.str_value}")
+
+    assert res == ['a', ' ', 'concept', ' ', 'name', ' ']
+
+
+def test_i_can_get_the_next_token_when_initialised_with_tokens():
+    tokens = list(Tokenizer(" def concept a as 'xyz' "))
+    parser_input = ParserInput(" def concept a as 'xyz' ", tokens).reset()
+    res = []
+    while parser_input.next_token():
+        res.append(f"{parser_input.token.str_value}")
+
+    assert res == ['def', 'concept', 'a', 'as', "'xyz'", '']
+
+    tokens = list(Tokenizer(" def concept a as 'xyz' ", yield_eof=False))
+    parser_input = ParserInput(" def concept a as 'xyz' ", tokens).reset()
+    res = []
+    while parser_input.next_token():
+        res.append(f"{parser_input.token.str_value}")
+
+    assert res == ['def', 'concept', 'a', 'as', "'xyz'"]
+
+
+def test_i_can_parse_twice():
+    text = """
+    def concept a + b
+    where isinstance(a, int) and isinstance(b, int)
+    pre isinstance(a, int) and isinstance(b, int)
+    post isinstance(res, int)
+    as:
+        def func(x,y):
+            return x+y
+        func(a,b)
+                """
+
+    p1 = ParserInput(text).reset()
+    while p1.next_token():
+        pass
+
+    p1.reset()
+    p2 = ParserInput(text).reset()
+
+    while p1.next_token():
+        p2.next_token()
+        assert p1.token == p2.token
+
+    p1.reset()
+    p2 = ParserInput(text).reset()
+
+    while p2.next_token():
+        p1.next_token()
+        assert p1.token == p2.token

tests/core/test_sheerka_call_parsers.py

@@ -1,4 +1,5 @@
 from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept
+from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.BaseParser import BaseParser
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -22,7 +23,7 @@ class BaseTestParser(BaseParser):
 
     @staticmethod
     def _get_source(text_):
-        return text_ if isinstance(text_, str) else text_.body
+        return text_.as_text() if isinstance(text_, ParserInput) else text_.body
 
     def _out(self, name, priority, status, source):
         debug = f"name={name}"
@@ -33,7 +34,7 @@ class BaseTestParser(BaseParser):
 
     def parse(self, context, text):
         self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        value = self._get_name(self.name) + ":" + self._get_source(text)
         parser_result = ParserResultConcept(parser=self, value=value)
         return ReturnValueConcept(self, self.status, self.parser_result or parser_result)
 
@@ -54,8 +55,8 @@ class Enabled80MultipleFalseParser(BaseTestParser):
 
     def parse(self, context, text):
         self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
-        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        value1 = self._get_name(self.name) + ":" + self._get_source(text) + "_1"
+        value2 = self._get_name(self.name) + ":" + self._get_source(text) + "_2"
         return [
             ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)),
             ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)),
@@ -68,8 +69,8 @@ class Enabled80MultipleTrueParser(BaseTestParser):
 
     def parse(self, context, text):
         self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
-        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
-        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        value1 = self._get_name(self.name) + ":" + self._get_source(text) + "_1"
+        value2 = self._get_name(self.name) + ":" + self._get_source(text) + "_2"
         return [
             ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)),
             ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)),
@@ -90,7 +91,7 @@ class Enabled50TrueParser(BaseTestParser):
         status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world"
         self._out(self._get_name(self.name), self.priority, status, source)
 
-        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        value = self._get_name(self.name) + ":" + self._get_source(text)
         return_value = ParserResultConcept(parser=self, value=value)
         return ReturnValueConcept(self, status, return_value)
 

tests/evaluators/test_LexerNodeEvaluator.py

@@ -3,6 +3,7 @@ import ast
 import pytest
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.concept import Concept, ConceptParts, DoNotResolve
+from core.sheerka.services.SheerkaExecute import ParserInput
 from evaluators.LexerNodeEvaluator import LexerNodeEvaluator
 from parsers.BaseNodeParser import SourceCodeNode
 from parsers.BnfNodeParser import ConceptNode, BnfNodeParser, UnrecognizedTokensNode
@@ -17,7 +18,7 @@ class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka):
         parser = BnfNodeParser()
         parser.init_from_concepts(context, concepts)
 
-        ret_val = parser.parse(context, expression)
+        ret_val = parser.parse(context, ParserInput(expression))
         assert ret_val.status
         return ret_val
 

tests/evaluators/test_PythonEvaluator.py

@@ -2,6 +2,7 @@ import pytest
 
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_DEF
+from core.sheerka.services.SheerkaExecute import ParserInput
 from evaluators.PythonEvaluator import PythonEvaluator
 from parsers.PythonParser import PythonNode, PythonParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -31,7 +32,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_eval(self, text, expected):
         context = self.get_context()
-        parsed = PythonParser().parse(context, text)
+        parsed = PythonParser().parse(context, ParserInput(text))
 
         evaluated = PythonEvaluator().eval(context, parsed)
 
@@ -40,7 +41,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_eval_using_context(self):
         context = self.get_context()
-        parsed = PythonParser().parse(context, "test_using_context('value for param1', 10)")
+        parsed = PythonParser().parse(context, ParserInput("test_using_context('value for param1', 10)"))
 
         evaluated = PythonEvaluator().eval(context, parsed)
 
@@ -49,7 +50,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
 
     def test_i_can_eval_using_context_when_self_is_not_sheerka(self):
         sheerka, context = self.init_concepts()
-        parsed = PythonParser().parse(context, "create_new_concept(Concept('foo'))")
+        parsed = PythonParser().parse(context, ParserInput("create_new_concept(Concept('foo'))"))
 
         evaluated = PythonEvaluator().eval(context, parsed)
 
@@ -66,7 +67,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         context.sheerka.add_in_cache(Concept("foo"))
 
-        parsed = PythonParser().parse(context, "foo")
+        parsed = PythonParser().parse(context, ParserInput("foo"))
         evaluated = PythonEvaluator().eval(context, parsed)
 
         assert not evaluated.status
@@ -80,7 +81,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         context.sheerka.add_in_cache(Concept("foo", body="1"))
 
-        parsed = PythonParser().parse(context, "foo + 2")
+        parsed = PythonParser().parse(context, ParserInput("foo + 2"))
         evaluated = PythonEvaluator().eval(context, parsed)
 
         assert evaluated.status
@@ -94,7 +95,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         context.sheerka.add_in_cache(Concept("foo"))
 
-        parsed = PythonParser().parse(context, "def a(b):\n    return b\na(c:foo:)")
+        parsed = PythonParser().parse(context, ParserInput("def a(b):\n    return b\na(c:foo:)"))
         evaluated = PythonEvaluator().eval(context, parsed)
 
         assert evaluated.status
@@ -108,7 +109,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         context.sheerka.add_in_cache(Concept("foo", body="2"))
 
-        parsed = PythonParser().parse(context, "def a(b):\n    return b\na(foo)")
+        parsed = PythonParser().parse(context, ParserInput("def a(b):\n    return b\na(foo)"))
         evaluated = PythonEvaluator().eval(context, parsed)
 
         assert evaluated.status
@@ -118,7 +119,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         context.sheerka.add_in_cache(Concept("foo", body="2"))
 
-        parsed = PythonParser().parse(context, "get_concept_name(c:foo:)")
+        parsed = PythonParser().parse(context, ParserInput("get_concept_name(c:foo:)"))
         python_evaluator = PythonEvaluator()
         python_evaluator.locals["get_concept_name"] = get_concept_name
         evaluated = python_evaluator.eval(context, parsed)
@@ -127,7 +128,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.value == "foo"
 
         # sanity, does not work otherwise
-        parsed = PythonParser().parse(context, "get_concept_name(foo)")
+        parsed = PythonParser().parse(context, ParserInput("get_concept_name(foo)"))
         python_evaluator = PythonEvaluator()
         python_evaluator.locals["get_concept_name"] = get_concept_name
         evaluated = python_evaluator.eval(context, parsed)
@@ -141,7 +142,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
             self.from_def_concept("mult", "a mult b", ["a", "b"]),
         )
 
-        parsed = PythonParser().parse(context, "is_greater_than(BuiltinConcepts.PRECEDENCE, mult, plus)")
+        parsed = PythonParser().parse(context, ParserInput("is_greater_than(BuiltinConcepts.PRECEDENCE, mult, plus)"))
         python_evaluator = PythonEvaluator()
 
         evaluated = python_evaluator.eval(context, parsed)

tests/non_reg/test_sheerka_non_reg.py

@@ -100,7 +100,7 @@ as:
     def func(x,y):
         return x+y
     func(a,b)
-        """
+"""
 
         expected = self.get_default_concept()
         expected.metadata.id = "1001"
@@ -167,7 +167,7 @@ as:
     def func(x,y):
         return x+y
     func(a,b)
-            """
+"""
 
         sheerka = self.get_sheerka()
         sheerka.evaluate_user_input(text)

tests/parsers/test_AtomsParser.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_DEF
+from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.AtomNodeParser import AtomNodeParser
 from parsers.BaseNodeParser import cnode, utnode, CNC, SCN, CN
 
@@ -26,7 +27,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
     def test_i_cannot_parse_empty_string(self):
         sheerka, context, parser = self.init_parser({})
 
-        res = parser.parse(context, "")
+        res = parser.parse(context, ParserInput(""))
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
@@ -49,7 +50,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -72,7 +73,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -110,7 +111,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -137,7 +138,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -158,7 +159,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        list_of_res = parser.parse(context, text)
+        list_of_res = parser.parse(context, ParserInput(text))
         assert len(list_of_res) == len(expected)
 
         for i, res in enumerate(list_of_res):
@@ -180,7 +181,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
 
         text = "one two x$!# one two"
         sheerka, context, parser = self.init_parser(concepts_map)
-        list_of_res = parser.parse(context, text)
+        list_of_res = parser.parse(context, ParserInput(text))
 
         expected = [
             (False, ["one", "two", " x$!# ", ("one", 1), ("two", 1)]),
@@ -210,7 +211,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser(concepts_map)
         sheerka.set_isa(context, sheerka.new("one"), sheerka.new("number"))
 
-        res = parser.parse(context, "one")
+        res = parser.parse(context, ParserInput("one"))
         lexer_nodes = res.body.body
         expected_array = compute_expected_array(concepts_map, "one", ["one"])
         assert lexer_nodes == expected_array
@@ -223,16 +224,36 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
 
         sheerka, context, parser = self.init_parser(concepts_map)
 
-        res = parser.parse(context, "a special concept")
+        res = parser.parse(context, ParserInput("a special concept"))
         lexer_nodes = res.body.body
         expected_array = compute_expected_array(concepts_map, "a special concept", ["a special concept"])
         assert lexer_nodes == expected_array
 
-        res = parser.parse(context, "isa")
+        res = parser.parse(context, ParserInput("isa"))
         lexer_nodes = res.body.body
         expected_array = compute_expected_array(concepts_map, "isa", ["isa"])
         assert lexer_nodes == expected_array
 
+    def test_i_can_parse_concepts_when_sub_tokens(self):
+        concepts_map = {
+            "foo": Concept("foo"),
+            "bar": Concept("bar"),
+        }
+
+        sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
+        text = "not recognized foo bar not not recognized"
+        expected = ["foo", "bar"]
+        parser_input = ParserInput(text, start=3, end=7)
+        res = parser.parse(context, parser_input)
+        wrapper = res.body
+        lexer_nodes = res.body.body
+
+        assert res.status
+
+        expected_array = compute_expected_array(concepts_map, text, expected)
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert lexer_nodes == expected_array
+
     @pytest.mark.parametrize("text", [
         "foo",
         f"foo one",
@@ -249,7 +270,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@@ -270,7 +291,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map, create_new=True, singleton=False)
-        list_of_res = parser.parse(context, text)
+        list_of_res = parser.parse(context, ParserInput(text))
 
         assert len(list_of_res) == len(expected)
 
@@ -293,7 +314,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -321,7 +342,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         lexer_nodes = res.body.body
 
         assert res.status

tests/parsers/test_BaseParser.py

@@ -81,3 +81,4 @@ def test_i_can_test_split_iter_parser_indexes():
 ])
 def test_i_can_get_tokens_boundaries(tokens, expected):
     assert BaseParser.get_tokens_boundaries(tokens) == expected
+

tests/parsers/test_BnfNodeParser.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DoNotResolve, DEFINITION_TYPE_DEF, CC
+from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.BaseNodeParser import CNC, UTN, CN
 from parsers.BnfNodeParser import BnfNodeParser, StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
     Optional, ZeroOrMore, OneOrMore, ConceptExpression
@@ -95,7 +96,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         parser = BnfNodeParser()
         parser.init_from_concepts(context, updated)
-        parser.reset_parser(context, text)
+        parser.reset_parser(context, ParserInput(text))
 
         bnf_parsers_helpers = parser.get_concepts_sequences()
 
@@ -113,34 +114,11 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             my_map, text, expected, multiple_result, post_init_concepts
         )
         return sequences
-        # sheerka, context, *updated = self.init_concepts(*my_map.values(), create_new=False, singleton=True)
-        # if not multiple_result:
-        #     expected_array = [compute_expected_array(my_map, text, expected)]
-        # else:
-        #     expected_array = [compute_expected_array(my_map, text, e) for e in expected]
-        #
-        # if post_init_concepts:
-        #     post_init_concepts(sheerka, context)
-        #
-        # parser = BnfNodeParser()
-        # parser.init_from_concepts(context, updated)
-        # parser.reset_parser(context, text)
-        #
-        # bnf_parsers_helpers = parser.get_concepts_sequences()
-        #
-        # assert len(bnf_parsers_helpers) == len(expected_array)
-        # for parser_helper, expected_sequence in zip(bnf_parsers_helpers, expected_array):
-        #     assert parser_helper.sequence == expected_sequence
-        #
-        # if len(bnf_parsers_helpers) == 1:
-        #     return bnf_parsers_helpers[0].sequence
-        # else:
-        #     return [pe.sequence for pe in bnf_parsers_helpers]
 
     def test_i_cannot_parse_empty_strings(self):
         sheerka, context, parser = self.init_parser({}, singleton=True)
 
-        res = parser.parse(context, "")
+        res = parser.parse(context, ParserInput(""))
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@@ -738,7 +716,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser(my_map, singleton=True)
         parser.init_from_concepts(context, my_map.values())
 
-        parser.reset_parser(context, "one three")
+        parser.reset_parser(context, ParserInput("one three"))
         sequences = parser.get_concepts_sequences()
         sequence = parser.get_valid(sequences)
 
@@ -756,7 +734,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_parse(self, parser_input, expected_status, expected):
         sheerka, context, parser = self.init_parser(init_from_sheerka=True)
 
-        res = parser.parse(context, parser_input)
+        res = parser.parse(context, ParserInput(parser_input))
         expected_array = compute_expected_array(cmap, parser_input, expected)
         parser_result = res.value
         concepts_nodes = res.value.value
@@ -771,7 +749,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         parser_input = "def one"
         expected = [CNC("def number", source="def one", number="one", one="one")]
 
-        res = parser.parse(context, parser_input)
+        res = parser.parse(context, ParserInput(parser_input))
         expected_array = compute_expected_array(cmap, parser_input, expected)
         expected_array[0].compiled["def"] = cmap["def_only"]
 

tests/parsers/test_BnfParser.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_BNF
+from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
 from parsers.BaseNodeParser import cnode
 from parsers.BaseParser import UnexpectedTokenErrorNode
@@ -135,7 +136,6 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("1|", UnexpectedEndOfFileError()),
         ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.RPAR])),
         ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.IDENTIFIER])),
-        ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
     ])
     def test_i_can_detect_errors(self, expression, error):
         sheerka, context, parser = self.init_parser()
@@ -146,6 +146,17 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert ret_value[0] == error
 
+    def test_i_can_detect_lexer_error(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, Tokenizer("'name"))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert isinstance(res.body.body[0], LexerError)
+        assert res.body.body[0].message == "Missing Trailing quote"
+        assert res.body.body[0].text == "'name"
+
     def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text(self):
         sheerka, context, regex_parser, foo, bar = self.init_parser("foo", "bar")
 
@@ -158,15 +169,15 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         bnf_parser = BnfNodeParser()
         bnf_parser.init_from_concepts(context, [foo, bar])
 
-        res = bnf_parser.parse(context, "twenty two")
+        res = bnf_parser.parse(context, ParserInput("twenty two"))
         assert res.status
         assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
 
-        res = bnf_parser.parse(context, "thirty one")
+        res = bnf_parser.parse(context, ParserInput("thirty one"))
         assert res.status
         assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
 
-        res = bnf_parser.parse(context, "twenty")
+        res = bnf_parser.parse(context, ParserInput("twenty"))
         assert res.status
         assert res.value.body == [cnode("foo", 0, 0, "twenty")]
 

tests/parsers/test_DefaultParser.py

@@ -1,8 +1,10 @@
 import ast
+from dataclasses import dataclass
 
 import pytest
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
+from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch
 from parsers.BnfParser import BnfParser
@@ -39,7 +41,7 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition
 
 def get_concept_part(part):
     if isinstance(part, str):
-        node = PythonNode(part, ast.parse(part, mode="eval"))
+        node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval"))
         return ReturnValueConcept(
             who="parsers.Default",
             status=True,
@@ -48,6 +50,16 @@ def get_concept_part(part):
                 parser=PythonParser(),
                 value=node))
 
+    if isinstance(part, PN):
+        node = PythonNode(part.source.strip(), ast.parse(part.source.strip(), mode=part.mode))
+        return ReturnValueConcept(
+            who="parsers.Default",
+            status=True,
+            value=ParserResultConcept(
+                source=part.source,
+                parser=PythonParser(),
+                value=node))
+
     if isinstance(part, PythonNode):
         return ReturnValueConcept(
             who="parsers.Default",
@@ -61,6 +73,12 @@ def get_concept_part(part):
         return part
 
 
+@dataclass
+class PN:
+    source: str  # parser result source
+    mode: str  # compilation mode
+
+
 class TestDefaultParser(TestUsingMemoryBasedSheerka):
 
     def init_parser(self, *concepts):
@@ -82,7 +100,7 @@ class TestDefaultParser(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_parse_def_concept(self, text, expected):
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         node = res.value.value
 
         assert res.status
@@ -99,14 +117,14 @@ post isinstance(res, int)
 as res = a + b
     """
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         return_value = res.value
         expected_concept = get_def_concept(
             name="a plus b",
-            where="a,b",
-            pre="isinstance(a, int) and isinstance(b, float)",
-            post="isinstance(res, int)",
-            body=PythonNode("res = a + b", ast.parse("res = a + b", mode="exec"))
+            where="a,b\n",
+            pre="isinstance(a, int) and isinstance(b, float)\n",
+            post="isinstance(res, int)\n",
+            body=PN("res = a + b\n    ", "exec")
         )
 
         assert res.status
@@ -119,17 +137,15 @@ def concept add one to a as
 def func(x):
     return x+1
 func(a)
-        """
+"""
 
         expected_concept = get_def_concept(
             name="add one to a ",
-            body=PythonNode(
-                "def func(x):\n    return x+1\nfunc(a)",
-                ast.parse("def func(x):\n    return x+1\nfunc(a)", mode="exec"))
+            body=PN("def func(x):\n    return x+1\nfunc(a)\n", "exec")
         )
 
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         return_value = res.value
 
         assert res.status
@@ -141,8 +157,7 @@ func(a)
 def concept add one to a as:
     def func(x):
         return x+1
-    func(a)
-    """
+    func(a)"""
 
         expected_concept = get_def_concept(
             name="add one to a ",
@@ -152,7 +167,7 @@ def concept add one to a as:
         )
 
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         return_value = res.value
 
         assert res.status
@@ -168,7 +183,7 @@ func(a)
     """
 
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         return_value = res.value
 
         assert not res.status
@@ -184,7 +199,7 @@ def concept add one to a as
     func(a)
         """
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         return_value = res.value
 
         assert not res.status
@@ -194,7 +209,7 @@ def concept add one to a as
         text = "def concept as 'hello'"
 
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         return_value = res.value
 
         assert not res.status
@@ -205,9 +220,9 @@ def concept add one to a as
     def test_concept_keyword_is_mandatory_but_the_concept_is_recognized(self):
         text = "def hello as a where b pre c post d"
 
-        expected_concept = get_def_concept(name="hello", body="a", where="b", pre="c", post="d")
+        expected_concept = get_def_concept(name="hello", body="a ", where="b ", pre="c ", post="d")
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         return_value = res.value
 
         assert not res.status
@@ -225,7 +240,7 @@ def concept add one to a as
     ])
     def test_i_can_detect_error_in_declaration(self, text):
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         return_value = res.value
 
         assert not res.status
@@ -235,7 +250,7 @@ def concept add one to a as
         text = "def concept hello \n my friend as 'hello'"
 
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         return_value = res.value
 
         assert not res.status
@@ -244,7 +259,7 @@ def concept add one to a as
     def test_i_can_parse_def_concept_from_bnf(self):
         text = "def concept name from bnf a_concept | 'a_string' as __definition[0]"
         sheerka, context, parser, a_concept = self.init_parser("a_concept")
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         node = res.value.value
         definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
@@ -260,7 +275,7 @@ def concept add one to a as
     def test_i_can_parse_def_concept_where_bnf_references_itself(self):
         text = "def concept name from bnf 'a' + name?"
         sheerka, context, parser, a_concept = self.init_parser("a_concept")
-        parser.parse(context, text)
+        parser.parse(context, ParserInput(text))
 
         assert not parser.has_error
 
@@ -271,7 +286,7 @@ def concept add one to a as
     ])
     def test_i_can_detect_empty_bnf_declaration(self, text):
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         assert not res.status
         assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration")
@@ -281,7 +296,7 @@ def concept add one to a as
         "def concept addition from def a plus b as a + b"])
     def test_i_can_def_concept_from_definition(self, text):
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         expected = get_def_concept("addition", definition="a plus b", body="a + b")
         node = res.value.value
 
@@ -294,7 +309,7 @@ def concept add one to a as
     def test_i_can_detect_not_for_me(self):
         text = "hello world"
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         assert not res.status
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
@@ -303,7 +318,7 @@ def concept add one to a as
     def test_i_can_parse_is_a(self):
         text = "the name of my 'concept' isa the name of the set"
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         expected = IsaConceptNode([],
                                   concept=NameNode(list(Tokenizer("the name of my 'concept'"))),
                                   set=NameNode(list(Tokenizer("the name of the set"))))
@@ -323,7 +338,7 @@ def concept add one to a as
     ])
     def test_i_cannot_parse_invalid_entries(self, text):
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         assert not res.status
         assert isinstance(res.body, ParserResultConcept)
@@ -339,10 +354,10 @@ def concept add one to a as
     ])
     def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text):
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         assert not res.status
-        assert isinstance(res.body, ParserResultConcept)
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert isinstance(res.body.body[0], LexerError)
         assert res.body.body[0].message == error_msg
         assert res.body.body[0].text == error_text
@@ -350,7 +365,7 @@ def concept add one to a as
     def test_i_cannot_parse_bnf_definition_referencing_unknown_concept(self):
         text = "def concept name from bnf unknown"
         sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         assert not res.status
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)

tests/parsers/test_ExactConceptParser.py

@@ -1,5 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, CMV
+from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
 from parsers.ExactConceptParser import ExactConceptParser
 
@@ -55,7 +56,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         concept = self.create_and_add_in_cache_concept(sheerka, "hello world")
 
         source = "hello world"
-        results = ExactConceptParser().parse(context, source)
+        results = ExactConceptParser().parse(context, ParserInput(source))
         concept_found = results[0].value.value
 
         assert len(results) == 1
@@ -71,7 +72,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         self.create_and_add_in_cache_concept(sheerka, "hello a", variables=["a"])
 
         source = "hello world"
-        results = ExactConceptParser().parse(context, source)
+        results = ExactConceptParser().parse(context, ParserInput(source))
 
         assert len(results) == 2
         results = sorted(results, key=lambda x: x.value.value.name)  # because of the usage of sets
@@ -91,7 +92,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         concept = self.create_and_add_in_cache_concept(sheerka, "a + b", ["a", "b"])
 
         source = "10 + 5"
-        results = ExactConceptParser().parse(context, source)
+        results = ExactConceptParser().parse(context, ParserInput(source))
 
         assert len(results) == 1
         assert results[0].status
@@ -107,7 +108,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         concept = self.create_and_add_in_cache_concept(sheerka, "a + b + a", ["a", "b"])
 
         source = "10 + 5 + 10"
-        results = ExactConceptParser(max_word_size=5).parse(context, source)
+        results = ExactConceptParser(max_word_size=5).parse(context, ParserInput(source))
 
         assert len(results) == 1
         assert results[0].status
@@ -122,7 +123,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         )
 
         source = "10 + 5"
-        results = ExactConceptParser().parse(context, source)
+        results = ExactConceptParser().parse(context, ParserInput(source))
         concept_found = results[0].value.value
 
         assert len(results) == 1
@@ -135,7 +136,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         sheerka, context, foo = self.init_concepts("foo")
 
         source = "c:foo:"
-        results = ExactConceptParser().parse(context, source)
+        results = ExactConceptParser().parse(context, ParserInput(source))
         concept_found = results[0].value.value
 
         assert len(results) == 1
@@ -152,7 +153,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         )
 
         source = "c:one: plus c:two:"
-        results = ExactConceptParser().parse(context, source)
+        results = ExactConceptParser().parse(context, ParserInput(source))
         concept_found = results[0].value.value
 
         assert len(results) == 1
@@ -168,7 +169,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         )
 
         source = "z is a concept"
-        results = ExactConceptParser().parse(context, source)
+        results = ExactConceptParser().parse(context, ParserInput(source))
         concept_found = results[0].value.value
 
         assert len(results) == 1
@@ -178,7 +179,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert not concept_found.metadata.is_evaluated
 
         source = "def concept z"
-        results = ExactConceptParser().parse(context, source)
+        results = ExactConceptParser().parse(context, ParserInput(source))
         concept_found = results[0].value.value
 
         assert len(results) == 1
@@ -190,7 +191,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
     def test_i_can_manage_unknown_concept(self):
         context = self.get_context(self.get_sheerka(singleton=True))
         source = "def concept hello"  # this is not a concept by itself
-        res = ExactConceptParser().parse(context, source)
+        res = ExactConceptParser().parse(context, ParserInput(source))
 
         assert not res.status
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
@@ -199,7 +200,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
     def test_i_can_detect_concepts_too_long(self):
         context = self.get_context(self.get_sheerka(singleton=True))
         source = "a very but finally too long concept"
-        res = ExactConceptParser().parse(context, source)
+        res = ExactConceptParser().parse(context, ParserInput(source))
 
         assert not res.status
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
@@ -207,15 +208,15 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka):
         assert res.value.reason.body == source
         assert res.value.body == source
 
-    def test_i_can_detect_concept_from_tokens(self):
-        context = self.get_context(self.get_sheerka(singleton=True))
-        concept = get_concept("hello world", [])
-        context.sheerka.add_in_cache(concept)
-
-        source = "hello world"
-        results = ExactConceptParser().parse(context, list(Tokenizer(source)))
-
-        assert len(results) == 1
-        assert results[0].status
-        assert results[0].value.value == concept
-        assert not results[0].value.value.metadata.need_validation
+    # def test_i_can_detect_concept_from_tokens(self):
+    #     context = self.get_context(self.get_sheerka(singleton=True))
+    #     concept = get_concept("hello world", [])
+    #     context.sheerka.add_in_cache(concept)
+    #
+    #     source = "hello world"
+    #     results = ExactConceptParser().parse(context, list(Tokenizer(source)))
+    #
+    #     assert len(results) == 1
+    #     assert results[0].status
+    #     assert results[0].value.value == concept
+    #     assert not results[0].value.value.metadata.need_validation

tests/parsers/test_PythonParser.py

@@ -3,7 +3,8 @@ import ast
 import core.utils
 import pytest
 from core.builtin_concepts import ParserResultConcept, NotForMeConcept
-from core.tokenizer import Tokenizer, LexerError
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import LexerError
 from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -17,21 +18,7 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_parse_a_simple_expression(self, text, expected):
         parser = PythonParser()
-        res = parser.parse(self.get_context(), text)
-
-        assert res.status
-        assert res.who == parser.name
-        assert isinstance(res.value, ParserResultConcept)
-        assert res.value.value == expected
-
-    @pytest.mark.parametrize("text, expected", [
-        ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))),
-        ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))),
-    ])
-    def test_i_can_parse_from_tokens(self, text, expected):
-        parser = PythonParser()
-        tokens = list(Tokenizer(text))
-        res = parser.parse(self.get_context(), tokens)
+        res = parser.parse(self.get_context(), ParserInput(text))
 
         assert res.status
         assert res.who == parser.name
@@ -40,12 +27,11 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("text", [
         "1+",
-        "'name",
-        "foo = 'name"
+        "&#é",
     ])
-    def test_i_can_detect_error(self, text):
+    def test_i_can_detect_python_error(self, text):
         parser = PythonParser()
-        res = parser.parse(self.get_context(), text)
+        res = parser.parse(self.get_context(), ParserInput(text))
 
         assert not res.status
         assert res.who == parser.name
@@ -61,7 +47,7 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_detect_lexer_errors(self, text, error_msg, error_text):
         parser = PythonParser()
-        res = parser.parse(self.get_context(), text)
+        res = parser.parse(self.get_context(), ParserInput(text))
 
         assert not res.status
         assert isinstance(res.value, NotForMeConcept)
@@ -76,12 +62,13 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
         text = "c:name|id: + 1"
 
         parser = PythonParser()
-        res = parser.parse(self.get_context(), text)
+        res = parser.parse(self.get_context(), ParserInput(text))
         encoded = core.utils.encode_concept(("name", "id"))
 
         assert res
+        assert res.value.source == "c:name|id: + 1"
         assert res.value.value == PythonNode(
-            "c:name|id: + 1",
+            "__C__KEY_name__ID_id__C__ + 1",
             ast.parse(encoded + "+1", mode="eval"))
         assert res.value.value.concepts == {
             encoded: ("name", "id")

tests/parsers/test_PythonWithConceptsParser.py

@@ -3,6 +3,7 @@ import ast
 import pytest
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept
+from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Token, TokenKind, Tokenizer
 from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode
 from parsers.PythonParser import PythonNode
@@ -38,6 +39,7 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("text, interested", [
         ("not parser result", False),
+        (ParserInput("not parser result"), False),
         (ParserResultConcept(parser="not multiple_concepts_parser"), False),
         (ParserResultConcept(parser=unrecognized_nodes_parser, value=[UnrecognizedTokensNode(0, 0, [])]), True),
     ])

tests/parsers/test_SyaNodeParser.py

@@ -2,6 +2,7 @@ import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, CC
 from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager
+from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
 from parsers.BaseNodeParser import utnode, ConceptNode, cnode, short_cnode, UnrecognizedTokensNode, \
     SCWC, CNC, UTN
@@ -198,7 +199,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_post_fix_simple_infix_concepts(self, expression, expected_sequences):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_sequences)
         for res_i, expected in zip(res, expected_sequences):
@@ -224,7 +225,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         }
         sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_sequences)
         for res_i, expected in zip(res, expected_sequences):
@@ -248,7 +249,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_post_fix_simple_prefixed_concepts(self, expression, expected_sequences):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_sequences)
         for res_i, expected in zip(res, expected_sequences):
@@ -291,7 +292,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         }
         sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_sequences)
         for res_i, expected in zip(res, expected_sequences):
@@ -315,7 +316,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_post_fix_simple_suffixed_concepts(self, expression, expected_sequences):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_sequences)
         for res_i, expected in zip(res, expected_sequences):
@@ -337,7 +338,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         }
         sheerka, context, parser = self.init_parser(concepts_map, None)
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(concepts_map, expression, expected)
         assert len(res) == 1
         assert res[0].out == expected_array
@@ -377,7 +378,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         sheerka, context, parser = self.init_parser()
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_sequences)
         for res_i, expected in zip(res, expected_sequences):
@@ -409,7 +410,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         }
         sheerka, context, parser = self.init_parser(concepts_map, create_new=True, init_from_sheerka=True)
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_sequences)
         for res_i, expected in zip(res, expected_sequences):
@@ -430,7 +431,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         }
         sheerka, context, parser = self.init_parser(concepts_map, None)
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(concepts_map, expression, expected)
 
         assert len(res) == 1
@@ -453,7 +454,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         }
         sheerka, context, parser = self.init_parser(concepts_map, sya_def)
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(concepts_map, expression, expected)
 
         assert len(res) == 1
@@ -468,7 +469,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_post_fix_binary_with_precedence(self, expression, expected):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(cmap, expression, expected)
 
         assert len(res) == 1
@@ -489,7 +490,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         expression = "suffixed a prefixed"
         expected = ["a", "prefixed", "suffixed"]
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(concepts_map, expression, expected)
 
         assert len(res) == 1
@@ -504,7 +505,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         expression = "suffixed a prefixed"
         expected = ["a", "suffixed", "prefixed"]
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(concepts_map, expression, expected)
 
         assert len(res) == 1
@@ -524,7 +525,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser(concepts_map, sya_def)
 
         expression = "one equals two equals three"
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         expected = ["one", "two", "three", ("equals", 1), "equals"]
         expected_array = compute_expected_array(concepts_map, expression, expected)
@@ -546,7 +547,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser(concepts_map, sya_def)
 
         expression = "one plus two plus three"
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         expected = ["one", "two", "plus", "three", ("plus", 1)]
         expected_array = compute_expected_array(concepts_map, expression, expected)
@@ -576,7 +577,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         }
         sheerka, context, parser = self.init_parser(concepts_map, sya_def)
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(concepts_map, expression, expected)
         assert len(res) == 1
         assert res[0].out == expected_array
@@ -603,7 +604,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         }
         sheerka, context, parser = self.init_parser(concepts_map, sya_def)
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(concepts_map, expression, expected)
         assert len(res) == 1
         assert res[0].out == expected_array
@@ -617,7 +618,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser(concepts_map, None)
 
         expression = "foo bar baz"
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_sequences = [
             [UTN("bar "), "foo", "baz"],
             ["baz", "foo bar"]
@@ -664,7 +665,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_pos_fix_when_parenthesis(self, expression, expected):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(cmap, expression, expected)
 
         assert len(res) == 1
@@ -719,7 +720,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_post_fix_when_parenthesis_and_unknown(self, expression, expected_sequences):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == len(expected_sequences)
         for res_i, expected in zip(res, expected_sequences):
@@ -745,7 +746,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_detect_parenthesis_mismatch_error_when_post_fixing(self, expression, expected):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == 1
         assert res[0].errors == [expected]
@@ -756,7 +757,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_detected_when_too_many_parameters(self, expression, expected):
         sheerka, context, parser = self.init_parser(cmap, None)
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         assert len(res) == 1
         assert len(res[0].errors) == 1
@@ -790,7 +791,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_post_fix_sequences(self, expression, expected):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(cmap, expression, expected)
 
         assert len(res) == 1
@@ -808,7 +809,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser()
 
         expected = ["one", "two", "three", "?"]
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = compute_expected_array(cmap, expression, expected)
 
         assert len(res) == 1
@@ -830,7 +831,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser(concepts_map, sya_def)
 
         expression = "a plus plus equals b"
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
         expected_array = tests.parsers.parsers_utils.compute_debug_array(res)
         assert expected_array == [
             ["T(a)", "C(a plus b)", "C(a plus b)", "T(equals)", "T(b)"],
@@ -859,9 +860,9 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser(concepts_map, sya_def)
 
         expression = "a plus complex infix b"
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
-        res = parser.parse(context, expression)
+        res = parser.parse(context, ParserInput(expression))
         pass
 
     def test_i_can_use_string_instead_of_identifier(self):
@@ -874,7 +875,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         sheerka, context, parser = self.init_parser(concepts_map, None)
 
-        res = parser.infix_to_postfix(context, "one ? ? two '::' three")
+        res = parser.infix_to_postfix(context, ParserInput("one ? ? two '::' three"))
         assert len(res) == 1
         assert res[0].out == [
             cnode("one", start=0, end=0, source="one"),
@@ -896,7 +897,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
 
         sheerka, context, parser = self.init_parser(concepts_map, sya_def)
 
-        res = parser.infix_to_postfix(context, "one less than two less than three")
+        res = parser.infix_to_postfix(context, ParserInput("one less than two less than three"))
         assert len(res) == 1
         assert res[0].errors == [NoneAssociativeSequenceErrorNode(concepts_map["less than"], 2, 8)]
 
@@ -909,7 +910,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser()
 
         expression = "suffixed twenties"
-        res = parser.infix_to_postfix(context, expression)
+        res = parser.infix_to_postfix(context, ParserInput(expression))
 
         expected = [cnode("twenties", 2, 2, "twenties"), "suffixed"]
         expected_array = compute_expected_array(cmap, expression, expected)
@@ -920,7 +921,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser()
 
         text = "one plus two mult three"
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -939,7 +940,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser()
 
         text = "suffixed 1 + 1"
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -962,7 +963,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser()
 
         text = "suffixed twenty one"
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         assert len(res) == 2
         assert res[1].status
 
@@ -981,7 +982,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser()
 
         text = "one plus 1 + 1 suffixed two"
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -1020,7 +1021,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_can_parse_when_one_result(self, text, expected_status, expected_result):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -1043,7 +1044,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         """
         sheerka, context, parser = self.init_parser()
 
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@@ -1068,7 +1069,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         """
         sheerka, context, parser = self.init_parser()
 
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         wrapper = res.body
         lexer_nodes = res.body.body
@@ -1088,7 +1089,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_cannot_parse_when_unrecognized(self, text, expected_concept, expected_unrecognized):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
         expected_end = len(list(Tokenizer(text))) - 2
@@ -1108,7 +1109,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_cannot_parse_when_part_of_the_sequence_is_not_recognized(self, text, expected):
         sheerka, context, parser = self.init_parser()
 
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -1132,7 +1133,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         """
         sheerka, context, parser = self.init_parser()
 
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@@ -1141,7 +1142,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
     def test_i_cannot_parse_empty_string(self):
         sheerka, context, parser = self.init_parser({}, None)
 
-        res = parser.parse(context, "")
+        res = parser.parse(context, ParserInput(""))
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)

tests/parsers/test_UnrecognizedNodeParser.py

@@ -135,7 +135,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(concept.compiled["d"][0], BuiltinConcepts.RETURN_VALUE)
         assert concept.compiled["d"][0].status
         assert concept.compiled["d"][0].who == "parsers.Python"
-        assert concept.compiled["d"][0].body.source == "1 + 2"
+        assert concept.compiled["d"][0].body.source == " 1 + 2 "
 
         assert len(concept.compiled["e"]) == 1
         assert sheerka.isinstance(concept.compiled["e"][0], BuiltinConcepts.RETURN_VALUE)
@@ -171,13 +171,13 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
         assert len(res.body.concept.compiled["a"]) == 1
         assert res.body.concept.compiled["a"][0].status
         assert res.body.concept.compiled["a"][0].who == "parsers.Python"
-        assert res.body.concept.compiled["a"][0].body.source == "1"
+        assert res.body.concept.compiled["a"][0].body.source == "1 "
 
         assert res.body.concept.compiled["b"] == concepts_map["mult"]
         assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["a"][0], BuiltinConcepts.RETURN_VALUE)
         assert res.body.concept.compiled["b"].compiled["a"][0].status
         assert res.body.concept.compiled["b"].compiled["a"][0].who == "parsers.Python"
-        assert res.body.concept.compiled["b"].compiled["a"][0].body.source == "2"
+        assert res.body.concept.compiled["b"].compiled["a"][0].body.source == " 2 "
 
         assert sheerka.isinstance(res.body.concept.compiled["b"].compiled["b"][0], BuiltinConcepts.RETURN_VALUE)
         assert res.body.concept.compiled["b"].compiled["b"][0].status