Introduced ParserInput

tests/parsers/test_AtomsParser.py

@@ -1,6 +1,7 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_DEF
+from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.AtomNodeParser import AtomNodeParser
 from parsers.BaseNodeParser import cnode, utnode, CNC, SCN, CN
 
@@ -26,7 +27,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
     def test_i_cannot_parse_empty_string(self):
         sheerka, context, parser = self.init_parser({})
 
-        res = parser.parse(context, "")
+        res = parser.parse(context, ParserInput(""))
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
@@ -49,7 +50,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -72,7 +73,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -110,7 +111,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -137,7 +138,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -158,7 +159,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        list_of_res = parser.parse(context, text)
+        list_of_res = parser.parse(context, ParserInput(text))
         assert len(list_of_res) == len(expected)
 
         for i, res in enumerate(list_of_res):
@@ -180,7 +181,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
 
         text = "one two x$!# one two"
         sheerka, context, parser = self.init_parser(concepts_map)
-        list_of_res = parser.parse(context, text)
+        list_of_res = parser.parse(context, ParserInput(text))
 
         expected = [
             (False, ["one", "two", " x$!# ", ("one", 1), ("two", 1)]),
@@ -210,7 +211,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         sheerka, context, parser = self.init_parser(concepts_map)
         sheerka.set_isa(context, sheerka.new("one"), sheerka.new("number"))
 
-        res = parser.parse(context, "one")
+        res = parser.parse(context, ParserInput("one"))
         lexer_nodes = res.body.body
         expected_array = compute_expected_array(concepts_map, "one", ["one"])
         assert lexer_nodes == expected_array
@@ -223,16 +224,36 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
 
         sheerka, context, parser = self.init_parser(concepts_map)
 
-        res = parser.parse(context, "a special concept")
+        res = parser.parse(context, ParserInput("a special concept"))
         lexer_nodes = res.body.body
         expected_array = compute_expected_array(concepts_map, "a special concept", ["a special concept"])
         assert lexer_nodes == expected_array
 
-        res = parser.parse(context, "isa")
+        res = parser.parse(context, ParserInput("isa"))
         lexer_nodes = res.body.body
         expected_array = compute_expected_array(concepts_map, "isa", ["isa"])
         assert lexer_nodes == expected_array
 
+    def test_i_can_parse_concepts_when_sub_tokens(self):
+        concepts_map = {
+            "foo": Concept("foo"),
+            "bar": Concept("bar"),
+        }
+
+        sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
+        text = "not recognized foo bar not not recognized"
+        expected = ["foo", "bar"]
+        parser_input = ParserInput(text, start=3, end=7)
+        res = parser.parse(context, parser_input)
+        wrapper = res.body
+        lexer_nodes = res.body.body
+
+        assert res.status
+
+        expected_array = compute_expected_array(concepts_map, text, expected)
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert lexer_nodes == expected_array
+
     @pytest.mark.parametrize("text", [
         "foo",
         f"foo one",
@@ -249,7 +270,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
@@ -270,7 +291,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map, create_new=True, singleton=False)
-        list_of_res = parser.parse(context, text)
+        list_of_res = parser.parse(context, ParserInput(text))
 
         assert len(list_of_res) == len(expected)
 
@@ -293,7 +314,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         wrapper = res.body
         lexer_nodes = res.body.body
 
@@ -321,7 +342,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         }
 
         sheerka, context, parser = self.init_parser(concepts_map, create_new=True, use_sheerka=True)
-        res = parser.parse(context, text)
+        res = parser.parse(context, ParserInput(text))
         lexer_nodes = res.body.body
 
         assert res.status