Enhanced AtomNode parsing by name

src/parsers/BaseParser.py

@@ -153,17 +153,17 @@ class BaseParser:
 
         return parser_input
 
-    def get_input_as_tokens(self, parser_input):
+    def get_input_as_tokens(self, parser_input, strip_eof=False):
         if isinstance(parser_input, list):
-            return self.add_eof_if_needed(parser_input)
+            return self.manage_eof(parser_input, strip_eof)
 
         if isinstance(parser_input, ParserResultConcept):
             if parser_input.tokens:
-                return self.add_eof_if_needed(parser_input.tokens)
+                return self.manage_eof(parser_input.tokens, strip_eof)
             else:
                 return Tokenizer(parser_input.source)
 
-        return Tokenizer(parser_input)
+        return Tokenizer(parser_input, yield_eof=not strip_eof)
 
     def get_input_as_lexer_nodes(self, parser_input, expected_parser=None):
         if not isinstance(parser_input, ParserResultConcept):
@@ -183,7 +183,12 @@ class BaseParser:
         return parser_input.value
 
     @staticmethod
-    def add_eof_if_needed(lst):
+    def manage_eof(lst, strip_eof):
+        if strip_eof:
+            if len(lst) and lst[-1].type == TokenKind.EOF:
+                lst.pop()
+            return lst
+
         if len(lst) == 0 or not lst[-1].type == TokenKind.EOF:
             lst.append(Token(TokenKind.EOF, "", -1, -1, -1))
         return lst
@@ -210,6 +215,30 @@ class BaseParser:
             res += value
         return res
 
+    @staticmethod
+    def get_tokens_boundaries(tokens):
+        """
+        Returns the first and the last valid index of the tokens
+        a valid index is a token that is not a whitespace nor and EOF
+        :param tokens:
+        :return:
+        """
+        if tokens is None:
+            return None
+
+        if len(tokens) == 0:
+            return 0, 0
+
+        if tokens[0].type == TokenKind.EOF:
+            return 0, 0
+
+        start = 1 if tokens[0].type == TokenKind.WHITESPACE else 0
+        end = len(tokens) - 1
+        while tokens[end].type in (TokenKind.WHITESPACE, TokenKind.EOF):
+            end -= 1
+
+        return start, end
+
 
 class BaseTokenizerIterParser(BaseParser):