import pytest

from core.tokenizer import Tokenizer
from parsers.BaseParser import BaseParser, BaseParserInputParser


@pytest.mark.parametrize("tokens, expected", [
    (None, None),
    ([], (0, 0)),
    (list(Tokenizer("")), (0, 0)),
    (list(Tokenizer("", yield_eof=False)), (0, 0)),
    (list(Tokenizer(" a")), (1, 1)),
    (list(Tokenizer(" a", yield_eof=False)), (1, 1)),
    (list(Tokenizer("a ")), (0, 0)),
    (list(Tokenizer("a ", yield_eof=False)), (0, 0)),
    (list(Tokenizer(" a ")), (1, 1)),
    (list(Tokenizer(" a ", yield_eof=False)), (1, 1)),
])
def test_i_can_get_tokens_boundaries(tokens, expected):
    assert BaseParserInputParser.get_tokens_boundaries(tokens) == expected