import pytest from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind, Tokenizer @pytest.mark.parametrize("text, start, end, expected", [ ("def concept a", None, None, "def concept a"), ("&é#(-è_çà)='string'", None, None, "&é#(-è_çà)='string'"), ("def concept a", 2, None, "concept a"), ("def concept a", 0, 2, "def concept"), ]) def test_i_can_use_parser_input(text, start, end, expected): parser_input = ParserInput(text, start=start, end=end).reset() assert parser_input.as_text() == expected def test_i_can_get_the_next_token_when_yield_eof_is_activated(): parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=True).reset() res = [] parser_input.next_token() while True: res.append(f"{parser_input.token.repr_value}") if parser_input.token.type == TokenKind.EOF: break parser_input.next_token() expected = ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", ''] assert res == expected def test_i_can_get_the_next_token_when_yield_eof_is_deactivated(): parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=False).reset() res = [] while parser_input.next_token(): res.append(f"{parser_input.token.repr_value}") expected = ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'"] assert res == expected def test_i_can_get_the_next_token_when_start_and_end_are_provided(): parser_input = ParserInput("def concept a concept name from bnf 'xyz' as 'xyz'", start=4, end=9).reset() res = [] while parser_input.next_token(skip_whitespace=False): res.append(f"{parser_input.token.repr_value}") assert res == ['a', '', 'concept', '', 'name', ''] def test_i_can_get_next_token_when_yield_eof_is_false(): parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=False).reset() res = [] while parser_input.next_token(): res.append(f"{parser_input.token.repr_value}") assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'"] def test_i_can_override_yield_oef_within_reset(): parser_input = ParserInput("def concept a from bnf 'xyz' as 'xyz'", yield_oef=False).reset(yield_oef=True) res = [] parser_input.next_token() while True: res.append(f"{parser_input.token.repr_value}") if parser_input.token.type == TokenKind.EOF: break parser_input.next_token() assert res == ['def', 'concept', 'a', 'from', 'bnf', "'xyz'", 'as', "'xyz'", ""] assert not parser_input.yield_oef @pytest.mark.parametrize("list_has_eof, parser_has_eof, reset_has_eof", [ (True, True, True), (True, False, True), (False, True, True), (False, False, True), (True, True, False), (True, False, False), (False, True, False), (False, False, False), ]) def test_i_can_get_the_next_token_when_initialised_with_tokens(list_has_eof, parser_has_eof, reset_has_eof): tokens = list(Tokenizer(" def concept a as 'xyz' ", yield_eof=list_has_eof)) parser_input = ParserInput(" def concept a as 'xyz' ", tokens, yield_oef=parser_has_eof).reset() parser_input.reset(reset_has_eof) res = [] while parser_input.next_token(): res.append(f"{parser_input.token.repr_value}") expected = ['def', 'concept', 'a', 'as', "'xyz'"] if reset_has_eof: expected.append("") assert res == expected def test_i_can_parse_twice(): text = """ def concept a + b where isinstance(a, int) and isinstance(b, int) pre isinstance(a, int) and isinstance(b, int) post isinstance(res, int) as: def func(x,y): return x+y func(a,b) """ p1 = ParserInput(text).reset() while p1.next_token(): pass p1.reset() p2 = ParserInput(text).reset() while p1.next_token(): p2.next_token() assert p1.token == p2.token p1.reset() p2 = ParserInput(text).reset() while p2.next_token(): p1.next_token() assert p1.token == p2.token @pytest.mark.parametrize("text, skip_whitespace, expected", [ ("first second", True, "second"), ("first second", False, ""), ("first", True, ""), ("first", False, ""), ("first ", True, ""), ("first ", False, ""), ("first:", True, ":"), ("first:", False, ":"), ]) def test_i_can_get_the_token_after(text, skip_whitespace, expected): parser_input = ParserInput(text).reset() parser_input.next_token() assert parser_input.the_token_after(skip_whitespace).repr_value == expected def test_i_can_define_a_sub_part(): text = "Hello Koffi the great guy." tokens = list(Tokenizer(text)) parser_input = ParserInput(None, tokens, 2, 6) assert repr(parser_input) == "ParserInput(from_tokens'Koffi the great')" parser_input.reset() assert repr(parser_input) == "ParserInput(from_tokens'Koffi the great')" res = [] while parser_input.next_token(): res.append(f"{parser_input.token.repr_value}") assert res == ["Koffi", "the", "great"] assert repr(parser_input) == "ParserInput(from_tokens'Koffi the great')" def test_i_can_define_parse_input_from_tokens(): text = "Hello Koffi the great guy." tokens = list(Tokenizer(text)) parser_input = ParserInput(None, tokens) assert repr(parser_input) == "ParserInput(from_tokens'Hello Koffi the great guy.')" parser_input.reset() assert repr(parser_input) == "ParserInput(from_tokens'Hello Koffi the great guy.')" res = [] while parser_input.next_token(): res.append(f"{parser_input.token.repr_value}") assert res == ["Hello", "Koffi", "the", "great", "guy", "."] assert repr(parser_input) == "ParserInput(from_tokens'Hello Koffi the great guy.')"