import pytest from base import BaseTest from conftest import NewOntology, comparable_tokens from evaluators.base_evaluator import MultipleChoices from helpers import _mt, get_concept, get_concepts, get_parser_input from parsers.SyaConceptsParser import SyaConceptsParser from parsers.tokenizer import Tokenizer class TestSyaConceptsParser(BaseTest): @pytest.fixture() def parser(self): return SyaConceptsParser() @pytest.mark.parametrize("concept_key, expected_list", [ ["a long token name", [("a long token name", 0)]], ["__var__0 __var__1 __var__2", [("", 3)]], ["__var__0 __var__1 prefixed", [(" prefixed", 2)]], ["suffixed __var__0 __var__1", [("suffixed ", 0), ["", 2]]], ["__var__0 __var__1 infixed __var__0 __var__1", [(" infixed ", 2), ["", 2]]], ["if __var__0 __var__1 then __var__2 end", [("if ", 0), (" then ", 2), (" end", 1)]] ]) def test_i_can_initialize_expected_parameters(self, parser, concept_key, expected_list): resolved_expected_list = [(list(Tokenizer(source, yield_eof=False)), nb) for source, nb in expected_list] actual = parser._get_expected_tokens(concept_key) with comparable_tokens(): assert actual == resolved_expected_list def test_i_can_parse_a_simple_case(self, context, parser): with NewOntology(context, "test_i_can_parse_a_simple_case"): get_concepts(context, get_concept("a plus b", variables=["a", "b"]), use_sheerka=True) pi = get_parser_input("1 plus 2") res = parser.parse(context, pi) expected = [_mt("1001", a="1 ", b=" 2")] assert res == MultipleChoices([expected]) assert not parser.error_sink