from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute from parsers.BaseParser import BaseParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka def get_ret_val(text, who="who"): return ReturnValueConcept(who, True, UserInputConcept(text, "user_name")) class BaseTestParser(BaseParser): debug_out = [] def __init__(self, name, priority, status=None, parser_result=None): super().__init__(name, priority) self.status = status self.parser_result = parser_result @staticmethod def _get_name(name): return name[8:] if name.startswith("parsers.") else name @staticmethod def _get_source(text_): return text_.as_text() if isinstance(text_, ParserInput) else text_.body def _out(self, name, priority, status, source): debug = f"name={name}" debug += f", priority={priority}" debug += f", status={status}" debug += f", source={source}" self.debug_out.append(debug) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) value = self._get_name(self.name) + ":" + self._get_source(text) parser_result = ParserResultConcept(parser=self, value=value) return ReturnValueConcept(self, self.status, self.parser_result or parser_result) class Enabled90FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled90False", 90, False) class Enabled80FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled80False", 80, False) class Enabled80MultipleFalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled80MultipleFalse", 80, False) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) value1 = self._get_name(self.name) + ":" + self._get_source(text) + "_1" value2 = self._get_name(self.name) + ":" + self._get_source(text) + "_2" return [ ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)), ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)), ] class Enabled80MultipleTrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled80MultipleTrue", 80) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) value1 = self._get_name(self.name) + ":" + self._get_source(text) + "_1" value2 = self._get_name(self.name) + ":" + self._get_source(text) + "_2" return [ ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)), ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)), ] class Enabled70FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled70False", 70, False, "Not a ParserResult") class Enabled50TrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled50True", 50, True) def parse(self, context, text): source = self._get_source(text) status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world" self._out(self._get_name(self.name), self.priority, status, source) value = self._get_name(self.name) + ":" + self._get_source(text) return_value = ParserResultConcept(parser=self, value=value) return ReturnValueConcept(self, status, return_value) class Enabled50bisTrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled50BisTrue", 50, True) class Enabled50FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled50False", 50, False) class Enabled10TrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled10True", 10, True) class DisabledParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Disabled", 90, True) self.enabled = False class NoneParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("None", 90, True, None) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) return None class ListOfNoneParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("ListOfNone", 90, True, None) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) return [None, None] class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): @classmethod def teardown_class(cls): # At the end of the tests, sheerka singleton instance will be corrupted # Ask for a new one TestUsingMemoryBasedSheerka.sheerka = None def test_i_can_get_parser_when_context_is_not_altered(self): sheerka, context = self.init_concepts() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, } service = SheerkaExecute(sheerka) service.reset_registered_parsers() parsers_key, groups, sorted_priorities = service.get_parsers(context) assert parsers_key == "__default" assert groups == {80: [Enabled80FalseParser()], 90: [Enabled90FalseParser()]} assert sorted_priorities == [90, 80] def test_i_can_get_selected_parsers(self): sheerka, context = self.init_concepts() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled70False": Enabled70FalseParser, "Enabled50True": Enabled50TrueParser, "Disabled": Enabled50TrueParser, # <= this one is disabled. It can't be used } service = SheerkaExecute(sheerka) service.reset_registered_parsers() parsers_names = ["Enabled50True", "Enabled70False", "Disabled"] context.preprocess_parsers = parsers_names parsers_key, groups, sorted_priorities = service.get_parsers(context) assert parsers_key == "Enabled50True|Enabled70False|Disabled" assert groups == {50: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]} assert sorted_priorities == [70, 50] # Disabled parser does not appear key = "|".join(parsers_names) assert key in service.grouped_parsers_cache groups, sorted_priorities = service.grouped_parsers_cache[key] assert groups == {50: [Enabled50TrueParser], 70: [Enabled70FalseParser]} assert sorted_priorities == [70, 50] def test_i_can_get_altered_parsers(self): sheerka, context = self.init_concepts() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled70False": Enabled70FalseParser, "Enabled50True": Enabled50TrueParser, "Disabled": Enabled50TrueParser, # <= this one is disabled. It can't be used } service = SheerkaExecute(sheerka) service.reset_registered_parsers() parsers_names = ["Enabled90False", "Enabled50True", "Enabled70False", "Disabled"] context.preprocess_parsers = parsers_names context.add_preprocess(BaseParser.get_name("Enabled90False"), enabled=False) context.add_preprocess(BaseParser.get_name("Enabled50True"), priority=80) parsers_key, groups, sorted_priorities = service.get_parsers(context) assert parsers_key is None assert groups == {80: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]} assert sorted_priorities == [80, 70] # Disabled parsers does not appear key = "|".join(parsers_names) assert key not in service.grouped_parsers_cache # not saved in cache def test_disabled_parsers_are_not_executed(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled": Enabled10TrueParser, "Disabled": DisabledParser } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == ['name=Enabled10True, priority=10, status=True, source=hello world'] def test_parser_are_executed_by_priority(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled90False, priority=90, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', ] def test_parsing_stop_at_the_first_success(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled80False": Enabled80FalseParser, "Enabled50bisTrue": Enabled50bisTrueParser, "Enabled10True": Enabled10TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=Enabled80False:hello world', ] def test_parsing_stop_at_the_first_success_2(self): """ Same test than before, but Enabled50True takes more time to find a match :return: """ sheerka = self.get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, "Enabled10True": Enabled10TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled90False, priority=90, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', ] def test_all_parsers_of_a_given_priority_are_executed(self): """ Make sure that all parsers with priority 50 are executed :return: """ sheerka = self.get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, "Enabled50bisTrue": Enabled50bisTrueParser, "Enabled50False": Enabled50FalseParser, "Enabled10True": Enabled10TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled90False, priority=90, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=Enabled90False:hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=Enabled80False:hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', 'name=Enabled50False, priority=50, status=False, source=hello world', 'name=Enabled50False, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50False, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50False, priority=50, status=False, source=Enabled80False:Enabled90False:hello world' ] def test_a_parser_has_access_to_the_output_of_its_predecessors(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] assert res_as_tuple == [ ('Enabled90False', False, 'Enabled90False:hello world'), ('Enabled80False', False, 'Enabled80False:hello world'), ('Enabled80False', False, 'Enabled80False:Enabled90False:hello world'), ('Enabled50True', False, 'Enabled50True:hello world'), ('Enabled50True', False, 'Enabled50True:Enabled90False:hello world'), ('Enabled50True', False, 'Enabled50True:Enabled80False:hello world'), ('Enabled50True', True, 'Enabled50True:Enabled80False:Enabled90False:hello world'), ] def test_none_return_values_are_discarded(self): sheerka = self.get_sheerka() sheerka.parsers = { "NoneParser": NoneParser, "ListOfNone": ListOfNoneParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert res == [] assert BaseTestParser.debug_out == [ 'name=None, priority=90, status=True, source=hello world', 'name=ListOfNone, priority=90, status=True, source=hello world' ] def test_following_priorities_can_only_see_parser_result_return_values(self): """ Normally, lower priority parsers can see the result of the higher priority parsers This is true only if the higher priority parser return a ParserResultConcept :return: """ sheerka = self.get_sheerka() sheerka.parsers = { "Enabled80False": Enabled80FalseParser, "Enabled70False": Enabled70FalseParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled70False, priority=70, status=False, source=hello world', 'name=Enabled70False, priority=70, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', ] def test_i_can_manage_parser_with_multiple_results(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled80MultipleFalse": Enabled80MultipleFalseParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80MultipleFalse, priority=80, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_1', 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_2', ] res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] assert res_as_tuple == [ ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_1'), ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_2'), ('Enabled50True', False, 'Enabled50True:hello world'), ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_1'), ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'), ] def test_i_can_manage_parser_with_multiple_results_and_a_sucess(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled80MultipleTrue": Enabled80MultipleTrueParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80MultipleTrue, priority=80, status=None, source=hello world', ] res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] assert res_as_tuple == [ ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'), ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'), ]