from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept from core.sheerka import Sheerka, ExecutionContext from parsers.BaseParser import BaseParser from sdp.sheerkaDataProvider import Event def get_sheerka(): sheerka = Sheerka() sheerka.initialize("mem://") return sheerka def get_context(sheerka): return ExecutionContext("test", Event(), sheerka) def get_ret_val(text, who="who"): return ReturnValueConcept(who, True, UserInputConcept(text, "user_name")) class BaseTestParser(BaseParser): debug_out = [] def __init__(self, name, priority, status=None, parser_result=None): super().__init__(name, priority) self.status = status self.parser_result = parser_result @staticmethod def _get_name(name): return name[8:] if name.startswith("parsers.") else name @staticmethod def _get_source(text_): return text_ if isinstance(text_, str) else text_.body def _out(self, name, priority, status, source): debug = f"name={name}" debug += f", priority={priority}" debug += f", status={status}" debug += f", source={source}" self.debug_out.append(debug) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) parser_result = ParserResultConcept(parser=self, value=value) return ReturnValueConcept(self, self.status, self.parser_result or parser_result) class Enabled90FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled90False", 90, False) class Enabled80FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled80False", 80, False) class Enabled80MultipleFalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled80MultipleFalse", 80, False) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1" value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2" return [ ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)), ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)), ] class Enabled80MultipleTrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled80MultipleTrue", 80) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1" value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2" return [ ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)), ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)), ] class Enabled70FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled70False", 70, False, "Not a ParserResult") class Enabled50TrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled50True", 50, True) def parse(self, context, text): source = self._get_source(text) status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world" self._out(self._get_name(self.name), self.priority, status, source) value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) return_value = ParserResultConcept(parser=self, value=value) return ReturnValueConcept(self, status, return_value) class Enabled50bisTrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled50BisTrue", 50, True) class Enabled50FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled50False", 50, False) class Enabled10TrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled10True", 10, True) class DisabledParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Disabled", 90, True) self.enabled = False class NoneParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("None", 90, True, None) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) return None class ListOfNoneParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("ListOfNone", 90, True, None) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) return [None, None] def test_disabled_parsers_are_not_executed(): sheerka = get_sheerka() sheerka.parsers = { "Enabled": Enabled10TrueParser, "Disabled": DisabledParser } user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == ['name=Enabled10True, priority=10, status=True, source=hello world'] def test_parser_are_executed_by_priority(): sheerka = get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, } user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled90False, priority=90, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', ] def test_parsing_stop_at_the_first_success(): sheerka = get_sheerka() sheerka.parsers = { "Enabled80False": Enabled80FalseParser, "Enabled50bisTrue": Enabled50bisTrueParser, "Enabled10True": Enabled10TrueParser, } user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', ] def test_parsing_stop_at_the_first_success_2(): """ Same test than before, but Enabled50True takes more time to find a match :return: """ sheerka = get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, "Enabled10True": Enabled10TrueParser, } user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled90False, priority=90, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', ] def test_all_parsers_of_a_given_priority_are_executed(): """ Make sure that all parsers with priority 50 are executed :return: """ sheerka = get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, "Enabled50bisTrue": Enabled50bisTrueParser, "Enabled50False": Enabled50FalseParser, "Enabled10True": Enabled10TrueParser, } user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled90False, priority=90, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', 'name=Enabled50False, priority=50, status=False, source=hello world', 'name=Enabled50False, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50False, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50False, priority=50, status=False, source=Enabled80False:Enabled90False:hello world', ] def test_a_parser_has_access_to_the_output_of_its_predecessors(): sheerka = get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, } user_input = [get_ret_val("hello world")] res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] assert res_as_tuple == [ ('Enabled90False', False, 'Enabled90False:hello world'), ('Enabled80False', False, 'Enabled80False:hello world'), ('Enabled80False', False, 'Enabled80False:Enabled90False:hello world'), ('Enabled50True', False, 'Enabled50True:hello world'), ('Enabled50True', False, 'Enabled50True:Enabled90False:hello world'), ('Enabled50True', False, 'Enabled50True:Enabled80False:hello world'), ('Enabled50True', True, 'Enabled50True:Enabled80False:Enabled90False:hello world'), ] def test_none_return_values_are_discarded(): sheerka = get_sheerka() sheerka.parsers = { "NoneParser": NoneParser, "ListOfNone": ListOfNoneParser, } user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert res == [] assert BaseTestParser.debug_out == [ 'name=None, priority=90, status=True, source=hello world', 'name=ListOfNone, priority=90, status=True, source=hello world' ] def test_following_priorities_can_only_see_parser_result_return_values(): """ Normally, lower priority parsers can see the result of the higher priority parsers This is true only if the higher priority parser return a ParserResultConcept :return: """ sheerka = get_sheerka() sheerka.parsers = { "Enabled80False": Enabled80FalseParser, "Enabled70False": Enabled70FalseParser, "Enabled50True": Enabled50TrueParser, } user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled70False, priority=70, status=False, source=hello world', 'name=Enabled70False, priority=70, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', ] def test_i_can_manage_parser_with_multiple_results(): sheerka = get_sheerka() sheerka.parsers = { "Enabled80MultipleFalse": Enabled80MultipleFalseParser, "Enabled50True": Enabled50TrueParser, } user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80MultipleFalse, priority=80, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_1', 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_2', ] res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] assert res_as_tuple == [ ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_1'), ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_2'), ('Enabled50True', False, 'Enabled50True:hello world'), ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_1'), ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'), ] def test_i_can_manage_parser_with_multiple_results_and_a_sucess(): sheerka = get_sheerka() sheerka.parsers = { "Enabled80MultipleTrue": Enabled80MultipleTrueParser, "Enabled50True": Enabled50TrueParser, } user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80MultipleTrue, priority=80, status=None, source=hello world', ] res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] assert res_as_tuple == [ ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'), ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'), ]