import pytest from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept from core.concept import Concept from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute, DEFAULT, PARSE_STEPS from parsers.BaseExpressionParser import ExprNode from parsers.BaseParser import BaseParser from parsers.BnfNodeParser import BnfNodeParser from parsers.SequenceNodeParser import SequenceNodeParser from parsers.SyaNodeParser import SyaNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka def get_user_input(text, who="who"): return ReturnValueConcept(who, True, UserInputConcept(text, "user_name")) def check_same_results(result1, result2, user_input): assert len(result1) == len(result2) for previous, current in zip(result1, result2): assert current.parents == user_input assert current.who == previous.who assert current.status == previous.status assert id(current.body) == id(previous.body) def check_same_values(sheerka, context, ret_val1, ret_val2): previous = sheerka.execute(context, ret_val1, [BuiltinConcepts.EVALUATION]) current = sheerka.execute(context, ret_val2, [BuiltinConcepts.EVALUATION]) assert len(previous) == len(current) for p, c in zip(previous, current): assert p == c class BaseTestParser(BaseParser): debug_out = [] def __init__(self, name, priority, status=None, parser_result=None): super().__init__(name, priority) self.status = status self.parser_result = parser_result @staticmethod def _get_name(name): return name[8:] if name.startswith("parsers.") else name @staticmethod def _get_source(text_): return text_.as_text() if isinstance(text_, ParserInput) else text_.body def _out(self, name, priority, status, source): debug = f"name={name}" debug += f", priority={priority}" debug += f", status={status}" debug += f", source={source}" self.debug_out.append(debug) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) value = self._get_name(self.name) + ":" + self._get_source(text) parser_result = ParserResultConcept(parser=self, value=value) return ReturnValueConcept(self, self.status, self.parser_result or parser_result) class Enabled90FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled90False", 90, False) class Enabled80FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled80False", 80, False) class Enabled80MultipleFalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled80MultipleFalse", 80, False) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) value1 = self._get_name(self.name) + ":" + self._get_source(text) + "_1" value2 = self._get_name(self.name) + ":" + self._get_source(text) + "_2" return [ ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)), ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)), ] class Enabled80MultipleTrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled80MultipleTrue", 80) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) value1 = self._get_name(self.name) + ":" + self._get_source(text) + "_1" value2 = self._get_name(self.name) + ":" + self._get_source(text) + "_2" return [ ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)), ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)), ] class Enabled70FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled70False", 70, False, "Not a ParserResult") class Enabled50TrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled50True", 50, True) def parse(self, context, text): source = self._get_source(text) status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world" self._out(self._get_name(self.name), self.priority, status, source) value = self._get_name(self.name) + ":" + self._get_source(text) return_value = ParserResultConcept(parser=self, value=value) return ReturnValueConcept(self, status, return_value) class Enabled50bisTrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled50BisTrue", 50, True) class Enabled50FalseParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled50False", 50, False) class Enabled10TrueParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Enabled10True", 10, True) class DisabledParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("Disabled", 90, True) self.enabled = False class NoneParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("None", 90, True, None) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) return None class ListOfNoneParser(BaseTestParser): def __init__(self, **kwargs): super().__init__("ListOfNone", 90, True, None) def parse(self, context, text): self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) return [None, None] class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): default_parsers = None @classmethod def setup_class(cls): cls.default_parsers = cls().get_sheerka().parsers @classmethod def teardown_class(cls): # At the end of the tests, sheerka singleton instance will be corrupted # Ask for a new one TestUsingMemoryBasedSheerka.sheerka = None def reset_parsers(self, sheerka): sheerka.parsers = self.default_parsers service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() def test_i_can_get_parser_when_context_is_not_altered(self): sheerka, context = self.init_concepts() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, } service = SheerkaExecute(sheerka) service.reset_registered_parsers() parsers_key, groups, sorted_priorities = service.get_parsers(context) assert parsers_key == ("__default", "__default") assert groups == {80: [Enabled80FalseParser()], 90: [Enabled90FalseParser()]} assert sorted_priorities == [90, 80] def test_i_can_get_selected_parsers(self): sheerka, context = self.init_concepts() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled70False": Enabled70FalseParser, "Enabled50True": Enabled50TrueParser, "Disabled": Enabled50TrueParser, # <= this one is disabled. It can't be used } service = SheerkaExecute(sheerka) service.reset_registered_parsers() parsers_names = ["Enabled50True", "Enabled70False", "Disabled"] context.preprocess_parsers = parsers_names parsers_key, groups, sorted_priorities = service.get_parsers(context) assert parsers_key == (DEFAULT, "Enabled50True|Enabled70False|Disabled") assert groups == {50: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]} assert sorted_priorities == [70, 50] # Disabled parser does not appear key = "|".join(parsers_names) assert (DEFAULT, key) in service.grouped_parsers_cache groups, sorted_priorities = service.grouped_parsers_cache[(DEFAULT, key)] assert groups == {50: [Enabled50TrueParser], 70: [Enabled70FalseParser]} assert sorted_priorities == [70, 50] def test_i_can_get_altered_parsers(self): sheerka, context = self.init_concepts() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled70False": Enabled70FalseParser, "Enabled50True": Enabled50TrueParser, "Disabled": Enabled50TrueParser, # <= this one is disabled. It can't be used } service = SheerkaExecute(sheerka) service.reset_registered_parsers() parsers_names = ["Enabled90False", "Enabled50True", "Enabled70False", "Disabled"] context.preprocess_parsers = parsers_names context.add_preprocess(BaseParser.get_name("Enabled90False"), enabled=False) context.add_preprocess(BaseParser.get_name("Enabled50True"), priority=80) parsers_key, groups, sorted_priorities = service.get_parsers(context) assert parsers_key is None assert groups == {80: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]} assert sorted_priorities == [80, 70] # Disabled parsers does not appear key = (DEFAULT, "|".join(parsers_names)) assert key not in service.grouped_parsers_cache # not saved in cache def test_disabled_parsers_are_not_executed(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled": Enabled10TrueParser, "Disabled": DisabledParser } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == ['name=Enabled10True, priority=10, status=True, source=hello world'] def test_parser_are_executed_by_priority(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled90False, priority=90, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', ] def test_parsing_stop_at_the_first_success(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled80False": Enabled80FalseParser, "Enabled50bisTrue": Enabled50bisTrueParser, "Enabled10True": Enabled10TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=Enabled80False:hello world', ] def test_parsing_stop_at_the_first_success_2(self): """ Same test than before, but Enabled50True takes more time to find a match :return: """ sheerka = self.get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, "Enabled10True": Enabled10TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled90False, priority=90, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', ] def test_all_parsers_of_a_given_priority_are_executed(self): """ Make sure that all parsers with priority 50 are executed :return: """ sheerka = self.get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, "Enabled50bisTrue": Enabled50bisTrueParser, "Enabled50False": Enabled50FalseParser, "Enabled10True": Enabled10TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled90False, priority=90, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=Enabled90False:hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=Enabled80False:hello world', 'name=Enabled50BisTrue, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', 'name=Enabled50False, priority=50, status=False, source=hello world', 'name=Enabled50False, priority=50, status=False, source=Enabled90False:hello world', 'name=Enabled50False, priority=50, status=False, source=Enabled80False:hello world', 'name=Enabled50False, priority=50, status=False, source=Enabled80False:Enabled90False:hello world' ] def test_a_parser_has_access_to_the_output_of_its_predecessors(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled90False": Enabled90FalseParser, "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] assert res_as_tuple == [ ('Enabled90False', False, 'Enabled90False:hello world'), ('Enabled80False', False, 'Enabled80False:hello world'), ('Enabled80False', False, 'Enabled80False:Enabled90False:hello world'), ('Enabled50True', False, 'Enabled50True:hello world'), ('Enabled50True', False, 'Enabled50True:Enabled90False:hello world'), ('Enabled50True', False, 'Enabled50True:Enabled80False:hello world'), ('Enabled50True', True, 'Enabled50True:Enabled80False:Enabled90False:hello world'), ] def test_none_return_values_are_discarded(self): sheerka = self.get_sheerka() sheerka.parsers = { "NoneParser": NoneParser, "ListOfNone": ListOfNoneParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert res == [] assert BaseTestParser.debug_out == [ 'name=None, priority=90, status=True, source=hello world', 'name=ListOfNone, priority=90, status=True, source=hello world' ] def test_following_priorities_can_only_see_parser_result_return_values(self): """ Normally, lower priority parsers can see the result of the higher priority parsers This is true only if the higher priority parser return a ParserResultConcept :return: """ sheerka = self.get_sheerka() sheerka.parsers = { "Enabled80False": Enabled80FalseParser, "Enabled70False": Enabled70FalseParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80False, priority=80, status=False, source=hello world', 'name=Enabled70False, priority=70, status=False, source=hello world', 'name=Enabled70False, priority=70, status=False, source=Enabled80False:hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', ] def test_i_can_manage_parser_with_multiple_results(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled80MultipleFalse": Enabled80MultipleFalseParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80MultipleFalse, priority=80, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=hello world', 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_1', 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_2', ] res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] assert res_as_tuple == [ ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_1'), ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_2'), ('Enabled50True', False, 'Enabled50True:hello world'), ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_1'), ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'), ] def test_i_can_manage_parser_with_multiple_results_and_a_success(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled80MultipleTrue": Enabled80MultipleTrueParser, "Enabled50True": Enabled50TrueParser, } service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) assert BaseTestParser.debug_out == [ 'name=Enabled80MultipleTrue, priority=80, status=None, source=hello world', ] res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] assert res_as_tuple == [ ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'), ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'), ] def test_parser_calls_are_put_in_cache(self): sheerka, context = self.init_test().unpack() service = sheerka.services[SheerkaExecute.NAME] self.reset_parsers(sheerka) user_input = [get_user_input("1")] res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) assert len(res) > 0 assert len(service.parsers_cache.cache) == 1 assert (('__default', '__default'), '1') in service.parsers_cache def test_parser_calls_are_put_in_cache_when_question(self): sheerka, context = self.init_test().unpack() service = sheerka.services[SheerkaExecute.NAME] self.reset_parsers(sheerka) context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) user_input = [get_user_input("1")] res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) assert len(res) > 0 assert len(service.parsers_cache.cache) == 2 # one for EVAL_QUESTION, one to compile the Python assert ((BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT), '1') in service.parsers_cache def test_parser_calls_are_not_put_in_cache_when_preprocess_parsers(self): sheerka, context = self.init_test().unpack() service = sheerka.services[SheerkaExecute.NAME] self.reset_parsers(sheerka) context.preprocess_parsers = [SequenceNodeParser.NAME, BnfNodeParser.NAME, SyaNodeParser.NAME] user_input = [get_user_input("1")] res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) expected_key = (DEFAULT, "|".join([SequenceNodeParser.NAME, BnfNodeParser.NAME, SyaNodeParser.NAME])) assert len(res) == 3 assert len(service.parsers_cache.cache) == 1 assert (expected_key, '1') in service.parsers_cache def test_parser_calls_are_not_put_in_cache_when_preprocess(self): sheerka, context = self.init_test().unpack() service = sheerka.services[SheerkaExecute.NAME] self.reset_parsers(sheerka) context.add_preprocess(BaseParser.PREFIX + "parser_name", some_attribute='some_value') user_input = [get_user_input("1")] res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) assert len(res) > 0 assert len(service.parsers_cache.cache) == 0 def test_i_can_use_parser_memoization_on_python(self): sheerka, context = self.init_test().unpack() self.reset_parsers(sheerka) user_input = [get_user_input("1 + 1")] first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) user_input_2 = [get_user_input("1 + 1")] second_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) check_same_results(first_request, second_request, user_input_2) check_same_values(sheerka, context, first_request, second_request) @pytest.mark.parametrize("concept", { Concept("foo"), Concept("foo", body="1"), }) def test_i_can_use_parser_memoization_when_exact_concept(self, concept): sheerka, context, foo = self.init_test(eval_body=True, eval_where=True).with_concepts(concept).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("foo")] first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r.who for r in first_request if r.status] assert successful == ['parsers.ExactConcept'] user_input_2 = [get_user_input("foo")] second_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) check_same_results(first_request, second_request, user_input_2) check_same_values(sheerka, context, first_request, second_request) def test_i_can_use_parser_memoization_when_multiple_results(self): sheerka, context, *foo = self.init_test(eval_body=True, eval_where=True).with_concepts( Concept("foo", body="1"), Concept("foo", body="2")).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("foo")] first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r.who for r in first_request if r.status] assert successful == ['parsers.ExactConcept', 'parsers.ExactConcept'] user_input_2 = [get_user_input("foo")] second_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) check_same_results(first_request, second_request, user_input_2) check_same_values(sheerka, context, first_request, second_request) def test_i_can_use_parser_memoization_when_bnf_node(self): sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( Concept("one", body="1"), Concept("two", body="2"), Concept("twenties", definition="'twenty' (one|two)=unit", body="20 + unit").def_var("unit"), create_new=True).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("twenty one")] first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r.who for r in first_request if r.status] assert successful == ['parsers.Bnf'] user_input_2 = [get_user_input("twenty one")] second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) check_same_results(first_request, second_request, user_input_2) check_same_values(sheerka, context, first_request, second_request) def test_i_can_use_parser_memoization_when_sequence_node(self): sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( Concept("one", body="1"), Concept("two", body="2"), create_new=True).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("one two")] first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r.who for r in first_request if r.status] assert successful == ['parsers.Sequence'] user_input_2 = [get_user_input("one two")] second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) check_same_results(first_request, second_request, user_input_2) check_same_values(sheerka, context, first_request, second_request) def test_i_can_use_parser_memoization_when_sya_node(self): sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( Concept("one", body="1"), Concept("twenty two", body="22"), Concept("a plus b", body="a + b").def_var("a").def_var("b"), create_new=True).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("one plus twenty two")] first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r.who for r in first_request if r.status] assert successful == ['parsers.Sya'] user_input_2 = [get_user_input("one plus twenty two")] second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) check_same_results(first_request, second_request, user_input_2) check_same_values(sheerka, context, first_request, second_request) def test_i_can_use_parser_memoization_when_not_parser_input_from_sequence_node(self): # In the test, 'one two' will be partially parsed by the SequenceNodeParser # We test that the results of the UnrecognizedNodeParser and the PythonWithConceptParser # which both do not used ParsingInput as an input (but a ParserResult) # are added to the list of results sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( Concept("one", body="1")).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("one two")] first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r.who for r in first_request if r.status] assert successful == [] user_input_2 = [get_user_input("one two")] second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) check_same_results(first_request, second_request, user_input_2) check_same_values(sheerka, context, first_request, second_request) def test_i_can_use_parser_memoization_when_not_parser_input_from_bnf_node(self): # check the comment from test_i_can_use_parser_memoization_when_not_parser_input_from_sequence_node # for more information on this test sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( Concept("one", body="1"), Concept("two", body="2"), Concept("twenties", definition="'twenty' (one|two)=unit", body="20 + unit"), create_new=True).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("twenty two foo")] first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r.who for r in first_request if r.status] assert successful == [] user_input_2 = [get_user_input("twenty two foo")] second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) check_same_results(first_request, second_request, user_input_2) check_same_values(sheerka, context, first_request, second_request) def test_i_can_use_parser_memoization_when_not_parser_input_from_sy_node(self): # check the comment from test_i_can_use_parser_memoization_when_not_parser_input_from_sequence_node # for more information on this test sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( Concept("one", body="1"), Concept("twenty two", body="22"), Concept("a plus b", body="a + b").def_var("a").def_var("b"), create_new=True).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("twenty two plus one foo")] first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r.who for r in first_request if r.status] assert successful == [] user_input_2 = [get_user_input("twenty two plus one foo")] second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) check_same_results(first_request, second_request, user_input_2) check_same_values(sheerka, context, first_request, second_request) def test_cache_is_reset_on_concept_creation(self): sheerka, context, *concepts = self.init_test().with_concepts( Concept("foo", body="1"), create_new=True).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("foo")] res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r for r in res if r.status] assert len(successful) == 1 sheerka.create_new_concept(context, Concept("foo", body="2")) res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r for r in res if r.status] assert len(successful) == 2 def test_cache_is_reset_on_concept_deletion(self): sheerka, context, foo1, foo2 = self.init_test().with_concepts( Concept("foo", body="1"), Concept("foo", body="2"), create_new=True).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("foo")] res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r for r in res if r.status] assert len(successful) == 2 sheerka.remove_concept(context, foo2) res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r for r in res if r.status] assert len(successful) == 1 def test_cache_is_reset_on_concept_modification(self): sheerka, context, foo1, foo2 = self.init_test().with_concepts( Concept("foo", body="1"), Concept("foo", body="2"), create_new=True).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("foo")] res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r for r in res if r.status] assert len(successful) == 2 sheerka.modify_concept(context, foo2, to_add={'meta': {"name": "bar"}}) res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) successful = [r for r in res if r.status] assert len(successful) == 1 def test_i_do_not_use_short_term_memory_when_not_requested(self): sheerka, context, foo = self.init_test().with_concepts( Concept("foo"), create_new=True).unpack() self.reset_parsers(sheerka) user_input = [get_user_input("foo")] # put something is STM context.add_to_short_term_memory("foo", "value") res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) assert len(res) == 1 assert res[0].body == "value" # specify the parsers to use context.preprocess_parsers = ["ExactConcept"] res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) assert len(res) == 1 assert res[0].body.body == foo # I can force STM context.preprocess_parsers = ["ExactConcept", "ShortTermMemory"] # order in not relevant for STM parser res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) assert len(res) == 1 assert res[0].body == "value" def test_i_can_compute_parsers_key(self): sheerka = self.get_sheerka() context = self.get_context(sheerka) key = SheerkaExecute.get_parsers_key(context) assert key == (DEFAULT, DEFAULT) context = self.get_context(sheerka) context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) key = SheerkaExecute.get_parsers_key(context) assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT) context = self.get_context(sheerka) context.preprocess_parsers = ["foo", "bar", "baz"] key = SheerkaExecute.get_parsers_key(context) assert key == (DEFAULT, 'foo|bar|baz') context = self.get_context(sheerka) context.preprocess_parsers = ["foo", "bar", "baz"] context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) key = SheerkaExecute.get_parsers_key(context) assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, 'foo|bar|baz') context = self.get_context(sheerka) context.add_preprocess(BaseParser.PREFIX + "foo", key="some_value") key = SheerkaExecute.get_parsers_key(context) assert key is None context = self.get_context(sheerka) context.add_preprocess(BaseParser.PREFIX + "foo", key="some_value") context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) key = SheerkaExecute.get_parsers_key(context) assert key is None context = self.get_context(sheerka) context.add_preprocess("foo", key="some_value") key = SheerkaExecute.get_parsers_key(context) assert key == (DEFAULT, DEFAULT) context = self.get_context(sheerka) context.add_preprocess("foo", key="some_value") context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) key = SheerkaExecute.get_parsers_key(context) assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT) context = self.get_context(sheerka) context.preprocess_parsers = ["foo", "bar", "baz"] context.add_preprocess("foo", key="some_value") key = SheerkaExecute.get_parsers_key(context) assert key == (DEFAULT, 'foo|bar|baz') context = self.get_context(sheerka) context.preprocess_parsers = ["foo", "bar", "baz"] context.add_preprocess("foo", key="some_value") context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) key = SheerkaExecute.get_parsers_key(context) assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, 'foo|bar|baz') def test_i_use_expression_parser_when_needed(self): sheerka, context, one, number, isa_q, isa = self.init_concepts( "one", "number", Concept("x is a y ", pre="is_question()").def_var("x").def_var("y"), Concept("x is a y ").def_var("x").def_var("y"), ) user_input = get_user_input("one is a number") res = sheerka.execute(context, [user_input], PARSE_STEPS) res = self.successful_return_values(res) assert len(res) == 1 assert res[0].status assert isinstance(res[0].body.body, Concept) context = self.get_context(sheerka) context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) res = sheerka.execute(context, [user_input], PARSE_STEPS) assert len(res) == 1 assert res[0].status assert isinstance(res[0].body.body, ExprNode)