From c489a38ebc48bcf7ab931df9ed1f7d21b3fcd8bc Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Wed, 22 Jan 2020 17:49:28 +0100 Subject: [PATCH] Refactored sheerka class: splitted to use sub handlers. Refactored unit tests to use classes. --- core/sheerka.py | 1329 ----------------- main.py | 2 +- setup.py | 40 + {core => src/core}/__init__.py | 0 {core => src/core}/ast/__init__.py | 0 {core => src/core}/ast/nodes.py | 0 {core => src/core}/ast/visitors.py | 0 {core => src/core}/builtin_concepts.py | 0 {core => src/core}/builtin_helpers.py | 0 {core => src/core}/concept.py | 0 src/core/sheerka/ExecutionContext.py | 203 +++ src/core/sheerka/Sheerka.py | 600 ++++++++ src/core/sheerka/SheerkaCreateNewConcept.py | 99 ++ src/core/sheerka/SheerkaDump.py | 44 + src/core/sheerka/SheerkaEvaluateConcept.py | 195 +++ src/core/sheerka/SheerkaExecute.py | 254 ++++ src/core/sheerka/SheerkaSetsManager.py | 83 + {evaluators => src/core/sheerka}/__init__.py | 0 {core => src/core}/sheerka_logger.py | 0 {core => src/core}/sheerka_transform.py | 2 +- {core => src/core}/tokenizer.py | 0 {core => src/core}/utils.py | 0 .../evaluators}/AddConceptEvaluator.py | 0 .../evaluators}/AddConceptInSetEvaluator.py | 0 .../evaluators}/BaseEvaluator.py | 2 +- .../evaluators}/ConceptEvaluator.py | 0 .../evaluators}/EvalEvaluator.py | 0 .../evaluators}/LexerNodeEvaluator.py | 0 .../MutipleSameSuccessEvaluator.py | 0 .../evaluators}/OneErrorEvaluator.py | 0 .../evaluators}/OneSuccessEvaluator.py | 0 .../evaluators}/PrepareEvalEvaluator.py | 0 .../evaluators}/PythonEvaluator.py | 6 +- .../evaluators}/TooManySuccessEvaluator.py | 0 {parsers => src/evaluators}/__init__.py | 0 {parsers => src/parsers}/BaseParser.py | 0 {parsers => src/parsers}/BnfParser.py | 2 +- .../parsers}/ConceptLexerParser.py | 0 .../parsers}/ConceptsWithConceptsParser.py | 0 {parsers => src/parsers}/DefaultParser.py | 2 +- {parsers => src/parsers}/EmptyStringParser.py | 0 .../parsers}/ExactConceptParser.py | 0 .../parsers}/MultipleConceptsParser.py | 0 {parsers => src/parsers}/PythonParser.py | 0 .../parsers}/PythonWithConceptsParser.py | 0 {sdp => src/parsers}/__init__.py | 0 src/sdp/__init__.py | 0 {sdp => src/sdp}/readme.md | 0 {sdp => src/sdp}/sheerkaDataProvider.py | 0 {sdp => src/sdp}/sheerkaDataProviderIO.py | 0 {sdp => src/sdp}/sheerkaSerializer.py | 2 +- tests/BaseTest.py | 58 + tests/TestUsingFileBasedSheerka.py | 34 + tests/TestUsingMemoryBasedSheerka.py | 10 + tests/core/__init__.py | 0 tests/{ => core}/test_ExecutionContext.py | 2 +- tests/core/test_SheerkaCreateNewConcept.py | 180 +++ tests/core/test_SheerkaEvaluateConcept.py | 332 ++++ tests/core/test_SheerkaSetsManager.py | 124 ++ tests/{ => core}/test_ast.py | 2 +- tests/core/test_builtin_helpers.py | 145 ++ tests/{ => core}/test_concept.py | 0 tests/core/test_sheerka.py | 244 +++ tests/core/test_sheerka_call_evaluators.py | 375 +++++ tests/core/test_sheerka_call_parsers.py | 363 +++++ tests/core/test_sheerka_transform.py | 318 ++++ tests/{ => core}/test_tokenizer.py | 0 tests/{ => core}/test_utils.py | 0 tests/evaluators/__init__.py | 0 tests/evaluators/test_AddConceptEvaluator.py | 155 ++ .../test_AddConceptInSetEvaluator.py | 102 ++ tests/evaluators/test_ConceptEvaluator.py | 117 ++ tests/evaluators/test_EvalEvaluator.py | 72 + tests/evaluators/test_LexerNodeEvaluator.py | 104 ++ .../test_MultipleSameSuccessEvaluator.py | 200 +++ tests/evaluators/test_OneErrorEvaluator.py | 73 + tests/evaluators/test_OneSuccessEvaluator.py | 72 + tests/evaluators/test_PrepareEvalEvaluator.py | 49 + tests/evaluators/test_PythonEvaluator.py | 138 ++ .../evaluators/test_TooManySucessEvaluator.py | 105 ++ tests/non_reg/__init__.py | 0 tests/non_reg/test_sheerka_non_reg.py | 625 ++++++++ tests/parsers/__init__.py | 0 tests/{ => parsers}/test_BaseParser.py | 2 +- tests/parsers/test_BnfParser.py | 176 +++ tests/parsers/test_ConceptLexerParser.py | 1133 ++++++++++++++ .../test_ConceptsWithConceptsParser.py | 193 +++ tests/parsers/test_DefaultParser.py | 345 +++++ tests/parsers/test_ExactConceptParser.py | 138 ++ tests/parsers/test_MultipleConceptsParser.py | 215 +++ tests/parsers/test_PythonParser.py | 77 + .../parsers/test_PythonWithConceptsParser.py | 138 ++ tests/sdp/__init__.py | 0 tests/{ => sdp}/test_sheerkaDataProvider.py | 10 +- tests/{ => sdp}/test_sheerkaSerializer.py | 4 +- tests/test_AddConceptEvaluator.py | 194 --- tests/test_AddConceptInSetEvaluator.py | 112 -- tests/test_BnfParser.py | 185 --- tests/test_ConceptEvaluator.py | 139 -- tests/test_ConceptLexerParser.py | 1199 --------------- tests/test_ConceptsWithConceptsParser.py | 204 --- tests/test_DefaultParser.py | 368 ----- tests/test_EvalEvaluator.py | 80 - tests/test_ExactConceptParser.py | 151 -- tests/test_LexerNodeEvaluator.py | 115 -- tests/test_MultipleConceptsParser.py | 231 --- tests/test_MultipleSameSuccessEvaluator.py | 216 --- tests/test_OneErrorEvaluator.py | 80 - tests/test_OneSuccessEvaluator.py | 79 - tests/test_PrepareEvalEvaluator.py | 55 - tests/test_PythonEvaluator.py | 151 -- tests/test_PythonParser.py | 87 -- tests/test_PythonWithConceptsParser.py | 147 -- tests/test_TooManySucessEvaluator.py | 112 -- tests/test_builtin_helpers.py | 171 --- tests/test_sheerka.py | 955 ------------ tests/test_sheerka_call_evaluators.py | 399 ----- tests/test_sheerka_call_parsers.py | 380 ----- tests/test_sheerka_non_reg.py | 696 --------- tests/test_sheerka_transform.py | 336 ----- 120 files changed, 7947 insertions(+), 8190 deletions(-) delete mode 100644 core/sheerka.py create mode 100644 setup.py rename {core => src/core}/__init__.py (100%) rename {core => src/core}/ast/__init__.py (100%) rename {core => src/core}/ast/nodes.py (100%) rename {core => src/core}/ast/visitors.py (100%) rename {core => src/core}/builtin_concepts.py (100%) rename {core => src/core}/builtin_helpers.py (100%) rename {core => src/core}/concept.py (100%) create mode 100644 src/core/sheerka/ExecutionContext.py create mode 100644 src/core/sheerka/Sheerka.py create mode 100644 src/core/sheerka/SheerkaCreateNewConcept.py create mode 100644 src/core/sheerka/SheerkaDump.py create mode 100644 src/core/sheerka/SheerkaEvaluateConcept.py create mode 100644 src/core/sheerka/SheerkaExecute.py create mode 100644 src/core/sheerka/SheerkaSetsManager.py rename {evaluators => src/core/sheerka}/__init__.py (100%) rename {core => src/core}/sheerka_logger.py (100%) rename {core => src/core}/sheerka_transform.py (98%) rename {core => src/core}/tokenizer.py (100%) rename {core => src/core}/utils.py (100%) rename {evaluators => src/evaluators}/AddConceptEvaluator.py (100%) rename {evaluators => src/evaluators}/AddConceptInSetEvaluator.py (100%) rename {evaluators => src/evaluators}/BaseEvaluator.py (96%) rename {evaluators => src/evaluators}/ConceptEvaluator.py (100%) rename {evaluators => src/evaluators}/EvalEvaluator.py (100%) rename {evaluators => src/evaluators}/LexerNodeEvaluator.py (100%) rename {evaluators => src/evaluators}/MutipleSameSuccessEvaluator.py (100%) rename {evaluators => src/evaluators}/OneErrorEvaluator.py (100%) rename {evaluators => src/evaluators}/OneSuccessEvaluator.py (100%) rename {evaluators => src/evaluators}/PrepareEvalEvaluator.py (100%) rename {evaluators => src/evaluators}/PythonEvaluator.py (97%) rename {evaluators => src/evaluators}/TooManySuccessEvaluator.py (100%) rename {parsers => src/evaluators}/__init__.py (100%) rename {parsers => src/parsers}/BaseParser.py (100%) rename {parsers => src/parsers}/BnfParser.py (99%) rename {parsers => src/parsers}/ConceptLexerParser.py (100%) rename {parsers => src/parsers}/ConceptsWithConceptsParser.py (100%) rename {parsers => src/parsers}/DefaultParser.py (99%) rename {parsers => src/parsers}/EmptyStringParser.py (100%) rename {parsers => src/parsers}/ExactConceptParser.py (100%) rename {parsers => src/parsers}/MultipleConceptsParser.py (100%) rename {parsers => src/parsers}/PythonParser.py (100%) rename {parsers => src/parsers}/PythonWithConceptsParser.py (100%) rename {sdp => src/parsers}/__init__.py (100%) create mode 100644 src/sdp/__init__.py rename {sdp => src/sdp}/readme.md (100%) rename {sdp => src/sdp}/sheerkaDataProvider.py (100%) rename {sdp => src/sdp}/sheerkaDataProviderIO.py (100%) rename {sdp => src/sdp}/sheerkaSerializer.py (99%) create mode 100644 tests/BaseTest.py create mode 100644 tests/TestUsingFileBasedSheerka.py create mode 100644 tests/TestUsingMemoryBasedSheerka.py create mode 100644 tests/core/__init__.py rename tests/{ => core}/test_ExecutionContext.py (97%) create mode 100644 tests/core/test_SheerkaCreateNewConcept.py create mode 100644 tests/core/test_SheerkaEvaluateConcept.py create mode 100644 tests/core/test_SheerkaSetsManager.py rename tests/{ => core}/test_ast.py (99%) create mode 100644 tests/core/test_builtin_helpers.py rename tests/{ => core}/test_concept.py (100%) create mode 100644 tests/core/test_sheerka.py create mode 100644 tests/core/test_sheerka_call_evaluators.py create mode 100644 tests/core/test_sheerka_call_parsers.py create mode 100644 tests/core/test_sheerka_transform.py rename tests/{ => core}/test_tokenizer.py (100%) rename tests/{ => core}/test_utils.py (100%) create mode 100644 tests/evaluators/__init__.py create mode 100644 tests/evaluators/test_AddConceptEvaluator.py create mode 100644 tests/evaluators/test_AddConceptInSetEvaluator.py create mode 100644 tests/evaluators/test_ConceptEvaluator.py create mode 100644 tests/evaluators/test_EvalEvaluator.py create mode 100644 tests/evaluators/test_LexerNodeEvaluator.py create mode 100644 tests/evaluators/test_MultipleSameSuccessEvaluator.py create mode 100644 tests/evaluators/test_OneErrorEvaluator.py create mode 100644 tests/evaluators/test_OneSuccessEvaluator.py create mode 100644 tests/evaluators/test_PrepareEvalEvaluator.py create mode 100644 tests/evaluators/test_PythonEvaluator.py create mode 100644 tests/evaluators/test_TooManySucessEvaluator.py create mode 100644 tests/non_reg/__init__.py create mode 100644 tests/non_reg/test_sheerka_non_reg.py create mode 100644 tests/parsers/__init__.py rename tests/{ => parsers}/test_BaseParser.py (94%) create mode 100644 tests/parsers/test_BnfParser.py create mode 100644 tests/parsers/test_ConceptLexerParser.py create mode 100644 tests/parsers/test_ConceptsWithConceptsParser.py create mode 100644 tests/parsers/test_DefaultParser.py create mode 100644 tests/parsers/test_ExactConceptParser.py create mode 100644 tests/parsers/test_MultipleConceptsParser.py create mode 100644 tests/parsers/test_PythonParser.py create mode 100644 tests/parsers/test_PythonWithConceptsParser.py create mode 100644 tests/sdp/__init__.py rename tests/{ => sdp}/test_sheerkaDataProvider.py (99%) rename tests/{ => sdp}/test_sheerkaSerializer.py (92%) delete mode 100644 tests/test_AddConceptEvaluator.py delete mode 100644 tests/test_AddConceptInSetEvaluator.py delete mode 100644 tests/test_BnfParser.py delete mode 100644 tests/test_ConceptEvaluator.py delete mode 100644 tests/test_ConceptLexerParser.py delete mode 100644 tests/test_ConceptsWithConceptsParser.py delete mode 100644 tests/test_DefaultParser.py delete mode 100644 tests/test_EvalEvaluator.py delete mode 100644 tests/test_ExactConceptParser.py delete mode 100644 tests/test_LexerNodeEvaluator.py delete mode 100644 tests/test_MultipleConceptsParser.py delete mode 100644 tests/test_MultipleSameSuccessEvaluator.py delete mode 100644 tests/test_OneErrorEvaluator.py delete mode 100644 tests/test_OneSuccessEvaluator.py delete mode 100644 tests/test_PrepareEvalEvaluator.py delete mode 100644 tests/test_PythonEvaluator.py delete mode 100644 tests/test_PythonParser.py delete mode 100644 tests/test_PythonWithConceptsParser.py delete mode 100644 tests/test_TooManySucessEvaluator.py delete mode 100644 tests/test_builtin_helpers.py delete mode 100644 tests/test_sheerka.py delete mode 100644 tests/test_sheerka_call_evaluators.py delete mode 100644 tests/test_sheerka_call_parsers.py delete mode 100644 tests/test_sheerka_non_reg.py delete mode 100644 tests/test_sheerka_transform.py diff --git a/core/sheerka.py b/core/sheerka.py deleted file mode 100644 index 040390a..0000000 --- a/core/sheerka.py +++ /dev/null @@ -1,1329 +0,0 @@ -from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \ - UnknownConcept -from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW, DoNotResolve -from parsers.BaseParser import BaseParser -from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderDuplicateKeyError -import core.utils -import core.builtin_helpers - -from core.sheerka_logger import console_handler - -import logging -import time - -CONCEPT_EVALUATION_STEPS = [ - BuiltinConcepts.BEFORE_EVALUATION, - BuiltinConcepts.EVALUATION, - BuiltinConcepts.AFTER_EVALUATION] - -CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser" -DEBUG_TAB_SIZE = 4 -GROUP_PREFIX = 'All_' - - -class Sheerka(Concept): - """ - Main controller for the project - """ - - CONCEPTS_ENTRY = "All_Concepts" # to store all the concepts - CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID" - CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions" # to store definitions (bnf) of concepts - BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts" # sequential key for builtin concepts - USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts - - def __init__(self, skip_builtins_in_db=False, debug=False, loggers=None): - self.init_logging(debug, loggers) - - super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA) - self.log.debug("Starting Sheerka.") - - # cache of the most used concepts - # Note that these are only templates - # They are used as a footprint for instantiation - # Except of source when the concept is supposed to be unique - # key is the key of the concept (not the name or the id) - self.cache_by_key = {} - self.cache_by_id = {} - - # cache for concept definitions, - # Primarily used for unit test that does not have access to sdp - self.concepts_definition_cache = {} - - # - # cache for concepts grammars - # a grammar is a resolved BNF - self.concepts_grammars = {} - - # a concept can be instantiated - # ex: File is a concept, but File('foo.txt') is an instance - # TODO: manage contexts - self.instances = [] - - # List of the known rules by the system - # ex: hello => say('hello') - self.rules = [] - - self.sdp: SheerkaDataProvider = None # SheerkaDataProvider - self.builtin_cache = {} # cache for builtin concepts - self.parsers = {} # cache for builtin parsers - self.evaluators = [] # cache for builtin evaluators - - self.evaluators_prefix: str = None - self.parsers_prefix: str = None - - self.skip_builtins_in_db = skip_builtins_in_db - - def initialize(self, root_folder: str = None): - """ - Starting Sheerka - Loads the current configuration - Notes that when it's the first time, it also create the needed working folders - :param root_folder: root configuration folder - :return: ReturnValue(Success or Error) - """ - - try: - self.sdp = SheerkaDataProvider(root_folder) - if self.sdp.first_time: - self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000) - - event = Event("Initializing Sheerka.") - self.sdp.save_event(event) - exec_context = ExecutionContext(self.key, event, self) - - self.initialize_builtin_concepts() - self.initialize_builtin_parsers() - self.initialize_builtin_evaluators() - self.initialize_concepts_definitions(exec_context) - - except IOError as e: - return ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e) - - return ReturnValueConcept(self, True, self) - - def initialize_builtin_concepts(self): - """ - Initializes the builtin concepts - :return: None - """ - self.init_log.debug("Initializing builtin concepts") - builtins_classes = self.get_builtins_classes_as_dict() - - # this all initialization of the builtins seems to be little bit complicated - # why do we need to update it from DB ? - for key in BuiltinConcepts: - concept = self if key == BuiltinConcepts.SHEERKA \ - else builtins_classes[str(key)]() if str(key) in builtins_classes \ - else Concept(key, True, False, key) - - if key in BuiltinUnique: - concept.metadata.is_unique = True - concept.metadata.is_evaluated = True - - if not concept.metadata.is_unique and str(key) in builtins_classes: - self.builtin_cache[key] = builtins_classes[str(key)] - - if not self.skip_builtins_in_db: - from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.metadata.key) - if from_db is None: - self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.") - self.set_id_if_needed(concept, True) - self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True) - else: - self.init_log.debug(f"Found concept '{from_db}' in db. Updating.") - concept.update_from(from_db) - - self.add_in_cache(concept) - - def initialize_builtin_parsers(self): - """ - Init the parsers - :return: - """ - core.utils.init_package_import("parsers") - base_class = core.utils.get_class("parsers.BaseParser.BaseParser") - for parser in core.utils.get_sub_classes("parsers", base_class): - if parser.__module__ == base_class.__module__: - continue - - self.init_log.debug(f"Adding builtin parser '{parser.__name__}'") - self.parsers[core.utils.get_full_qualified_name(parser)] = parser - - def initialize_builtin_evaluators(self): - """ - Init the evaluators - :return: - """ - core.utils.init_package_import("evaluators") - for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator"): - self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'") - self.evaluators.append(evaluator) - - for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator"): - self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'") - self.evaluators.append(evaluator) - - def initialize_concepts_definitions(self, execution_context): - self.init_log.debug("Initializing concepts definitions") - definitions = self.sdp.get_safe(self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False) - - if definitions is None: - self.init_log.debug("No BNF defined") - return - - lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]() - ret_val = lexer_parser.initialize(execution_context, definitions) - if not ret_val.status: - self.init_log.error("Failed to initialize concepts definitions " + str(ret_val.body)) - return - - self.concepts_grammars = lexer_parser.concepts_grammars - - def reset_cache(self, filter_to_use=None): - """ - reset the different cache that exists - :param filter_to_use: - :return: - """ - if filter_to_use is None: - self.cache_by_key = {} - self.cache_by_id = {} - else: - raise NotImplementedError() - - return self - - def evaluate_user_input(self, text: str, user_name="kodjo"): - """ - Note to KSI: If you try to add execution context to this function, - You may end in an infinite loop - :param text: - :param user_name: - :return: - """ - self.log.debug(f"Processing user input '{text}', {user_name=}.") - event = Event(text, user_name) - evt_digest = self.sdp.save_event(event) - self.log.debug(f"{evt_digest=}") - - with ExecutionContext(self.key, event, self, f"Evaluating '{text}'") as execution_context: - user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name)) - reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED)) - - steps = [ - BuiltinConcepts.BEFORE_PARSING, - BuiltinConcepts.PARSING, - BuiltinConcepts.AFTER_PARSING, - BuiltinConcepts.BEFORE_EVALUATION, - BuiltinConcepts.EVALUATION, - BuiltinConcepts.AFTER_EVALUATION - ] - - ret = self.execute(execution_context, [user_input, reduce_requested], steps) - execution_context.add_values(return_values=ret) - - if not self.skip_builtins_in_db: - self.sdp.save_result(execution_context) - return ret - - def _call_parsers(self, execution_context, return_values, logger=None): - - # return_values must be a list - if not isinstance(return_values, list): - return_values = [return_values] - - # first make the distinguish between what is for the parsers and what is not - result = [] - to_process = [] - for r in return_values: - if not r.status or not self.isinstance(r.body, BuiltinConcepts.USER_INPUT): - result.append(r) - else: - to_process.append(r) - - if not to_process: - return result - - # keep track of the originals user inputs, as they need to be removed at the end - user_inputs = to_process[:] - - # group the parsers by priorities - instantiated_parsers = [parser(sheerka=self) for parser in self.parsers.values()] - grouped_parsers = {} - for parser in [p for p in instantiated_parsers if p.enabled]: - if logger: - parser.log = logger - grouped_parsers.setdefault(parser.priority, []).append(parser) - sorted_priorities = sorted(grouped_parsers.keys(), reverse=True) - - stop_processing = False - for priority in sorted_priorities: - inputs_for_this_group = to_process[:] - - for parser in grouped_parsers[priority]: - - return_value_success_found = False - for return_value in inputs_for_this_group: - - to_parse = return_value.body.body \ - if self.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \ - else return_value.body - - # if self.log.isEnabledFor(logging.DEBUG): - # debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \ - # else "'" + BaseParser.get_text_from_tokens(to_parse) + "' as tokens" - # execution_context.log(logger or self.log, f"Parsing {debug_text}") - - with execution_context.push(desc=f"Parsing using {parser.name}") as sub_context: - sub_context.add_inputs(to_parse=to_parse) - res = parser.parse(sub_context, to_parse) - if res is not None: - if hasattr(res, "__iter__"): - for r in res: - if r is None: - continue - r.parents = [return_value] - result.append(r) - if self.isinstance(r.body, BuiltinConcepts.PARSER_RESULT): - to_process.append(r) - if r.status: - return_value_success_found = True - - else: - res.parents = [return_value] - result.append(res) - if self.isinstance(res.body, BuiltinConcepts.PARSER_RESULT): - to_process.append(res) - if res.status: - return_value_success_found = True - sub_context.add_values(return_values=res) - - if return_value_success_found: - stop_processing = True - break # Stop the other return_values (but not the other parsers with the same priority) - - if stop_processing: - break # Do not try the other priorities if a match is found - - result = core.utils.remove_list_from_list(result, user_inputs) - return result - - def _call_evaluators(self, execution_context, return_values, process_step, evaluation_context=None, logger=None): - - def _preprocess_evaluators(context, evaluators): - if not context.preprocess: - return evaluators - - if not hasattr(evaluators, "__iter__"): - single_one = True - evaluators = [evaluators] - else: - single_one = False - - for preprocess in context.preprocess: - for e in evaluators: - if preprocess.props["name"].value == e.name: - for prop, value in preprocess.props.items(): - if prop == "name": - continue - if hasattr(e, prop): - setattr(e, prop, value.value) - return evaluators[0] if single_one else evaluators - - # return_values must be a list - if not isinstance(return_values, list): - return_values = [return_values] - - # Evaluation context are contexts that may modify the behaviour of the execution - # For example, a concept to indicate that the value is not wanted - # Or a concept to indicate that we want the letter form of the response - # But first, they need to be transformed into return values - if evaluation_context is None: - evaluation_return_values = [] - else: - evaluation_return_values = [self.ret(execution_context.who, True, c) for c in evaluation_context] - - # add the current step as part as the evaluation context - evaluation_return_values.append(self.ret(execution_context.who, True, self.new(process_step))) - - # the pool of return values are the mix - return_values.extend(evaluation_return_values) - - # group the evaluators by priority and sort them - # The first one to be applied will be the one with the highest priority - grouped_evaluators = {} - instantiated_evaluators = [e_class() for e_class in self.evaluators] - - # pre-process evaluators if needed - instantiated_evaluators = _preprocess_evaluators(execution_context, instantiated_evaluators) - - for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]: - if logger: - evaluator.log = logger - grouped_evaluators.setdefault(evaluator.priority, []).append(evaluator) - - # order the groups by priority, the higher first - sorted_priorities = sorted(grouped_evaluators.keys(), reverse=True) - - # process - iteration = 0 - while True: - with execution_context.push(desc=f"iteration #{iteration}", iteration=iteration) as iteration_context: - simple_digest = return_values[:] - iteration_context.add_inputs(return_values=simple_digest) - - for priority in sorted_priorities: - - original_items = return_values[:] - evaluated_items = [] - to_delete = [] - for evaluator in grouped_evaluators[priority]: - evaluator = _preprocess_evaluators(execution_context, evaluator.__class__()) # fresh copy - - sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})" - with iteration_context.push(desc=sub_context_desc) as sub_context: - sub_context.add_inputs(return_values=original_items) - - # process evaluators that work on one simple return value at the time - from evaluators.BaseEvaluator import OneReturnValueEvaluator - if isinstance(evaluator, OneReturnValueEvaluator): - debug_result = [] - for item in original_items: - if evaluator.matches(sub_context, item): - result = evaluator.eval(sub_context, item) - if result is None: - debug_result.append({"input": item, "return_value": None}) - continue - - to_delete.append(item) - if isinstance(result, list): - evaluated_items.extend(result) - elif isinstance(result, ReturnValueConcept): - evaluated_items.append(result) - else: - error = self.new(BuiltinConcepts.INVALID_RETURN_VALUE, body=result, - evaluator=evaluator) - result = self.ret("sheerka.process", False, error, parents=[item]) - evaluated_items.append(result) - debug_result.append({"input": item, "return_value": result}) - else: - debug_result.append({"input": item, "return_value": "** No Match **"}) - sub_context.add_values(return_values=debug_result) - - # process evaluators that work on all return values - else: - if evaluator.matches(sub_context, original_items): - results = evaluator.eval(sub_context, original_items) - if results is None: - continue - if not isinstance(results, list): - results = [results] - for result in results: - evaluated_items.append(result) - to_delete.extend(result.parents) - sub_context.add_values(return_values=results) - else: - sub_context.add_values(return_values="** No Match **") - - return_values = evaluated_items - return_values.extend([item for item in original_items if item not in to_delete]) - - iteration_context.add_values(return_values=return_values[:]) - - # have we done something ? - to_compare = return_values[:] - if simple_digest == to_compare: - break - - # inc the iteration and continue - iteration += 1 - - # remove all evaluation context that are not reduced - return_values = core.utils.remove_list_from_list(return_values, evaluation_return_values) - return return_values - - def execute(self, execution_context, return_values, execution_steps, logger=None): - """ - Executes process for all initial contexts - :param execution_context: - :param return_values: - :param execution_steps: - :param logger: logger to use (if not directly called by sheerka) - :return: - """ - - for step in execution_steps: - copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values] - with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context: - sub_context.log(logger or self.log, f"{step=}, context='{sub_context}'") - - if step == BuiltinConcepts.PARSING: - return_values = self._call_parsers(sub_context, return_values, logger) - else: - return_values = self._call_evaluators(sub_context, return_values, step, None, logger) - - if copy != return_values: - sub_context.log_result(logger or self.log, return_values) - - sub_context.add_values(return_values=return_values) - - return return_values - - def set_id_if_needed(self, obj: Concept, is_builtin: bool): - """ - Set the key for the concept if needed - For test purpose only !!!!! - :param obj: - :param is_builtin: - :return: - """ - if obj.metadata.id is not None: - return - obj.metadata.id = self.sdp.get_next_key(self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS) - self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.") - - def create_new_concept(self, context, concept: Concept, logger=None): - """ - Adds a new concept to the system - :param context: - :param concept: DefConceptNode - :param logger - :return: digest of the new concept - """ - - logger = logger or self.log - - concept.init_key() - concepts_definitions = None - init_ret_value = None - - # checks for duplicate concepts - # TODO checks if it exists in cache first - if self.sdp.exists(self.CONCEPTS_ENTRY, concept.key, concept.get_digest()): - error = SheerkaDataProviderDuplicateKeyError(self.CONCEPTS_ENTRY + "." + concept.key, concept) - return self.ret( - self.create_new_concept.__name__, - False, - self.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept), - error.args[0]) - - # set id before saving in db - self.set_id_if_needed(concept, False) - - # add the BNF if known - if concept.bnf: - concepts_definitions = self.get_concept_definition() - concepts_definitions[concept] = concept.bnf - - # check if it's a valid BNF or whether it breaks the known rules - concept_lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]() - with context.push(self.name, desc=f"Initializing concept definition for {concept}") as sub_context: - sub_context.concepts[concept.key] = concept # the concept is not in the real cache yet - sub_context.log_new(logger) - init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions) - sub_context.add_values(return_values=init_ret_value) - if not init_ret_value.status: - return self.ret(self.create_new_concept.__name__, False, ErrorConcept(init_ret_value.value)) - - # save the new concept in sdp - try: - # TODO : needs to make these calls atomic (or at least one single call) - self.sdp.add(context.event.get_digest(), self.CONCEPTS_ENTRY, concept, use_ref=True) - self.sdp.add(context.event.get_digest(), - self.CONCEPTS_BY_ID_ENTRY, - {concept.id: concept.get_digest()}, - is_ref=True) - if concepts_definitions is not None: - self.sdp.set(context.event.get_digest(), - self.CONCEPTS_DEFINITIONS_ENTRY, - concepts_definitions, use_ref=True) - except SheerkaDataProviderDuplicateKeyError as error: - context.log_error(logger, "Failed to create a new concept.", who=self.create_new_concept.__name__) - return self.ret( - self.create_new_concept.__name__, - False, - self.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept), - error.args[0]) - - # Updates the caches - self.cache_by_key[concept.key] = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.key) # reset from sdp - self.cache_by_id[concept.id] = concept # no need to reset - if init_ret_value is not None and init_ret_value.status: - self.concepts_grammars = init_ret_value.body - - # process the return in needed - ret = self.ret(self.create_new_concept.__name__, True, self.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) - return ret - - def add_concept_to_set(self, context, concept, concept_set, logger=None): - """ - Add an entry in sdp to tell that concept isa concept_set - :param context: - :param concept: - :param concept_set: - :param logger: - :return: - """ - logger = logger or self.log - - context.log(logger, f"Adding concept {concept} to set {concept_set}", who=self.add_concept_to_set.__name__) - - assert concept.id - assert concept_set.id - - try: - ret = self.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id) - if ret == (None, None): # concept already in set - return self.ret( - self.add_concept_to_set.__name__, - False, - self.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set)) - else: - return self.ret(self.add_concept_to_set.__name__, True, self.new(BuiltinConcepts.SUCCESS)) - except Exception as error: - context.log_error(logger, "Failed to add to set.", who=self.add_concept_to_set.__name__) - return self.ret(self.create_new_concept.__name__, False, ErrorConcept(error), error.args[0]) - - def get_set_elements(self, concept): - """ - Concept is supposed to be a set - Returns all elements if the set - :param concept: - :return: - """ - - assert concept.id - - ids = self.sdp.get_safe(GROUP_PREFIX + concept.id) - if ids is None: - return self.new(BuiltinConcepts.NOT_A_SET, body=concept) - - elements = [self.get_by_id(element_id) for element_id in ids] - return elements - - def initialize_concept_asts(self, context, concept: Concept, logger=None): - """ - Updates the codes of the newly created concept - Basically, it runs the parsers on all parts - :param concept: - :param context: - :param logger: - :return: - """ - steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING] - for part_key in ConceptParts: - if part_key in concept.compiled: - continue - - source = getattr(concept.metadata, part_key.value) - if source is None or not isinstance(source, str): - continue - - if source.strip() == "": - concept.compiled[part_key] = DoNotResolve(source) - else: - with context.push(desc=f"Initializing compiled for {part_key}") as sub_context: - sub_context.log_new(logger) - sub_context.add_inputs(source=source) - to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=source)) - res = self.execute(sub_context, to_parse, steps, logger) - concept.compiled[part_key] = res - sub_context.add_values(return_values=res) - - for prop, default_value in concept.metadata.props: - if prop in concept.compiled: - continue - - if default_value is None or not isinstance(default_value, str): - continue - - if default_value.strip() == "": - concept.compiled[prop] = DoNotResolve(default_value) - else: - with context.push(desc=f"Initializing AST for property {prop}") as sub_context: - sub_context.log_new(logger) - sub_context.add_inputs(source=default_value) - to_parse = self.ret(context.who, True, self.new(BuiltinConcepts.USER_INPUT, body=default_value)) - res = self.execute(context, to_parse, steps) - concept.compiled[prop] = res - sub_context.add_values(return_values=res) - - # Updates the cache of concepts when possible - if concept.key in self.cache_by_key: - entry = self.cache_by_key[concept.key] - if isinstance(entry, list): - # TODO : manage when there are multiple entries - pass - else: - self.cache_by_key[concept.key].compiled = concept.compiled - - def evaluate_concept(self, context, concept: Concept, logger=None): - """ - Evaluation a concept - It means that if the where clause is True, will evaluate the body - :param context: - :param concept: - :param logger: - :return: value of the evaluation or error - """ - - logger = logger or self.log - - if concept.metadata.is_evaluated: - return concept - - def _resolve(to_resolve, current_prop, current_concept): - if isinstance(to_resolve, DoNotResolve): - return to_resolve.value - - desc = f"Evaluating {current_prop} (concept={current_concept})" - context.log(logger, desc, self.evaluate_concept.__name__) - with context.push(desc=desc, obj=current_concept) as sub_context: - sub_context.log_new(logger) - - # when it's a concept, evaluate it - if isinstance(to_resolve, Concept) and \ - not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE): - evaluated = self.evaluate_concept(sub_context, to_resolve) - sub_context.add_values(return_values=evaluated) - if evaluated.key == to_resolve.key: - return evaluated - else: - error = evaluated - - # otherwise, execute all return values to find out what is the value - else: - r = self.execute(sub_context, to_resolve, CONCEPT_EVALUATION_STEPS, logger) - one_r = core.builtin_helpers.expect_one(context, r) - sub_context.add_values(return_values=one_r) - if one_r.status: - return one_r.value - else: - error = one_r.value - - return self.new(BuiltinConcepts.CONCEPT_EVAL_ERROR, - body=error, - concept=concept, - property_name=prop_name) - - def _resolve_list(sheerka, list_to_resolve, current_prop, current_concept): - """When dealing with a list, there are two possibilities""" - # It may be a list of ReturnValueConcept to execute (always the case for metadata) - # or a list of single values (may be the case for properties) - # in this latter case, all values are to be processed one by one and a list should be returned - if len(list_to_resolve) == 0: - return [] - - if sheerka.isinstance(list_to_resolve[0], BuiltinConcepts.RETURN_VALUE): - return _resolve(list_to_resolve, current_prop, current_concept) - - res = [] - for to_resolve in list_to_resolve: - # sanity check - if sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE): - return self.new(BuiltinConcepts.CONCEPT_EVAL_ERROR, - body="Mix between real values and return values", - concept=concept, - property_name=prop_name) - - r = _resolve(to_resolve, current_prop, current_concept) - if sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR): - return r - res.append(r) - - return res - - # WHERE condition should already be validated by the parser. - # It's a mandatory condition for the concept before it can be recognized - - # - # TODO : Validate the PRE condition - # - - self.initialize_concept_asts(context, concept, logger) - - # to make sure of the order, it don't use ConceptParts.get_parts() - # props must be evaluated first - all_metadata_to_eval = ["props", "where", "pre", "post", "body"] - - for metadata_to_eval in all_metadata_to_eval: - if metadata_to_eval == "props": - for prop_name in (p for p in concept.props if p in concept.compiled): - prop_ast = concept.compiled[prop_name] - - if isinstance(prop_ast, list): - resolved = _resolve_list(context.sheerka, prop_ast, prop_name, None) - else: - resolved = _resolve(prop_ast, prop_name, None) - if context.sheerka.isinstance(resolved, BuiltinConcepts.CONCEPT_EVAL_ERROR): - return resolved - else: - concept.set_prop(prop_name, resolved) - else: - part_key = ConceptParts(metadata_to_eval) - if part_key in concept.compiled and concept.compiled[part_key] is not None: - metadata_ast = concept.compiled[part_key] - resolved = _resolve(metadata_ast, part_key, concept) - if context.sheerka.isinstance(resolved, BuiltinConcepts.CONCEPT_EVAL_ERROR): - return resolved - else: - concept.values[part_key] = resolved - - # - # TODO : Validate the POST condition - # - - concept.init_key() # only does it if needed - concept.metadata.is_evaluated = True - return concept - - def add_in_cache(self, concept: Concept): - """ - Adds a concept template in cache. - The cache is used as a proxy before looking at sdp - :param concept: - :return: - """ - - # sanity check - if concept.key is None: - concept.init_key() - - if concept.key is None: - raise KeyError() - - self.cache_by_key[concept.key] = concept - - if concept.id: - self.cache_by_id[concept.id] = concept - - return concept - - def get(self, concept_key, concept_id=None): - """ - Tries to find a concept - What is return must be used a template for another concept. - You must not modify the returned concept - :param concept_key: key of the concept - :param concept_id: when multiple concepts with the same key, use the id - :return: - """ - - if concept_key is None: - return ErrorConcept("Concept key is undefined.") - - if isinstance(concept_key, BuiltinConcepts): - concept_key = str(concept_key) - - # first search in cache - result = self.cache_by_key[concept_key] if concept_key in self.cache_by_key else \ - self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key) - - if result and (concept_id is None or not isinstance(result, list)): - return result - - if isinstance(result, list): - if concept_id: - for c in result: - if c.id == concept_id: - return c - else: - return result - - metadata = [("key", concept_key), ("id", concept_id)] if concept_id else ("key", concept_key) - return self._get_unknown(metadata) - - def get_by_id(self, concept_id): - if concept_id is None: - return ErrorConcept("Concept id is undefined.") - - # first search in cache - result = self.cache_by_id[concept_id] if concept_id in self.cache_by_id else \ - self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id) - - return result or self._get_unknown(('id', concept_id)) - - def new(self, concept_key, **kwargs): - """ - Returns an instance of a new concept - When the concept is supposed to be unique, returns the same instance - :param concept_key: - :param kwargs: - :return: - """ - if isinstance(concept_key, tuple): - concept_key, concept_id = concept_key[0], concept_key[1] - else: - concept_id = None - - template = self.get(concept_key, concept_id) - - # manage concept not found - if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \ - concept_key != BuiltinConcepts.UNKNOWN_CONCEPT: - return template - - if isinstance(template, list): - # if template is a list, it means that there a multiple concepts under the same key - concepts = [self.new_from_template(t, concept_key, **kwargs) for t in template] - return concepts - else: - return self.new_from_template(template, concept_key, **kwargs) - - def new_from_template(self, template, key, **kwargs): - # manage singleton - if template.metadata.is_unique: - return template - - # otherwise, create another instance - concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept() - concept.update_from(template) - - if len(kwargs) == 0: - return concept - - # update the properties, values, attributes - # Not quite sure that this is the correct process order - for k, v in kwargs.items(): - if k in concept.props: - concept.set_prop(k, v) - elif k in PROPERTIES_FOR_NEW: - concept.values[ConceptParts(k)] = v - elif hasattr(concept, k): - setattr(concept, k, v) - else: - return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept) - - # TODO : add the concept to the list of known concepts (self.instances) - concept.metadata.is_evaluated = True - return concept - - def ret(self, who: str, status: bool, value, message=None, parents=None): - """ - Creates and returns a ReturnValue concept - :param who: - :param status: - :param value: - :param message: - :param parents: - :return: - """ - return self.new( - BuiltinConcepts.RETURN_VALUE, - who=who, - status=status, - value=value, - message=message, - parents=parents) - - def value(self, obj, reduce_simple_list=False): - if obj is None: - return None - - if hasattr(obj, "get_value"): - return obj.get_value() - - if not isinstance(obj, Concept): - return obj - - if obj.body is None: - return obj - - if reduce_simple_list and (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1: - body_to_use = obj.body[0] - else: - body_to_use = obj.body - - return self.value(body_to_use) - - def get_values(self, objs): - if not (isinstance(objs, list) or - self.isinstance(objs, BuiltinConcepts.LIST) or - self.isinstance(objs, BuiltinConcepts.ENUMERATION)): - objs = [objs] - - return (self.value(obj) for obj in objs) - - def is_success(self, obj): - if isinstance(obj, bool): # quick win - return obj - - if isinstance(obj, ReturnValueConcept): - return obj.status - - if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors: - return False - - return obj - - def is_known(self, obj): - if not isinstance(obj, Concept): - return True - - return obj.key != str(BuiltinConcepts.UNKNOWN_CONCEPT) - - def isinstance(self, a, b): - """ - return true if the concept a is an instance of the concept b - :param a: - :param b: - :return: - """ - - if isinstance(a, BuiltinConcepts): # common KSI error ;-) - raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept") - - if not isinstance(a, Concept): - return False - - b_key = b.key if isinstance(b, Concept) else str(b) - - # TODO : manage when a is the list of all possible b - # for example, if a is a color, it will be found the entry 'All_Colors' - return a.key == b_key - - def isa(self, a, b): - """ - return true if the concept a is a b - Will handle when the keyword isa will be implemented - :param a: - :param b: - :return: - """ - - if isinstance(a, BuiltinConcepts): # common KSI error ;-) - raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept") - - assert isinstance(a, Concept) - assert isinstance(b, Concept) - - # TODO, first check the 'isa' property of a - - return self.sdp.exists(GROUP_PREFIX + b.id, a.id) - - def isagroup(self, concept): - """True if exists All_ in sdp""" - if not concept.id: - return None - - res = self.sdp.get_safe(GROUP_PREFIX + concept.id) - return res is not None - - def get_evaluator_name(self, name): - if self.evaluators_prefix is None: - base_evaluator_class = core.utils.get_class("evaluators.BaseEvaluator.BaseEvaluator") - self.evaluators_prefix = base_evaluator_class.PREFIX - - return self.evaluators_prefix + name - - def get_parser_name(self, name): - if self.parsers_prefix is None: - base_parser_class = core.utils.get_class("parsers.BaseParser.BaseParser") - self.parsers_prefix = base_parser_class.PREFIX - - return self.parsers_prefix + name - - def get_concept_definition(self): - if self.concepts_definition_cache: - return self.concepts_definition_cache - - self.concepts_definition_cache = self.sdp.get_safe(self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False) or {} - return self.concepts_definition_cache - - def concepts(self): - res = [] - lst = self.sdp.list(self.CONCEPTS_ENTRY) - for item in lst: - if isinstance(item, list): - res.extend(item) - else: - res.append(item) - - return sorted(res, key=lambda i: int(i.id)) - - def test(self): - return f"I have access to Sheerka !" - - def test_error(self): - raise Exception("I can raise an error") - - def dump_concepts(self): - lst = self.sdp.list(self.CONCEPTS_ENTRY) - for item in lst: - if hasattr(item, "__iter__"): - for i in item: - self.log.info(i) - else: - self.log.info(item) - - def dump_definitions(self): - defs = self.sdp.get(self.CONCEPTS_DEFINITIONS_ENTRY) - self.log.info(defs) - - def dump_desc(self, *concept_names): - first = True - for concept_name in concept_names: - if isinstance(concept_name, Concept): - concepts = concept_name - else: - concepts = self.get(concept_name) - if self.isinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT): - self.log.error(f"Concept '{concept_name}' is unknown") - return False - - if not hasattr(concepts, "__iter__"): - concepts = [concepts] - - for c in concepts: - if not first: - self.log.info("") - self.log.info(f"name : {c.name}") - self.log.info(f"bnf : {c.metadata.definition}") - self.log.info(f"key : {c.key}") - self.log.info(f"body : {c.body}") - self.log.info(f"digest : {c.get_digest()}") - first = False - - @staticmethod - def _get_unknown(metadata): - """ - Returns the concept 'UnknownConcept' for a requested id or key - Note that I don't call the new() method to prevent cyclic call - :param metadata: - :return: - """ - - # metadata is a list of tuple that contains the known metadata for this concept - # ex : (key, 'not_found) - # or - # (id, invalid_id) - # - # the metadata can be a list, if several attributes where given - # (key, 'not_found), (id, invalid_id) - - unknown_concept = UnknownConcept() - unknown_concept.set_metadata_value(ConceptParts.BODY, metadata) - for meta in (metadata if isinstance(metadata, list) else [metadata]): - unknown_concept.set_prop(meta[0], meta[1]) - unknown_concept.metadata.is_evaluated = True - return unknown_concept - - @staticmethod - def get_builtins_classes_as_dict(): - res = {} - for c in core.utils.get_classes("core.builtin_concepts"): - if issubclass(c, Concept) and c != Concept: - res[c().metadata.key] = c - - return res - - @staticmethod - def init_logging(debug, loggers): - core.sheerka_logger.set_enabled(loggers) - if debug: - # log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s" - log_format = "%(asctime)s [%(levelname)s] %(message)s" - log_level = logging.DEBUG - else: - log_format = "%(message)s" - log_level = logging.INFO - - logging.basicConfig(format=log_format, level=log_level, handlers=[console_handler]) - - -class ExecutionContext: - """ - To keep track of the execution of a request - """ - - def __init__(self, - who, - event: Event, - sheerka: Sheerka, - desc: str = None, - **kwargs): - - self._parent = None - self._id = ExecutionContextIdManager.get_id(event.get_digest()) - self._tab = "" - self._bag = {} # other variables - self._start = 0 - self._stop = 0 - - self.who = who # who is asking - self.event = event # what was the (original) trigger - self.sheerka = sheerka # sheerka - self.desc = desc # human description of what is going on - self.children = [] - self.preprocess = None - - self.inputs = {} # what was the parameters of the execution context - self.values = {} # what was produced by the execution context - - self.obj = kwargs.pop("obj", None) - self.concepts = kwargs.pop("concepts", {}) - # update the other elements - for k, v in kwargs.items(): - self._bag[k] = v - - @property - def elapsed(self): - if self._start == 0: - return 0 - - return (self._stop if self._stop > 0 else time.time_ns()) - self._start - - @property - def elapsed_str(self): - nano_sec = self.elapsed - dt = nano_sec / 1e6 - return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s" - - @property - def id(self): - return self._id - - def __getattr__(self, item): - if item in self._bag: - return self._bag[item] - - raise AttributeError(f"'ExecutionContext' object has no attribute '{item}'") - - def __enter__(self): - self._start = time.time_ns() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self._stop = time.time_ns() - - def __repr__(self): - msg = f"ExecutionContext(who={self.who}, id={self._id}" - if self.desc: - msg += f", desc='{self.desc}'" - msg += ")" - return msg - - def add_preprocess(self, name, **kwargs): - preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS) - preprocess.set_prop("name", name) - for k, v in kwargs.items(): - preprocess.set_prop(k, v) - - if not self.preprocess: - self.preprocess = set() - self.preprocess.add(preprocess) - return self - - def add_inputs(self, **kwargs): - for k, v in kwargs.items(): - self.inputs[k] = v - return self - - def add_values(self, **kwargs): - for k, v in kwargs.items(): - self.values[k] = v - return self - - def get_concept(self, key): - # search in obj - if isinstance(self.obj, Concept): - if self.obj.key == key: - return self.obj - for prop in self.obj.props: - if prop == key: - value = self.obj.props[prop].value - if isinstance(value, Concept): - return value - - # search in concepts - if self.concepts: - for k, c in self.concepts.items(): - if k == key: - return c - - return self.sheerka.get(key) - - def new_concept(self, key, **kwargs): - # search in obj - if self.obj: - if self.obj.key == key: - return self.sheerka.new_from_template(self.obj, key, **kwargs) - for prop in self.obj.props: - if prop == key: - value = self.obj.props[prop].value - if isinstance(value, Concept): - return self.sheerka.new_from_template(value, key, **kwargs) - else: - return value - - if self.concepts: - for k, c in self.concepts.items(): - if k == key: - return self.sheerka.new_from_template(c, key, **kwargs) - - return self.sheerka.new(key, **kwargs) - - def push(self, who=None, desc=None, **kwargs): - who = who or self.who - _kwargs = {"obj": self.obj, "concepts": self.concepts} - _kwargs.update(self._bag) - _kwargs.update(kwargs) - new = ExecutionContext( - who, - self.event, - self.sheerka, - desc, - **_kwargs, - ) - new._parent = self - new._tab = self._tab + " " * DEBUG_TAB_SIZE - new.preprocess = self.preprocess - - self.children.append(new) - return new - - def log_new(self, logger): - logger.debug(f"[{self._id:2}]" + self._tab + str(self)) - - def log(self, logger, message, who=None): - logger.debug(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message)) - - def log_error(self, logger, message, who=None): - logger.exception(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message)) - - def log_result(self, logger, return_values): - if not logger.isEnabledFor(logging.DEBUG): - return - - if len(return_values) == 0: - logger.debug(self._tab + "No return value") - - for r in return_values: - to_str = self.return_value_to_str(r) - logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str) - - def to_dict(self): - from core.sheerka_transform import SheerkaTransform - st = SheerkaTransform(self.sheerka) - return st.to_dict(self) - - @staticmethod - def return_value_to_str(r): - value = str(r.value) - if len(value) > 50: - value = value[:47] + "..." - to_str = f"ReturnValue(who={r.who}, status={r.status}, value={value})" - return to_str - - -class ExecutionContextIdManager: - ids = {} - - @staticmethod - def get_id(event_digest): - if event_digest in ExecutionContextIdManager.ids: - ExecutionContextIdManager.ids[event_digest] += 1 - else: - ExecutionContextIdManager.ids[event_digest] = 0 - return ExecutionContextIdManager.ids[event_digest] diff --git a/main.py b/main.py index a3eb633..fd083d8 100644 --- a/main.py +++ b/main.py @@ -2,8 +2,8 @@ import getopt import sys import logging -from core.sheerka import Sheerka import core.utils +from core.sheerka.Sheerka import Sheerka def usage(): diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c080e3f --- /dev/null +++ b/setup.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from __future__ import absolute_import +from __future__ import print_function + +from glob import glob +from os.path import basename +from os.path import splitext + +from setuptools import find_packages +from setuptools import setup + +setup( + name='sheerka', + version='0.1', + license='', + description='A human/computer communication interface', + long_description='', + author='Kodjo Sossouvi', + author_email='kodjo.sossouvi@gmail.com', + url='', + packages=find_packages('src'), + package_dir={'': 'src'}, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + zip_safe=False, + classifiers=[ + # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Operating System :: Unix', + 'Operating System :: POSIX', + 'Operating System :: Microsoft :: Windows', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Topic :: Utilities', + ], install_requires=['pytest'] +) diff --git a/core/__init__.py b/src/core/__init__.py similarity index 100% rename from core/__init__.py rename to src/core/__init__.py diff --git a/core/ast/__init__.py b/src/core/ast/__init__.py similarity index 100% rename from core/ast/__init__.py rename to src/core/ast/__init__.py diff --git a/core/ast/nodes.py b/src/core/ast/nodes.py similarity index 100% rename from core/ast/nodes.py rename to src/core/ast/nodes.py diff --git a/core/ast/visitors.py b/src/core/ast/visitors.py similarity index 100% rename from core/ast/visitors.py rename to src/core/ast/visitors.py diff --git a/core/builtin_concepts.py b/src/core/builtin_concepts.py similarity index 100% rename from core/builtin_concepts.py rename to src/core/builtin_concepts.py diff --git a/core/builtin_helpers.py b/src/core/builtin_helpers.py similarity index 100% rename from core/builtin_helpers.py rename to src/core/builtin_helpers.py diff --git a/core/concept.py b/src/core/concept.py similarity index 100% rename from core/concept.py rename to src/core/concept.py diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py new file mode 100644 index 0000000..de2bcee --- /dev/null +++ b/src/core/sheerka/ExecutionContext.py @@ -0,0 +1,203 @@ +import logging +import time + +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from sdp.sheerkaDataProvider import Event + +DEBUG_TAB_SIZE = 4 + + +class ExecutionContext: + """ + To keep track of the execution of a request + """ + + ids = {} + + @staticmethod + def get_id(event_digest): + if event_digest in ExecutionContext.ids: + ExecutionContext.ids[event_digest] += 1 + else: + ExecutionContext.ids[event_digest] = 0 + return ExecutionContext.ids[event_digest] + + def __init__(self, + who, + event: Event, + sheerka, + desc: str = None, + **kwargs): + + self._parent = None + self._id = ExecutionContext.get_id(event.get_digest()) + self._tab = "" + self._bag = {} # other variables + self._start = 0 + self._stop = 0 + + self.who = who # who is asking + self.event = event # what was the (original) trigger + self.sheerka = sheerka # sheerka + self.desc = desc # human description of what is going on + self.children = [] + self.preprocess = None + + self.inputs = {} # what was the parameters of the execution context + self.values = {} # what was produced by the execution context + + self.obj = kwargs.pop("obj", None) + self.concepts = kwargs.pop("concepts", {}) + # update the other elements + for k, v in kwargs.items(): + self._bag[k] = v + + @property + def elapsed(self): + if self._start == 0: + return 0 + + return (self._stop if self._stop > 0 else time.time_ns()) - self._start + + @property + def elapsed_str(self): + nano_sec = self.elapsed + dt = nano_sec / 1e6 + return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s" + + @property + def id(self): + return self._id + + def __getattr__(self, item): + if item in self._bag: + return self._bag[item] + + raise AttributeError(f"'ExecutionContext' object has no attribute '{item}'") + + def __enter__(self): + self._start = time.time_ns() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._stop = time.time_ns() + + def __repr__(self): + msg = f"ExecutionContext(who={self.who}, id={self._id}" + if self.desc: + msg += f", desc='{self.desc}'" + msg += ")" + return msg + + def add_preprocess(self, name, **kwargs): + preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS) + preprocess.set_prop("name", name) + for k, v in kwargs.items(): + preprocess.set_prop(k, v) + + if not self.preprocess: + self.preprocess = set() + self.preprocess.add(preprocess) + return self + + def add_inputs(self, **kwargs): + for k, v in kwargs.items(): + self.inputs[k] = v + return self + + def add_values(self, **kwargs): + for k, v in kwargs.items(): + self.values[k] = v + return self + + def get_concept(self, key): + # search in obj + if isinstance(self.obj, Concept): + if self.obj.key == key: + return self.obj + for prop in self.obj.props: + if prop == key: + value = self.obj.props[prop].value + if isinstance(value, Concept): + return value + + # search in concepts + if self.concepts: + for k, c in self.concepts.items(): + if k == key: + return c + + return self.sheerka.get(key) + + def new_concept(self, key, **kwargs): + # search in obj + if self.obj: + if self.obj.key == key: + return self.sheerka.new_from_template(self.obj, key, **kwargs) + for prop in self.obj.props: + if prop == key: + value = self.obj.props[prop].value + if isinstance(value, Concept): + return self.sheerka.new_from_template(value, key, **kwargs) + else: + return value + + if self.concepts: + for k, c in self.concepts.items(): + if k == key: + return self.sheerka.new_from_template(c, key, **kwargs) + + return self.sheerka.new(key, **kwargs) + + def push(self, who=None, desc=None, **kwargs): + who = who or self.who + _kwargs = {"obj": self.obj, "concepts": self.concepts} + _kwargs.update(self._bag) + _kwargs.update(kwargs) + new = ExecutionContext( + who, + self.event, + self.sheerka, + desc, + **_kwargs, + ) + new._parent = self + new._tab = self._tab + " " * DEBUG_TAB_SIZE + new.preprocess = self.preprocess + + self.children.append(new) + return new + + def log_new(self, logger): + logger.debug(f"[{self._id:2}]" + self._tab + str(self)) + + def log(self, logger, message, who=None): + logger.debug(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message)) + + def log_error(self, logger, message, who=None): + logger.exception(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message)) + + def log_result(self, logger, return_values): + if not logger.isEnabledFor(logging.DEBUG): + return + + if len(return_values) == 0: + logger.debug(self._tab + "No return value") + + for r in return_values: + to_str = self.return_value_to_str(r) + logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str) + + def to_dict(self): + from core.sheerka_transform import SheerkaTransform + st = SheerkaTransform(self.sheerka) + return st.to_dict(self) + + @staticmethod + def return_value_to_str(r): + value = str(r.value) + if len(value) > 50: + value = value[:47] + "..." + to_str = f"ReturnValue(who={r.who}, status={r.status}, value={value})" + return to_str diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py new file mode 100644 index 0000000..8614e14 --- /dev/null +++ b/src/core/sheerka/Sheerka.py @@ -0,0 +1,600 @@ +from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \ + UnknownConcept +from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW +from core.sheerka.ExecutionContext import ExecutionContext +from core.sheerka.SheerkaCreateNewConcept import SheerkaCreateNewConcept +from core.sheerka.SheerkaDump import SheerkaDump +from core.sheerka.SheerkaEvaluateConcept import SheerkaEvaluateConcept +from core.sheerka.SheerkaExecute import SheerkaExecute +from core.sheerka.SheerkaSetsManager import SheerkaSetsManager +from sdp.sheerkaDataProvider import SheerkaDataProvider, Event +import core.utils +import core.builtin_helpers + +from core.sheerka_logger import console_handler + +import logging + +# CONCEPT_EVALUATION_STEPS = [ +# BuiltinConcepts.BEFORE_EVALUATION, +# BuiltinConcepts.EVALUATION, +# BuiltinConcepts.AFTER_EVALUATION] + +CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser" + + +class Sheerka(Concept): + """ + Main controller for the project + """ + + CONCEPTS_ENTRY = "All_Concepts" # to store all the concepts + CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID" + CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions" # to store definitions (bnf) of concepts + BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts" # sequential key for builtin concepts + USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts + + def __init__(self, skip_builtins_in_db=False, debug=False, loggers=None): + self.init_logging(debug, loggers) + + super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA) + self.log.debug("Starting Sheerka.") + + # cache of the most used concepts + # Note that these are only templates + # They are used as a footprint for instantiation + # Except of source when the concept is supposed to be unique + # key is the key of the concept (not the name or the id) + self.cache_by_key = {} + self.cache_by_id = {} + + # cache for concept definitions, + # Primarily used for unit test that does not have access to sdp + self.concepts_definition_cache = {} + + # + # cache for concepts grammars + # a grammar is a resolved BNF + self.concepts_grammars = {} + + # a concept can be instantiated + # ex: File is a concept, but File('foo.txt') is an instance + # TODO: manage contexts + self.instances = [] + + # List of the known rules by the system + # ex: hello => say('hello') + self.rules = [] + + self.sdp: SheerkaDataProvider = None # SheerkaDataProvider + self.builtin_cache = {} # cache for builtin concepts + self.parsers = {} # cache for builtin parsers + self.evaluators = [] # cache for builtin evaluators + + self.evaluators_prefix: str = None + self.parsers_prefix: str = None + + self.skip_builtins_in_db = skip_builtins_in_db + + self.execute_handler = SheerkaExecute(self) + self.create_new_concept_handler = SheerkaCreateNewConcept(self) + self.dump_handler = SheerkaDump(self) + self.sets_handler = SheerkaSetsManager(self) + self.evaluate_concept_handler = SheerkaEvaluateConcept(self) + + def initialize(self, root_folder: str = None): + """ + Starting Sheerka + Loads the current configuration + Notes that when it's the first time, it also create the needed working folders + :param root_folder: root configuration folder + :return: ReturnValue(Success or Error) + """ + + try: + self.sdp = SheerkaDataProvider(root_folder) + if self.sdp.first_time: + self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000) + + event = Event("Initializing Sheerka.") + self.sdp.save_event(event) + exec_context = ExecutionContext(self.key, event, self) + + self.initialize_builtin_concepts() + self.initialize_builtin_parsers() + self.initialize_builtin_evaluators() + self.initialize_concepts_definitions(exec_context) + + except IOError as e: + return ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e) + + return ReturnValueConcept(self, True, self) + + def initialize_builtin_concepts(self): + """ + Initializes the builtin concepts + :return: None + """ + self.init_log.debug("Initializing builtin concepts") + builtins_classes = self.get_builtins_classes_as_dict() + + # this all initialization of the builtins seems to be little bit complicated + # why do we need to update it from DB ? + for key in BuiltinConcepts: + concept = self if key == BuiltinConcepts.SHEERKA \ + else builtins_classes[str(key)]() if str(key) in builtins_classes \ + else Concept(key, True, False, key) + + if key in BuiltinUnique: + concept.metadata.is_unique = True + concept.metadata.is_evaluated = True + + if not concept.metadata.is_unique and str(key) in builtins_classes: + self.builtin_cache[key] = builtins_classes[str(key)] + + if not self.skip_builtins_in_db: + from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.metadata.key) + if from_db is None: + self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.") + self.set_id_if_needed(concept, True) + self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True) + else: + self.init_log.debug(f"Found concept '{from_db}' in db. Updating.") + concept.update_from(from_db) + + self.add_in_cache(concept) + + def initialize_builtin_parsers(self): + """ + Init the parsers + :return: + """ + core.utils.init_package_import("parsers") + base_class = core.utils.get_class("parsers.BaseParser.BaseParser") + for parser in core.utils.get_sub_classes("parsers", base_class): + if parser.__module__ == base_class.__module__: + continue + + self.init_log.debug(f"Adding builtin parser '{parser.__name__}'") + self.parsers[core.utils.get_full_qualified_name(parser)] = parser + + def initialize_builtin_evaluators(self): + """ + Init the evaluators + :return: + """ + core.utils.init_package_import("evaluators") + for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator"): + self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'") + self.evaluators.append(evaluator) + + for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator"): + self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'") + self.evaluators.append(evaluator) + + def initialize_concepts_definitions(self, execution_context): + self.init_log.debug("Initializing concepts definitions") + definitions = self.sdp.get_safe(self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False) + + if definitions is None: + self.init_log.debug("No BNF defined") + return + + lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]() + ret_val = lexer_parser.initialize(execution_context, definitions) + if not ret_val.status: + self.init_log.error("Failed to initialize concepts definitions " + str(ret_val.body)) + return + + self.concepts_grammars = lexer_parser.concepts_grammars + + def reset_cache(self, filter_to_use=None): + """ + reset the different cache that exists + :param filter_to_use: + :return: + """ + if filter_to_use is None: + self.cache_by_key = {} + self.cache_by_id = {} + else: + raise NotImplementedError() + + return self + + def evaluate_user_input(self, text: str, user_name="kodjo"): + """ + Note to KSI: If you try to add execution context to this function, + You may end in an infinite loop + :param text: + :param user_name: + :return: + """ + self.log.debug(f"Processing user input '{text}', {user_name=}.") + event = Event(text, user_name) + evt_digest = self.sdp.save_event(event) + self.log.debug(f"{evt_digest=}") + + with ExecutionContext(self.key, event, self, f"Evaluating '{text}'") as execution_context: + user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name)) + reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED)) + + steps = [ + BuiltinConcepts.BEFORE_PARSING, + BuiltinConcepts.PARSING, + BuiltinConcepts.AFTER_PARSING, + BuiltinConcepts.BEFORE_EVALUATION, + BuiltinConcepts.EVALUATION, + BuiltinConcepts.AFTER_EVALUATION + ] + + ret = self.execute(execution_context, [user_input, reduce_requested], steps) + execution_context.add_values(return_values=ret) + + if not self.skip_builtins_in_db: + self.sdp.save_result(execution_context) + return ret + + def execute(self, execution_context, return_values, execution_steps, logger=None): + """ + Executes process for all initial contexts + :param execution_context: + :param return_values: + :param execution_steps: + :param logger: logger to use (if not directly called by sheerka) + :return: + """ + return self.execute_handler.execute(execution_context, return_values, execution_steps, logger) + + def set_id_if_needed(self, obj: Concept, is_builtin: bool): + """ + Set the key for the concept if needed + For test purpose only !!!!! + :param obj: + :param is_builtin: + :return: + """ + if obj.metadata.id is not None: + return + + entry = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS + obj.metadata.id = self.sdp.get_next_key(entry) + self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.") + + def create_new_concept(self, context, concept: Concept, logger=None): + """ + Adds a new concept to the system + :param context: + :param concept: DefConceptNode + :param logger + :return: digest of the new concept + """ + + return self.create_new_concept_handler.create_new_concept(context, concept, logger) + + def add_concept_to_set(self, context, concept, concept_set, logger=None): + """ + Add an entry in sdp to tell that concept isa concept_set + :param context: + :param concept: + :param concept_set: + :param logger: + :return: + """ + return self.sets_handler.add_concept_to_set(context, concept, concept_set, logger) + + def get_set_elements(self, concept): + """ + Concept is supposed to be a set + Returns all elements if the set + :param concept: + :return: + """ + + return self.sets_handler.get_set_elements(concept) + + def evaluate_concept(self, context, concept: Concept, logger=None): + """ + Evaluation a concept + It means that if the where clause is True, will evaluate the body + :param context: + :param concept: + :param logger: + :return: value of the evaluation or error + """ + return self.evaluate_concept_handler.evaluate_concept(context, concept, logger) + + def add_in_cache(self, concept: Concept): + """ + Adds a concept template in cache. + The cache is used as a proxy before looking at sdp + :param concept: + :return: + """ + + # sanity check + if concept.key is None: + concept.init_key() + + if concept.key is None: + raise KeyError() + + self.cache_by_key[concept.key] = concept + + if concept.id: + self.cache_by_id[concept.id] = concept + + return concept + + def get(self, concept_key, concept_id=None): + """ + Tries to find a concept + What is return must be used a template for another concept. + You must not modify the returned concept + :param concept_key: key of the concept + :param concept_id: when multiple concepts with the same key, use the id + :return: + """ + + if concept_key is None: + return ErrorConcept("Concept key is undefined.") + + if isinstance(concept_key, BuiltinConcepts): + concept_key = str(concept_key) + + # first search in cache + result = self.cache_by_key[concept_key] if concept_key in self.cache_by_key else \ + self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key) + + if result and (concept_id is None or not isinstance(result, list)): + return result + + if isinstance(result, list): + if concept_id: + for c in result: + if c.id == concept_id: + return c + else: + return result + + metadata = [("key", concept_key), ("id", concept_id)] if concept_id else ("key", concept_key) + return self._get_unknown(metadata) + + def get_by_id(self, concept_id): + if concept_id is None: + return ErrorConcept("Concept id is undefined.") + + # first search in cache + result = self.cache_by_id[concept_id] if concept_id in self.cache_by_id else \ + self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id) + + return result or self._get_unknown(('id', concept_id)) + + def get_concept_definition(self): + if self.concepts_definition_cache: + return self.concepts_definition_cache + + self.concepts_definition_cache = self.sdp.get_safe( + self.CONCEPTS_DEFINITIONS_ENTRY, + load_origin=False) or {} + return self.concepts_definition_cache + + def new(self, concept_key, **kwargs): + """ + Returns an instance of a new concept + When the concept is supposed to be unique, returns the same instance + :param concept_key: + :param kwargs: + :return: + """ + if isinstance(concept_key, tuple): + concept_key, concept_id = concept_key[0], concept_key[1] + else: + concept_id = None + + template = self.get(concept_key, concept_id) + + # manage concept not found + if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \ + concept_key != BuiltinConcepts.UNKNOWN_CONCEPT: + return template + + if isinstance(template, list): + # if template is a list, it means that there a multiple concepts under the same key + concepts = [self.new_from_template(t, concept_key, **kwargs) for t in template] + return concepts + else: + return self.new_from_template(template, concept_key, **kwargs) + + def new_from_template(self, template, key, **kwargs): + # manage singleton + if template.metadata.is_unique: + return template + + # otherwise, create another instance + concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept() + concept.update_from(template) + + if len(kwargs) == 0: + return concept + + # update the properties, values, attributes + # Not quite sure that this is the correct process order + for k, v in kwargs.items(): + if k in concept.props: + concept.set_prop(k, v) + elif k in PROPERTIES_FOR_NEW: + concept.values[ConceptParts(k)] = v + elif hasattr(concept, k): + setattr(concept, k, v) + else: + return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept) + + # TODO : add the concept to the list of known concepts (self.instances) + concept.metadata.is_evaluated = True + return concept + + def ret(self, who: str, status: bool, value, message=None, parents=None): + """ + Creates and returns a ReturnValue concept + :param who: + :param status: + :param value: + :param message: + :param parents: + :return: + """ + return self.new( + BuiltinConcepts.RETURN_VALUE, + who=who, + status=status, + value=value, + message=message, + parents=parents) + + def value(self, obj, reduce_simple_list=False): + if obj is None: + return None + + if hasattr(obj, "get_value"): + return obj.get_value() + + if not isinstance(obj, Concept): + return obj + + if obj.body is None: + return obj + + if reduce_simple_list and (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1: + body_to_use = obj.body[0] + else: + body_to_use = obj.body + + return self.value(body_to_use) + + def get_values(self, objs): + if not (isinstance(objs, list) or + self.isinstance(objs, BuiltinConcepts.LIST) or + self.isinstance(objs, BuiltinConcepts.ENUMERATION)): + objs = [objs] + + return (self.value(obj) for obj in objs) + + def is_success(self, obj): + if isinstance(obj, bool): # quick win + return obj + + if isinstance(obj, ReturnValueConcept): + return obj.status + + if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors: + return False + + return obj + + def is_known(self, obj): + if not isinstance(obj, Concept): + return True + + return obj.key != str(BuiltinConcepts.UNKNOWN_CONCEPT) + + def isinstance(self, a, b): + """ + return true if the concept a is an instance of the concept b + :param a: + :param b: + :return: + """ + + if isinstance(a, BuiltinConcepts): # common KSI error ;-) + raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept") + + if not isinstance(a, Concept): + return False + + b_key = b.key if isinstance(b, Concept) else str(b) + + return a.key == b_key + + def isa(self, a, b): + return self.sets_handler.isa(a, b) + + def isagroup(self, concept): + return self.sets_handler.isagroup(concept) + + def get_evaluator_name(self, name): + if self.evaluators_prefix is None: + base_evaluator_class = core.utils.get_class("evaluators.BaseEvaluator.BaseEvaluator") + self.evaluators_prefix = base_evaluator_class.PREFIX + + return self.evaluators_prefix + name + + def get_parser_name(self, name): + if self.parsers_prefix is None: + base_parser_class = core.utils.get_class("parsers.BaseParser.BaseParser") + self.parsers_prefix = base_parser_class.PREFIX + + return self.parsers_prefix + name + + def concepts(self): + res = [] + lst = self.sdp.list(self.CONCEPTS_ENTRY) + for item in lst: + if isinstance(item, list): + res.extend(item) + else: + res.append(item) + + return sorted(res, key=lambda i: int(i.id)) + + def test(self): + return f"I have access to Sheerka !" + + def test_error(self): + raise Exception("I can raise an error") + + @staticmethod + def _get_unknown(metadata): + """ + Returns the concept 'UnknownConcept' for a requested id or key + Note that I don't call the new() method to prevent cyclic call + :param metadata: + :return: + """ + + # metadata is a list of tuple that contains the known metadata for this concept + # ex : (key, 'not_found) + # or + # (id, invalid_id) + # + # the metadata can be a list, if several attributes where given + # (key, 'not_found), (id, invalid_id) + + unknown_concept = UnknownConcept() + unknown_concept.set_metadata_value(ConceptParts.BODY, metadata) + for meta in (metadata if isinstance(metadata, list) else [metadata]): + unknown_concept.set_prop(meta[0], meta[1]) + unknown_concept.metadata.is_evaluated = True + return unknown_concept + + @staticmethod + def get_builtins_classes_as_dict(): + res = {} + for c in core.utils.get_classes("core.builtin_concepts"): + if issubclass(c, Concept) and c != Concept: + res[c().metadata.key] = c + + return res + + @staticmethod + def init_logging(debug, loggers): + core.sheerka_logger.set_enabled(loggers) + if debug: + # log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s" + log_format = "%(asctime)s [%(levelname)s] %(message)s" + log_level = logging.DEBUG + else: + log_format = "%(message)s" + log_level = logging.INFO + + logging.basicConfig(format=log_format, level=log_level, handlers=[console_handler]) diff --git a/src/core/sheerka/SheerkaCreateNewConcept.py b/src/core/sheerka/SheerkaCreateNewConcept.py new file mode 100644 index 0000000..65bd19e --- /dev/null +++ b/src/core/sheerka/SheerkaCreateNewConcept.py @@ -0,0 +1,99 @@ +from core.builtin_concepts import BuiltinConcepts, ErrorConcept +from core.concept import Concept +from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError + +CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser" + + +class SheerkaCreateNewConcept: + """ + Manage the creation of a new concept + """ + + def __init__(self, sheerka): + self.sheerka = sheerka + self.logger_name = self.create_new_concept.__name__ + + def create_new_concept(self, context, concept: Concept, logger=None): + """ + Adds a new concept to the system + :param context: + :param concept: DefConceptNode + :param logger + :return: digest of the new concept + """ + + logger = logger or self.sheerka.log + + concept.init_key() + concepts_definitions = None + init_ret_value = None + + # checks for duplicate concepts + # TODO checks if it exists in cache first + + if self.sheerka.sdp.exists(self.sheerka.CONCEPTS_ENTRY, concept.key, concept.get_digest()): + error = SheerkaDataProviderDuplicateKeyError(self.sheerka.CONCEPTS_ENTRY + "." + concept.key, concept) + return self.sheerka.ret( + self.logger_name, + False, + self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept), + error.args[0]) + + # set id before saving in db + self.sheerka.set_id_if_needed(concept, False) + + # add the BNF if known + if concept.bnf: + concepts_definitions = self.sheerka.get_concept_definition() + concepts_definitions[concept] = concept.bnf + + # check if it's a valid BNF or whether it breaks the known rules + concept_lexer_parser = self.sheerka.parsers[CONCEPT_LEXER_PARSER_CLASS]() + with context.push(self.sheerka.name, desc=f"Initializing concept definition for {concept}") as sub_context: + sub_context.concepts[concept.key] = concept # the concept is not in the real cache yet + sub_context.log_new(logger) + init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions) + sub_context.add_values(return_values=init_ret_value) + if not init_ret_value.status: + return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value)) + + # save the new concept in sdp + try: + # TODO : needs to make these calls atomic (or at least one single call) + self.sheerka.sdp.add( + context.event.get_digest(), + self.sheerka.CONCEPTS_ENTRY, + concept, + use_ref=True) + self.sheerka.sdp.add( + context.event.get_digest(), + self.sheerka.CONCEPTS_BY_ID_ENTRY, + {concept.id: concept.get_digest()}, + is_ref=True) + if concepts_definitions is not None: + self.sheerka.sdp.set( + context.event.get_digest(), + self.sheerka.CONCEPTS_DEFINITIONS_ENTRY, + concepts_definitions, + use_ref=True) + except SheerkaDataProviderDuplicateKeyError as error: + context.log_error(logger, "Failed to create a new concept.", who=self.logger_name) + return self.sheerka.ret( + self.logger_name, + False, + self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept), + error.args[0]) + + # Updates the caches + self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key) + self.sheerka.cache_by_id[concept.id] = concept + if init_ret_value is not None and init_ret_value.status: + self.sheerka.concepts_grammars = init_ret_value.body + + # process the return in needed + ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) + return ret + + + diff --git a/src/core/sheerka/SheerkaDump.py b/src/core/sheerka/SheerkaDump.py new file mode 100644 index 0000000..315006b --- /dev/null +++ b/src/core/sheerka/SheerkaDump.py @@ -0,0 +1,44 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept + + +class SheerkaDump: + def __init__(self, sheerka): + self.sheerka = sheerka + + def dump_concepts(self): + lst = self.sheerkasdp.list(self.sheerkaCONCEPTS_ENTRY) + for item in lst: + if hasattr(item, "__iter__"): + for i in item: + self.sheerkalog.info(i) + else: + self.sheerkalog.info(item) + + def dump_definitions(self): + defs = self.sheerkasdp.get(self.sheerkaCONCEPTS_DEFINITIONS_ENTRY) + self.sheerkalog.info(defs) + + def dump_desc(self, *concept_names): + first = True + for concept_name in concept_names: + if isinstance(concept_name, Concept): + concepts = concept_name + else: + concepts = self.sheerkaget(concept_name) + if self.sheerkaisinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT): + self.sheerkalog.error(f"Concept '{concept_name}' is unknown") + return False + + if not hasattr(concepts, "__iter__"): + concepts = [concepts] + + for c in concepts: + if not first: + self.sheerkalog.info("") + self.sheerkalog.info(f"name : {c.name}") + self.sheerkalog.info(f"bnf : {c.metadata.definition}") + self.sheerkalog.info(f"key : {c.key}") + self.sheerkalog.info(f"body : {c.body}") + self.sheerkalog.info(f"digest : {c.get_digest()}") + first = False diff --git a/src/core/sheerka/SheerkaEvaluateConcept.py b/src/core/sheerka/SheerkaEvaluateConcept.py new file mode 100644 index 0000000..7229867 --- /dev/null +++ b/src/core/sheerka/SheerkaEvaluateConcept.py @@ -0,0 +1,195 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, DoNotResolve, ConceptParts +import core.builtin_helpers + +CONCEPT_EVALUATION_STEPS = [ + BuiltinConcepts.BEFORE_EVALUATION, + BuiltinConcepts.EVALUATION, + BuiltinConcepts.AFTER_EVALUATION] + + +class SheerkaEvaluateConcept: + def __init__(self, sheerka): + self.sheerka = sheerka + self.logger_name = self.evaluate_concept.__name__ + + def initialize_concept_asts(self, context, concept: Concept, logger=None): + """ + Updates the codes of the newly created concept + Basically, it runs the parsers on all parts + :param concept: + :param context: + :param logger: + :return: + """ + steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING] + for part_key in ConceptParts: + if part_key in concept.compiled: + continue + + source = getattr(concept.metadata, part_key.value) + if source is None or not isinstance(source, str): + continue + + if source.strip() == "": + concept.compiled[part_key] = DoNotResolve(source) + else: + with context.push(desc=f"Initializing compiled for {part_key}") as sub_context: + sub_context.log_new(logger) + sub_context.add_inputs(source=source) + to_parse = self.sheerka.ret(context.who, True, + self.sheerka.new(BuiltinConcepts.USER_INPUT, body=source)) + res = self.sheerka.execute(sub_context, to_parse, steps, logger) + concept.compiled[part_key] = res + sub_context.add_values(return_values=res) + + for prop, default_value in concept.metadata.props: + if prop in concept.compiled: + continue + + if default_value is None or not isinstance(default_value, str): + continue + + if default_value.strip() == "": + concept.compiled[prop] = DoNotResolve(default_value) + else: + with context.push(desc=f"Initializing AST for property {prop}") as sub_context: + sub_context.log_new(logger) + sub_context.add_inputs(source=default_value) + to_parse = self.sheerka.ret(context.who, True, + self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value)) + res = self.sheerka.execute(context, to_parse, steps) + concept.compiled[prop] = res + sub_context.add_values(return_values=res) + + # Updates the cache of concepts when possible + if concept.key in self.sheerka.cache_by_key: + entry = self.sheerka.cache_by_key[concept.key] + if isinstance(entry, list): + # TODO : manage when there are multiple entries + pass + else: + self.sheerka.cache_by_key[concept.key].compiled = concept.compiled + + def resolve(self, context, to_resolve, current_prop, current_concept, logger): + if isinstance(to_resolve, DoNotResolve): + return to_resolve.value + + desc = f"Evaluating {current_prop} (concept={current_concept})" + context.log(logger, desc, self.logger_name) + with context.push(desc=desc, obj=current_concept) as sub_context: + sub_context.log_new(logger) + + # when it's a concept, evaluate it + if isinstance(to_resolve, Concept) and \ + not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE): + evaluated = self.evaluate_concept(sub_context, to_resolve, logger) + sub_context.add_values(return_values=evaluated) + if evaluated.key == to_resolve.key: + return evaluated + else: + error = evaluated + + # otherwise, execute all return values to find out what is the value + else: + r = self.sheerka.execute(sub_context, to_resolve, CONCEPT_EVALUATION_STEPS, logger) + one_r = core.builtin_helpers.expect_one(context, r) + sub_context.add_values(return_values=one_r) + if one_r.status: + return one_r.value + else: + error = one_r.value + + return self.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR, + body=error, + concept=current_concept, + property_name=current_prop) + + def resolve_list(self, context, list_to_resolve, current_prop, current_concept, logger): + """When dealing with a list, there are two possibilities""" + # It may be a list of ReturnValueConcept to execute (always the case for metadata) + # or a list of single values (may be the case for properties) + # in this latter case, all values are to be processed one by one and a list should be returned + if len(list_to_resolve) == 0: + return [] + + if self.sheerka.isinstance(list_to_resolve[0], BuiltinConcepts.RETURN_VALUE): + return self.resolve(context, list_to_resolve, current_prop, current_concept, logger) + + res = [] + for to_resolve in list_to_resolve: + # sanity check + if self.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE): + return self.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR, + body="Mix between real values and return values", + concept=current_concept, + property_name=current_prop) + + r = self.resolve(context, to_resolve, current_prop, current_concept, logger) + if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR): + return r + res.append(r) + + return res + + def evaluate_concept(self, context, concept: Concept, logger=None): + """ + Evaluation a concept + It means that if the where clause is True, will evaluate the body + :param context: + :param concept: + :param logger: + :return: value of the evaluation or error + """ + + logger = logger or self.sheerka.log + + if concept.metadata.is_evaluated: + return concept + + # WHERE condition should already be validated by the parser. + # It's a mandatory condition for the concept before it can be recognized + + # + # TODO : Validate the PRE condition + # + + self.initialize_concept_asts(context, concept, logger) + + # to make sure of the order, it don't use ConceptParts.get_parts() + # props must be evaluated first + all_metadata_to_eval = ["props", "where", "pre", "post", "body"] + + for metadata_to_eval in all_metadata_to_eval: + if metadata_to_eval == "props": + for prop_name in (p for p in concept.props if p in concept.compiled): + prop_ast = concept.compiled[prop_name] + + if isinstance(prop_ast, list): + # Do not send the current concept for the properties + resolved = self.resolve_list(context, prop_ast, prop_name, None, logger) + else: + # Do not send the current concept for the properties + resolved = self.resolve(context, prop_ast, prop_name, None, logger) + if context.sheerka.isinstance(resolved, BuiltinConcepts.CONCEPT_EVAL_ERROR): + resolved.set_prop("concept", concept) # since current concept was not sent + return resolved + else: + concept.set_prop(prop_name, resolved) + else: + part_key = ConceptParts(metadata_to_eval) + if part_key in concept.compiled and concept.compiled[part_key] is not None: + metadata_ast = concept.compiled[part_key] + resolved = self.resolve(context, metadata_ast, part_key, concept, logger) + if context.sheerka.isinstance(resolved, BuiltinConcepts.CONCEPT_EVAL_ERROR): + return resolved + else: + concept.values[part_key] = resolved + + # + # TODO : Validate the POST condition + # + + concept.init_key() # only does it if needed + concept.metadata.is_evaluated = True + return concept diff --git a/src/core/sheerka/SheerkaExecute.py b/src/core/sheerka/SheerkaExecute.py new file mode 100644 index 0000000..f3fc236 --- /dev/null +++ b/src/core/sheerka/SheerkaExecute.py @@ -0,0 +1,254 @@ +from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept +import core.utils + + +class SheerkaExecute: + """ + Manage the execution of a process flow + """ + + def __init__(self, sheerka): + self.sheerka = sheerka + + def call_parsers(self, execution_context, return_values, logger=None): + + # return_values must be a list + if not isinstance(return_values, list): + return_values = [return_values] + + # first make the distinguish between what is for the parsers and what is not + result = [] + to_process = [] + for r in return_values: + if not r.status or not self.sheerka.isinstance(r.body, BuiltinConcepts.USER_INPUT): + result.append(r) + else: + to_process.append(r) + + if not to_process: + return result + + # keep track of the originals user inputs, as they need to be removed at the end + user_inputs = to_process[:] + + # group the parsers by priorities + instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()] + grouped_parsers = {} + for parser in [p for p in instantiated_parsers if p.enabled]: + if logger: + parser.log = logger + grouped_parsers.setdefault(parser.priority, []).append(parser) + sorted_priorities = sorted(grouped_parsers.keys(), reverse=True) + + stop_processing = False + for priority in sorted_priorities: + inputs_for_this_group = to_process[:] + + for parser in grouped_parsers[priority]: + + return_value_success_found = False + for return_value in inputs_for_this_group: + + to_parse = return_value.body.body \ + if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \ + else return_value.body + + # if self.sheerka.log.isEnabledFor(logging.DEBUG): + # debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \ + # else "'" + BaseParser.get_text_from_tokens(to_parse) + "' as tokens" + # execution_context.log(logger or self.sheerka.log, f"Parsing {debug_text}") + + with execution_context.push(desc=f"Parsing using {parser.name}") as sub_context: + sub_context.add_inputs(to_parse=to_parse) + res = parser.parse(sub_context, to_parse) + if res is not None: + if hasattr(res, "__iter__"): + for r in res: + if r is None: + continue + r.parents = [return_value] + result.append(r) + if self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT): + to_process.append(r) + if r.status: + return_value_success_found = True + + else: + res.parents = [return_value] + result.append(res) + if self.sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT): + to_process.append(res) + if res.status: + return_value_success_found = True + sub_context.add_values(return_values=res) + + if return_value_success_found: + stop_processing = True + break # Stop the other return_values (but not the other parsers with the same priority) + + if stop_processing: + break # Do not try the other priorities if a match is found + + result = core.utils.remove_list_from_list(result, user_inputs) + return result + + def call_evaluators(self, execution_context, return_values, process_step, evaluation_context=None, logger=None): + + # return_values must be a list + if not isinstance(return_values, list): + return_values = [return_values] + + # Evaluation context are contexts that may modify the behaviour of the execution + # For example, a concept to indicate that the value is not wanted + # Or a concept to indicate that we want the letter form of the response + # But first, they need to be transformed into return values + if evaluation_context is None: + evaluation_return_values = [] + else: + evaluation_return_values = [self.sheerka.ret(execution_context.who, True, c) for c in evaluation_context] + + # add the current step as part as the evaluation context + evaluation_return_values.append(self.sheerka.ret(execution_context.who, True, self.sheerka.new(process_step))) + + # the pool of return values are the mix + return_values.extend(evaluation_return_values) + + # group the evaluators by priority and sort them + # The first one to be applied will be the one with the highest priority + grouped_evaluators = {} + instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators] + + # pre-process evaluators if needed + instantiated_evaluators = self._preprocess_evaluators(execution_context, instantiated_evaluators) + + for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]: + if logger: + evaluator.log = logger + grouped_evaluators.setdefault(evaluator.priority, []).append(evaluator) + + # order the groups by priority, the higher first + sorted_priorities = sorted(grouped_evaluators.keys(), reverse=True) + + # process + iteration = 0 + while True: + with execution_context.push(desc=f"iteration #{iteration}", iteration=iteration) as iteration_context: + simple_digest = return_values[:] + iteration_context.add_inputs(return_values=simple_digest) + + for priority in sorted_priorities: + + original_items = return_values[:] + evaluated_items = [] + to_delete = [] + for evaluator in grouped_evaluators[priority]: + evaluator = self._preprocess_evaluators(execution_context, evaluator.__class__()) # fresh copy + + sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})" + with iteration_context.push(desc=sub_context_desc) as sub_context: + sub_context.add_inputs(return_values=original_items) + + # process evaluators that work on one simple return value at the time + from evaluators.BaseEvaluator import OneReturnValueEvaluator + if isinstance(evaluator, OneReturnValueEvaluator): + debug_result = [] + for item in original_items: + if evaluator.matches(sub_context, item): + result = evaluator.eval(sub_context, item) + if result is None: + debug_result.append({"input": item, "return_value": None}) + continue + + to_delete.append(item) + if isinstance(result, list): + evaluated_items.extend(result) + elif isinstance(result, ReturnValueConcept): + evaluated_items.append(result) + else: + error = self.sheerka.new(BuiltinConcepts.INVALID_RETURN_VALUE, body=result, + evaluator=evaluator) + result = self.sheerka.ret("sheerka.process", False, error, parents=[item]) + evaluated_items.append(result) + debug_result.append({"input": item, "return_value": result}) + else: + debug_result.append({"input": item, "return_value": "** No Match **"}) + sub_context.add_values(return_values=debug_result) + + # process evaluators that work on all return values + else: + if evaluator.matches(sub_context, original_items): + results = evaluator.eval(sub_context, original_items) + if results is None: + continue + if not isinstance(results, list): + results = [results] + for result in results: + evaluated_items.append(result) + to_delete.extend(result.parents) + sub_context.add_values(return_values=results) + else: + sub_context.add_values(return_values="** No Match **") + + return_values = evaluated_items + return_values.extend([item for item in original_items if item not in to_delete]) + + iteration_context.add_values(return_values=return_values[:]) + + # have we done something ? + to_compare = return_values[:] + if simple_digest == to_compare: + break + + # inc the iteration and continue + iteration += 1 + + # remove all evaluation context that are not reduced + return_values = core.utils.remove_list_from_list(return_values, evaluation_return_values) + return return_values + + def execute(self, execution_context, return_values, execution_steps, logger=None): + """ + Executes process for all initial contexts + :param execution_context: + :param return_values: + :param execution_steps: + :param logger: logger to use (if not directly called by sheerka) + :return: + """ + + for step in execution_steps: + copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values] + with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context: + sub_context.log(logger or self.sheerka.log, f"{step=}, context='{sub_context}'") + + if step == BuiltinConcepts.PARSING: + return_values = self.call_parsers(sub_context, return_values, logger) + else: + return_values = self.call_evaluators(sub_context, return_values, step, None, logger) + + if copy != return_values: + sub_context.log_result(logger or self.sheerka.log, return_values) + + sub_context.add_values(return_values=return_values) + + return return_values + + def _preprocess_evaluators(self, context, evaluators): + if not context.preprocess: + return evaluators + + if not hasattr(evaluators, "__iter__"): + single_one = True + evaluators = [evaluators] + else: + single_one = False + + for preprocess in context.preprocess: + for e in evaluators: + if preprocess.props["name"].value == e.name: + for prop, value in preprocess.props.items(): + if prop == "name": + continue + if hasattr(e, prop): + setattr(e, prop, value.value) + return evaluators[0] if single_one else evaluators diff --git a/src/core/sheerka/SheerkaSetsManager.py b/src/core/sheerka/SheerkaSetsManager.py new file mode 100644 index 0000000..5065e1e --- /dev/null +++ b/src/core/sheerka/SheerkaSetsManager.py @@ -0,0 +1,83 @@ +from core.builtin_concepts import BuiltinConcepts, ErrorConcept +from core.concept import Concept + +GROUP_PREFIX = 'All_' + + +class SheerkaSetsManager: + def __init__(self, sheerka): + self.sheerka = sheerka + self.logger_name = self.add_concept_to_set.__name__ + + def add_concept_to_set(self, context, concept, concept_set, logger=None): + """ + Add an entry in sdp to tell that concept isa concept_set + :param context: + :param concept: + :param concept_set: + :param logger: + :return: + """ + logger = logger or self.sheerka.log + + context.log(logger, f"Adding concept {concept} to set {concept_set}", who=self.logger_name) + + assert concept.id + assert concept_set.id + + try: + ret = self.sheerka.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id) + if ret == (None, None): # concept already in set + return self.sheerka.ret( + self.logger_name, + False, + self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set)) + else: + return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + except Exception as error: + context.log_error(logger, "Failed to add to set.", who=self.logger_name) + return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0]) + + def get_set_elements(self, concept): + """ + Concept is supposed to be a set + Returns all elements if the set + :param concept: + :return: + """ + + assert concept.id + + ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id) + if ids is None: + return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept) + + elements = [self.sheerka.get_by_id(element_id) for element_id in ids] + return elements + + def isa(self, a, b): + """ + return true if the concept a is a b + Will handle when the keyword isa will be implemented + :param a: + :param b: + :return: + """ + + if isinstance(a, BuiltinConcepts): # common KSI error ;-) + raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept") + + assert isinstance(a, Concept) + assert isinstance(b, Concept) + + # TODO, first check the 'isa' property of a + + return self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id) + + def isagroup(self, concept): + """True if exists All_ in sdp""" + if not concept.id: + return None + + res = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id) + return res is not None diff --git a/evaluators/__init__.py b/src/core/sheerka/__init__.py similarity index 100% rename from evaluators/__init__.py rename to src/core/sheerka/__init__.py diff --git a/core/sheerka_logger.py b/src/core/sheerka_logger.py similarity index 100% rename from core/sheerka_logger.py rename to src/core/sheerka_logger.py diff --git a/core/sheerka_transform.py b/src/core/sheerka_transform.py similarity index 98% rename from core/sheerka_transform.py rename to src/core/sheerka_transform.py index 85415ae..45fd829 100644 --- a/core/sheerka_transform.py +++ b/src/core/sheerka_transform.py @@ -2,7 +2,7 @@ import dataclasses from enum import Enum from core.concept import Concept, PROPERTIES_TO_SERIALIZE -from core.sheerka import ExecutionContext +from core.sheerka.Sheerka import ExecutionContext from core.tokenizer import Token from evaluators.BaseEvaluator import BaseEvaluator from parsers.BaseParser import BaseParser, Node diff --git a/core/tokenizer.py b/src/core/tokenizer.py similarity index 100% rename from core/tokenizer.py rename to src/core/tokenizer.py diff --git a/core/utils.py b/src/core/utils.py similarity index 100% rename from core/utils.py rename to src/core/utils.py diff --git a/evaluators/AddConceptEvaluator.py b/src/evaluators/AddConceptEvaluator.py similarity index 100% rename from evaluators/AddConceptEvaluator.py rename to src/evaluators/AddConceptEvaluator.py diff --git a/evaluators/AddConceptInSetEvaluator.py b/src/evaluators/AddConceptInSetEvaluator.py similarity index 100% rename from evaluators/AddConceptInSetEvaluator.py rename to src/evaluators/AddConceptInSetEvaluator.py diff --git a/evaluators/BaseEvaluator.py b/src/evaluators/BaseEvaluator.py similarity index 96% rename from evaluators/BaseEvaluator.py rename to src/evaluators/BaseEvaluator.py index a17f3eb..870ac7f 100644 --- a/evaluators/BaseEvaluator.py +++ b/src/evaluators/BaseEvaluator.py @@ -1,4 +1,4 @@ -from core.sheerka import ExecutionContext +from core.sheerka.Sheerka import ExecutionContext from core.sheerka_logger import get_logger diff --git a/evaluators/ConceptEvaluator.py b/src/evaluators/ConceptEvaluator.py similarity index 100% rename from evaluators/ConceptEvaluator.py rename to src/evaluators/ConceptEvaluator.py diff --git a/evaluators/EvalEvaluator.py b/src/evaluators/EvalEvaluator.py similarity index 100% rename from evaluators/EvalEvaluator.py rename to src/evaluators/EvalEvaluator.py diff --git a/evaluators/LexerNodeEvaluator.py b/src/evaluators/LexerNodeEvaluator.py similarity index 100% rename from evaluators/LexerNodeEvaluator.py rename to src/evaluators/LexerNodeEvaluator.py diff --git a/evaluators/MutipleSameSuccessEvaluator.py b/src/evaluators/MutipleSameSuccessEvaluator.py similarity index 100% rename from evaluators/MutipleSameSuccessEvaluator.py rename to src/evaluators/MutipleSameSuccessEvaluator.py diff --git a/evaluators/OneErrorEvaluator.py b/src/evaluators/OneErrorEvaluator.py similarity index 100% rename from evaluators/OneErrorEvaluator.py rename to src/evaluators/OneErrorEvaluator.py diff --git a/evaluators/OneSuccessEvaluator.py b/src/evaluators/OneSuccessEvaluator.py similarity index 100% rename from evaluators/OneSuccessEvaluator.py rename to src/evaluators/OneSuccessEvaluator.py diff --git a/evaluators/PrepareEvalEvaluator.py b/src/evaluators/PrepareEvalEvaluator.py similarity index 100% rename from evaluators/PrepareEvalEvaluator.py rename to src/evaluators/PrepareEvalEvaluator.py diff --git a/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py similarity index 97% rename from evaluators/PythonEvaluator.py rename to src/evaluators/PythonEvaluator.py index 4baf886..1ec6206 100644 --- a/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -61,9 +61,9 @@ class PythonEvaluator(OneReturnValueEvaluator): def get_locals(self, context, node): my_locals = { "sheerka": context.sheerka, - "desc": context.sheerka.dump_desc, - "concepts": context.sheerka.dump_concepts, - "definitions": context.sheerka.dump_definitions, + "desc": context.sheerka.dump_handler.dump_desc, + "concepts": context.sheerka.dump_handler.dump_concepts, + "definitions": context.sheerka.dump_handler.dump_definitions, } if context.obj: context.log(self.verbose_log, diff --git a/evaluators/TooManySuccessEvaluator.py b/src/evaluators/TooManySuccessEvaluator.py similarity index 100% rename from evaluators/TooManySuccessEvaluator.py rename to src/evaluators/TooManySuccessEvaluator.py diff --git a/parsers/__init__.py b/src/evaluators/__init__.py similarity index 100% rename from parsers/__init__.py rename to src/evaluators/__init__.py diff --git a/parsers/BaseParser.py b/src/parsers/BaseParser.py similarity index 100% rename from parsers/BaseParser.py rename to src/parsers/BaseParser.py diff --git a/parsers/BnfParser.py b/src/parsers/BnfParser.py similarity index 99% rename from parsers/BnfParser.py rename to src/parsers/BnfParser.py index 3b5a03a..be53c74 100644 --- a/parsers/BnfParser.py +++ b/src/parsers/BnfParser.py @@ -2,7 +2,7 @@ from dataclasses import dataclass import core.utils from core.builtin_concepts import BuiltinConcepts -from core.sheerka import ExecutionContext +from core.sheerka.Sheerka import ExecutionContext from core.tokenizer import Tokenizer, Token, TokenKind, LexerError from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode from parsers.ConceptLexerParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, ConceptExpression, StrMatch diff --git a/parsers/ConceptLexerParser.py b/src/parsers/ConceptLexerParser.py similarity index 100% rename from parsers/ConceptLexerParser.py rename to src/parsers/ConceptLexerParser.py diff --git a/parsers/ConceptsWithConceptsParser.py b/src/parsers/ConceptsWithConceptsParser.py similarity index 100% rename from parsers/ConceptsWithConceptsParser.py rename to src/parsers/ConceptsWithConceptsParser.py diff --git a/parsers/DefaultParser.py b/src/parsers/DefaultParser.py similarity index 99% rename from parsers/DefaultParser.py rename to src/parsers/DefaultParser.py index e4df972..5837227 100644 --- a/parsers/DefaultParser.py +++ b/src/parsers/DefaultParser.py @@ -6,7 +6,7 @@ from parsers.BaseParser import BaseParser, Node, ErrorNode, NotInitializedNode from core.tokenizer import Tokenizer, TokenKind, Token, Keywords from dataclasses import dataclass, field from parsers.BnfParser import BnfParser -from core.sheerka import ExecutionContext +from core.sheerka.Sheerka import ExecutionContext @dataclass() diff --git a/parsers/EmptyStringParser.py b/src/parsers/EmptyStringParser.py similarity index 100% rename from parsers/EmptyStringParser.py rename to src/parsers/EmptyStringParser.py diff --git a/parsers/ExactConceptParser.py b/src/parsers/ExactConceptParser.py similarity index 100% rename from parsers/ExactConceptParser.py rename to src/parsers/ExactConceptParser.py diff --git a/parsers/MultipleConceptsParser.py b/src/parsers/MultipleConceptsParser.py similarity index 100% rename from parsers/MultipleConceptsParser.py rename to src/parsers/MultipleConceptsParser.py diff --git a/parsers/PythonParser.py b/src/parsers/PythonParser.py similarity index 100% rename from parsers/PythonParser.py rename to src/parsers/PythonParser.py diff --git a/parsers/PythonWithConceptsParser.py b/src/parsers/PythonWithConceptsParser.py similarity index 100% rename from parsers/PythonWithConceptsParser.py rename to src/parsers/PythonWithConceptsParser.py diff --git a/sdp/__init__.py b/src/parsers/__init__.py similarity index 100% rename from sdp/__init__.py rename to src/parsers/__init__.py diff --git a/src/sdp/__init__.py b/src/sdp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sdp/readme.md b/src/sdp/readme.md similarity index 100% rename from sdp/readme.md rename to src/sdp/readme.md diff --git a/sdp/sheerkaDataProvider.py b/src/sdp/sheerkaDataProvider.py similarity index 100% rename from sdp/sheerkaDataProvider.py rename to src/sdp/sheerkaDataProvider.py diff --git a/sdp/sheerkaDataProviderIO.py b/src/sdp/sheerkaDataProviderIO.py similarity index 100% rename from sdp/sheerkaDataProviderIO.py rename to src/sdp/sheerkaDataProviderIO.py diff --git a/sdp/sheerkaSerializer.py b/src/sdp/sheerkaSerializer.py similarity index 99% rename from sdp/sheerkaSerializer.py rename to src/sdp/sheerkaSerializer.py index 3e2d2be..8c5c9de 100644 --- a/sdp/sheerkaSerializer.py +++ b/src/sdp/sheerkaSerializer.py @@ -252,7 +252,7 @@ class ExecutionContextSerializer(BaseSerializer): BaseSerializer.__init__(self, "R", 1) def matches(self, obj): - return core.utils.get_full_qualified_name(obj) == "core.sheerka.ExecutionContext" + return core.utils.get_full_qualified_name(obj) == "core.sheerka.ExecutionContext.ExecutionContext" def dump(self, stream, obj, context): as_json = obj.to_dict() diff --git a/tests/BaseTest.py b/tests/BaseTest.py new file mode 100644 index 0000000..d3ae48a --- /dev/null +++ b/tests/BaseTest.py @@ -0,0 +1,58 @@ +import ast + +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept +from core.concept import Concept +from core.sheerka.ExecutionContext import ExecutionContext +from sdp.sheerkaDataProvider import Event + + +class BaseTest: + def get_sheerka(self): + pass + + def get_context(self, sheerka=None): + return ExecutionContext("test", Event(), sheerka or self.get_sheerka()) + + def get_default_concept(self): + concept = Concept( + name="a + b", + where="isinstance(a, int) and isinstance(b, int)", + pre="isinstance(a, int) and isinstance(b, int)", + post="isinstance(res, int)", + body="def func(x,y):\n return x+y\nfunc(a,b)", + desc="specific description") + concept.def_prop("a", "value1") + concept.def_prop("b", "value2") + + return concept + + def dump_ast(self, node): + dump = ast.dump(node) + for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]: + dump = dump.replace(to_remove, "") + return dump + + @staticmethod + def retval(obj, who="who", status=True): + """ret_val""" + return ReturnValueConcept.ret(who, status, obj) + + @staticmethod + def tretval(sheerka, obj, who="who"): + """True ret_val + add concept in cache""" + if isinstance(obj, Concept): + obj.init_key() + if obj.key not in sheerka.cache_by_key: + sheerka.cache_by_key[obj.key] = obj + return sheerka.ret(who, True, obj) + + @staticmethod + def pretval(concept, source=None, parser="parsers.name"): + """ParserResult ret_val""" + return ReturnValueConcept( + "some_name", + True, + ParserResultConcept(parser=parser, + source=source or concept.name, + value=concept, + try_parsed=concept)) diff --git a/tests/TestUsingFileBasedSheerka.py b/tests/TestUsingFileBasedSheerka.py new file mode 100644 index 0000000..cd3e620 --- /dev/null +++ b/tests/TestUsingFileBasedSheerka.py @@ -0,0 +1,34 @@ +import os +import shutil +from os import path + +import pytest +from core.sheerka.Sheerka import Sheerka +from tests.BaseTest import BaseTest + + +class TestUsingFileBasedSheerka(BaseTest): + + tests_root = path.abspath("../../build/tests") + root_folder = "init_folder" + + @pytest.fixture(autouse=True) + def init_test(self): + if path.exists(self.tests_root): + shutil.rmtree(self.tests_root) + + if not path.exists(self.tests_root): + os.makedirs(self.tests_root) + current_pwd = os.getcwd() + os.chdir(self.tests_root) + + yield None + + os.chdir(current_pwd) + + def get_sheerka(self, use_dict=True, skip_builtins_in_db=True): + root = "mem://" if use_dict else self.root_folder + sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db) + sheerka.initialize(root) + + return sheerka diff --git a/tests/TestUsingMemoryBasedSheerka.py b/tests/TestUsingMemoryBasedSheerka.py new file mode 100644 index 0000000..8c9b37e --- /dev/null +++ b/tests/TestUsingMemoryBasedSheerka.py @@ -0,0 +1,10 @@ +from core.sheerka.Sheerka import Sheerka +from tests.BaseTest import BaseTest + + +class TestUsingMemoryBasedSheerka(BaseTest): + + def get_sheerka(self, skip_builtins_in_db=True): + sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db) + sheerka.initialize("mem://") + return sheerka diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_ExecutionContext.py b/tests/core/test_ExecutionContext.py similarity index 97% rename from tests/test_ExecutionContext.py rename to tests/core/test_ExecutionContext.py index eea3dd9..4797697 100644 --- a/tests/test_ExecutionContext.py +++ b/tests/core/test_ExecutionContext.py @@ -2,7 +2,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept -from core.sheerka import ExecutionContext +from core.sheerka.Sheerka import ExecutionContext from sdp.sheerkaDataProvider import Event diff --git a/tests/core/test_SheerkaCreateNewConcept.py b/tests/core/test_SheerkaCreateNewConcept.py new file mode 100644 index 0000000..b8cfbd3 --- /dev/null +++ b/tests/core/test_SheerkaCreateNewConcept.py @@ -0,0 +1,180 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import PROPERTIES_TO_SERIALIZE +from sdp.sheerkaDataProvider import SheerkaDataProvider + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): + def test_i_can_add_a_concept(self): + sheerka = self.get_sheerka() + concept = self.get_default_concept() + + res = sheerka.create_new_concept(self.get_context(sheerka), concept) + + assert res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT) + + concept_found = res.value.body + for prop in PROPERTIES_TO_SERIALIZE: + assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop) + + assert concept_found.key == "__var__0 + __var__1" + assert concept_found.id == "1001" + + assert concept.key in sheerka.cache_by_key + assert concept.id in sheerka.cache_by_id + assert sheerka.sdp.io.exists( + sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest())) + + def test_i_cannot_add_the_same_concept_twice(self): + """ + Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp + :return: + """ + sheerka = self.get_sheerka() + concept = self.get_default_concept() + + sheerka.create_new_concept(self.get_context(sheerka), concept) + res = sheerka.create_new_concept(self.get_context(sheerka), concept) + + assert not res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED) + assert res.value.body == concept + + def test_i_can_get_a_newly_created_concept(self): + sheerka = self.get_sheerka() + concept = self.get_default_concept() + + sheerka.create_new_concept(self.get_context(sheerka), concept) + + from_cache = sheerka.get(concept.key) + assert from_cache is not None + assert from_cache == concept + + from_cache = sheerka.get_by_id(concept.id) + assert from_cache is not None + assert from_cache == concept + + def test_i_first_look_in_local_cache(self): + sheerka = self.get_sheerka() + concept = self.get_default_concept() + + sheerka.create_new_concept(self.get_context(sheerka), concept) + sheerka.cache_by_key[concept.key].pre = "I have modified the concept in cache" + + from_cache = sheerka.get(concept.key) + assert from_cache is not None + assert from_cache.key == concept.key + assert from_cache.pre == "I have modified the concept in cache" + + def test_i_can_get_a_known_concept_when_not_in_cache(self): + """ + When not in cache, uses sdp + :return: + """ + sheerka = self.get_sheerka() + concept = self.get_default_concept() + sheerka.create_new_concept(self.get_context(sheerka), concept) + + sheerka.cache_by_key = {} # reset the cache + loaded = sheerka.get(concept.key) + + assert loaded is not None + assert loaded == concept + + # I can also get it by its id + loaded = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, concept.id) + assert loaded is not None + assert loaded == concept + + def test_i_can_get_a_concept_by_its_id(self): + sheerka = self.get_sheerka() + concept = self.get_default_concept() + sheerka.create_new_concept(self.get_context(sheerka), concept) + + sheerka.cache_by_key = {} # reset the cache + loaded = sheerka.get_by_id(concept.id) + + assert loaded is not None + assert loaded == concept + + def test_i_can_get_list_of_concept_when_same_key_when_no_cache(self): + sheerka = self.get_sheerka() + concept1 = self.get_default_concept() + concept2 = self.get_default_concept() + concept2.metadata.body = "a+b" + + res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1) + res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2) + + assert res1.value.body.key == res2.value.body.key # same key + + sheerka.cache_by_key = {} # reset the cache + + result = sheerka.get(concept1.key) + assert len(result) == 2 + assert result[0] == concept1 + assert result[1] == concept2 + + def test_i_can_get_list_of_concept_when_same_key_when_cache(self): + sheerka = self.get_sheerka() + concept1 = self.get_default_concept() + concept2 = self.get_default_concept() + concept2.metadata.body = "a+b" + + res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1) + res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2) + + assert res1.value.body.key == res2.value.body.key # same key + + # sheerka.cache_by_key = {} # Do not reset the cache + + result = sheerka.get(concept1.key) + assert len(result) == 2 + assert result[0] == concept1 + assert result[1] == concept2 + + def test_i_can_get_the_correct_concept_using_the_id_when_same_key_when_no_cache(self): + sheerka = self.get_sheerka() + concept1 = self.get_default_concept() + concept2 = self.get_default_concept() + concept2.metadata.body = "a+b" + + res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1) + res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2) + + assert res1.value.body.key == res2.value.body.key # same key + + result = sheerka.get(concept1.key, res2.body.body.id) + assert result.name == "a + b" + assert result.metadata.body == "a+b" + + def test_i_can_get_the_correct_concept_using_the_id__when_same_key_when_cache(self): + sheerka = self.get_sheerka() + concept1 = self.get_default_concept() + concept2 = self.get_default_concept() + concept2.metadata.body = "a+b" + + res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1) + res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2) + + assert res1.value.body.key == res2.value.body.key # same key + + result = sheerka.get(concept1.key, res2.body.body.id) + assert result.name == "a + b" + assert result.metadata.body == "a+b" + + def test_i_cannot_get_the_correct_concept_id_the_id_is_wrong(self): + sheerka = self.get_sheerka() + concept1 = self.get_default_concept() + concept2 = self.get_default_concept() + concept2.metadata.body = "a+b" + + res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1) + res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2) + + assert res1.value.body.key == res2.value.body.key # same key + + result = sheerka.get(concept1.key, "wrong id") + assert sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT) diff --git a/tests/core/test_SheerkaEvaluateConcept.py b/tests/core/test_SheerkaEvaluateConcept.py new file mode 100644 index 0000000..5120f1a --- /dev/null +++ b/tests/core/test_SheerkaEvaluateConcept.py @@ -0,0 +1,332 @@ +import pytest +from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept +from core.concept import Concept, simplec, DoNotResolve, ConceptParts, Property +from parsers.PythonParser import PythonNode + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("body, expected", [ + # (None, None), + # ("", ""), + ("1", 1), + ("1+1", 2), + ("'one'", "one"), + ("'one' + 'two'", "onetwo"), + ("True", True), + ("1 > 2", False), + ]) + def test_i_can_evaluate_a_concept_with_simple_body(self,body, expected): + sheerka = self.get_sheerka() + + concept = Concept("foo", body=body) + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.key + assert evaluated.body == expected + assert evaluated.metadata.body == body + assert evaluated.metadata.pre is None + assert evaluated.metadata.post is None + assert evaluated.metadata.where is None + assert evaluated.props == {} + assert evaluated.metadata.is_evaluated + assert len(evaluated.values) == 0 if body is None else 1 + + @pytest.mark.parametrize("expr, expected", [ + ("", ""), + ("1", 1), + ("1+1", 2), + ("'one'", "one"), + ("'one' + 'two'", "onetwo"), + ("True", True), + ("1 > 2", False), + ]) + def test_i_can_evaluate_the_other_metadata(self,expr, expected): + """ + I only test WHERE, it's the same for the others + :param expr: + :param expected: + :return: + """ + + sheerka = self.get_sheerka() + + concept = Concept("foo", where=expr) + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.key + assert evaluated.metadata.body is None + assert evaluated.metadata.pre is None + assert evaluated.metadata.post is None + assert evaluated.metadata.where == expr + assert evaluated.get_metadata_value(ConceptParts.WHERE) == expected + assert evaluated.props == {} + assert evaluated.metadata.is_evaluated + assert len(evaluated.values) == 0 if expr is None else 1 + + @pytest.mark.parametrize("expr, expected", [ + (None, None), + ("", ""), + ("1", 1), + ("1+1", 2), + ("'one'", "one"), + ("'one' + 'two'", "onetwo"), + ("True", True), + ("1 > 2", False), + ]) + def test_i_can_evaluate_a_concept_with_prop(self, expr, expected): + sheerka = self.get_sheerka() + + concept = Concept("foo").def_prop("a", expr) + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.key + assert evaluated.metadata.pre is None + assert evaluated.metadata.pre is None + assert evaluated.metadata.post is None + assert evaluated.metadata.where is None + assert evaluated.props == {"a": Property("a", expected)} + assert evaluated.metadata.is_evaluated + + def test_i_can_evaluate_metadata_using_do_not_resolve(self): + sheerka = self.get_sheerka() + concept = Concept("foo") + concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve") + + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.body == "do not resolve" + assert evaluated.metadata.is_evaluated + + def test_i_can_evaluate_property_using_do_not_resolve(self): + sheerka = self.get_sheerka() + concept = Concept("foo").def_prop("a") + concept.compiled["a"] = DoNotResolve("do not resolve") + + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.get_prop("a") == "do not resolve" + assert evaluated.metadata.is_evaluated + + def test_original_value_is_overridden_when_using_do_no_resolve(self): + sheerka = self.get_sheerka() + concept = Concept("foo", body="original value").set_prop("a", "original value") + concept.compiled["a"] = DoNotResolve("do not resolve") + concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve") + + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.body == "do not resolve" + assert evaluated.get_prop("a") == "do not resolve" + assert evaluated.metadata.is_evaluated + + def test_props_are_evaluated_before_body(self): + sheerka = self.get_sheerka() + + concept = Concept("foo", body="a+1").def_prop("a", "10").init_key() + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.key + assert evaluated.body == 11 + + def test_i_can_evaluate_when_another_concept_is_referenced(self): + sheerka = self.get_sheerka() + concept_a = Concept("a") + sheerka.add_in_cache(concept_a) + + concept = Concept("foo", body="a").init_key() + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated == simplec("foo", simplec("a", None)) + assert id(evaluated.body) != id(concept_a) + assert evaluated.metadata.is_evaluated + assert evaluated.body.metadata.is_evaluated + + def test_i_can_evaluate_when_the_referenced_concept_has_a_body(self): + sheerka = self.get_sheerka() + concept_a = Concept("a", body="1") + sheerka.add_in_cache(concept_a) + + concept = Concept("foo", body="a") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.key + assert evaluated.body == simplec("a", 1) + assert not concept_a.metadata.is_evaluated + assert evaluated.metadata.is_evaluated + + def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(self): + sheerka = self.get_sheerka() + sheerka.add_in_cache(Concept(name="a", body="'a'")) + sheerka.add_in_cache(Concept(name="b", body="a")) + sheerka.add_in_cache(Concept(name="c", body="b")) + concept_d = sheerka.add_in_cache(Concept(name="d", body="c")) + + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept_d) + + assert evaluated.key == concept_d.key + expected = simplec("c", simplec("b", simplec("a", "a"))) + assert evaluated.body == expected + assert sheerka.value(evaluated) == 'a' + assert evaluated.metadata.is_evaluated + + def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(self): + sheerka = self.get_sheerka() + sheerka.add_in_cache(Concept(name="a")) + sheerka.add_in_cache(Concept(name="b", body="a")) + sheerka.add_in_cache(Concept(name="c", body="b")) + concept_d = sheerka.add_in_cache(Concept(name="d", body="c")) + + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept_d) + + assert evaluated.key == concept_d.key + expected = simplec("c", simplec("b", simplec("a", None))) + assert evaluated.body == expected + assert sheerka.value(evaluated) == Concept(name="a").init_key() + assert evaluated.metadata.is_evaluated + + def test_i_can_evaluate_concept_when_properties_reference_others_concepts(self): + sheerka = self.get_sheerka() + concept_a = sheerka.add_in_cache(Concept(name="a").init_key()) + + concept = Concept("foo", body="a").def_prop("a", "a").init_key() + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + # first prop a is evaluated to concept_a + # then body is evaluated to prop a -> concept_a + assert evaluated.key == concept.key + assert evaluated.body == concept_a + + def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2(self): + """ + Same test, + but the name of the property and the name of the concept are different + :return: + """ + sheerka = self.get_sheerka() + concept_a = sheerka.add_in_cache(Concept(name="a")) + + concept = Concept("foo", body="concept_a").def_prop("concept_a", "a") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.key + assert evaluated.body == concept_a + + def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body(self): + sheerka = self.get_sheerka() + sheerka.add_in_cache(Concept(name="a", body="1")) + sheerka.add_in_cache(Concept(name="b", body="2")) + + concept = Concept("foo", body="propA + propB").def_prop("propA", "a").def_prop("propB", "b") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.key + assert evaluated.body == 3 + + def test_i_can_evaluate_concept_when_properties_is_a_concept(self): + sheerka = self.get_sheerka() + concept_a = sheerka.add_in_cache(Concept(name="a", body="'a'").init_key()) + + concept = Concept("foo").def_prop("a") + concept.compiled["a"] = concept_a + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.key + assert evaluated.get_prop("a") == simplec("a", "a") + + def test_i_can_evaluate_when_property_asts_is_a_list(self): + sheerka = self.get_sheerka() + foo = Concept("foo", body="1") + + concept = Concept("to_eval").def_prop("prop") + concept.compiled["prop"] = [foo, DoNotResolve("1")] + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + props = evaluated.get_prop("prop") + assert len(props) == 2 + assert props[0] == simplec("foo", 1) + assert props[1] == "1" + + def test_i_can_evaluate_when_compiled_is_set_up_with_return_value(self): + sheerka = self.get_sheerka() + + python_node = PythonNode("1 +1 ") + parser_result = ParserResultConcept(parser="who", value=python_node) + + concept = Concept("to_eval").def_prop("prop") + concept.compiled["prop"] = [ReturnValueConcept("who", True, parser_result)] + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert evaluated.get_prop("prop") == 2 + + # also works when only one return value + concept = Concept("to_eval").def_prop("prop") + concept.compiled["prop"] = ReturnValueConcept("who", True, parser_result) + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert evaluated.get_prop("prop") == 2 + + def test_i_can_reference_sheerka(self): + sheerka = self.get_sheerka() + + concept = Concept("foo", body="sheerka.test()").init_key() + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.key + assert evaluated.body == sheerka.test() + + def test_properties_values_takes_precedence_over_the_outside_world(self): + sheerka = self.get_sheerka() + sheerka.add_in_cache(Concept(name="a", body="'concept_a'")) + sheerka.add_in_cache(Concept(name="b", body="'concept_b'")) + + concept = Concept("foo", body="a") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert evaluated.key == concept.key + assert evaluated.body == simplec("a", "concept_a") # this test was already done + + # so check this one. + concept = Concept("foo", body="a").def_prop("a", "'property_a'") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert evaluated.key == concept.key + assert evaluated.body == 'property_a' + + # or this one. + concept = Concept("foo", body="a").def_prop("a", "b") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert evaluated.key == concept.key + assert evaluated.body == simplec(name="b", body="concept_b") + + def test_properties_values_takes_precedence(self): + sheerka = self.get_sheerka() + sheerka.add_in_cache(Concept(name="a", body="'concept_a'")) + sheerka.add_in_cache(Concept(name="b", body="'concept_b'")) + + concept = Concept("foo", body="a + b").def_prop("a", "'prop_a'") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert evaluated.key == concept.key + assert evaluated.body == 'prop_aconcept_b' + + def test_i_can_reference_sub_property_of_a_property(self): + sheerka = self.get_sheerka() + sheerka.add_in_cache(Concept(name="concept_a").def_prop("subProp", "'sub_a'")) + + concept = Concept("foo", body="a.props['subProp'].value").def_prop("a", "concept_a") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert evaluated == simplec(concept.key, "sub_a") + + def test_i_cannot_evaluate_concept_if_property_is_in_error(self): + sheerka = self.get_sheerka() + + concept = Concept(name="concept_a").def_prop("subProp", "undef_concept") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) + + def test_key_is_initialized_by_evaluation(self): + sheerka = self.get_sheerka() + + concept = Concept("foo") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + assert evaluated.key == concept.init_key().key + diff --git a/tests/core/test_SheerkaSetsManager.py b/tests/core/test_SheerkaSetsManager.py new file mode 100644 index 0000000..85cc95b --- /dev/null +++ b/tests/core/test_SheerkaSetsManager.py @@ -0,0 +1,124 @@ +import os +import shutil +from os import path + +import pytest +from core.builtin_concepts import ConceptAlreadyInSet, BuiltinConcepts +from core.concept import Concept + +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka + + +class TestSheerkaSetsManager(TestUsingFileBasedSheerka): + + + + def test_i_can_add_concept_to_set(self): + sheerka = self.get_sheerka(False, False) + + foo = Concept("foo") + sheerka.set_id_if_needed(foo, False) + + all_foos = Concept("all_foos") + sheerka.set_id_if_needed(all_foos, False) + + context = self.get_context(sheerka) + res = sheerka.add_concept_to_set(context, foo, all_foos) + + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) + + all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False) + assert len(all_entries) == 1 + assert foo.id in all_entries + + def test_i_can_add_several_concepts_to_set(self): + sheerka = self.get_sheerka(False, False) + + foo1 = Concept("foo1") + sheerka.set_id_if_needed(foo1, False) + + foo2 = Concept("foo1") + sheerka.set_id_if_needed(foo2, False) + + all_foos = Concept("all_foos") + sheerka.set_id_if_needed(all_foos, False) + + context = self.get_context(sheerka) + sheerka.add_concept_to_set(context, foo1, all_foos) + res = sheerka.add_concept_to_set(context, foo2, all_foos) + + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) + + all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False) + assert len(all_entries) == 2 + assert foo1.id in all_entries + assert foo2.id in all_entries + + def test_i_cannot_add_the_same_concept_twice_in_a_set(self): + sheerka = self.get_sheerka() + + foo = Concept("foo") + sheerka.set_id_if_needed(foo, False) + + all_foos = Concept("all_foos") + sheerka.set_id_if_needed(all_foos, False) + + context = self.get_context(sheerka) + sheerka.add_concept_to_set(context, foo, all_foos) + res = sheerka.add_concept_to_set(context, foo, all_foos) + + assert not res.status + assert res.body == ConceptAlreadyInSet(foo, all_foos) + + all_entries = sheerka.sdp.get("All_" + all_foos.id, None, False) + assert len(all_entries) == 1 + assert foo.id in all_entries + + def test_i_get_elements_from_a_set(self): + sheerka = self.get_sheerka() + + one = Concept("one") + two = Concept("two") + three = Concept("three") + number = Concept("number") + + for c in [one, two, three, number]: + sheerka.set_id_if_needed(c, False) + sheerka.add_in_cache(c) + + context = self.get_context(sheerka) + for c in [one, two, three]: + sheerka.add_concept_to_set(context, c, number) + + elements = sheerka.get_set_elements(number) + + assert set(elements) == {one, two, three} + + def test_i_cannot_get_elements_if_not_a_set(self): + sheerka = self.get_sheerka() + one = Concept("one") + sheerka.set_id_if_needed(one, False) + sheerka.add_in_cache(one) + + error = sheerka.get_set_elements(one) + + assert sheerka.isinstance(error, BuiltinConcepts.NOT_A_SET) + assert error.body == one + + def test_isa_and_isa_group(self): + sheerka = self.get_sheerka() + + group = Concept("group").init_key() + group.metadata.id = "1001" + assert not sheerka.isagroup(group) + + foo = Concept("foo").init_key() + foo.metadata.id = "1002" + assert not sheerka.isa(foo, group) + + context = self.get_context(sheerka) + sheerka.add_concept_to_set(context, foo, group) + assert sheerka.isagroup(group) + assert sheerka.isa(foo, group) diff --git a/tests/test_ast.py b/tests/core/test_ast.py similarity index 99% rename from tests/test_ast.py rename to tests/core/test_ast.py index 53ce5d5..2e73bd6 100644 --- a/tests/test_ast.py +++ b/tests/core/test_ast.py @@ -6,7 +6,7 @@ from core.ast.nodes import NodeParent, GenericNodeConcept import core.ast.nodes from core.ast.visitors import ConceptNodeVisitor, UnreferencedNamesVisitor from core.builtin_concepts import BuiltinConcepts -from core.sheerka import Sheerka +from core.sheerka.Sheerka import Sheerka def get_sheerka(): diff --git a/tests/core/test_builtin_helpers.py b/tests/core/test_builtin_helpers.py new file mode 100644 index 0000000..14dc593 --- /dev/null +++ b/tests/core/test_builtin_helpers.py @@ -0,0 +1,145 @@ +import ast +import pytest +import core.builtin_helpers + +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestBuiltinHelpers(TestUsingMemoryBasedSheerka): + + def test_i_can_use_expect_one_when_empty(self): + sheerka = self.get_sheerka() + + res = core.builtin_helpers.expect_one(self.get_context(sheerka), []) + assert not res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY) + + def test_i_can_use_expect_one_when_too_many_success(self): + sheerka = self.get_sheerka() + + items = [ + ReturnValueConcept("who", True, "value1"), + ReturnValueConcept("who", True, "value2"), + ] + res = core.builtin_helpers.expect_one(self.get_context(sheerka), items) + assert not res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS) + assert res.value.body == items + assert res.parents == items + + def test_i_can_use_expect_one_when_same_success(self): + sheerka = self.get_sheerka() + + items = [ + ReturnValueConcept("who", True, "value"), + ReturnValueConcept("who", True, "value"), + ] + res = core.builtin_helpers.expect_one(self.get_context(sheerka), items) + assert res.status + assert res.value == items[0].value + assert res.parents == items + + def test_i_can_use_expect_when_only_errors_1(self): + sheerka = self.get_sheerka() + + items = [ + ReturnValueConcept("who", False, sheerka.new(BuiltinConcepts.ERROR)), + ] + res = core.builtin_helpers.expect_one(self.get_context(sheerka), items) + assert not res.status + assert res.value, items[0] + assert res.parents == items + + def test_i_can_use_expect_when_only_errors_2(self): + sheerka = self.get_sheerka() + + items = [ + ReturnValueConcept("who", False, None), + ReturnValueConcept("who", False, None), + ] + res = core.builtin_helpers.expect_one(self.get_context(sheerka), items) + assert not res.status + assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS) + assert res.value.body == items + assert res.parents == items + + def test_i_can_use_expect_one_when_one_success_1(self): + sheerka = self.get_sheerka() + + items = [ + ReturnValueConcept("who", True, None), + ] + res = core.builtin_helpers.expect_one(self.get_context(sheerka), items) + assert res.status + assert res.body == items[0].body + assert res.parents == items + + def test_i_can_use_expect_one_when_one_success_2(self): + sheerka = self.get_sheerka() + + items = [ + ReturnValueConcept("who", False, None), + ReturnValueConcept("who", True, None), + ReturnValueConcept("who", False, None), + ] + res = core.builtin_helpers.expect_one(self.get_context(sheerka), items) + assert res.status + assert res.body == items[1].body + assert res.parents == items + + def test_i_can_use_expect_one_when_not_a_list_true(self): + sheerka = self.get_sheerka() + + item = ReturnValueConcept("who", True, None) + res = core.builtin_helpers.expect_one(self.get_context(sheerka), item) + assert res.status + assert res == item + + def test_i_can_use_expect_one_when_not_a_list_false(self): + sheerka = self.get_sheerka() + + item = ReturnValueConcept("who", False, None) + res = core.builtin_helpers.expect_one(self.get_context(sheerka), item) + + assert not res.status + assert res == item + + @pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [ + ("a == 1", [], [], []), + ("a == 1", ["a"], [], ["a == 1"]), + ("a == 1", [], ["a"], []), + ("predicate(a)", [], [], []), + ("predicate(a)", ["a"], [], ["predicate(a)"]), + ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]), + ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]), + ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]), + ("predicate(a, b)", ["a"], ["b"], []), + ("a + b == 1", [], [], []), + ("a + b == 1", ["a"], [], ["a + b == 1"]), + ("a + b == 1", ["a"], ["b"], []), + ("a + b == 1", ["b"], [], ["a + b == 1"]), + ("a + b == 1", ["a", "b"], [], ["a + b == 1"]), + ("a == 1 and b == 2", [], [], []), + ("a == 1 and b == 2", ["a"], [], ["a == 1"]), + ("a == 1 and b == 2", ["b"], [], ["b == 2"]), + ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]), + ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]), + ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]), + ("not(a == 1)", [], [], []), + ("not(a == 1)", ["a"], [], ["not(a==1)"]), + ("a == 1 or b == 2", [], [], []), + ("a == 1 or b == 2", ["a"], [], ["a == 1"]), + ("a == 1 or b == 2", ["b"], [], ["b == 2"]), + ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]), + ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]), + ]) + def test_i_can_extract_predicates(self, expression, vars_to_include, vars_to_exclude, expected_expr): + sheerka = self.get_sheerka() + expected = [ast.parse(expr, mode="eval") for expr in expected_expr] + + actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude) + assert len(actual) == len(expected) + for i in range(len(actual)): + assert self.dump_ast(actual[i]) == self.dump_ast(expected[i]) diff --git a/tests/test_concept.py b/tests/core/test_concept.py similarity index 100% rename from tests/test_concept.py rename to tests/core/test_concept.py diff --git a/tests/core/test_sheerka.py b/tests/core/test_sheerka.py new file mode 100644 index 0000000..0d93abb --- /dev/null +++ b/tests/core/test_sheerka.py @@ -0,0 +1,244 @@ +import pytest +import os + +from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept +from core.concept import Concept, PROPERTIES_TO_SERIALIZE +from core.sheerka.Sheerka import Sheerka + +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka + + +class ConceptWithGetValue(Concept): + def get_value(self): + return self.get_prop("my_prop") + + +class TestSheerka(TestUsingFileBasedSheerka): + + def test_root_folder_is_created_after_initialization(self): + return_value = Sheerka().initialize(self.root_folder) + assert return_value.status, "initialisation should be successful" + assert os.path.exists(self.root_folder), "init folder should be created" + + def test_i_can_list_builtin_concepts(self): + sheerka = self.get_sheerka() + builtins = list(sheerka.get_builtins_classes_as_dict()) + + assert str(BuiltinConcepts.ERROR) in builtins + assert str(BuiltinConcepts.RETURN_VALUE) in builtins + + def test_builtin_concepts_are_initialized(self): + sheerka = self.get_sheerka(skip_builtins_in_db=False) + assert len(sheerka.cache_by_key) == len(BuiltinConcepts) + for concept_name in BuiltinConcepts: + assert str(concept_name) in sheerka.cache_by_key + assert sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, str(concept_name)) is not None + + for key, concept_class in sheerka.get_builtins_classes_as_dict().items(): + assert isinstance(sheerka.cache_by_key[key], concept_class) + + def test_builtin_concepts_can_be_updated(self): + sheerka = self.get_sheerka(False, False) + loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA) + loaded_sheerka.metadata.desc = "I have a description" + sheerka.sdp.modify("Test", sheerka.CONCEPTS_ENTRY, loaded_sheerka.key, loaded_sheerka) + + sheerka = self.get_sheerka(False, False) + loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA) + + assert loaded_sheerka.metadata.desc == "I have a description" + + def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string(self): + """ + Checks that a concept can be found its name + even when there are variables in the name (ex 'hello + a' or 'a + b' ) + :return: + """ + sheerka = self.get_sheerka() + for key in sheerka.get_builtins_classes_as_dict(): + assert sheerka.get(key) is not None + assert sheerka.get(str(key)) is not None + + def test_i_cannot_get_when_key_is_none(self): + sheerka = self.get_sheerka() + + res = sheerka.get(None) + assert sheerka.isinstance(res, BuiltinConcepts.ERROR) + assert res.body == "Concept key is undefined." + + def test_unknown_concept_is_return_when_the_concept_key_is_not_found(self): + sheerka = self.get_sheerka() + + loaded = sheerka.get("key_that_does_not_exist") + + assert loaded is not None + assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT) + assert loaded.body == ("key", "key_that_does_not_exist") + assert loaded.metadata.is_evaluated + + def test_unknown_concept_is_return_when_the_concept_id_is_not_found(self): + sheerka = self.get_sheerka() + + loaded = sheerka.get_by_id("id_that_does_not_exist") + + assert loaded is not None + assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT) + assert loaded.body == ("id", "id_that_does_not_exist") + assert loaded.metadata.is_evaluated + + def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self): + sheerka = self.get_sheerka() + ret = sheerka.new(BuiltinConcepts.RETURN_VALUE, who="who", status="status", value="value", message="message") + + assert isinstance(ret, ReturnValueConcept) + assert ret.key == str(BuiltinConcepts.RETURN_VALUE) + assert ret.who == "who" + assert ret.status == "status" + assert ret.value == "value" + assert ret.message == "message" + + def test_i_can_instantiate_a_builtin_concept_when_no_specific_class(self): + sheerka = self.get_sheerka() + ret = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body="fake_concept") + + assert isinstance(ret, Concept) + assert ret.key == str(BuiltinConcepts.UNKNOWN_CONCEPT) + assert ret.body == "fake_concept" + + def test_i_can_instantiate_a_concept(self): + sheerka = self.get_sheerka() + concept = self.get_default_concept() + sheerka.create_new_concept(self.get_context(sheerka), concept) + + new = sheerka.new(concept.key, a=10, b="value") + + assert sheerka.isinstance(new, concept) + for prop in PROPERTIES_TO_SERIALIZE: + assert getattr(new.metadata, prop) == getattr(concept.metadata, prop) + + assert new.props["a"].value == 10 + assert new.props["b"].value == "value" + + def test_i_can_instantiate_with_the_name_and_the_id(self): + sheerka = self.get_sheerka() + sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1")) + sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2")) + + concepts = sheerka.new("foo") + assert len(concepts) == 2 + + foo1 = sheerka.new(("foo", "1001")) + assert foo1.metadata.body == "foo1" + + foo2 = sheerka.new(("foo", "1002")) + assert foo2.metadata.body == "foo2" + + def test_instances_are_different_when_asking_for_new(self): + sheerka = self.get_sheerka() + concept = self.get_default_concept() + sheerka.create_new_concept(self.get_context(sheerka), concept) + + new1 = sheerka.new(concept.key, a=10, b="value") + new2 = sheerka.new(concept.key, a=10, b="value") + + assert new1 == new2 + assert id(new1) != id(new2) + + def test_i_get_the_same_instance_when_is_unique_is_true(self): + sheerka = self.get_sheerka() + concept = Concept(name="unique", is_unique=True) + sheerka.create_new_concept(self.get_context(sheerka), concept) + + new1 = sheerka.new(concept.key, a=10, b="value") + new2 = sheerka.new(concept.key, a=10, b="value") + + assert new1 == new2 + assert id(new1) == id(new2) + + def test_i_cannot_instantiate_an_unknown_concept(self): + sheerka = self.get_sheerka() + + new = sheerka.new("fake_concept") + + assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT) + assert new.body == ('key', 'fake_concept') + + def test_i_cannot_instantiate_with_invalid_id(self): + sheerka = self.get_sheerka() + sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1")) + sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2")) + + new = sheerka.new(("foo", "invalid_id")) + + assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT) + assert new.body == [('key', 'foo'), ('id', 'invalid_id')] + + def test_i_cannot_instantiate_with_invalid_key(self): + sheerka = self.get_sheerka() + sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1")) + sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2")) + + new = sheerka.new(("invalid_key", "1001")) + + assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT) + assert new.body == [('key', 'invalid_key'), ('id', '1001')] + + def test_concept_id_is_irrelevant_when_only_one_concept(self): + sheerka = self.get_sheerka() + sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1")) + + new = sheerka.new(("foo", "invalid_id")) + + assert sheerka.isinstance(new, "foo") + assert new.metadata.body == "foo1" + + def test_i_cannot_instantiate_when_properties_are_not_recognized(self): + sheerka = self.get_sheerka() + concept = self.get_default_concept() + sheerka.create_new_concept(self.get_context(sheerka), concept) + + new = sheerka.new(concept.key, a=10, c="value") + + assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_PROPERTY) + assert new.property_name == "c" + assert sheerka.isinstance(new.concept, concept) + + @pytest.mark.parametrize("concept, reduce_simple_list, expected", [ + (None, False, None), + (3.14, False, 3.14), + ("foo", False, "foo"), + (True, False, True), + (Concept("name", body="foo"), False, "foo"), + (Concept("name"), False, Concept("name")), + (ConceptWithGetValue("name").set_prop("my_prop", "my_value"), False, "my_value"), + (ReturnValueConcept(value="return_value"), False, "return_value"), + (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"), + (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"), + (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]), + (Concept("name", body=["foo"]), True, "foo"), + (Concept("name", body=Concept("foo")), False, Concept("foo")), + (Concept("name", body=Concept("foo", body="value")), False, "value"), + (Concept("name", body=Concept("foo", body=Concept("bar", body="value"))), False, "value"), + (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"), + ]) + def test_i_can_get_value(self, concept, reduce_simple_list, expected): + sheerka = self.get_sheerka() + + # I use auto_init() instead of evaluate_concept() to be quicker + c = concept + while isinstance(c, Concept): + c.auto_init() + c = c.body + + assert sheerka.value(concept, reduce_simple_list) == expected + + def test_list_of_concept_is_sorted_by_id(self): + sheerka = self.get_sheerka(False, False) + concepts = sheerka.concepts() + + assert concepts[0].id < concepts[-1].id + + def test_builtin_error_concept_are_errors(self): + # only test a random one, it will be the same for the others + sheerka = self.get_sheerka() + assert not sheerka.is_success(sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS)) diff --git a/tests/core/test_sheerka_call_evaluators.py b/tests/core/test_sheerka_call_evaluators.py new file mode 100644 index 0000000..fa04e65 --- /dev/null +++ b/tests/core/test_sheerka_call_evaluators.py @@ -0,0 +1,375 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator + +from tests.BaseTest import BaseTest +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class Out: + debug_out = [] + + def out(self, method, name, context, return_value): + name = name[len(BaseEvaluator.PREFIX):] + if isinstance(return_value, list): + target = [str(r.body.key) for r in return_value] + else: + target = str(return_value.body.key) + step = str(context.step) + text = f"{step} [{context.iteration}] " + text += f"{name} - {method} - target={target}" + self.debug_out.append(text) + + def out_all(self, method, name, context, return_values): + name = name[len(BaseEvaluator.PREFIX):] + target = [str(r.body.key) for r in return_values] + step = str(context.step) + text = f"{step} [{context.iteration}] " + text += f"{name} - {method} - target={target}" + self.debug_out.append(text) + + +class OneReturnValueEvaluatorForTestingPurpose(OneReturnValueEvaluator, Out): + def __init__(self, name, steps, priority): + super().__init__(name, steps, priority) + + def matches(self, context, return_value): + self.out("matches", self.name, context, return_value) + return True + + def eval(self, context, return_value): + self.out("eval", self.name, context, return_value) + + +class AllReturnValueEvaluatorForTestingPurpose(AllReturnValuesEvaluator, Out): + def __init__(self, name, steps, priority): + super().__init__(name, steps, priority) + + def matches(self, context, return_values): + self.out("matches", self.name, context, return_values) + return True + + def eval(self, context, return_values): + self.out("eval", self.name, context, return_values) + + +class EvaluatorOneWithPriority(OneReturnValueEvaluatorForTestingPurpose): + def __init__(self, name, priority): + super().__init__(name, [BuiltinConcepts.EVALUATION], priority) + + +class EvaluatorOneWithPriority10(EvaluatorOneWithPriority): + def __init__(self): + super().__init__("priority10", 10) + + +class EvaluatorOneWithPriority15(EvaluatorOneWithPriority): + def __init__(self): + super().__init__("priority15", 15) + + +class EvaluatorOneWithPriority20(EvaluatorOneWithPriority): + def __init__(self): + super().__init__("priority20", 20) + + +class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose): + def __init__(self, name, priority): + super().__init__(name, [BuiltinConcepts.EVALUATION], priority) + + +class EvaluatorAllWithPriority10(EvaluatorAllWithPriority): + def __init__(self): + super().__init__("all_priority10", 10) + + +class EvaluatorAllWithPriority15(EvaluatorAllWithPriority): + def __init__(self): + super().__init__("all_priority15", 15) + + +class EvaluatorAllWithPriority20(EvaluatorAllWithPriority): + def __init__(self): + super().__init__("all_priority20", 20) + + +class EvaluatorOneModifyFoo(EvaluatorOneWithPriority): + def __init__(self): + super().__init__("modifyFoo", 10) + + def matches(self, context, return_value): + super().matches(context, return_value) + return context.sheerka.isinstance(return_value.body, "foo") + + def eval(self, context, return_value): + super().eval(context, return_value) + return BaseTest.tretval(context.sheerka, Concept("bar")) + + +class EvaluatorOneModifyBar(EvaluatorOneWithPriority): + def __init__(self): + super().__init__("modifyBar", 10) + + def matches(self, context, return_value): + super().matches(context, return_value) + return context.sheerka.isinstance(return_value.body, "bar") + + def eval(self, context, return_value): + super().eval(context, return_value) + return BaseTest.tretval(context.sheerka, Concept("baz")) + + +class EvaluatorOnePreEvaluation(OneReturnValueEvaluatorForTestingPurpose): + def __init__(self): + super().__init__("preEval", [BuiltinConcepts.BEFORE_EVALUATION], 10) + + +class EvaluatorOneMultiSteps(OneReturnValueEvaluatorForTestingPurpose): + def __init__(self): + super().__init__("multiStep", [BuiltinConcepts.EVALUATION, BuiltinConcepts.BEFORE_EVALUATION], 10) + + +class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority): + def __init__(self): + super().__init__("all_reduce_foobar", 10) + + def matches(self, context, return_values): + super().matches(context, return_values) + keys = [c.body.key for c in return_values] + return "foo" in keys and "bar" in keys + + def eval(self, context, return_values): + super().eval(context, return_values) + ret = BaseTest.tretval(context.sheerka, context.sheerka.new(BuiltinConcepts.SUCCESS)) + ret.parents = return_values + return ret + + +class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority): + + def __init__(self): + super().__init__("suppress", 100) + + def matches(self, context, return_values): + super().matches(context, return_values) + return True + + def eval(self, context, return_values): + super().eval(context, return_values) + foo = None + for ret in return_values: + if ret.body.name == "foo": + foo = ret + + if foo: + return context.sheerka.ret(self.name, False, Concept("does not matter"), parents=[foo]) + else: + return None + + +class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): + + def test_that_return_values_is_unchanged_when_no_evaluator(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [] + + entries = self.tretval(sheerka, Concept("foo")) + return_values = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) + + assert return_values == [entries] + + def test_i_can_use_a_list_as_input(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [] + + entries = [self.tretval(sheerka, Concept("foo"))] + return_values = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) + + assert return_values == entries + + def test_step_concept_is_removed_after_processing_if_not_reduced(self): + """ + No evaluator + """ + sheerka = self.get_sheerka() + sheerka.evaluators = [] + + entry = self.tretval(sheerka, Concept("foo")) + return_values = sheerka.execute(self.get_context(sheerka), entry, [BuiltinConcepts.EVALUATION]) + + assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values] + + def test_step_concept_is_removed_after_processing_if_not_reduced_2(self): + """ + This time the entry is modified by an evaluator, + nevertheless, step concept is removed + """ + sheerka = self.get_sheerka() + sheerka.evaluators = [EvaluatorOneModifyFoo] + + entry = self.tretval(sheerka, Concept("foo")) + return_values = sheerka.execute(self.get_context(sheerka), entry, [BuiltinConcepts.EVALUATION]) + + assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values] + + def test_that_higher_priority_evaluators_are_evaluated_first(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [ + EvaluatorAllWithPriority10, + EvaluatorOneWithPriority20, + EvaluatorAllWithPriority15, + EvaluatorOneWithPriority10, + EvaluatorOneWithPriority15, + EvaluatorAllWithPriority20, ] + + entries = [self.tretval(sheerka, Concept("foo"))] + Out.debug_out = [] + sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) + + assert Out.debug_out == [ + '__EVALUATION [0] priority20 - matches - target=foo', + '__EVALUATION [0] priority20 - eval - target=foo', + '__EVALUATION [0] priority20 - matches - target=__EVALUATION', + '__EVALUATION [0] priority20 - eval - target=__EVALUATION', + "__EVALUATION [0] all_priority20 - matches - target=['foo', '__EVALUATION']", + "__EVALUATION [0] all_priority20 - eval - target=['foo', '__EVALUATION']", + "__EVALUATION [0] all_priority15 - matches - target=['foo', '__EVALUATION']", + "__EVALUATION [0] all_priority15 - eval - target=['foo', '__EVALUATION']", + '__EVALUATION [0] priority15 - matches - target=foo', + '__EVALUATION [0] priority15 - eval - target=foo', + '__EVALUATION [0] priority15 - matches - target=__EVALUATION', + '__EVALUATION [0] priority15 - eval - target=__EVALUATION', + "__EVALUATION [0] all_priority10 - matches - target=['foo', '__EVALUATION']", + "__EVALUATION [0] all_priority10 - eval - target=['foo', '__EVALUATION']", + '__EVALUATION [0] priority10 - matches - target=foo', + '__EVALUATION [0] priority10 - eval - target=foo', + '__EVALUATION [0] priority10 - matches - target=__EVALUATION', + '__EVALUATION [0] priority10 - eval - target=__EVALUATION' + ] + + def test_that_predicate_is_checked_before_evaluation_for_one_return(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [EvaluatorOneModifyFoo] + + entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))] + Out.debug_out = [] + sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) + + assert Out.debug_out == [ + '__EVALUATION [0] modifyFoo - matches - target=foo', + '__EVALUATION [0] modifyFoo - eval - target=foo', + '__EVALUATION [0] modifyFoo - matches - target=baz', + '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION', + '__EVALUATION [1] modifyFoo - matches - target=bar', + '__EVALUATION [1] modifyFoo - matches - target=baz', + '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION' + ] + + def test_that_predicate_is_checked_before_evaluation_for_all_return(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [EvaluatorAllReduceFooBar] + + entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar"))] + Out.debug_out = [] + sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) + assert Out.debug_out == [ + "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', 'bar', '__EVALUATION']", + "__EVALUATION [0] all_reduce_foobar - eval - target=['foo', 'bar', '__EVALUATION']", + "__EVALUATION [1] all_reduce_foobar - matches - target=['__SUCCESS']" + ] + + entries = [self.tretval(sheerka, Concept("foo"))] + Out.debug_out = [] + sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) + assert Out.debug_out == [ + "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', '__EVALUATION']" + ] + + def test_evaluation_continue_until_no_more_modification(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar] + + entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))] + Out.debug_out = [] + sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) + + assert Out.debug_out == [ + '__EVALUATION [0] modifyFoo - matches - target=foo', + '__EVALUATION [0] modifyFoo - eval - target=foo', + '__EVALUATION [0] modifyFoo - matches - target=baz', + '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION', + '__EVALUATION [0] modifyBar - matches - target=foo', + '__EVALUATION [0] modifyBar - matches - target=baz', + '__EVALUATION [0] modifyBar - matches - target=__EVALUATION', + '__EVALUATION [1] modifyFoo - matches - target=bar', + '__EVALUATION [1] modifyFoo - matches - target=baz', + '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION', + '__EVALUATION [1] modifyBar - matches - target=bar', + '__EVALUATION [1] modifyBar - eval - target=bar', + '__EVALUATION [1] modifyBar - matches - target=baz', + '__EVALUATION [1] modifyBar - matches - target=__EVALUATION', + '__EVALUATION [2] modifyFoo - matches - target=baz', + '__EVALUATION [2] modifyFoo - matches - target=baz', + '__EVALUATION [2] modifyFoo - matches - target=__EVALUATION', + '__EVALUATION [2] modifyBar - matches - target=baz', + '__EVALUATION [2] modifyBar - matches - target=baz', + '__EVALUATION [2] modifyBar - matches - target=__EVALUATION' + ] + + def test_evaluation_steps_are_respected(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation] + + entries = [self.tretval(sheerka, Concept("foo"))] + Out.debug_out = [] + sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION]) + + assert Out.debug_out == [ + '__BEFORE_EVALUATION [0] preEval - matches - target=foo', + '__BEFORE_EVALUATION [0] preEval - eval - target=foo', + '__BEFORE_EVALUATION [0] preEval - matches - target=__BEFORE_EVALUATION', + '__BEFORE_EVALUATION [0] preEval - eval - target=__BEFORE_EVALUATION'] + + def test_evaluation_multi_steps_are_respected(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [EvaluatorOneMultiSteps] + + entries = [self.tretval(sheerka, Concept("foo"))] + Out.debug_out = [] + sheerka.execute(self.get_context(sheerka), entries, + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION]) + + assert Out.debug_out == [ + '__BEFORE_EVALUATION [0] multiStep - matches - target=foo', + '__BEFORE_EVALUATION [0] multiStep - eval - target=foo', + '__BEFORE_EVALUATION [0] multiStep - matches - target=__BEFORE_EVALUATION', + '__BEFORE_EVALUATION [0] multiStep - eval - target=__BEFORE_EVALUATION', + '__EVALUATION [0] multiStep - matches - target=foo', + '__EVALUATION [0] multiStep - eval - target=foo', + '__EVALUATION [0] multiStep - matches - target=__EVALUATION', + '__EVALUATION [0] multiStep - eval - target=__EVALUATION' + ] + + def test_evaluators_can_be_pre_processed(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [EvaluatorOneModifyFoo] + + entries = [self.tretval(sheerka, Concept("foo"))] + + # disable evaluator + context = self.get_context(sheerka) + context.add_preprocess(EvaluatorOneModifyFoo().name, enabled=False) # disabled for this exec context + Out.debug_out = [] + sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION]) + assert Out.debug_out == [] + + # other contextes are not impacted + Out.debug_out = [] + sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) + assert Out.debug_out == [ + '__EVALUATION [0] modifyFoo - matches - target=foo', + '__EVALUATION [0] modifyFoo - eval - target=foo', + '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION', + '__EVALUATION [1] modifyFoo - matches - target=bar', + '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION' + ] diff --git a/tests/core/test_sheerka_call_parsers.py b/tests/core/test_sheerka_call_parsers.py new file mode 100644 index 0000000..eff4549 --- /dev/null +++ b/tests/core/test_sheerka_call_parsers.py @@ -0,0 +1,363 @@ +from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept +from parsers.BaseParser import BaseParser + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def get_ret_val(text, who="who"): + return ReturnValueConcept(who, True, UserInputConcept(text, "user_name")) + + +class BaseTestParser(BaseParser): + debug_out = [] + + def __init__(self, name, priority, status=None, parser_result=None): + super().__init__(name, priority) + self.status = status + self.parser_result = parser_result + + @staticmethod + def _get_name(name): + return name[8:] if name.startswith("parsers.") else name + + @staticmethod + def _get_source(text_): + return text_ if isinstance(text_, str) else text_.body + + def _out(self, name, priority, status, source): + debug = f"name={name}" + debug += f", priority={priority}" + debug += f", status={status}" + debug += f", source={source}" + self.debug_out.append(debug) + + def parse(self, context, text): + self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) + value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + parser_result = ParserResultConcept(parser=self, value=value) + return ReturnValueConcept(self, self.status, self.parser_result or parser_result) + + +class Enabled90FalseParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Enabled90False", 90, False) + + +class Enabled80FalseParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Enabled80False", 80, False) + + +class Enabled80MultipleFalseParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Enabled80MultipleFalse", 80, False) + + def parse(self, context, text): + self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) + value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1" + value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2" + return [ + ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)), + ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)), + ] + + +class Enabled80MultipleTrueParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Enabled80MultipleTrue", 80) + + def parse(self, context, text): + self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) + value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1" + value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2" + return [ + ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)), + ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)), + ] + + +class Enabled70FalseParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Enabled70False", 70, False, "Not a ParserResult") + + +class Enabled50TrueParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Enabled50True", 50, True) + + def parse(self, context, text): + source = self._get_source(text) + status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world" + self._out(self._get_name(self.name), self.priority, status, source) + + value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + return_value = ParserResultConcept(parser=self, value=value) + return ReturnValueConcept(self, status, return_value) + + +class Enabled50bisTrueParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Enabled50BisTrue", 50, True) + + +class Enabled50FalseParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Enabled50False", 50, False) + + +class Enabled10TrueParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Enabled10True", 10, True) + + +class DisabledParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("Disabled", 90, True) + self.enabled = False + + +class NoneParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("None", 90, True, None) + + def parse(self, context, text): + self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) + return None + + +class ListOfNoneParser(BaseTestParser): + def __init__(self, **kwargs): + super().__init__("ListOfNone", 90, True, None) + + def parse(self, context, text): + self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) + return [None, None] + + +class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): + + def test_disabled_parsers_are_not_executed(self): + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled": Enabled10TrueParser, + "Disabled": DisabledParser + } + + user_input = [get_ret_val("hello world")] + BaseTestParser.debug_out = [] + sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + + assert BaseTestParser.debug_out == ['name=Enabled10True, priority=10, status=True, source=hello world'] + + def test_parser_are_executed_by_priority(self): + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled90False": Enabled90FalseParser, + "Enabled80False": Enabled80FalseParser, + "Enabled50True": Enabled50TrueParser, + } + + user_input = [get_ret_val("hello world")] + BaseTestParser.debug_out = [] + sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + + assert BaseTestParser.debug_out == [ + 'name=Enabled90False, priority=90, status=False, source=hello world', + 'name=Enabled80False, priority=80, status=False, source=hello world', + 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', + 'name=Enabled50True, priority=50, status=False, source=hello world', + 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', + 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', + 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', + ] + + def test_parsing_stop_at_the_first_success(self): + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled80False": Enabled80FalseParser, + "Enabled50bisTrue": Enabled50bisTrueParser, + "Enabled10True": Enabled10TrueParser, + } + + user_input = [get_ret_val("hello world")] + BaseTestParser.debug_out = [] + sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + + assert BaseTestParser.debug_out == [ + 'name=Enabled80False, priority=80, status=False, source=hello world', + 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', + ] + + def test_parsing_stop_at_the_first_success_2(self): + """ + Same test than before, but Enabled50True takes more time to find a match + :return: + """ + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled90False": Enabled90FalseParser, + "Enabled80False": Enabled80FalseParser, + "Enabled50True": Enabled50TrueParser, + "Enabled10True": Enabled10TrueParser, + } + + user_input = [get_ret_val("hello world")] + BaseTestParser.debug_out = [] + sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + + assert BaseTestParser.debug_out == [ + 'name=Enabled90False, priority=90, status=False, source=hello world', + 'name=Enabled80False, priority=80, status=False, source=hello world', + 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', + 'name=Enabled50True, priority=50, status=False, source=hello world', + 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', + 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', + 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', + ] + + def test_all_parsers_of_a_given_priority_are_executed(self): + """ + Make sure that all parsers with priority 50 are executed + :return: + """ + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled90False": Enabled90FalseParser, + "Enabled80False": Enabled80FalseParser, + "Enabled50True": Enabled50TrueParser, + "Enabled50bisTrue": Enabled50bisTrueParser, + "Enabled50False": Enabled50FalseParser, + "Enabled10True": Enabled10TrueParser, + } + + user_input = [get_ret_val("hello world")] + BaseTestParser.debug_out = [] + sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + + assert BaseTestParser.debug_out == [ + 'name=Enabled90False, priority=90, status=False, source=hello world', + 'name=Enabled80False, priority=80, status=False, source=hello world', + 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', + 'name=Enabled50True, priority=50, status=False, source=hello world', + 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', + 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', + 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', + 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', + 'name=Enabled50False, priority=50, status=False, source=hello world', + 'name=Enabled50False, priority=50, status=False, source=Enabled90False:hello world', + 'name=Enabled50False, priority=50, status=False, source=Enabled80False:hello world', + 'name=Enabled50False, priority=50, status=False, source=Enabled80False:Enabled90False:hello world', + ] + + def test_a_parser_has_access_to_the_output_of_its_predecessors(self): + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled90False": Enabled90FalseParser, + "Enabled80False": Enabled80FalseParser, + "Enabled50True": Enabled50TrueParser, + } + + user_input = [get_ret_val("hello world")] + + res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] + assert res_as_tuple == [ + ('Enabled90False', False, 'Enabled90False:hello world'), + ('Enabled80False', False, 'Enabled80False:hello world'), + ('Enabled80False', False, 'Enabled80False:Enabled90False:hello world'), + ('Enabled50True', False, 'Enabled50True:hello world'), + ('Enabled50True', False, 'Enabled50True:Enabled90False:hello world'), + ('Enabled50True', False, 'Enabled50True:Enabled80False:hello world'), + ('Enabled50True', True, 'Enabled50True:Enabled80False:Enabled90False:hello world'), + ] + + def test_none_return_values_are_discarded(self): + sheerka = self.get_sheerka() + sheerka.parsers = { + "NoneParser": NoneParser, + "ListOfNone": ListOfNoneParser, + } + + user_input = [get_ret_val("hello world")] + + BaseTestParser.debug_out = [] + res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + assert res == [] + assert BaseTestParser.debug_out == [ + 'name=None, priority=90, status=True, source=hello world', + 'name=ListOfNone, priority=90, status=True, source=hello world' + ] + + def test_following_priorities_can_only_see_parser_result_return_values(self): + """ + Normally, lower priority parsers can see the result of the higher priority parsers + This is true only if the higher priority parser return a ParserResultConcept + :return: + """ + + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled80False": Enabled80FalseParser, + "Enabled70False": Enabled70FalseParser, + "Enabled50True": Enabled50TrueParser, + } + + user_input = [get_ret_val("hello world")] + BaseTestParser.debug_out = [] + sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + + assert BaseTestParser.debug_out == [ + 'name=Enabled80False, priority=80, status=False, source=hello world', + 'name=Enabled70False, priority=70, status=False, source=hello world', + 'name=Enabled70False, priority=70, status=False, source=Enabled80False:hello world', + 'name=Enabled50True, priority=50, status=False, source=hello world', + 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', + ] + + def test_i_can_manage_parser_with_multiple_results(self): + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled80MultipleFalse": Enabled80MultipleFalseParser, + "Enabled50True": Enabled50TrueParser, + } + + user_input = [get_ret_val("hello world")] + BaseTestParser.debug_out = [] + res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + + assert BaseTestParser.debug_out == [ + 'name=Enabled80MultipleFalse, priority=80, status=False, source=hello world', + 'name=Enabled50True, priority=50, status=False, source=hello world', + 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_1', + 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_2', + ] + + res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] + assert res_as_tuple == [ + ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_1'), + ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_2'), + ('Enabled50True', False, 'Enabled50True:hello world'), + ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_1'), + ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'), + ] + + def test_i_can_manage_parser_with_multiple_results_and_a_sucess(self): + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled80MultipleTrue": Enabled80MultipleTrueParser, + "Enabled50True": Enabled50TrueParser, + } + + user_input = [get_ret_val("hello world")] + BaseTestParser.debug_out = [] + res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + + assert BaseTestParser.debug_out == [ + 'name=Enabled80MultipleTrue, priority=80, status=None, source=hello world', + ] + + res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] + assert res_as_tuple == [ + ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'), + ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'), + ] diff --git a/tests/core/test_sheerka_transform.py b/tests/core/test_sheerka_transform.py new file mode 100644 index 0000000..307b18b --- /dev/null +++ b/tests/core/test_sheerka_transform.py @@ -0,0 +1,318 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, ConceptParts +from core.sheerka.ExecutionContext import ExecutionContext +from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY +from sdp.sheerkaDataProvider import Event + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaTransform(TestUsingMemoryBasedSheerka): + def test_i_can_transform_an_unknown_concept(self): + sheerka = self.get_sheerka() + + foo = Concept("foo", body="body") + concept_with_sub = Concept("concept_with_sub", body=foo) + + concept = Concept( + name="concept_name", + is_builtin=True, + is_unique=True, + key="concept_key", + body=concept_with_sub, + where=[foo, 1, "1", True, 1.0], + pre=foo, + post=None, # will not appear + definition="it is a definition", + definition_type="def type", + desc="this this the desc" + ).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub") + + # add values and props + concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init() + concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0] + concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init() + concept.values[ConceptParts.POST] = "a value for POST" + concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub) + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(concept) + + assert to_dict == { + OBJ_TYPE_KEY: SheerkaTransformType.Concept, + OBJ_ID_KEY: 0, + 'meta.name': 'concept_name', + 'meta.key': 'concept_key', + 'meta.is_builtin': True, + 'meta.is_unique': True, + 'meta.definition': 'it is a definition', + 'meta.definition_type': 'def type', + 'meta.desc': 'this this the desc', + 'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept, + OBJ_ID_KEY: 1, + 'meta.body': 'body', + 'meta.name': 'foo'}, 1, '1', True, 1.0], + 'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, + 'meta.body': { + '__type__': SheerkaTransformType.Concept, + '__id__': 2, + 'meta.name': 'concept_with_sub', + 'meta.body': { + '__type__': SheerkaTransformType.Reference, + '__id__': 1}}, + 'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']], + 'pre': {'__type__': SheerkaTransformType.Concept, + '__id__': 4, + 'meta.body': 'body', + 'meta.name': 'foo', + 'body': 'body'}, + 'post': "a value for POST", + 'body': {'__type__': SheerkaTransformType.Concept, + '__id__': 3, + 'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}, + 'meta.name': 'concept_with_sub', + 'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}}, + 'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0], + 'props': [('a', 10), + ('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}), + ('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})], + } + + def test_i_can_transform_unknown_concept_with_almost_same_value(self): + sheerka = self.get_sheerka() + concept = Concept("foo") + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(concept) + + assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'} + + def test_i_can_transform_known_concept_when_the_values_are_the_same(self): + """ + Values are the same means that we are serializing a concept which has kept all its default values + There is not diff between the concept to serialize and the one which was registered with create_new_concept() + We serialize only the id of the concept + :return: + """ + sheerka = self.get_sheerka() + + concept = Concept( + name="concept_name", + is_builtin=True, + is_unique=False, + key="concept_key", + body="body definition", + where="where definition", + pre="pre definition", + post="post definition", + definition="it is a definition", + definition_type="def type", + desc="this this the desc" + ).def_prop("a").def_prop("b") + sheerka.create_new_concept(self.get_context(sheerka), concept) + + new_concept = sheerka.new(concept.key) + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(new_concept) + + assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"} + + def test_i_can_transform_known_concept_when_the_values_are_different(self): + """ + Values are the different means the concept was modified. + It's different from the one which was registered with create_new_concept() + We serialize only the differences + :return: + """ + sheerka = self.get_sheerka() + + concept = Concept( + name="concept_name", + is_builtin=True, + is_unique=False, + key="concept_key", + body="body definition", + where="where definition", + pre="pre definition", + post="post definition", + definition="it is a definition", + definition_type="def type", + desc="this this the desc" + ).def_prop("a").def_prop("b") + sheerka.create_new_concept(self.get_context(sheerka), concept) + + new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre") + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(new_concept) + + assert to_dict == { + OBJ_TYPE_KEY: SheerkaTransformType.Concept, + OBJ_ID_KEY: 0, + "id": "1001", + 'pre': 'another pre', + "body": "another", + 'props': [('a', 10)] + } + + def test_i_can_transform_concept_with_circular_reference(self): + sheerka = self.get_sheerka() + foo = Concept("foo") + bar = Concept("bar", body=foo) + foo.metadata.body = bar + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(foo) + + assert to_dict == { + OBJ_TYPE_KEY: SheerkaTransformType.Concept, + OBJ_ID_KEY: 0, + 'meta.name': 'foo', + 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept, + OBJ_ID_KEY: 1, + 'meta.name': 'bar', + 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, + OBJ_ID_KEY: 0}, + }, + } + + def test_i_can_transform_concept_with_circular_reference_2(self): + sheerka = self.get_sheerka() + foo = Concept("foo") + bar = Concept("foo", body=foo) + foo.metadata.body = bar + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(foo) + + assert to_dict == { + OBJ_TYPE_KEY: SheerkaTransformType.Concept, + OBJ_ID_KEY: 0, + 'meta.name': 'foo', + 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept, + OBJ_ID_KEY: 1, + 'meta.name': 'foo', + 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, + OBJ_ID_KEY: 0}, + }, + } + + def test_i_can_transform_the_unknown_concept(self): + sheerka = self.get_sheerka(False) + + unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(unknown) + + assert len(to_dict) == 3 + assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept + assert to_dict[OBJ_ID_KEY] == 0 + assert "id" in to_dict + + def test_i_can_transform_simple_execution_context(self): + sheerka = self.get_sheerka() + ExecutionContext.ids = {} + context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(context) + + assert to_dict == { + OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext, + OBJ_ID_KEY: 0, + '_parent': None, + '_id': 0, + '_tab': '', + '_bag': {}, + '_start': 0, + '_stop': 0, + 'who': 'requester', + 'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'}, + 'desc': 'this is the desc', + 'children': [], + 'preprocess': None, + 'inputs': {}, + 'values': {}, + 'obj': None, + 'concepts': {} + } + + def test_i_can_transform_list(self): + sheerka = self.get_sheerka() + ExecutionContext.ids = {} + context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict([context]) + + assert len(to_dict) == 1 + assert isinstance(to_dict, list) + assert to_dict[0]["who"] == "requester" + assert to_dict[0]["desc"] == "this is the desc" + + def test_i_can_transform_set(self): + sheerka = self.get_sheerka() + ExecutionContext.ids = {} + context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict({context}) + + assert len(to_dict) == 1 + assert isinstance(to_dict, list) + assert to_dict[0]["who"] == "requester" + assert to_dict[0]["desc"] == "this is the desc" + + def test_i_can_transform_dict(self): + sheerka = self.get_sheerka() + ExecutionContext.ids = {} + context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') + known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key() + sheerka.create_new_concept(self.get_context(sheerka), known_concept) + unknown_concept = Concept("bar") + known = sheerka.new("foo") + + bag = { + "context": context, + "known_concept": known_concept, + "unknown_concept": unknown_concept, + "True": True, + "Number": 1.1, + "String": "a string value", + "None": None, + unknown_concept: "hello", + known: "world" + } + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(bag) + + assert isinstance(to_dict, dict) + assert to_dict['Number'] == 1.1 + assert to_dict['String'] == 'a string value' + assert to_dict['True'] + assert to_dict['None'] is None + assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext + assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept + assert to_dict["known_concept"]["id"] == '1001' + assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept + assert to_dict["(None)bar"] == "hello" + assert to_dict["(1001)foo"] == "world" + + def test_i_can_transform_when_circular_references(self): + sheerka = self.get_sheerka() + ExecutionContext.ids = {} + context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') + context.push("another requester", "another desc") + + st = SheerkaTransform(sheerka) + to_dict = st.to_dict(context) + + assert isinstance(to_dict, dict) + assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext + assert len(to_dict["children"]) == 1 + assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext + assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference + assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0 + assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference + assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1 diff --git a/tests/test_tokenizer.py b/tests/core/test_tokenizer.py similarity index 100% rename from tests/test_tokenizer.py rename to tests/core/test_tokenizer.py diff --git a/tests/test_utils.py b/tests/core/test_utils.py similarity index 100% rename from tests/test_utils.py rename to tests/core/test_utils.py diff --git a/tests/evaluators/__init__.py b/tests/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/evaluators/test_AddConceptEvaluator.py b/tests/evaluators/test_AddConceptEvaluator.py new file mode 100644 index 0000000..13e7b54 --- /dev/null +++ b/tests/evaluators/test_AddConceptEvaluator.py @@ -0,0 +1,155 @@ +import ast +import pytest + +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts +from core.concept import VARIABLE_PREFIX, Concept +from core.tokenizer import Tokenizer +from evaluators.AddConceptEvaluator import AddConceptEvaluator +from parsers.BaseParser import BaseParser +from parsers.ConceptLexerParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression +from parsers.BnfParser import BnfParser +from parsers.DefaultParser import DefConceptNode, NameNode +from parsers.PythonParser import PythonNode, PythonParser + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): + + @staticmethod + def get_concept_part(part): + if isinstance(part, str): + node = PythonNode(part, ast.parse(part, mode="eval")) + return ReturnValueConcept( + who="parsers.Default", + status=True, + value=ParserResultConcept( + source=part, + parser=PythonParser(), + value=node)) + + if isinstance(part, PythonNode): + return ReturnValueConcept( + who="parsers.Default", + status=True, + value=ParserResultConcept( + source=part.source, + parser=PythonParser(), + value=part)) + + if isinstance(part, ReturnValueConcept): + return part + + @staticmethod + def get_return_value(source, parsing_expression): + return ReturnValueConcept( + who="Parsers:RegexParser", + status=True, + value=ParserResultConcept( + source=source, + parser=BnfParser(), + value=parsing_expression + ) + ) + + def get_def_concept(self, name, where=None, pre=None, post=None, body=None, definition=None): + concept = DefConceptNode([], name=NameNode(list(Tokenizer(name)))) + + if body: + concept.body = self.get_concept_part(body) + if where: + concept.where = self.get_concept_part(where) + if pre: + concept.pre = self.get_concept_part(pre) + if post: + concept.post = self.get_concept_part(post) + if definition: + concept.definition = definition + + return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=concept)) + + @pytest.mark.parametrize("ret_val, expected", [ + ( + ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))), + True), + (ReturnValueConcept(BaseParser.PREFIX + "some_name", False, ParserResultConcept(value=DefConceptNode([]))), + False), + (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not a ParserResultConcept"), False), + (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept()), False), + ]) + def test_i_can_match(self, ret_val, expected): + context = self.get_context() + assert AddConceptEvaluator().matches(context, ret_val) == expected + + def test_that_the_source_is_correctly_set(self): + context = self.get_context() + def_concept_return_value = self.get_def_concept( + name="hello a", + definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))), + where="isinstance(a, str )", + pre="a is not None", + body="print('hello' + a)") + + evaluated = AddConceptEvaluator().eval(context, def_concept_return_value) + + assert evaluated.status + assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT) + + created_concept = evaluated.body.body + assert created_concept.metadata.name == "hello a" + assert created_concept.metadata.where == "isinstance(a, str )" + assert created_concept.metadata.pre == "a is not None" + assert created_concept.metadata.post is None + assert created_concept.metadata.body == "print('hello' + a)" + assert created_concept.metadata.definition == "hello a" + + def test_that_the_new_concept_is_correctly_saved_in_db(self): + context = self.get_context() + def_concept_return_value = self.get_def_concept( + name="hello a", + definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))), + where="isinstance(a, str )", + pre="a is not None", + body="print('hello' + a)") + + # sanity. Make sure that the concept does not already exist + from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0") + assert context.sheerka.isinstance(from_db, BuiltinConcepts.UNKNOWN_CONCEPT) + + AddConceptEvaluator().eval(context, def_concept_return_value) + context.sheerka.concepts_cache = {} # reset cache + from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0") + + assert from_db.metadata.key == f"hello {VARIABLE_PREFIX}0" + assert from_db.metadata.name == "hello a" + assert from_db.metadata.where == "isinstance(a, str )" + assert from_db.metadata.pre == "a is not None" + assert from_db.metadata.post is None + assert from_db.metadata.body == "print('hello' + a)" + assert from_db.metadata.definition == "hello a" + assert len(from_db.metadata.props) == 1 + assert from_db.metadata.props[0] == ("a", None) + assert "a" in from_db.props + + assert from_db.compiled == {} # ast is not saved in db + + def test_i_can_get_props_from_python_node(self): + ret_val = self.get_concept_part("isinstance(a, str)") + context = self.get_context() + + assert AddConceptEvaluator.get_props(context.sheerka, ret_val, ["a"]) == ["a"] + + def test_i_can_get_props_from_another_concept(self): + concept = Concept("hello").def_prop("a").def_prop("b") + ret_val = ReturnValueConcept(who="some_parser", + status=True, + value=ParserResultConcept(value=concept)) + + assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["a", "b"] + + def test_i_can_get_props_from_definition(self): + parsing_expression = Sequence(ConceptExpression('mult'), + ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add")))) + ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression) + + assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["add", "mult"] diff --git a/tests/evaluators/test_AddConceptInSetEvaluator.py b/tests/evaluators/test_AddConceptInSetEvaluator.py new file mode 100644 index 0000000..b6ebbc2 --- /dev/null +++ b/tests/evaluators/test_AddConceptInSetEvaluator.py @@ -0,0 +1,102 @@ +import pytest + +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts +from core.concept import Concept +from core.tokenizer import Tokenizer +from evaluators.AddConceptInSetEvaluator import AddConceptInSetEvaluator +from parsers.DefaultParser import IsaConceptNode, NameNode +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def get_ret_val(concept_name, concept_set_name): + n1 = NameNode(list(Tokenizer(concept_name))) + n2 = NameNode(list(Tokenizer(concept_set_name))) + + return ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([], n1, n2))) + + +class TestAssConceptInSetEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("ret_val, expected", [ + (ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([]))), True), + (ReturnValueConcept("some_name", False, ParserResultConcept(value=IsaConceptNode([]))), False), + (ReturnValueConcept("some_name", True, "not a ParserResultConcept"), False), + (ReturnValueConcept("some_name", True, ParserResultConcept()), False), + ]) + def test_i_can_match(self, ret_val, expected): + context = self.get_context() + assert AddConceptInSetEvaluator().matches(context, ret_val) == expected + + def test_i_cannot_add_if_the_concept_does_not_exists(self): + context = self.get_context() + + ret_val = get_ret_val("foo", "bar") + res = AddConceptInSetEvaluator().eval(context, ret_val) + + assert not res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) + assert res.value.body == "foo" + + def test_i_cannot_add_if_the_set_does_not_exists(self): + context = self.get_context() + foo = Concept("foo") + context.sheerka.set_id_if_needed(foo, False) + context.sheerka.add_in_cache(foo) + + ret_val = get_ret_val("foo", "bar") + res = AddConceptInSetEvaluator().eval(context, ret_val) + + assert not res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) + assert res.value.body == "bar" + + def test_i_can_add_concept_to_a_set_of_concept(self): + context = self.get_context() + foo = Concept("foo") + context.sheerka.set_id_if_needed(foo, False) + context.sheerka.add_in_cache(foo) + + bar = Concept("bar") + context.sheerka.set_id_if_needed(bar, False) + context.sheerka.add_in_cache(bar) + + ret_val = get_ret_val("foo", "bar") + res = AddConceptInSetEvaluator().eval(context, ret_val) + + assert res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS) + + def test_i_can_add_concept_with_a_body_to_a_set_of_concept(self): + context = self.get_context() + foo = Concept("foo", body="1") + context.sheerka.set_id_if_needed(foo, False) + context.sheerka.add_in_cache(foo) + + bar = Concept("bar") + context.sheerka.set_id_if_needed(bar, False) + context.sheerka.add_in_cache(bar) + + ret_val = get_ret_val("foo", "bar") + res = AddConceptInSetEvaluator().eval(context, ret_val) + + assert res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS) + + def test_i_cannot_add_the_same_concept_twice(self): + context = self.get_context() + foo = Concept("foo") + context.sheerka.set_id_if_needed(foo, False) + context.sheerka.add_in_cache(foo) + + bar = Concept("bar") + context.sheerka.set_id_if_needed(bar, False) + context.sheerka.add_in_cache(bar) + + ret_val = get_ret_val("foo", "bar") + AddConceptInSetEvaluator().eval(context, ret_val) + res = AddConceptInSetEvaluator().eval(context, ret_val) + + assert not res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_IN_SET) + assert res.value.concept == foo + assert res.value.concept_set == bar diff --git a/tests/evaluators/test_ConceptEvaluator.py b/tests/evaluators/test_ConceptEvaluator.py new file mode 100644 index 0000000..b0bd27f --- /dev/null +++ b/tests/evaluators/test_ConceptEvaluator.py @@ -0,0 +1,117 @@ +import pytest + +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts +from core.concept import Concept, ConceptParts +from evaluators.ConceptEvaluator import ConceptEvaluator +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("ret_val, expected", [ + (ReturnValueConcept("some_name", True, ParserResultConcept(value=Concept())), True), + (ReturnValueConcept("some_name", False, ParserResultConcept(value=Concept())), False), + (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept")), False), + (ReturnValueConcept("some_name", True, Concept()), False), + ]) + def test_i_can_match(self, ret_val, expected): + context = self.get_context() + assert ConceptEvaluator().matches(context, ret_val) == expected + + def test_i_can_evaluate_concept(self): + context = self.get_context() + concept = Concept(name="foo", + where="1", + pre="2", + post="3").def_prop("a", "4").def_prop("b", "5") + + evaluator = ConceptEvaluator() + item = self.pretval(concept) + result = evaluator.eval(context, item) + + assert result.who == evaluator.name + assert result.status + assert result.value.name == "foo" + assert result.value.get_metadata_value(ConceptParts.WHERE) == 1 + assert result.value.get_metadata_value(ConceptParts.PRE) == 2 + assert result.value.get_metadata_value(ConceptParts.POST) == 3 + assert result.value.get_prop("a") == 4 + assert result.value.get_prop("b") == 5 + assert result.value.key == "foo" + assert result.parents == [item] + + def test_body_is_returned_when_defined_and_requested(self): + context = self.get_context() + concept = Concept(name="foo", + body="'I have a value'", + where="1", + pre="2", + post="3").set_prop("a", "4").set_prop("b", "5") + + evaluator = ConceptEvaluator(return_body=True) + item = self.pretval(concept) + result = evaluator.eval(context, item) + + assert result.who == evaluator.name + assert result.status + assert result.value == "I have a value" + assert result.parents == [item] + + def test_body_is_not_returned_if_not_requested(self): + context = self.get_context() + concept = Concept(name="foo", + body="'I have a value'", + where="1", + pre="2", + post="3").set_prop("a", "4").set_prop("b", "5") + + evaluator = ConceptEvaluator(return_body=False) # which is the default behaviour + item = self.pretval(concept) + result = evaluator.eval(context, item) + + assert result.who == evaluator.name + assert result.status + assert result.value == concept + assert result.parents == [item] + + def test_i_can_eval_if_with_the_same_name_is_defined_in_the_context(self): + # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'") + # ConceptEvaluator will be called to resolve 'a' while we know that 'a' refers to the string 'property_a' + + context = self.get_context() + context.obj = Concept("other").set_prop("foo", "'some_other_value'") + concept = Concept(name="foo") + + item = self.pretval(concept) + result = ConceptEvaluator().eval(context, item) + + assert result.status + assert result.value == "'some_other_value'" + + def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(self): + context = self.get_context() + context.sheerka.add_in_cache(Concept(name="one").init_key()) + concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b") + .def_prop("a", "one") + .def_prop("b", "two").init_key()) + + evaluator = ConceptEvaluator() + item = self.pretval(concept_plus) + result = evaluator.eval(context, item) + + assert not result.status + assert context.sheerka.isinstance(result.value, BuiltinConcepts.CONCEPT_EVAL_ERROR) + assert result.value.concept == concept_plus + assert result.value.property_name == "b" + assert context.sheerka.isinstance(result.value.error, BuiltinConcepts.TOO_MANY_ERRORS) + + def test_that_error_concepts_are_transformed_into_errors_only_if_the_key_is_different(self): + context = self.get_context() + + error_concept = context.sheerka.new(BuiltinConcepts.ERROR) + item = self.pretval(error_concept) + result = ConceptEvaluator().eval(context, item) + + assert not context.sheerka.is_success(error_concept) # it's indeed an error + assert result.status + assert result.value == error_concept diff --git a/tests/evaluators/test_EvalEvaluator.py b/tests/evaluators/test_EvalEvaluator.py new file mode 100644 index 0000000..d02f868 --- /dev/null +++ b/tests/evaluators/test_EvalEvaluator.py @@ -0,0 +1,72 @@ +import pytest + +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +from core.concept import Concept +from evaluators.EvalEvaluator import EvalEvaluator +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + +eval_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.CONCEPT_EVAL_REQUESTED)) + + +def retval(obj, who="who", status=True): + """ret_val""" + return ReturnValueConcept(who, status, obj) + + +class TestEvalEvaluator(TestUsingMemoryBasedSheerka): + def test_i_can_match_and_eval(self): + context = self.get_context() + + to_eval1 = ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init()) + to_eval2 = ReturnValueConcept("some_name", True, Concept(name="3", body="also to eval").auto_init()) + + return_values = [ + ReturnValueConcept("some_name", True, "not to eval"), + ReturnValueConcept("some_name", True, Concept(name="not to eval")), + ReturnValueConcept("some_name", False, Concept(name="1", body="'not to eval'")), + to_eval1, + to_eval2, + eval_requested + ] + + evaluator = EvalEvaluator() + assert evaluator.matches(context, return_values) + + evaluated = evaluator.eval(context, return_values) + assert len(evaluated) == 2 + assert evaluated[0].value == to_eval1.body.body + assert evaluated[0].parents == [to_eval1, eval_requested] + + assert evaluated[1].value == to_eval2.body.body + assert evaluated[1].parents == [to_eval2, eval_requested] + + @pytest.mark.parametrize("return_values, expected", [ + ([retval(Concept("foo", body="bar")), eval_requested], True), + ([retval(Concept("status is false", body="bar"), status=False), eval_requested], True), + ([retval("string_value"), eval_requested], True), + ([retval(Concept("no body")), eval_requested], True), + ([retval(Concept("eval requested missing", body="bar"))], False), + ]) + def test_i_cannot_match_if_eval_request_is_not_present(self, return_values, expected): + context = self.get_context() + assert EvalEvaluator().matches(context, return_values) == expected + + def test_concept_eval_requested_is_reduced_when_nothing_to_reduce(self): + context = self.get_context() + + return_values = [ + ReturnValueConcept("some_name", True, "not to eval"), + ReturnValueConcept("some_name", True, Concept(name="not to eval")), + ReturnValueConcept("some_name", False, Concept(name="1", body="not to eval")), + eval_requested + ] + + evaluator = EvalEvaluator() + assert evaluator.matches(context, return_values) + + evaluated = evaluator.eval(context, return_values) + assert evaluated == ReturnValueConcept( + "evaluators.Eval", + False, + context.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_REQUESTED)) + assert evaluated.parents == [eval_requested] diff --git a/tests/evaluators/test_LexerNodeEvaluator.py b/tests/evaluators/test_LexerNodeEvaluator.py new file mode 100644 index 0000000..39b1dc5 --- /dev/null +++ b/tests/evaluators/test_LexerNodeEvaluator.py @@ -0,0 +1,104 @@ +import ast +import pytest + +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts +from core.concept import Concept, ConceptParts, DoNotResolve +from evaluators.LexerNodeEvaluator import LexerNodeEvaluator +from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, StrMatch, UnrecognizedTokensNode, SourceCodeNode +from parsers.PythonParser import PythonNode +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka): + + def from_parsing(self, context, grammar, expression): + parser = ConceptLexerParser() + parser.initialize(context, grammar) + + ret_val = parser.parse(context, expression) + assert ret_val.status + return ret_val + + def from_fragments(self, *fragments): + nodes = [] + for fragment in fragments: + if isinstance(fragment, str): + node = PythonNode(fragment, ast.parse(fragment.strip(), mode="eval")) + nodes.append(SourceCodeNode(node, 0, 0, [], fragment)) + else: + nodes.append(ConceptNode(fragment, 0, 0, [], fragment.name)) + + return ReturnValueConcept("somme_name", True, ParserResultConcept(value=nodes)) + + def init(self, concept, grammar, text): + context = self.get_context() + if isinstance(concept, list): + for c in concept: + context.sheerka.add_in_cache(c) + else: + context.sheerka.add_in_cache(concept) + ret_val = self.from_parsing(context, grammar, text) + node = ret_val.value.value[0] + + return context, node + + @pytest.mark.parametrize("ret_val, expected", [ + (ReturnValueConcept("some_name", True, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), True), + (ReturnValueConcept("some_name", True, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), True), + (ReturnValueConcept("some_name", True, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), True), + (ReturnValueConcept("some_name", True, ParserResultConcept(value=SourceCodeNode(0, 0, []))), True), + (ReturnValueConcept("some_name", True, ParserResultConcept(value=[UnrecognizedTokensNode(0, 0, [])])), False), + (ReturnValueConcept("some_name", True, ParserResultConcept(value=UnrecognizedTokensNode(0, 0, []))), False), + (ReturnValueConcept("some_name", False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), False), + (ReturnValueConcept("some_name", False, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), False), + (ReturnValueConcept("some_name", False, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), False), + (ReturnValueConcept("some_name", False, ParserResultConcept(value=SourceCodeNode(0, 0, []))), False), + (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept node")), False), + (ReturnValueConcept("some_name", True, ParserResultConcept(value=["Not a concept node"])), False), + (ReturnValueConcept("some_name", True, [ConceptNode(Concept(), 0, 0)]), False), + (ReturnValueConcept("some_name", True, ConceptNode(Concept(), 0, 0)), False), + ]) + def test_i_can_match(self, ret_val, expected): + context = self.get_context() + assert LexerNodeEvaluator().matches(context, ret_val) == expected + + def test_concept_is_returned_when_only_one_in_the_list(self): + foo = Concept("foo") + context = self.get_context() + context.sheerka.add_in_cache(foo) + + ret_val = self.from_parsing(context, {foo: StrMatch("foo")}, "foo") + + evaluator = LexerNodeEvaluator() + result = evaluator.eval(context, ret_val) + wrapper = result.body + return_value = result.body.body + + assert result.who == evaluator.name + assert result.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert wrapper.parser == evaluator + assert wrapper.source == "foo" + assert return_value == Concept("foo").init_key() + assert return_value.compiled[ConceptParts.BODY] == DoNotResolve("foo") + assert result.parents == [ret_val] + + def test_concept_python_node_is_returned_when_source_code(self): + context = self.get_context() + foo = Concept("foo") + ret_val = self.from_fragments(foo, " + 1") + + evaluator = LexerNodeEvaluator() + result = evaluator.eval(context, ret_val) + wrapper = result.body + return_value = result.body.body + + assert result.who == evaluator.name + assert result.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert wrapper.parser == evaluator + assert wrapper.source == "foo + 1" + + assert return_value == PythonNode('foo + 1', ast.parse("__C__foo__C__ + 1", mode="eval")) + assert return_value.concepts == {"__C__foo__C__": foo} + assert result.parents == [ret_val] diff --git a/tests/evaluators/test_MultipleSameSuccessEvaluator.py b/tests/evaluators/test_MultipleSameSuccessEvaluator.py new file mode 100644 index 0000000..f97745a --- /dev/null +++ b/tests/evaluators/test_MultipleSameSuccessEvaluator.py @@ -0,0 +1,200 @@ +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +from core.concept import Concept +from evaluators.BaseEvaluator import BaseEvaluator +from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator +from parsers.BaseParser import BaseParser +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestMultipleSameSuccessEvaluator(TestUsingMemoryBasedSheerka): + + def test_i_can_match_and_eval(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + evaluator = MultipleSameSuccessEvaluator() + assert evaluator.matches(context, return_values) + + evaluated = evaluator.eval(context, return_values) + assert evaluated.status + assert evaluated.value == Concept(name="1", body="value").auto_init() # the first concept is returned + + def test_i_can_match_and_eval_when_no_body(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + evaluator = MultipleSameSuccessEvaluator() + assert evaluator.matches(context, return_values) + + evaluated = evaluator.eval(context, return_values) + assert evaluated.status + assert evaluated.value == Concept(name="1").auto_init() + + def test_i_can_match_and_eval_when_value_is_not_a_concept(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + evaluator = MultipleSameSuccessEvaluator() + assert evaluator.matches(context, return_values) + + evaluated = evaluator.eval(context, return_values) + assert evaluated.status + assert evaluated.value == "value" + + def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_concept_evaluator_first(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), + ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"), + ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"), + ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="2", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "Concept", True, Concept(name="1", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="3", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + evaluator = MultipleSameSuccessEvaluator() + assert evaluator.matches(context, return_values) + + evaluated = evaluator.eval(context, return_values) + assert evaluated.status + assert evaluated.value == Concept(name="1", body="value").auto_init() # the concept is returned, not the value + + def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_python_evaluator_first(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), + ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"), + ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"), + ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="2", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="1", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="3", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + evaluator = MultipleSameSuccessEvaluator() + assert evaluator.matches(context, return_values) + + evaluated = evaluator.eval(context, return_values) + assert evaluated.status + assert evaluated.value == Concept(name="1", body="value").auto_init() # the concept is returned, not the value + + def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value2").auto_init()), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + evaluator = MultipleSameSuccessEvaluator() + assert evaluator.matches(context, return_values) + assert evaluator.eval(context, return_values) is None + + def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail_when_no_body(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2").auto_init()), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + evaluator = MultipleSameSuccessEvaluator() + assert evaluator.matches(context, return_values) + assert evaluator.eval(context, return_values) is None + + def test_i_can_match_if_no_parser(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + assert MultipleSameSuccessEvaluator().matches(context, return_values) + + def test_i_cannot_match_if_not_reduced_requested(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), + ReturnValueConcept("some_name", True, Concept(name="2", body="value")), + # ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + assert not MultipleSameSuccessEvaluator().matches(context, return_values) + + def test_i_cannot_match_if_only_one_successful_evaluator(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), + # ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), + ReturnValueConcept("some_name", True, Concept(name="2", body="value")), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + assert not MultipleSameSuccessEvaluator().matches(context, return_values) + + def test_i_cannot_match_if_at_least_one_parser_is_successful(self): + context = self.get_context() + sheerka = context.sheerka + + return_values = [ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), + ReturnValueConcept(BaseParser.PREFIX + "some_name2", True, "Not relevant"), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), + ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) + ] + + assert not MultipleSameSuccessEvaluator().matches(context, return_values) diff --git a/tests/evaluators/test_OneErrorEvaluator.py b/tests/evaluators/test_OneErrorEvaluator.py new file mode 100644 index 0000000..8d1728d --- /dev/null +++ b/tests/evaluators/test_OneErrorEvaluator.py @@ -0,0 +1,73 @@ +import pytest + +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +from core.concept import Concept +from evaluators.OneErrorEvaluator import OneErrorEvaluator + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def r(value, status=True): + return ReturnValueConcept(value, status, value) + + +reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED)) + + +class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka): + @pytest.mark.parametrize("return_values, expected", [ + ([r("evaluators.one error", False), reduce_requested], True), + ([r("evaluators.one error", False), r("failed", False), r("failed", False), reduce_requested], True), + ([r("evaluators.error", False), r("not a parser in success"), reduce_requested], True), + ([r("evaluators.no reduce required", False), r("failed", False), r("failed", False)], False), + ([r("evaluators.no reduce required", False)], False), + ([r("evaluators.error", False), r("evaluators.success"), reduce_requested], False), + ([r("evaluators.error", False), r("parsers.success"), reduce_requested], False), + ([r("evaluators.success"), r("not an evaluator in error", False), reduce_requested], False), + ([r("evaluators.error", False), r("evaluators.another error", False), reduce_requested], False), + ]) + def test_i_can_match(self, return_values, expected): + context = self.get_context() + assert OneErrorEvaluator().matches(context, return_values) == expected + + def test_i_can_eval(self): + context = self.get_context() + + return_values = [ + r("evaluators.one error", False), + r("parsers.failed", False), + r("parsers.failed", False), + reduce_requested + ] + + evaluator = OneErrorEvaluator() + evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert not res.status + assert res.body == "evaluators.one error" + assert len(res.parents) == 4 + + def test_unwanted_return_values_are_not_eaten(self): + context = self.get_context() + + a_successful_concept = r("successful concept") + a_concept_in_error = r("concept in error", False) + return_values = [ + r("evaluators.one error", False), + r("parsers.failed", False), + r("parsers.failed", False), + a_successful_concept, + a_concept_in_error, + reduce_requested + ] + + evaluator = OneErrorEvaluator() + evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert not res.status + assert res.body == "evaluators.one error" + assert len(res.parents) == 4 + + assert a_successful_concept not in res.parents diff --git a/tests/evaluators/test_OneSuccessEvaluator.py b/tests/evaluators/test_OneSuccessEvaluator.py new file mode 100644 index 0000000..f303c0a --- /dev/null +++ b/tests/evaluators/test_OneSuccessEvaluator.py @@ -0,0 +1,72 @@ +import pytest + +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +from core.concept import Concept +from evaluators.OneSuccessEvaluator import OneSuccessEvaluator +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def r(value, status=True): + return ReturnValueConcept(value, status, value) + + +reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED)) + + +class TestOneSuccessEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("return_values, expected", [ + ([r("evaluators.one success"), reduce_requested], True), + ([r("evaluators.one success"), r("failed", False), r("failed", False), reduce_requested], True), + ([r("evaluators.no reduce required"), r("failed", False), r("failed", False)], False), + ([r("evaluators.no reduce required")], False), + ([r("evaluators.failed", False), r("failed", False), r("failed", False), reduce_requested], False), + ([r("evaluators.failed", False), r("not evaluator success"), reduce_requested], False), + ([r("evaluators.success"), r("evaluators.another success"), reduce_requested], False), + ([r("evaluators.one success"), r("other success"), r("failed", False), reduce_requested], True), + ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False), + ]) + def test_i_can_match(self, return_values, expected): + context = self.get_context() + assert OneSuccessEvaluator().matches(context, return_values) == expected + + def test_i_can_eval(self): + context = self.get_context() + + return_values = [ + r("evaluators.one success"), + r("failed", False), + r("failed", False), + reduce_requested + ] + + evaluator = OneSuccessEvaluator() + matches = evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert matches + assert res.status + assert res.body == "evaluators.one success" + assert len(res.parents) == 4 + + def test_i_do_not_eat_the_other_success(self): + context = self.get_context() + + not_a_parser_success = r("other success") + return_values = [ + r("evaluators.one success"), + not_a_parser_success, + r("failed", False), + reduce_requested + ] + + evaluator = OneSuccessEvaluator() + matches = evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert matches + assert res.status + assert res.body == "evaluators.one success" + assert len(res.parents) == 3 + + assert not_a_parser_success not in res.parents diff --git a/tests/evaluators/test_PrepareEvalEvaluator.py b/tests/evaluators/test_PrepareEvalEvaluator.py new file mode 100644 index 0000000..d0abee1 --- /dev/null +++ b/tests/evaluators/test_PrepareEvalEvaluator.py @@ -0,0 +1,49 @@ +import pytest + +from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept +from core.concept import Concept +from evaluators.PrepareEvalEvaluator import PrepareEvalEvaluator + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + +r = ReturnValueConcept + + +class TestPrepareEvalEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("ret_val, expected", [ + (r("name", True, UserInputConcept("eval 1 + 1")), True), + (r("name", True, UserInputConcept(" eval 1 + 1")), True), + (r("name", True, UserInputConcept("eval")), False), + (r("name", True, UserInputConcept("1+1")), False), + (r("name", True, UserInputConcept("")), False), + (r("name", True, UserInputConcept("eval ")), False), + (r("name", True, UserInputConcept([])), False), + (r("name", True, Concept("foo")), False), + (r("name", True, "not a concept"), False), + (r("name", False, UserInputConcept("eval 1 + 1")), False), + (r("name", False, UserInputConcept(" eval 1 + 1")), False), + ]) + def test_i_can_match(self, ret_val, expected): + context = self.get_context() + assert PrepareEvalEvaluator().matches(context, ret_val) == expected + + @pytest.mark.parametrize("ret_val, expected", [ + (r("name", True, UserInputConcept("eval 1 + 1")), "1 + 1"), + (r("name", True, UserInputConcept(" eval 1 + 1")), "1 + 1"), + ]) + def test_i_can_eval(self, ret_val, expected): + context = self.get_context() + sheerka = context.sheerka + + prepare_evaluator = PrepareEvalEvaluator() + prepare_evaluator.matches(context, ret_val) + res = prepare_evaluator.eval(context, ret_val) + + assert len(res) == 2 + assert res[0].status + assert sheerka.isinstance(res[0].body, BuiltinConcepts.USER_INPUT) + assert res[0].body.body == expected + + assert res[1].status + assert sheerka.isinstance(res[1].body, BuiltinConcepts.CONCEPT_EVAL_REQUESTED) diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py new file mode 100644 index 0000000..b1bc306 --- /dev/null +++ b/tests/evaluators/test_PythonEvaluator.py @@ -0,0 +1,138 @@ +import pytest + +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts +from core.concept import Concept +from evaluators.PythonEvaluator import PythonEvaluator +from parsers.PythonParser import PythonNode, PythonParser +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def get_context_name(context): + return context.name + + +class TestPythonEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("ret_val, expected", [ + (ReturnValueConcept("some_name", True, ParserResultConcept(value=PythonNode("", None))), True), + (ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False), + (ReturnValueConcept("some_name", False, "not relevant"), False), + (ReturnValueConcept("some_name", True, Concept()), False) + ]) + def test_i_can_match(self, ret_val, expected): + context = self.get_context() + assert PythonEvaluator().matches(context, ret_val) == expected + + @pytest.mark.parametrize("text, expected", [ + ("1 + 1", 2), + ("sheerka.test()", "I have access to Sheerka !"), + ("a=10\na", 10), + ]) + def test_i_can_eval(self, text, expected): + context = self.get_context() + parsed = PythonParser().parse(context, text) + + evaluated = PythonEvaluator().eval(context, parsed) + + assert evaluated.status + assert evaluated.value == expected + + @pytest.mark.parametrize("concept", [ + Concept("foo"), + Concept("foo", body="2"), + Concept("foo").set_prop("prop", "'a'"), + Concept("foo", body="bar") + ]) + def test_i_cannot_eval_simple_concept(self, concept): + context = self.get_context() + context.sheerka.add_in_cache(Concept("foo")) + + parsed = PythonParser().parse(context, "foo") + evaluated = PythonEvaluator().eval(context, parsed) + + assert not evaluated.status + assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.NOT_FOR_ME) + + def test_i_can_eval_expression_with_that_references_concepts(self): + """ + I can test modules with variables + :return: + """ + context = self.get_context() + context.sheerka.add_in_cache(Concept("foo", body="1")) + + parsed = PythonParser().parse(context, "foo + 2") + evaluated = PythonEvaluator().eval(context, parsed) + + assert evaluated.status + assert evaluated.value == 3 + + def test_i_can_eval_module_with_that_references_concepts(self): + """ + I can test modules with variables + :return: + """ + context = self.get_context() + context.sheerka.add_in_cache(Concept("foo")) + + parsed = PythonParser().parse(context, "def a(b):\n return b\na(c:foo:)") + evaluated = PythonEvaluator().eval(context, parsed) + + assert evaluated.status + assert evaluated.value == Concept("foo").init_key() + + def test_i_can_eval_module_with_that_references_concepts_with_body(self): + """ + I can test modules with variables + :return: + """ + context = self.get_context() + context.sheerka.add_in_cache(Concept("foo", body="2")) + + parsed = PythonParser().parse(context, "def a(b):\n return b\na(foo)") + evaluated = PythonEvaluator().eval(context, parsed) + + assert evaluated.status + assert evaluated.value == 2 + + def test_i_can_eval_concept_token(self): + context = self.get_context() + context.sheerka.add_in_cache(Concept("foo", body="2")) + + parsed = PythonParser().parse(context, "get_context_name(c:foo:)") + python_evaluator = PythonEvaluator() + python_evaluator.locals["get_context_name"] = get_context_name + evaluated = python_evaluator.eval(context, parsed) + + assert evaluated.status + assert evaluated.value == "foo" + + # sanity, does not work otherwise + parsed = PythonParser().parse(context, "get_context_name(foo)") + python_evaluator = PythonEvaluator() + python_evaluator.locals["get_context_name"] = get_context_name + evaluated = python_evaluator.eval(context, parsed) + + assert not evaluated.status + assert evaluated.body.body.args[0] == "'int' object has no attribute 'name'" + + @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [ + ("__C__key__C__", "key", None, False), + ("__C__key__id__C__", "key", "id", False), + ("__C__USE_CONCEPT__key__id__C__", "key", "id", True), + ("__C__USE_CONCEPT__key__id__C__", "key", "id", True), + ]) + def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept): + context = self.get_context() + assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept) + + @pytest.mark.parametrize("text", [ + "__C__", + "__C__key", + "__C__key____", + "__C____", + "__C__USE_CONCEPT__", + ]) + def test_i_cannot_resolve_name(self, text): + context = self.get_context() + assert PythonEvaluator().resolve_name(context, text) is None diff --git a/tests/evaluators/test_TooManySucessEvaluator.py b/tests/evaluators/test_TooManySucessEvaluator.py new file mode 100644 index 0000000..98f78d3 --- /dev/null +++ b/tests/evaluators/test_TooManySucessEvaluator.py @@ -0,0 +1,105 @@ +import pytest + +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +from core.concept import Concept +from evaluators.TooManySuccessEvaluator import TooManySuccessEvaluator + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def r(name, status=True, value=None): + value = value or name + return ReturnValueConcept(name, status, value) + + +reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED)) + + +class TestTooManySuccessEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("return_values, expected", [ + ([r("evaluators.success1"), r("evaluators.success2"), reduce_requested], True), + ([r("evaluators.success1"), r("evaluators.success2"), r("other"), reduce_requested], True), + ([r("evaluators.success1"), r("evaluators.success2"), r("other", False), reduce_requested], True), + ([r("evaluators.a", value=Concept("c1", body="1")), + r("evaluators.a", value=Concept("c2", body="1")), + r("other"), + reduce_requested], True), + ([r("evaluators.a", value=Concept("c1", body="1")), + r("evaluators.a", value=Concept("c2", body="1")), + r("parsers.other", False), + reduce_requested], True), + ([r("evaluators.a", value=Concept("c1", body="1")), + r("evaluators.a", value=Concept("c2", body="1")), + r("parsers.other"), + reduce_requested], False), + ([r("evaluators.success1"), reduce_requested], False), + ([reduce_requested], False), + ([r("evaluators.success1"), r("evaluators.success2")], False), + ]) + def test_i_can_match(self, return_values, expected): + context = self.get_context() + assert TooManySuccessEvaluator().matches(context, return_values) == expected + + def test_i_can_eval(self): + context = self.get_context() + + value1 = r("evaluators.a", value=Concept("c1", body="1")) + value2 = r("evaluators.a", value=Concept("c2", body="2")) + return_values = [ + value1, + value2, + r("other", False), + reduce_requested + ] + + evaluator = TooManySuccessEvaluator() + matches = evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert matches + assert not res.status + assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS) + assert res.body.body == [value1, value2] + assert len(res.parents) == 4 + + def test_i_can_eval_when_same_success(self): + context = self.get_context() + + return_values = [ + r("evaluators.a", value=Concept("c1", body="1").auto_init()), + r("evaluators.a", value=Concept("c2", body="1").auto_init()), + r("other", False), + reduce_requested + ] + + evaluator = TooManySuccessEvaluator() + matches = evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert matches + assert res is None + + def test_other_success_are_not_reduced(self): + context = self.get_context() + + value1 = r("evaluators.a", value=Concept("c1", body="1").auto_init()) + value2 = r("evaluators.a", value=Concept("c2", body="2").auto_init()) + other_success = r("other") + return_values = [ + value1, + value2, + other_success, + reduce_requested + ] + + evaluator = TooManySuccessEvaluator() + matches = evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert matches + assert not res.status + assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS) + assert res.body.body == [value1, value2] + assert len(res.parents) == 3 + assert other_success not in res.parents diff --git a/tests/non_reg/__init__.py b/tests/non_reg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py new file mode 100644 index 0000000..c8e3471 --- /dev/null +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -0,0 +1,625 @@ +import pytest + +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, PROPERTIES_TO_SERIALIZE, Property, simplec +from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator +from parsers.ConceptLexerParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression +from sdp.sheerkaDataProvider import SheerkaDataProvider + +from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka + + +class TestSheerkaNonReg(TestUsingFileBasedSheerka): + + @pytest.mark.parametrize("text, expected", [ + ("1 + 1", 2), + ("sheerka.test()", 'I have access to Sheerka !') + ]) + def test_i_can_eval_python_expressions_with_no_variable(self, text, expected): + sheerka = self.get_sheerka() + + res = sheerka.evaluate_user_input(text) + + assert len(res) == 1 + assert res[0].status + assert res[0].value == expected + + def test_i_can_eval_concept_with_python_body(self): + sheerka = self.get_sheerka() + concept = Concept(name="one", body="1") + sheerka.add_in_cache(concept) + + text = "one" + res = sheerka.evaluate_user_input(text) + assert len(res) == 1 + assert res[0].status + assert res[0].value == simplec("one", 1) # by default, the concept is returned + + def test_i_can_eval_concept_with_concept_body(self): + sheerka = self.get_sheerka() + concept_one = Concept(name="one") + concept_un = Concept(name="un", body="one") + sheerka.add_in_cache(concept_one) + sheerka.add_in_cache(concept_un) + + res = sheerka.evaluate_user_input("un") + return_value = res[0].value + assert len(res) == 1 + assert res[0].status + assert return_value == simplec("un", simplec("one", None)) + + def test_i_can_eval_concept_with_no_body(self): + sheerka = self.get_sheerka() + concept = Concept(name="one") + sheerka.add_in_cache(concept) + + text = "one" + res = sheerka.evaluate_user_input(text) + assert len(res) == 1 + assert res[0].status + assert res[0].value == concept + assert id(res[0].value) != id(concept) + + def test_is_unique_property_is_used_when_evaluating(self): + sheerka = self.get_sheerka() + concept = Concept(name="one", is_unique=True) + sheerka.add_in_cache(concept) + + text = "one" + res = sheerka.evaluate_user_input(text) + assert len(res) == 1 + assert res[0].status + assert res[0].value == concept + assert id(res[0].value) == id(concept) + + def test_i_can_eval_def_concept_request(self): + text = """ +def concept a + b +where isinstance(a, int) and isinstance(b, int) +pre isinstance(a, int) and isinstance(b, int) +post isinstance(res, int) +as: + def func(x,y): + return x+y + func(a,b) + """ + + expected = self.get_default_concept() + expected.metadata.id = "1001" + expected.metadata.desc = None + expected.metadata.props = [("a", None), ("b", None)] + expected.init_key() + + sheerka = self.get_sheerka() + res = sheerka.evaluate_user_input(text) + + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT) + + concept_saved = res[0].value.body + + for prop in PROPERTIES_TO_SERIALIZE: + assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop) + + assert concept_saved.key in sheerka.cache_by_key + assert concept_saved.id in sheerka.cache_by_id + assert sheerka.sdp.io.exists( + sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) + + def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept(self): + """ + In this test, we test that the properties of 'concept a xx b' (which are 'a' and 'b') + are correctly detected, thanks to the source code 'a plus b' in its body + :return: + """ + sheerka = self.get_sheerka() + + # concept 'a plus b' is known + concept_a_plus_b = Concept(name="a plus b").def_prop("a").def_prop("b") + sheerka.add_in_cache(concept_a_plus_b) + + res = sheerka.evaluate_user_input("def concept a xx b as a plus b") + expected = Concept(name="a xx b", body="a plus b").def_prop("a").def_prop("b").init_key() + expected.metadata.id = "1001" + + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT) + + concept_saved = res[0].value.body + + for prop in PROPERTIES_TO_SERIALIZE: + assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop) + + assert concept_saved.key in sheerka.cache_by_key + assert concept_saved.id in sheerka.cache_by_id + assert sheerka.sdp.io.exists( + sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) + + def test_i_cannot_eval_the_same_def_concept_twice(self): + text = """ +def concept a + b +where isinstance(a, int) and isinstance(b, int) +pre isinstance(a, int) and isinstance(b, int) +post isinstance(res, int) +as: + def func(x,y): + return x+y + func(a,b) + """ + + sheerka = self.get_sheerka() + sheerka.evaluate_user_input(text) + res = sheerka.evaluate_user_input(text) + + assert len(res) == 1 + assert not res[0].status + assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED) + + @pytest.mark.parametrize("text", [ + "", + " ", + "\n", + ]) + def test_i_can_eval_a_empty_input(self, text): + sheerka = self.get_sheerka() + + res = sheerka.evaluate_user_input(text) + + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].value, BuiltinConcepts.NOP) + + def test_i_can_eval_concept_with_variable(self): + sheerka = self.get_sheerka() + concept_hello = Concept(name="hello a").def_prop("a") + concept_foo = Concept(name="foo") + sheerka.add_in_cache(concept_hello) + sheerka.add_in_cache(concept_foo) + + res = sheerka.evaluate_user_input("hello foo") + return_value = res[0].value + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(return_value, concept_hello) + assert return_value.props["a"].value == concept_foo + + def test_i_can_eval_concept_with_variable_and_python_as_body(self): + sheerka = self.get_sheerka() + hello_a = sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a")) + sheerka.add_in_cache(Concept(name="foo", body="'foo'")) + + res = sheerka.evaluate_user_input("hello foo") + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].value, hello_a) + assert res[0].value.body == "hello foo" + assert res[0].value.metadata.is_evaluated + assert res[0].value.props["a"].value == simplec("foo", "foo") + assert res[0].value.props["a"].value.metadata.is_evaluated + + def test_i_can_eval_duplicate_concepts_with_same_value(self): + sheerka = self.get_sheerka() + + sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a")) + sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'")) + sheerka.add_in_cache(Concept(name="foo", body="'foo'")) + + res = sheerka.evaluate_user_input("hello foo") + assert len(res) == 1 + assert res[0].status + assert res[0].value.body == "hello foo" + assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME) + + def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different(self): + sheerka = self.get_sheerka() + + sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a")) + sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'")) + sheerka.add_in_cache(Concept(name="foo", body="'another value'")) + + res = sheerka.evaluate_user_input("hello foo") + assert len(res) == 1 + assert not res[0].status + assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS) + + concepts = res[0].value.body + assert len(concepts) == 2 + sorted_values = sorted(concepts, key=lambda x: x.value.body) + assert sorted_values[0].value.body == "hello another value" + assert sorted_values[1].value.body == "hello foo" + + def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(self): + sheerka = self.get_sheerka() + context = self.get_context(sheerka) + + sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").def_prop("a")) + sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").def_prop("b")) + + res = sheerka.evaluate_user_input("hello 'foo'") + assert len(res) == 1 + assert res[0].status + assert res[0].value.body == "hello foo" # I don't know yet the one to choose + assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME) + + def test_i_can_create_concepts_with_python_code_as_body(self): + sheerka = self.get_sheerka() + context = self.get_context(sheerka) + + sheerka.create_new_concept(context, Concept(name="concepts", body="sheerka.concepts()")) + res = sheerka.evaluate_user_input("concepts") + + assert len(res) == 1 + assert res[0].status + assert isinstance(res[0].value.body, list) + + def test_i_can_create_concept_with_bnf_definition(self): + sheerka = self.get_sheerka(False, False) + a = Concept("a") + sheerka.add_in_cache(a) + sheerka.concepts_definition_cache = {a: OrderedChoice("one", "two")} + + res = sheerka.evaluate_user_input("def concept plus from bnf a ('plus' plus)?") + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT) + + saved_concept = sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, "plus") + assert saved_concept.key == "plus" + assert saved_concept.metadata.definition == "a ('plus' plus)?" + assert "a" in saved_concept.props + assert "plus" in saved_concept.props + + saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY) + expected_bnf = Sequence( + a, Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus")))) + assert saved_definitions[saved_concept] == expected_bnf + + new_concept = res[0].value.body + assert new_concept.metadata.name == "plus" + assert new_concept.metadata.definition == "a ('plus' plus)?" + assert new_concept.bnf == expected_bnf + assert "a" in new_concept.props + assert "plus" in new_concept.props + + def test_i_can_eval_bnf_definitions(self): + sheerka = self.get_sheerka() + concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body + + res = sheerka.evaluate_user_input("one") + + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].value, concept_a) + + def test_i_can_eval_bnf_definitions_with_variables(self): + sheerka = self.get_sheerka() + concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body + concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body + + res = sheerka.evaluate_user_input("one three") + + assert len(res) == 1 + assert res[0].status + return_value = res[0].value + + assert sheerka.isinstance(return_value, concept_b) + assert return_value.body == "one three" + assert return_value.metadata.is_evaluated + + assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one").init_key()) + assert return_value.props["a"].value.metadata.is_evaluated + + def test_i_can_eval_bnf_definitions_from_separate_instances(self): + """ + Same test then before, + but make sure that the BNF are correctly persisted and loaded + """ + sheerka = self.get_sheerka(False) + concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' 'two'")[0].body.body + + res = self.get_sheerka(False).evaluate_user_input("one two") + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].value, concept_a) + + # add another bnf definition + concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body + + res = self.get_sheerka(False).evaluate_user_input("one two") # previous one still works + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].value, concept_a) + + res = self.get_sheerka(False).evaluate_user_input("one two three") # new one works + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].value, concept_b) + assert res[0].value.body == "one two three" + assert res[0].value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key()) + + @pytest.mark.parametrize("desc, definitions", [ + ("Simple form", [ + "def concept one as 1", + "def concept two as 2", + "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit" + ]), + ("When twenty is a concept", [ + "def concept one as 1", + "def concept two as 2", + "def concept twenty as 20", + "def concept twenties from bnf twenty (one|two)=unit as twenty + unit" + ]), + ("When digit is a concept", [ + "def concept one as 1", + "def concept two as 2", + "def concept twenty as 20", + "def concept digit from bnf one|two", + "def concept twenties from bnf twenty digit as twenty + digit" + ]), + ("When using isa and concept twenty", [ + "def concept one as 1", + "def concept two as 2", + "def concept number", + "one isa number", + "two isa number", + "def concept twenties from bnf 'twenty' number as 20 + number" + ]), + ("When using isa and concept twenty", [ + "def concept one as 1", + "def concept two as 2", + "def concept twenty as 20", + "def concept number", + "one isa number", + "two isa number", + "def concept twenties from bnf twenty number as 20 + number" + ]), + ]) + def test_i_can_mix_concept_with_python_to_define_numbers(self, desc, definitions): + sheerka = self.get_sheerka() + + for definition in definitions: + sheerka.evaluate_user_input(definition) + + res = sheerka.evaluate_user_input("twenty one") + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].body, "twenties") + assert res[0].body.body == 21 + + res = sheerka.evaluate_user_input("twenty one + 1") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 22 + + res = sheerka.evaluate_user_input("twenty one + one") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 22 + + res = sheerka.evaluate_user_input("twenty one + twenty two") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 43 + + res = sheerka.evaluate_user_input("1 + twenty one") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 22 + + res = sheerka.evaluate_user_input("1 + 1 + twenty one") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 23 + + def test_i_can_mix_bnf_and_isa(self): + """ + if 'one' isa 'number, twenty number should be recognized + :return: + """ + sheerka = self.get_sheerka() + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("def concept two as 2") + sheerka.evaluate_user_input("def concept number") + sheerka.evaluate_user_input("one isa number") + sheerka.evaluate_user_input("two isa number") + sheerka.evaluate_user_input("def concept twenties from bnf 'twenty' number as 20 + number") + + res = sheerka.evaluate_user_input("twenty one") + assert len(res) == 1 + assert res[0].status + assert res[0].body == simplec("twenties", 21) + + res = sheerka.evaluate_user_input("twenty one + 1") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 22 + + res = sheerka.evaluate_user_input("twenty one + one") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 22 + + res = sheerka.evaluate_user_input("twenty one + twenty two") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 43 + + res = sheerka.evaluate_user_input("1 + twenty one") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 22 + + res = sheerka.evaluate_user_input("1 + 1 + twenty one") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 23 + + def test_i_can_mix_concept_of_concept(self): + sheerka = self.get_sheerka() + + definitions = [ + "def concept one as 1", + "def concept two as 2", + "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit", + "def concept a plus b as a + b" + ] + + for definition in definitions: + sheerka.evaluate_user_input(definition) + + res = sheerka.evaluate_user_input("1 plus 2") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 3 + + res = sheerka.evaluate_user_input("1 plus one") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 2 + + res = sheerka.evaluate_user_input("1 + 1 plus 1") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 3 + + res = sheerka.evaluate_user_input("1 plus twenty one") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 22 + + res = sheerka.evaluate_user_input("one plus 1") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 2 + + res = sheerka.evaluate_user_input("one plus two") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 3 + + res = sheerka.evaluate_user_input("one plus twenty one") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 22 + + res = sheerka.evaluate_user_input("twenty one plus 1") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 22 + + res = sheerka.evaluate_user_input("twenty one plus one") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 22 + + res = sheerka.evaluate_user_input("twenty one plus twenty two") + assert len(res) == 1 + assert res[0].status + assert res[0].body.body == 43 + + @pytest.mark.xfail + def test_i_can_evaluate_concept_of_concept_when_multiple_choices(self): + sheerka = self.get_sheerka() + + definitions = [ + "def concept little a where a", + "def concept blue a where a", + "def concept little blue a where a", + "def concept house" + ] + + for definition in definitions: + sheerka.evaluate_user_input(definition) + + ### CAUTION #### + # this test cannot work !! + # it is just to hint the result that I would like to achieve + + res = sheerka.evaluate_user_input("little blue house") + assert len(res) == 2 + assert res[0].status + assert res[0].body == "little(blue(house))" + + assert res[1].status + assert res[1].body == "little blue(house)" + + def test_i_can_say_that_a_concept_isa_another_concept(self): + sheerka = self.get_sheerka() + sheerka.evaluate_user_input("def concept foo") + sheerka.evaluate_user_input("def concept bar") + + res = sheerka.evaluate_user_input("foo isa bar") + assert len(res) == 1 + assert res[0].status + assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) + + def test_eval_does_not_break_valid_result(self): + sheerka = self.get_sheerka() + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("def concept two as 2") + + res = sheerka.evaluate_user_input("one + two") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 3 + + res = sheerka.evaluate_user_input("eval one + two") + assert len(res) == 1 + assert res[0].status + assert res[0].body == 3 + + @pytest.mark.parametrize("text", [ + "'hello", + '"foo" + "string', + "c::", + "c:foo\nbar:", + "c:foo", + "def concept 'name", + "def concept name from bnf 'name" + ]) + def test_i_can_manage_tokenizer_error(self, text): + sheerka = self.get_sheerka() + sheerka.add_in_cache(Concept("foo")) + + res = sheerka.evaluate_user_input(text) + + assert len(res) > 1 + for r in [r for r in res if r.who.startswith("parsers.")]: + assert not r.status + + def test_i_can_recognize_concept_from_string(self): + sheerka = self.get_sheerka() + sheerka.add_in_cache(Concept("one", body="1")) + + res = sheerka.evaluate_user_input("'one'") + + assert len(res) == 1 + assert res[0].status + assert res[0].body == "one" + + res = sheerka.evaluate_user_input("eval 'one'") + + assert len(res) == 1 + assert res[0].status + assert res[0].body == "one" + + @pytest.mark.parametrize("expression", [ + "def concept twenties from bnf 'twenty' (one | two)=unit as 20 + unit", + "def concept twenties from bnf 'twenty' (one | two)=unit as twenty + unit", + "def concept twenties from bnf twenty (one | two)=unit as 20 + unit", + "def concept twenties from bnf twenty (one | two)=unit as twenty + unit", + ]) + def test_i_can_evaluate_bnf_concepts(self, expression): + sheerka = self.get_sheerka() + + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("def concept two as 2") + sheerka.evaluate_user_input("def concept twenty as 20") + sheerka.evaluate_user_input(expression) + res = sheerka.evaluate_user_input("eval twenty one") + + assert len(res) == 1 + assert res[0].status + assert res[0].body == 21 diff --git a/tests/parsers/__init__.py b/tests/parsers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_BaseParser.py b/tests/parsers/test_BaseParser.py similarity index 94% rename from tests/test_BaseParser.py rename to tests/parsers/test_BaseParser.py index 3b78576..31cf9bf 100644 --- a/tests/test_BaseParser.py +++ b/tests/parsers/test_BaseParser.py @@ -1,6 +1,6 @@ import pytest -from core.tokenizer import Tokenizer, Token, TokenKind +from core.tokenizer import Tokenizer, TokenKind from parsers.BaseParser import BaseParser diff --git a/tests/parsers/test_BnfParser.py b/tests/parsers/test_BnfParser.py new file mode 100644 index 0000000..90163da --- /dev/null +++ b/tests/parsers/test_BnfParser.py @@ -0,0 +1,176 @@ +import pytest + +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from core.tokenizer import Tokenizer, TokenKind, LexerError +from parsers.BaseParser import UnexpectedTokenErrorNode +from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError +from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \ + ConceptLexerParser, ConceptExpression, cnode + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class ClassWithName: + def __init__(self, name): + self.name = name + + +class TestBnfParser(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("expression, expected", [ + ("'str'", StrMatch("str")), + ("1", StrMatch("1")), + (" 1", StrMatch("1")), + (",", StrMatch(",")), + ("'foo'?", Optional(StrMatch("foo"))), + ("'foo'*", ZeroOrMore(StrMatch("foo"))), + ("'foo'+", OneOrMore(StrMatch("foo"))), + ("1 | 2 | 3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))), + ("1|2|3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))), + ("1'|' 2 '|' 3", Sequence(StrMatch("1"), StrMatch("|"), StrMatch("2"), StrMatch("|"), StrMatch("3"))), + ("1 2 'foo'", Sequence(StrMatch("1"), StrMatch("2"), StrMatch("foo"))), + ("1 2 | 3 4+", OrderedChoice( + Sequence(StrMatch("1"), StrMatch("2")), + Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))), + ( + "1 (2 | 3) 4+", + Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))), + ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))), + ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))), + ("1 *", Sequence(StrMatch("1"), StrMatch("*"))), + ("1 ?", Sequence(StrMatch("1"), StrMatch("?"))), + ("1 +", Sequence(StrMatch("1"), StrMatch("+"))), + ("(1|*) +", Sequence(OrderedChoice(StrMatch("1"), StrMatch("*")), StrMatch("+"))), + ("1, :&", Sequence(StrMatch("1"), StrMatch(","), StrMatch(":"), StrMatch("&"))), + ("(1 )", StrMatch("1")), + ("'str'=var", StrMatch("str", rule_name="var")), + ("'foo'?=var", Optional(StrMatch("foo"), rule_name="var")), + ("('foo'?)=var", Optional(StrMatch("foo"), rule_name="var")), + ("'foo'*=var", ZeroOrMore(StrMatch("foo"), rule_name="var")), + ("('foo'*)=var", ZeroOrMore(StrMatch("foo"), rule_name="var")), + ("'foo'+=var", OneOrMore(StrMatch("foo"), rule_name="var")), + ("('foo'+)=var", OneOrMore(StrMatch("foo"), rule_name="var")), + ("'foo'=var?", Optional(StrMatch("foo", rule_name="var"))), + ("('foo'=var)?", Optional(StrMatch("foo", rule_name="var"))), + ("'foo'=var*", ZeroOrMore(StrMatch("foo", rule_name="var"))), + ("('foo'=var)*", ZeroOrMore(StrMatch("foo", rule_name="var"))), + ("'foo'=var+", OneOrMore(StrMatch("foo", rule_name="var"))), + ("('foo'=var)+", OneOrMore(StrMatch("foo", rule_name="var"))), + ("(1 | 2 | 3)=var", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"), rule_name="var")), + ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")), + ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")), + ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))), + ]) + def test_i_can_parse_regex(self, expression, expected): + parser = BnfParser() + res = parser.parse(self.get_context(), Tokenizer(expression)) + + assert not parser.has_error + assert res.status + assert res.value.value == expected + assert res.value.source == expression + + @pytest.mark.parametrize("expression, expected", [ + ("foo", Concept("foo").init_key()), + ("foo*", ZeroOrMore(Concept("foo").init_key())), + ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))), + ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))), + ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())), + ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())), + ]) + def test_i_can_parse_regex_with_concept(self, expression, expected): + foo = Concept("foo") + bar = Concept("bar") + var = Concept("var") + context = self.get_context() + + for c in (foo, bar, var): + context.sheerka.add_in_cache(c) + parser = BnfParser() + res = parser.parse(context, Tokenizer(expression)) + + assert not parser.has_error + assert res.status + assert res.value.value == expected + assert res.value.source == expression + + def test_i_can_parse_regex_with_concept_when_the_concept_is_still_under_definition(self): + expression = "foo" + expected = ConceptExpression("foo") + + context = self.get_context() + context.obj = ClassWithName("foo") + + parser = BnfParser() + res = parser.parse(context, Tokenizer(expression)) + + assert not parser.has_error + assert res.status + assert res.value.value == expected + assert res.value.source == expression + + @pytest.mark.parametrize("expression, error", [ + ("1 ", UnexpectedEndOfFileError()), + ("1|", UnexpectedEndOfFileError()), + ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token()'", [TokenKind.RPAR])), + ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token()'", [TokenKind.IDENTIFIER])), + ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6)) + ]) + def test_i_can_detect_errors(self, expression, error): + parser = BnfParser() + res = parser.parse(self.get_context(), Tokenizer(expression)) + ret_value = res.value.value + assert parser.has_error + assert not res.status + assert ret_value[0] == error + + def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + context = self.get_context() + context.sheerka.add_in_cache(foo) + context.sheerka.add_in_cache(bar) + + regex_parser = BnfParser() + foo_definition = regex_parser.parse(context, "'twenty' | 'thirty'").value.value + bar_definition = regex_parser.parse(context, "foo ('one' | 'two')").value.value + + concepts = {bar: bar_definition, foo: foo_definition} + concept_parser = ConceptLexerParser() + concept_parser.initialize(context, concepts) + + res = concept_parser.parse(context, "twenty two") + assert res.status + assert res.value.body == [cnode("bar", 0, 2, "twenty two")] + + res = concept_parser.parse(context, "thirty one") + assert res.status + assert res.value.body == [cnode("bar", 0, 2, "thirty one")] + + res = concept_parser.parse(context, "twenty") + assert res.status + assert res.value.body == [cnode("foo", 0, 0, "twenty")] + + def test_i_cannot_parse_when_too_many_concepts(self): + foo1 = Concept(name="foo", body="1") + foo2 = Concept(name="foo", body="2") + context = self.get_context() + context.sheerka.cache_by_key["foo"] = [foo1, foo2] + + regex_parser = BnfParser() + res = regex_parser.parse(context, "foo") + + assert not res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT) + assert res.value.body == ('key', 'foo') + + def test_i_cannot_parse_when_unknown_concept(self): + context = self.get_context() + + regex_parser = BnfParser() + res = regex_parser.parse(self.get_context(), "foo") + + assert not res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) + assert res.value.body == ('key', 'foo') diff --git a/tests/parsers/test_ConceptLexerParser.py b/tests/parsers/test_ConceptLexerParser.py new file mode 100644 index 0000000..9723d12 --- /dev/null +++ b/tests/parsers/test_ConceptLexerParser.py @@ -0,0 +1,1133 @@ +import pytest +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, ConceptParts, DoNotResolve +from core.tokenizer import Tokenizer, TokenKind, Token +from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \ + ParsingExpressionVisitor, TerminalNode, NonTerminalNode, ZeroOrMore, OneOrMore, \ + UnrecognizedTokensNode, cnode, short_cnode + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class ConceptVisitor(ParsingExpressionVisitor): + def __init__(self): + self.concepts = set() + + def visit_ConceptExpression(self, node): + self.concepts.add(node.concept) + + +def u(parsing_expression, start, end, children=None): + """ + u stands for underlying + :param parsing_expression: + :param start: + :param end: + :param children: + :return: + """ + if isinstance(parsing_expression, str): + parsing_expression = StrMatch(parsing_expression) + + if isinstance(parsing_expression, StrMatch): + return TerminalNode(parsing_expression, start, end, parsing_expression.to_match) + + return NonTerminalNode(parsing_expression, start, end, [], children) + + +def evaluated(concept): + c = Concept(name=concept.name, body=concept.name) + + +def t(text): + if text.startswith("'") or text.startswith('"'): + return Token(TokenKind.STRING, text, 0, 0, 0) + + if text.startswith(" "): + return Token(TokenKind.WHITESPACE, text, 0, 0, 0) + + return Token(TokenKind.IDENTIFIER, text, 0, 0, 0) + + +def get_expected(concept, text=None): + c = Concept(name=concept.name) + c.compiled[ConceptParts.BODY] = DoNotResolve(text or concept.name) + c.init_key() + c.metadata.id = concept.id + return c + + +def cbody(concept): + """cbody stands for compiled body""" + if not ConceptParts.BODY in concept.compiled: + return None + return concept.compiled[ConceptParts.BODY] + + +def cprop(concept, prop_name): + """cbody stands for compiled property""" + return concept.compiled[prop_name] + + +class TestConceptLexerParser(TestUsingMemoryBasedSheerka): + + def init(self, concepts, grammar): + context = self.get_context() + for c in concepts: + context.sheerka.add_in_cache(c) + + parser = ConceptLexerParser() + parser.initialize(context, grammar) + + return context, parser + + def execute(self, concepts, grammar, text): + context, parser = self.init(concepts, grammar) + + res = parser.parse(context, text) + wrapper = res.value + return_value = res.value.value + + return context, res, wrapper, return_value + + @pytest.mark.parametrize("match, text", [ + ("foo", "foo"), + ("'foo'", "'foo'"), + ("1", "1"), + ("3.14", "3.14"), + ("+", "+"), + (StrMatch("foo"), "foo"), + (StrMatch("'foo'"), "'foo'"), + (StrMatch("1"), "1"), + (StrMatch("3.14"), "3.14"), + (StrMatch("+"), "+"), + ]) + def test_i_can_match_simple_tokens(self, match, text): + foo = Concept(name="foo") + grammar = {foo: match} + + context, res, wrapper, return_value = self.execute([foo], grammar, text) + + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [ConceptNode(get_expected(foo, text), 0, 0, source=text, underlying=u(match, 0, 0))] + + def test_i_can_match_multiple_concepts_in_one_input(self): + one = Concept(name="one") + two = Concept(name="two") + grammar = {one: "one", two: "two"} + + context, res, wrapper, return_value = self.execute([one, two], grammar, "one two one") + + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [ + ConceptNode(get_expected(one), 0, 0, source="one", underlying=u("one", 0, 0)), + ConceptNode(get_expected(two), 2, 2, source="two", underlying=u("two", 2, 2)), + ConceptNode(get_expected(one), 4, 4, source="one", underlying=u("one", 4, 4)), + ] + + def test_i_can_match_sequence(self): + foo = Concept(name="foo") + grammar = {foo: Sequence("one", "two", "three")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one two three") + + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [ + ConceptNode( + get_expected(foo, "one two three"), + 0, + 4, + source="one two three", + underlying=u(grammar[foo], 0, 4, [ + u("one", 0, 0), + u("two", 2, 2), + u("three", 4, 4)]))] + + def test_i_always_choose_the_longest_match(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = {bar: Sequence("one", "two"), foo: Sequence("one", "two", "three")} + + context, res, wrapper, return_value = self.execute([foo, bar], grammar, "one two three") + + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [cnode("foo", 0, 4, "one two three")] + + def test_i_can_match_several_sequences(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = {bar: Sequence("one", "two"), foo: Sequence("one", "two", "three")} + + context, res, wrapper, return_value = self.execute([foo, bar], grammar, "one two three one two") + + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [ + cnode("foo", 0, 4, "one two three"), + cnode("bar", 6, 8, "one two"), + ] + + def test_i_can_match_ordered_choice(self): + foo = Concept(name="foo") + grammar = {foo: OrderedChoice("one", "two")} + context, parser = self.init([foo], grammar) + + res1 = parser.parse(context, "one") + assert res1.status + assert context.sheerka.isinstance(res1.value, BuiltinConcepts.PARSER_RESULT) + assert res1.value.body == [cnode("foo", 0, 0, "one")] + assert res1.value.body[0].underlying == u(grammar[foo], 0, 0, [u("one", 0, 0)]) + + res2 = parser.parse(context, "two") + assert res2.status + assert context.sheerka.isinstance(res2.value, BuiltinConcepts.PARSER_RESULT) + assert res2.value.body == [cnode("foo", 0, 0, "two")] + assert res2.value.body[0].underlying == u(grammar[foo], 0, 0, [u("two", 0, 0)]) + + res3 = parser.parse(context, "three") + assert not res3.status + assert context.sheerka.isinstance(res3.value, BuiltinConcepts.PARSER_RESULT) + assert res3.value.value == [ + UnrecognizedTokensNode(0, 0, [t("three")]) + ] + + def test_i_cannot_match_ordered_choice_with_empty_alternative(self): + foo = Concept(name="foo") + grammar = {foo: Sequence(OrderedChoice("one", ""), "two")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "ok") + + assert not res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [ + UnrecognizedTokensNode(0, 0, [t("ok")]) + ] + + def test_i_can_mix_sequences_and_ordered_choices(self): + foo = Concept(name="foo") + grammar = {foo: Sequence(OrderedChoice("twenty", "thirty"), "one", "ok")} + + context, parser = self.init([foo], grammar) + + res1 = parser.parse(context, "twenty one ok") + assert res1.status + assert context.sheerka.isinstance(res1.value, BuiltinConcepts.PARSER_RESULT) + assert res1.value.body == [ConceptNode(get_expected(foo, "twenty one ok"), 0, 4, source="twenty one ok", + underlying=u(grammar[foo], 0, 4, [ + u(OrderedChoice("twenty", "thirty"), 0, 0, [u("twenty", 0, 0)]), + u("one", 2, 2), + u("ok", 4, 4)]))] + + res2 = parser.parse(context, "thirty one ok") + assert res2.status + assert context.sheerka.isinstance(res2.value, BuiltinConcepts.PARSER_RESULT) + assert res2.value.body == [ConceptNode(get_expected(foo, "thirty one ok"), 0, 4, source="thirty one ok", + underlying=u(grammar[foo], 0, 4, [ + u(OrderedChoice("twenty", "thirty"), 0, 0, [u("thirty", 0, 0)]), + u("one", 2, 2), + u("ok", 4, 4)]))] + + res3 = parser.parse(context, "twenty one") + assert not res3.status + assert context.sheerka.isinstance(res2.value, BuiltinConcepts.PARSER_RESULT) + assert res3.value.value == [ + UnrecognizedTokensNode(0, 2, [t("twenty"), t(" "), t("one")]) + ] + + def test_i_can_mix_ordered_choices_and_sequences(self): + foo = Concept(name="foo") + grammar = {foo: OrderedChoice(Sequence("twenty", "thirty"), "one")} + + context, parser = self.init([foo], grammar) + + res = parser.parse(context, "twenty thirty") + assert res.status + assert res.value.value == [cnode("foo", 0, 2, "twenty thirty")] + + res = parser.parse(context, "one") + assert res.status + assert res.value.value == [cnode("foo", 0, 0, "one")] + + def test_i_cannot_parse_empty_optional(self): + foo = Concept(name="foo") + grammar = {foo: Optional("one")} + context, parser = self.init([foo], grammar) + + res = parser.parse(context, "") + return_value = res.value + + assert not res.status + assert context.sheerka.isinstance(return_value, BuiltinConcepts.IS_EMPTY) + + def test_i_can_parse_optional(self): + foo = Concept(name="foo") + grammar = {foo: Optional("one")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one") + + assert res.status + assert return_value == [ConceptNode(get_expected(foo, "one"), 0, 0, source="one", + underlying=u(grammar[foo], 0, 0, [u("one", 0, 0)]))] + + def test_i_can_parse_sequence_starting_with_optional(self): + foo = Concept(name="foo") + grammar = {foo: Sequence(Optional("twenty"), "one")} + context, parser = self.init([foo], grammar) + + res = parser.parse(context, "twenty one") + assert res.status + assert res.value.body == [ConceptNode( + get_expected(foo, "twenty one"), 0, 2, + source="twenty one", + underlying=u(grammar[foo], 0, 2, + [ + u(Optional("twenty"), 0, 0, [u("twenty", 0, 0)]), + u("one", 2, 2)] + ))] + + res = parser.parse(context, "one") + assert res.status + assert res.value.body == [ConceptNode(get_expected(foo, "one"), 0, 0, source="one", + underlying=u(grammar[foo], 0, 0, [u("one", 0, 0)]))] + + def test_i_can_parse_sequence_ending_with_optional(self): + foo = Concept(name="foo") + grammar = {foo: Sequence("one", "two", Optional("three"))} + + context, parser = self.init([foo], grammar) + + res = parser.parse(context, "one two three") + assert res.status + assert res.value.body == [cnode("foo", 0, 4, "one two three")] + + res = parser.parse(context, "one two") + assert res.status + assert res.value.body == [cnode("foo", 0, 2, "one two")] + + def test_i_can_parse_sequence_with_optional_in_between(self): + foo = Concept(name="foo") + + grammar = {foo: Sequence("one", Optional("two"), "three")} + + context, parser = self.init([foo], grammar) + + res = parser.parse(context, "one two three") + assert res.status + assert res.value.body == [cnode("foo", 0, 4, "one two three")] + + res = parser.parse(context, "one three") + assert res.status + assert res.value.body == [cnode("foo", 0, 2, "one three")] + + def test_i_cannot_parse_wrong_input_with_optional(self): + foo = Concept(name="foo") + grammar = {foo: Optional("one")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "two") + + assert not res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [ + UnrecognizedTokensNode(0, 0, [t("two")]) + ] + + def test_i_can_use_reference(self): + # when there are multiple matches for the same input + # Do I need to create a choice concept ? + # No, create a return value for every possible graph + + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = {foo: Sequence("one", "two"), bar: foo} + context, parser = self.init([foo, bar], grammar) + res = parser.parse(context, "one two") + + assert len(res) == 2 + + assert res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [cnode("foo", 0, 2, "one two")] + concept_found_1 = res[0].value.body[0].concept + assert cbody(concept_found_1) == DoNotResolve("one two") + + assert res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [cnode("bar", 0, 2, "one two")] + concept_found_2 = res[1].value.body[0].concept + # the body and the prop['foo'] are the same concept 'foo' + assert cbody(concept_found_2) == get_expected(foo, "one two") + assert id(cprop(concept_found_2, "foo")) == id(cbody(concept_found_2)) + + def test_i_can_use_a_reference_with_a_body(self): + """ + Same test than before (test_i_can_use_reference()) + but this time, the concept 'foo' already has a body. + :return: + """ + + foo = Concept(name="foo", body="'foo'") + bar = Concept(name="bar") + grammar = {foo: Sequence("one", "two"), bar: foo} + context, parser = self.init([foo, bar], grammar) + res = parser.parse(context, "one two") + + assert len(res) == 2 + + assert res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [cnode("foo", 0, 2, "one two")] + concept_found_1 = res[0].value.body[0].concept + assert concept_found_1.metadata.body == "'foo'" + assert cbody(concept_found_1) is None + + assert res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [cnode("bar", 0, 2, "one two")] + concept_found_2 = res[1].value.body[0].concept + assert cbody(concept_found_2) == foo + # the body and the prop['foo'] are the same concept 'foo' + assert id(cprop(concept_found_2, "foo")) == id(cbody(concept_found_2)) + + def test_i_can_use_context_reference_with_multiple_levels(self): + """ + Same than previous one, but with reference of reference + :return: + """ + + foo = Concept(name="foo") + bar = Concept(name="bar") + baz = Concept(name="baz") + grammar = {foo: Sequence("one", "two"), bar: foo, baz: bar} + context, parser = self.init([foo, bar, baz], grammar) + + res = parser.parse(context, "one two") + assert len(res) == 3 + + assert res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [cnode("foo", 0, 2, "one two")] + concept_found_1 = res[0].value.body[0].concept + assert cbody(concept_found_1) == DoNotResolve("one two") + + assert res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [cnode("bar", 0, 2, "one two")] + concept_found_2 = res[1].value.body[0].concept + assert cbody(concept_found_2) == get_expected(foo, "one two") + assert id(cprop(concept_found_2, "foo")) == id(cbody(concept_found_2)) + + assert res[2].status + assert context.sheerka.isinstance(res[2].value, BuiltinConcepts.PARSER_RESULT) + assert res[2].value.body == [cnode("baz", 0, 2, "one two")] + concept_found_3 = res[2].value.body[0].concept + expected_foo = get_expected(foo, "one two") + assert cbody(concept_found_3) == get_expected(bar, expected_foo) + assert cprop(concept_found_3, "foo") == expected_foo + assert id(cprop(concept_found_3, "bar")) == id(cbody(concept_found_3)) + + def test_order_is_not_important_when_using_references(self): + """ + Same test than test_i_can_use_reference(), + but this time, 'bar' is declared before 'foo' + So the order of the result is different + :return: + """ + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = {bar: foo, foo: Sequence("one", "two")} + context, parser = self.init([foo, bar], grammar) + + res = parser.parse(context, "one two") + assert len(res) == 2 + assert res[0].value.body == [cnode("bar", 0, 2, "one two")] + assert res[1].value.body == [cnode("foo", 0, 2, "one two")] + + def test_i_can_parse_when_reference(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = {bar: Sequence(foo, OrderedChoice("one", "two")), foo: OrderedChoice("twenty", "thirty")} + context, parser = self.init([foo, bar], grammar) + + res = parser.parse(context, "twenty two") + assert res.status + assert res.value.body == [cnode("bar", 0, 2, "twenty two")] + concept_found = res.value.body[0].concept + assert cbody(concept_found) == DoNotResolve("twenty two") + assert cprop(concept_found, "foo") == get_expected(foo, "twenty") + + res = parser.parse(context, "thirty one") + assert res.status + assert res.value.body == [cnode("bar", 0, 2, "thirty one")] + concept_found = res.value.body[0].concept + assert cbody(concept_found) == DoNotResolve("thirty one") + assert cprop(concept_found, "foo") == get_expected(foo, "thirty") + + res = parser.parse(context, "twenty") + assert res.status + assert res.value.body == [cnode("foo", 0, 0, "twenty")] + concept_found = res.value.body[0].concept + assert cbody(concept_found) == DoNotResolve("twenty") + + def test_i_can_parse_when_reference_has_a_body(self): + foo = Concept(name="foo", body="'one'") + bar = Concept(name="bar") + grammar = {bar: Sequence(foo, OrderedChoice("one", "two")), foo: OrderedChoice("twenty", "thirty")} + context, parser = self.init([foo, bar], grammar) + + res = parser.parse(context, "twenty two") + assert res.status + assert res.value.body == [cnode("bar", 0, 2, "twenty two")] + concept_found = res.value.body[0].concept + assert cbody(concept_found) == DoNotResolve("twenty two") + assert cprop(concept_found, "foo") == foo + + res = parser.parse(context, "twenty") + assert res.status + assert res.value.body == [cnode("foo", 0, 0, "twenty")] + concept_found = res.value.body[0].concept + assert concept_found.metadata.body == "'one'" + + def test_i_can_parse_multiple_results(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = { + bar: Sequence("one", "two"), + foo: Sequence("one", OrderedChoice("two", "three")) + } + context, parser = self.init([foo, bar], grammar) + + res = parser.parse(context, "one two") + assert len(res) == 2 + assert res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [cnode("bar", 0, 2, "one two")] + concept_found_0 = res[0].value.body[0].concept + assert cbody(concept_found_0) == DoNotResolve("one two") + assert len(concept_found_0.props) == 0 + + assert res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [cnode("foo", 0, 2, "one two")] + concept_found_1 = res[1].value.body[0].concept + assert cbody(concept_found_1) == DoNotResolve("one two") + assert len(concept_found_1.props) == 0 + + def test_i_can_parse_multiple_results_times_two(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = { + bar: Sequence("one", "two"), + foo: Sequence("one", OrderedChoice("two", "three")) + } + context, parser = self.init([foo, bar], grammar) + + res = parser.parse(context, "one two one two") + assert len(res) == 4 + assert res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [short_cnode("bar", "one two"), short_cnode("bar", "one two")] + + assert res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [short_cnode("foo", "one two"), short_cnode("bar", "one two")] + + assert res[2].status + assert context.sheerka.isinstance(res[2].value, BuiltinConcepts.PARSER_RESULT) + assert res[2].value.body == [short_cnode("bar", "one two"), short_cnode("foo", "one two")] + + assert res[3].status + assert context.sheerka.isinstance(res[3].value, BuiltinConcepts.PARSER_RESULT) + assert res[3].value.body == [short_cnode("foo", "one two"), short_cnode("foo", "one two")] + + def test_i_can_parse_multiple_results_when_reference(self): + """ + TODO : There should no be two answer, has the one with bar is totally useless + Note that bar = Sequence(foo, OrderedChoice("one", "two")) does not match + + :return: + """ + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = { + bar: Sequence(foo, Optional(OrderedChoice("one", "two"))), + foo: OrderedChoice("twenty", "thirty") + } + context, parser = self.init([foo, bar], grammar) + + res = parser.parse(context, "twenty") + assert len(res) == 2 + assert res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [cnode("bar", 0, 0, "twenty")] + + assert res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [cnode("foo", 0, 0, "twenty")] + + def test_i_can_parse_concept_reference_that_is_not_in_grammar(self): + one = Concept(name="one") + two = Concept(name="two") + foo = Concept(name="foo") + grammar = {foo: Sequence("twenty", OrderedChoice(one, two))} + context, parser = self.init([one, two, foo], grammar) + + res = parser.parse(context, "twenty two") + assert res.status + assert res.value.body == [cnode("foo", 0, 2, "twenty two")] + concept_found = res.value.body[0].concept + assert cbody(concept_found) == DoNotResolve("twenty two") + assert cprop(concept_found, "two") == get_expected(two, "two") + + res = parser.parse(context, "twenty one") + assert res.status + assert res.value.body == [cnode("foo", 0, 2, "twenty one")] + + def test_i_can_parse_concept_reference_that_is_group(self): + """ + if one is number, then number is a 'group' + a group can be found under the sdp entry 'all_' + """ + + context = self.get_context() + one = Concept(name="one") + two = Concept(name="two") + number = Concept(name="number") + foo = Concept(name="foo") + for c in [one, two, number, foo]: + context.sheerka.set_id_if_needed(c, False) + context.sheerka.add_in_cache(c) + + context.sheerka.add_concept_to_set(context, one, number) + context.sheerka.add_concept_to_set(context, two, number) + + grammar = {foo: Sequence("twenty", number)} + + parser = ConceptLexerParser() + parser.initialize(context, grammar) + + res = parser.parse(context, "twenty two") + assert res.status + assert res.value.body == [cnode("foo", 0, 2, "twenty two")] + concept_found = res.value.body[0].concept + assert cbody(concept_found) == DoNotResolve("twenty two") + assert cprop(concept_found, "two") == get_expected(two, "two") + assert cprop(concept_found, "number") == get_expected(number, get_expected(two, "two")) + + res = parser.parse(context, "twenty one") + assert res.status + assert res.value.body == [cnode("foo", 0, 2, "twenty one")] + concept_found = res.value.body[0].concept + assert cbody(concept_found) == DoNotResolve("twenty one") + assert cprop(concept_found, "one") == get_expected(one, "one") + assert cprop(concept_found, "number") == get_expected(number, get_expected(one, "one")) + + def test_i_can_parse_zero_or_more(self): + foo = Concept(name="foo") + grammar = {foo: ZeroOrMore("one")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one one") + + assert res.status + assert return_value == [cnode("foo", 0, 2, "one one")] + assert return_value[0].underlying == u(grammar[foo], 0, 2, [u("one", 0, 0), u("one", 2, 2)]) + + concept_found = return_value[0].concept + assert cbody(concept_found) == DoNotResolve("one one") + + def test_i_can_parse_sequence_and_zero_or_more(self): + foo = Concept(name="foo") + grammar = {foo: Sequence(ZeroOrMore("one"), "two")} + context, parser = self.init([foo], grammar) + + res = parser.parse(context, "one one two") + assert res.status + assert res.value.value == [cnode("foo", 0, 4, "one one two")] + + res = parser.parse(context, "two") + assert res.status + assert res.value.value == [cnode("foo", 0, 0, "two")] + + def test_i_cannot_parse_zero_and_more_when_wrong_entry(self): + foo = Concept(name="foo") + grammar = {foo: ZeroOrMore("one")} + context, parser = self.init([foo], grammar) + + parser = ConceptLexerParser() + parser.initialize(context, grammar) + + res = parser.parse(context, "one two") + assert not res.status + assert res.value.value == [ + cnode("foo", 0, 0, "one"), + UnrecognizedTokensNode(1, 2, [t(" "), t("two")]) + ] + + res = parser.parse(context, "two") + assert not res.status + assert res.value.value == [ + UnrecognizedTokensNode(0, 0, [t("two")]) + ] + + def test_i_can_parse_zero_and_more_with_separator(self): + foo = Concept(name="foo") + grammar = {foo: ZeroOrMore("one", sep=",")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one, one , one") + + assert res.status + assert return_value == [cnode("foo", 0, 7, "one, one , one")] + + def test_that_zero_and_more_is_greedy(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = {foo: ZeroOrMore("one"), bar: "one"} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one one one") + + assert res.status + assert return_value == [cnode("foo", 0, 4, "one one one")] + + def test_i_can_parse_one_and_more(self): + foo = Concept(name="foo") + grammar = {foo: OneOrMore("one")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one one") + + assert res.status + assert return_value == [cnode("foo", 0, 2, "one one")] + assert return_value[0].underlying == u(grammar[foo], 0, 2, [ + u("one", 0, 0), + u("one", 2, 2)]) + + def test_i_can_parse_sequence_and_one_or_more(self): + foo = Concept(name="foo") + grammar = {foo: Sequence(OneOrMore("one"), "two")} + context, parser = self.init([foo], grammar) + + res = parser.parse(context, "one one two") + assert res.status + assert res.value.value == [cnode("foo", 0, 4, "one one two")] + + res = parser.parse(context, "two") + assert not res.status + assert res.value.value == [ + UnrecognizedTokensNode(0, 0, [t("two")]) + ] + + def test_i_can_parse_one_and_more_with_separator(self): + foo = Concept(name="foo") + grammar = {foo: OneOrMore("one", sep=",")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one, one , one") + + assert res.status + assert return_value == [cnode("foo", 0, 7, "one, one , one")] + assert return_value[0].underlying == u(grammar[foo], 0, 7, [ + u("one", 0, 0), + u("one", 3, 3), + u("one", 7, 7)]) + + def test_that_one_and_more_is_greedy(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = {foo: OneOrMore("one"), bar: "one"} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one one one") + + assert res.status + assert return_value == [cnode("foo", 0, 4, "one one one")] + + def test_i_can_detect_infinite_recursion(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + + grammar = { + bar: foo, + foo: bar + } + parser = ConceptLexerParser() + parser.initialize(self.get_context(), grammar) + + assert bar not in parser.concepts_grammars + assert foo not in parser.concepts_grammars + + def test_i_can_detect_indirect_infinite_recursion_with_ordered_choice(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = { + bar: foo, + foo: OrderedChoice(bar, "foo") + } + + parser = ConceptLexerParser() + parser.initialize(self.get_context(), grammar) + + assert foo not in parser.concepts_grammars # removed because of the infinite recursion + assert bar not in parser.concepts_grammars # removed because of the infinite recursion + + # the other way around is possible + grammar = { + bar: foo, + foo: OrderedChoice("foo", bar) + } + context, parser = self.init([foo, bar], grammar) + + assert foo in parser.concepts_grammars + assert bar in parser.concepts_grammars + + res = parser.parse(context, "foo") + assert len(res) == 2 + assert res[0].status + assert res[0].value.body == [cnode("bar", 0, 0, "foo")] + assert res[1].status + assert res[1].value.body == [cnode("foo", 0, 0, "foo")] + + def test_i_can_detect_indirect_infinite_recursion_with_sequence(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + + grammar = { + bar: foo, + foo: Sequence("one", bar, "two") + } + parser = ConceptLexerParser() + parser.initialize(self.get_context(), grammar) + + assert foo not in parser.concepts_grammars # removed because of the infinite recursion + assert bar not in parser.concepts_grammars # removed because of the infinite recursion + + def test_i_can_detect_indirect_infinite_recursion_with_sequence_or_ordered_choice(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + + grammar = { + bar: foo, + foo: Sequence("one", OrderedChoice(bar, "other"), "two") + } + parser = ConceptLexerParser() + parser.initialize(self.get_context(), grammar) + + assert foo not in parser.concepts_grammars # removed because of the infinite recursion + assert bar not in parser.concepts_grammars # removed because of the infinite recursion + + def test_infinite_recursion_does_not_fail_if_a_concept_is_missing(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + + grammar = { + foo: bar + } + parser = ConceptLexerParser() + parser.initialize(self.get_context(), grammar) + + assert foo in parser.concepts_grammars + + def test_i_can_detect_indirect_infinite_recursion_with_optional(self): + # TODO infinite recursion with optional + pass + + def test_i_can_detect_indirect_infinite_recursion_with_zero_and_more(self): + # TODO infinite recursion with optional + pass + + def test_i_can_detect_indirect_infinite_recursion_with_one_and_more(self): + # TODO infinite recursion with optional + pass + + def test_i_can_visit_parsing_expression(self): + mult = Concept(name="mult") + add = Concept(name="add") + + visitor = ConceptVisitor() + visitor.visit(Sequence(mult, Optional(Sequence("+", add)))) + + assert sorted(list(visitor.concepts)) == ["add", "mult"] + + def test_i_can_initialize_rule_names(self): + context = self.get_context() + foo = Concept(name="foo") + bar = Concept(name="bar") + + grammar = {foo: Sequence("one", "two"), bar: foo} + parser = ConceptLexerParser() + ret = parser.initialize(context, grammar) + return_value = ret.body + + assert return_value[foo].rule_name == "" + assert return_value[bar].rule_name == "foo" + + @pytest.mark.parametrize("text, end_position", [ + ("foo", 0), + ("foo bar", 2), + ("foo bar ", 3), + (" foo bar ", 4) + ]) + def test_cannot_parser_unknown_concepts(self, text, end_position): + context, res, wrapper, return_value = self.execute([], {}, text) + tokens = list(Tokenizer(text))[:-1] + + assert not res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [UnrecognizedTokensNode(0, end_position, tokens)] + + def test_i_cannot_parse_when_part_of_the_input_is_unrecognized(self): + one = Concept(name="one") + two = Concept(name="two") + grammar = {one: "one", two: "two"} + + context, res, wrapper, return_value = self.execute([one, two], grammar, "one two three") + + assert not res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [ + ConceptNode(get_expected(one, "one"), 0, 0, source="one", underlying=u("one", 0, 0)), + ConceptNode(get_expected(two, "two"), 2, 2, source="two", underlying=u("two", 2, 2)), + UnrecognizedTokensNode(3, 4, [t(" "), t("three")]) + ] + + def test_i_cannot_parse_when_wrong_sequence(self): + foo = Concept(name="foo") + grammar = {foo: Sequence("one", "two", "three")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one two three one") + + assert not res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [ + short_cnode("foo", "one two three"), + UnrecognizedTokensNode(5, 6, [t(" "), t("one")]) + ] + + def test_i_cannot_parse_when_sequence_cannot_match_because_of_end_of_file(self): + foo = Concept(name="foo") + grammar = {foo: Sequence("one", "two", "three")} + + context, res, wrapper, return_value = self.execute([foo], grammar, "one two") + + assert not res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [ + UnrecognizedTokensNode(0, 2, [t("one"), t(" "), t("two")]) + ] + + def test_i_cannot_parse_multiple_results_when_unknown_tokens_at_the_end(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = { + bar: Sequence("one", "two"), + foo: Sequence("one", OrderedChoice("two", "three")) + } + context, parser = self.init([foo, bar], grammar) + + res = parser.parse(context, "one two four five") + + assert len(res) == 2 + assert not res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [ + cnode("bar", 0, 2, "one two"), + UnrecognizedTokensNode(3, 6, [t(" "), t("four"), t(" "), t("five")]) + ] + + assert not res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [ + cnode("foo", 0, 2, "one two"), + UnrecognizedTokensNode(3, 6, [t(" "), t("four"), t(" "), t("five")]) + ] + + def test_i_cannot_parse_multiple_results_when_beginning_by_unknown_tokens(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = { + bar: Sequence("one", "two"), + foo: Sequence("one", OrderedChoice("two", "three")) + } + context, parser = self.init([foo, bar], grammar) + + res = parser.parse(context, "four five one two") + + assert len(res) == 2 + assert not res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [ + UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]), + cnode("bar", 4, 6, "one two"), + ] + + assert not res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [ + UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]), + cnode("foo", 4, 6, "one two"), + ] + + def test_i_cannot_parse_multiple_results_when_surrounded_by_unknown_tokens(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = { + bar: Sequence("one", "two"), + foo: Sequence("one", OrderedChoice("two", "three")) + } + context, parser = self.init([foo, bar], grammar) + + res = parser.parse(context, "four five one two six seven") + assert len(res) == 2 + assert not res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [ + UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]), + cnode("bar", 4, 6, "one two"), + UnrecognizedTokensNode(7, 10, [t(" "), t("six"), t(" "), t("seven")]), + ] + + assert not res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [ + UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]), + cnode("foo", 4, 6, "one two"), + UnrecognizedTokensNode(7, 10, [t(" "), t("six"), t(" "), t("seven")]), + ] + + def test_i_cannot_parse_multiple_results_when_unknown_tokens_in_the_middle(self): + context = self.get_context() + foo = Concept(name="foo") + bar = Concept(name="bar") + baz = Concept(name="baz") + grammar = { + bar: Sequence("one", "two"), + foo: Sequence("one", OrderedChoice("two", "three")), + baz: StrMatch("six"), + } + context, parser = self.init([foo, bar, baz], grammar) + + res = parser.parse(context, "one two four five six") + assert len(res) == 2 + assert not res[0].status + assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) + assert res[0].value.body == [ + cnode("bar", 0, 2, "one two"), + UnrecognizedTokensNode(3, 7, [t(" "), t("four"), t(" "), t("five"), t(" ")]), + cnode("baz", 8, 8, "six"), + ] + + assert not res[1].status + assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) + assert res[1].value.body == [ + cnode("foo", 0, 2, "one two"), + UnrecognizedTokensNode(3, 7, [t(" "), t("four"), t(" "), t("five"), t(" ")]), + cnode("baz", 8, 8, "six"), + ] + + def test_i_can_get_the_inner_concept_when_possible(self): + foo = Concept(name="foo") + one = Concept(name="one") + grammar = {foo: Sequence(Optional(ZeroOrMore(one)), ZeroOrMore("one"))} + + context, res, wrapper, return_value = self.execute([foo, one], grammar, "one") + + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [cnode("foo", 0, 0, "one")] + concept_found = return_value[0].concept + assert cbody(concept_found) == get_expected(one, "one") + assert id(cprop(concept_found, "one")) == id(cbody(concept_found)) + + def test_i_can_get_the_inner_concept_when_possible_with_rule_name(self): + foo = Concept(name="foo") + one = Concept(name="one") + grammar = {foo: Sequence( + Optional(ZeroOrMore(one, rule_name="zero"), rule_name="opt"), + ZeroOrMore("one"), rule_name="seq")} + + context, res, wrapper, return_value = self.execute([foo, one], grammar, "one") + + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [cnode("foo", 0, 0, "one")] + concept_found = return_value[0].concept + assert cbody(concept_found) == get_expected(one, "one") + assert id(cprop(concept_found, "one")) == id(cbody(concept_found)) + assert id(cprop(concept_found, "zero")) == id(cbody(concept_found)) + assert id(cprop(concept_found, "opt")) == id(cbody(concept_found)) + assert id(cprop(concept_found, "seq")) == id(cbody(concept_found)) + + def test_i_get_multiple_props_when_zero_or_more(self): + foo = Concept(name="foo") + one = Concept(name="one") + grammar = {foo: ZeroOrMore(one)} + + context, res, wrapper, return_value = self.execute([foo, one], grammar, "one one one") + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [cnode("foo", 0, 4, "one one one")] + concept_found = return_value[0].concept + assert cbody(concept_found) == DoNotResolve("one one one") + assert len(concept_found.compiled["one"]) == 3 + assert cprop(concept_found, "one")[0] == get_expected(one) + assert cprop(concept_found, "one")[1] == get_expected(one) + assert cprop(concept_found, "one")[2] == get_expected(one) + assert id(cprop(concept_found, "one")[0]) != id(cprop(concept_found, "one")[1]) + assert id(cprop(concept_found, "one")[1]) != id(cprop(concept_found, "one")[2]) + assert id(cprop(concept_found, "one")[2]) != id(cprop(concept_found, "one")[0]) + + def test_i_get_multiple_props_when_zero_or_more_and_different_values(self): + foo = Concept(name="foo") + one = Concept(name="one") + grammar = {foo: ZeroOrMore(Sequence(one, "ok", rule_name="seq")), one: OrderedChoice("one", "un", "uno")} + + context, res, wrapper, return_value = self.execute([foo, one], grammar, "one ok un ok uno ok") + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert return_value == [short_cnode("foo", "one ok un ok uno ok")] + concept_found = return_value[0].concept + assert cprop(concept_found, "one")[0] == get_expected(one, "one") + assert cprop(concept_found, "one")[1] == get_expected(one, "un") + assert cprop(concept_found, "one")[2] == get_expected(one, "uno") + assert cprop(concept_found, "seq")[0] == DoNotResolve("one ok") + assert cprop(concept_found, "seq")[1] == DoNotResolve("un ok") + assert cprop(concept_found, "seq")[2] == DoNotResolve("uno ok") + + # + # def test_i_can_parse_basic_arithmetic_operations_and_resolve_properties(self): + # context = self.get_context() + # add = Concept(name="add") + # mult = Concept(name="mult") + # atom = Concept(name="atom") + # + # grammar = { + # add: Sequence(mult, Optional(Sequence(OrderedChoice('+', '-', rule_name="sign"), add))), + # mult: Sequence(atom, Optional(Sequence(OrderedChoice('*', '/'), mult))), + # atom: OrderedChoice(OrderedChoice('1', '2', '3'), Sequence('(', add, ')')), + # } + # + # parser = ConceptLexerParser() + # parser.register(grammar) + # + # # res = parser.parse(context, "1") + # # assert len(res) == 3 # add, mult, atom + # # + # # res = parser.parse(context, "1 * 2") + # # assert len(res) == 2 # add and mult + # # + # # res = parser.parse(context, "1 + 2") + # # assert res.status + # # assert return_value == [ConceptNode(add, 0, 4, source="1 + 2")] + # + # res = parser.parse(context, "1 * 2 + 3") + # assert res.status + # assert return_value == [ConceptNode(add, 0, 4, source="1 + 2 + 3")] + + def test_i_can_register_concepts_with_the_same_name(self): + # TODO : concepts are registered by name, + # what when two concepts have the same name ? + pass + + def test_i_can_parse_very_very_long_input(self): + # TODO: In the current implementation, all the tokens are loaded in memory + # It's clearly not the good approach + pass diff --git a/tests/parsers/test_ConceptsWithConceptsParser.py b/tests/parsers/test_ConceptsWithConceptsParser.py new file mode 100644 index 0000000..8bfd1b3 --- /dev/null +++ b/tests/parsers/test_ConceptsWithConceptsParser.py @@ -0,0 +1,193 @@ +import ast + +import pytest + +from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts +from core.concept import Concept +from core.tokenizer import Token, TokenKind, Tokenizer +from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode +from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser +from parsers.MultipleConceptsParser import MultipleConceptsParser +from parsers.PythonParser import PythonNode + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + +multiple_concepts_parser = MultipleConceptsParser() + + +def ret_val(*args): + result = [] + index = 0 + source = "" + for item in args: + if isinstance(item, Concept): + tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)] + result.append(ConceptNode(item, index, index, tokens, item.name)) + index += 1 + source += item.name + elif isinstance(item, PythonNode): + tokens = list(Tokenizer(item.source))[:-1] # strip trailing EOF + result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source)) + index += len(tokens) + source += item.source + else: + tokens = list(Tokenizer(item))[:-1] # strip trailing EOF + result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens)) + index += len(tokens) + source += item + + return ReturnValueConcept( + "who", + False, + ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source)) + + +class TestConceptsWithConceptsParser(TestUsingMemoryBasedSheerka): + + def init(self, concepts, inputs): + context = self.get_context() + for concept in concepts: + context.sheerka.create_new_concept(context, concept) + + return context, ret_val(*inputs) + + def execute(self, concepts, inputs): + context, input_return_values = self.init(concepts, inputs) + + parser = ConceptsWithConceptsParser() + result = parser.parse(context, input_return_values.body) + + wrapper = result.body + return_value = result.body.body + + return context, parser, result, wrapper, return_value + + @pytest.mark.parametrize("text, interested", [ + ("not parser result", False), + (ParserResultConcept(parser="not multiple_concepts_parser"), False), + (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True), + ]) + def test_not_interested(self, text, interested): + context = self.get_context() + + res = ConceptsWithConceptsParser().parse(context, text) + if interested: + assert res is not None + else: + assert res is None + + def test_i_can_parse_composition_of_concepts(self): + foo = Concept("foo") + bar = Concept("bar") + plus = Concept("a plus b").def_prop("a").def_prop("b") + + context, parser, result, wrapper, return_value = self.execute([foo, bar, plus], [foo, " plus ", bar]) + + assert result.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert result.who == wrapper.parser.name + assert wrapper.source == "foo plus bar" + assert context.sheerka.isinstance(return_value, plus) + + assert return_value.compiled["a"] == foo + assert return_value.compiled["b"] == bar + + # sanity check, I can evaluate the result + evaluated = context.sheerka.evaluate_concept(context, return_value) + assert evaluated.key == return_value.key + assert evaluated.get_prop("a") == foo.init_key() + assert evaluated.get_prop("b") == bar.init_key() + + def test_i_can_parse_when_composition_of_source_code(self): + plus = Concept("a plus b", body="a + b").def_prop("a").def_prop("b") + left = PythonNode("1+1", ast.parse("1+1", mode="eval")) + right = PythonNode("2+2", ast.parse("2+2", mode="eval")) + context, parser, result, wrapper, return_value = self.execute([plus], [left, " plus ", right]) + + assert result.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert result.who == wrapper.parser.name + assert wrapper.source == "1+1 plus 2+2" + assert context.sheerka.isinstance(return_value, plus) + + left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left) + right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right) + assert return_value.compiled["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)] + assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)] + + # sanity check, I can evaluate the result + evaluated = context.sheerka.evaluate_concept(context, return_value) + assert evaluated.key == return_value.key + assert evaluated.get_prop("a") == 2 + assert evaluated.get_prop("b") == 4 + assert evaluated.body == 6 + + def test_i_can_parse_when_mix_of_concept_and_code(self): + plus = Concept("a plus b").def_prop("a").def_prop("b") + code = PythonNode("1+1", ast.parse("1+1", mode="eval")) + foo = Concept("foo") + context, parser, result, wrapper, return_value = self.execute([plus, foo], [foo, " plus ", code]) + + assert result.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert result.who == wrapper.parser.name + assert wrapper.source == "foo plus 1+1" + assert context.sheerka.isinstance(return_value, plus) + + code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code) + assert return_value.compiled["a"] == foo + assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)] + + # sanity check, I can evaluate the result + evaluated = context.sheerka.evaluate_concept(context, return_value) + assert evaluated.key == return_value.key + assert evaluated.get_prop("a") == foo.init_key() + assert evaluated.get_prop("b") == 2 + + def test_i_can_parse_when_multiple_concepts_are_recognized(self): + foo = Concept("foo") + bar = Concept("bar") + plus_1 = Concept("a plus b", body="body1").def_prop("a").def_prop("b") + plus_2 = Concept("a plus b", body="body2").def_prop("a").def_prop("b") + + context, input_return_values = self.init([foo, bar, plus_1, plus_2], [foo, " plus ", bar]) + parser = ConceptsWithConceptsParser() + result = parser.parse(context, input_return_values.body) + + assert len(result) == 2 + + res = result[0] + wrapper = res.value + return_value = res.value.value + assert res.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert res.who == wrapper.parser.name + assert wrapper.source == "foo plus bar" + assert context.sheerka.isinstance(return_value, plus_1) + assert return_value.compiled["a"] == foo + assert return_value.compiled["b"] == bar + + res = result[1] + wrapper = res.value + return_value = res.value.value + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert res.who == wrapper.parser.name + assert wrapper.source == "foo plus bar" + assert context.sheerka.isinstance(return_value, plus_2) + assert return_value.compiled["a"] == foo + assert return_value.compiled["b"] == bar + + def test_i_cannot_parse_when_unknown_concept(self): + foo = Concept("foo") + bar = Concept("bar") + + context, input_return_values = self.init([foo, bar], [foo, " plus ", bar]) + parser = ConceptsWithConceptsParser() + result = parser.parse(context, input_return_values.body) + wrapper = result.body + return_value = result.body.body + + assert not result.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME) + assert result.who == parser.name + assert return_value == input_return_values.body.body diff --git a/tests/parsers/test_DefaultParser.py b/tests/parsers/test_DefaultParser.py new file mode 100644 index 0000000..c23f942 --- /dev/null +++ b/tests/parsers/test_DefaultParser.py @@ -0,0 +1,345 @@ +import pytest +import ast + +from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept +from core.concept import Concept +from parsers.ConceptLexerParser import OrderedChoice, StrMatch +from parsers.PythonParser import PythonParser, PythonNode +from core.tokenizer import Keywords, Tokenizer, LexerError +from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode +from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode +from parsers.BnfParser import BnfParser + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None): + def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name)))) + + if body: + def_concept.body = get_concept_part(body) + if where: + def_concept.where = get_concept_part(where) + if pre: + def_concept.pre = get_concept_part(pre) + if post: + def_concept.post = get_concept_part(post) + if definition: + def_concept.definition = ReturnValueConcept( + "parsers.Bnf", + True, + definition) + + return def_concept + + +def get_concept_part(part): + if isinstance(part, str): + node = PythonNode(part, ast.parse(part, mode="eval")) + return ReturnValueConcept( + who="parsers.Default", + status=True, + value=ParserResultConcept( + source=part, + parser=PythonParser(), + value=node)) + + if isinstance(part, PythonNode): + return ReturnValueConcept( + who="parsers.Default", + status=True, + value=ParserResultConcept( + source=part.source, + parser=PythonParser(), + value=part)) + + if isinstance(part, ReturnValueConcept): + return part + + +class TestDefaultParser(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("text, expected", [ + ("def concept hello", get_def_concept(name="hello")), + ("def concept hello ", get_def_concept(name="hello")), + ("def concept a + b", get_def_concept(name="a + b")), + ("def concept a+b", get_def_concept(name="a + b")), + ("def concept 'a+b'+c", get_def_concept(name="'a+b' + c")), + ("def concept 'as if'", get_def_concept(name="'as if'")), + ("def concept 'as' if", get_def_concept(name="'as if'")), + ("def concept hello as 'hello'", get_def_concept(name="hello", body="'hello'")), + ("def concept hello as 1", get_def_concept(name="hello", body="1")), + ("def concept hello as 1 + 1", get_def_concept(name="hello", body="1 + 1")), + ]) + def test_i_can_parse_def_concept(self, text, expected): + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + node = res.value.value + + assert res.status + assert res.who == parser.name + assert res.value.source == text + assert isinstance(res.value, ParserResultConcept) + assert node == expected + + def test_i_can_parse_complex_def_concept_statement(self): + text = """def concept a plus b +where a,b +pre isinstance(a, int) and isinstance(b, float) +post isinstance(res, int) +as res = a + b + """ + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + return_value = res.value + expected_concept = get_def_concept( + name="a plus b", + where="a,b", + pre="isinstance(a, int) and isinstance(b, float)", + post="isinstance(res, int)", + body=PythonNode("res = a + b", ast.parse("res = a + b", mode="exec")) + ) + + assert res.status + assert isinstance(return_value, ParserResultConcept) + assert return_value.value == expected_concept + + def test_i_can_have_mutilines_declarations(self): + text = """ +def concept add one to a as +def func(x): + return x+1 +func(a) + """ + + expected_concept = get_def_concept( + name="add one to a ", + body=PythonNode( + "def func(x):\n return x+1\nfunc(a)", + ast.parse("def func(x):\n return x+1\nfunc(a)", mode="exec")) + ) + + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + return_value = res.value + + assert res.status + assert isinstance(return_value, ParserResultConcept) + assert return_value.value == expected_concept + + def test_i_can_use_colon_to_use_indentation(self): + text = """ +def concept add one to a as: + def func(x): + return x+1 + func(a) + """ + + expected_concept = get_def_concept( + name="add one to a ", + body=PythonNode( + "def func(x):\n return x+1\nfunc(a)", + ast.parse("def func(x):\n return x+1\nfunc(a)", mode="exec")) + ) + + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + return_value = res.value + + assert res.status + assert isinstance(return_value, ParserResultConcept) + assert return_value.value == expected_concept + + def test_indentation_is_mandatory_after_a_colon(self): + text = """ +def concept add one to a as: +def func(x): + return x+1 +func(a) + """ + + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + return_value = res.value + + assert not res.status + assert isinstance(return_value, ParserResultConcept) + assert isinstance(return_value.value[0], SyntaxErrorNode) + assert return_value.value[0].message == "Indentation not found." + + def test_indentation_is_not_allowed_if_the_colon_is_missing(self): + text = """ +def concept add one to a as + def func(x): + return x+1 + func(a) + """ + context = self.get_context() + + parser = DefaultParser() + res = parser.parse(context, text) + return_value = res.value + + assert not res.status + assert context.sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS) + + def test_name_is_mandatory(self): + text = "def concept as 'hello'" + + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + return_value = res.value + + assert not res.status + assert isinstance(return_value, ParserResultConcept) + assert isinstance(return_value.value[0], SyntaxErrorNode) + assert return_value.value[0].message == "Name is mandatory" + + def test_concept_keyword_is_mandatory_but_the_concept_is_recognized(self): + text = "def hello as a where b pre c post d" + + expected_concept = get_def_concept(name="hello", body="a", where="b", pre="c", post="d") + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + return_value = res.value + + assert not res.status + assert isinstance(return_value, ParserResultConcept) + assert isinstance(return_value.value[0], UnexpectedTokenErrorNode) + assert return_value.value[0].message == "Syntax error." + assert return_value.value[0].expected_tokens == [Keywords.CONCEPT] + assert return_value.try_parsed == expected_concept + + @pytest.mark.parametrize("text", [ + "def concept hello where 1+", + "def concept hello pre 1+", + "def concept hello post 1+", + "def concept hello as 1+" + ]) + def test_i_can_detect_error_in_declaration(self, text): + context = self.get_context() + sheerka = context.sheerka + + parser = DefaultParser() + res = parser.parse(context, text) + return_value = res.value + + assert not res.status + assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS) + + def test_new_line_is_not_allowed_in_the_name(self): + text = "def concept hello \n my friend as 'hello'" + + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + return_value = res.value + + assert not res.status + assert return_value.value == [SyntaxErrorNode([], "Newline are not allowed in name.")] + + def test_i_can_parse_def_concept_from_regex(self): + context = self.get_context() + a_concept = Concept("a_concept") + context.sheerka.add_in_cache(a_concept) + + text = "def concept name from bnf a_concept | 'a_string' as __definition[0]" + parser = DefaultParser() + res = parser.parse(context, text) + node = res.value.value + definition = OrderedChoice(a_concept, StrMatch("a_string")) + parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition) + expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result) + + assert res.status + assert res.who == parser.name + assert res.value.source == text + assert isinstance(res.value, ParserResultConcept) + assert node == expected + + def test_i_can_parse_def_concept_where_bnf_references_itself(self): + context = self.get_context() + a_concept = Concept("a_concept") + context.sheerka.add_in_cache(a_concept) + + text = "def concept name from bnf 'a' + name?" + parser = DefaultParser() + parser.parse(context, text) + + assert not parser.has_error + + def test_i_can_detect_empty_bnf_declaration(self): + text = "def concept name from bnf as __definition[0]" + + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + + assert not res.status + assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration") + + def test_i_can_detect_not_for_me(self): + text = "hello world" + context = self.get_context() + parser = DefaultParser() + res = parser.parse(context, text) + + assert not res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME) + assert isinstance(res.value.body[0], CannotHandleErrorNode) + + def test_i_can_parse_is_a(self): + parser = DefaultParser() + text = "the name of my 'concept' isa the name of the set" + res = parser.parse(self.get_context(), text) + expected = IsaConceptNode([], + concept=NameNode(list(Tokenizer("the name of my 'concept'"))), + set=NameNode(list(Tokenizer("the name of the set")))) + + assert res.status + assert res.who == parser.name + assert res.value.source == text + assert isinstance(res.value, ParserResultConcept) + assert res.value.value == expected + + @pytest.mark.parametrize("text", [ + "concept", + "isa number", + "name isa", + "def", + "def concept_name" + ]) + def test_i_cannot_parse_invalid_entries(self, text): + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + + assert not res.status + assert isinstance(res.body, ParserResultConcept) + assert isinstance(res.body.body[0], UnexpectedTokenErrorNode) + + @pytest.mark.parametrize("text, error_msg, error_text", [ + ("'name", "Missing Trailing quote", "'name"), + ("foo isa 'name", "Missing Trailing quote", "'name"), + ("def concept 'name", "Missing Trailing quote", "'name"), + ("def concept name as 'body", "Missing Trailing quote", "'body"), + ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"), + ("def concept c::", "Concept name not found", ""), + ]) + def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text): + parser = DefaultParser() + res = parser.parse(self.get_context(), text) + + assert not res.status + assert isinstance(res.body, ParserResultConcept) + assert isinstance(res.body.body[0], LexerError) + assert res.body.body[0].message == error_msg + assert res.body.body[0].text == error_text + + def test_i_cannot_parse_bnf_definition_referencing_unknown_concept(self): + context = self.get_context() + text = "def concept name from bnf unknown" + + parser = DefaultParser() + res = parser.parse(context, text) + + assert not res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) + assert res.value.body == ("key", "unknown") diff --git a/tests/parsers/test_ExactConceptParser.py b/tests/parsers/test_ExactConceptParser.py new file mode 100644 index 0000000..8547077 --- /dev/null +++ b/tests/parsers/test_ExactConceptParser.py @@ -0,0 +1,138 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from core.tokenizer import Tokenizer +from parsers.ExactConceptParser import ExactConceptParser + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def metadata_prop(concept, prop_name): + for name, value in concept.metadata.props: + if name == prop_name: + return value + + return None + + +def get_concept(name, variables): + c = Concept(name=name) + if variables: + for v in variables: + c.def_prop(v) + c.init_key() + return c + + +class TestExactConceptParser(TestUsingMemoryBasedSheerka): + + def test_i_can_compute_combinations(self): + parser = ExactConceptParser() + res = parser.combinations(["foo", "bar", "baz"]) + + assert res == {('foo', 'bar', 'baz'), + ('__var__0', 'bar', 'baz'), + ('foo', '__var__0', 'baz'), + ('foo', 'bar', '__var__0'), + ('__var__0', '__var__1', 'baz'), + ('__var__0', 'bar', '__var__1'), + ('foo', '__var__0', '__var__1'), + ('__var__0', '__var__1', '__var__2')} + + def test_i_can_compute_combinations_with_duplicates(self): + parser = ExactConceptParser() + res = parser.combinations(["foo", "bar", "foo"]) + + assert res == {('foo', 'bar', 'foo'), + ('__var__0', 'bar', '__var__0'), + ('foo', '__var__0', 'foo'), + ('__var__0', '__var__1', '__var__0'), + ('__var__1', '__var__0', '__var__1')} + # TODO: the last tuple is not possible, so the algo can be improved + + def test_i_can_recognize_a_simple_concept(self): + context = self.get_context() + concept = get_concept("hello world", []) + context.sheerka.add_in_cache(concept) + + source = "hello world" + results = ExactConceptParser().parse(context, source) + + assert len(results) == 1 + assert results[0].status + assert results[0].value.value == concept + + def test_i_can_recognize_concepts_defined_several_times(self): + context = self.get_context() + context.sheerka.add_in_cache(get_concept("hello world", [])) + context.sheerka.add_in_cache(get_concept("hello a", ["a"])) + + source = "hello world" + results = ExactConceptParser().parse(context, source) + + assert len(results) == 2 + results = sorted(results, key=lambda x: x.value.value.name) # because of the usage of sets + + assert results[0].status + assert results[0].value.value.name == "hello a" + assert metadata_prop(results[0].value.value, "a") == "world" + + assert results[1].status + assert results[1].value.value.name == "hello world" + + def test_i_can_recognize_a_concept_with_variables(self): + context = self.get_context() + concept = get_concept("a + b", ["a", "b"]) + context.sheerka.add_in_cache(concept) + source = "10 + 5" + results = ExactConceptParser().parse(context, source) + + assert len(results) == 1 + assert results[0].status + concept_found = results[0].value.value + assert concept_found.key == concept.key + assert metadata_prop(concept_found, "a") == "10" + assert metadata_prop(concept_found, "b") == "5" + + def test_i_can_recognize_a_concept_with_duplicate_variables(self): + context = self.get_context() + concept = get_concept("a + b + a", ["a", "b"]) + context.sheerka.cache_by_key[concept.key] = concept + source = "10 + 5 + 10" + results = ExactConceptParser().parse(context, source) + + assert len(results) == 1 + assert results[0].status + concept_found = results[0].value.value + assert concept_found.key == concept.key + assert metadata_prop(concept_found, "a") == "10" + assert metadata_prop(concept_found, "b") == "5" + + def test_i_can_manage_unknown_concept(self): + context = self.get_context() + source = "def concept hello world" # this is not a concept by itself + res = ExactConceptParser().parse(context, source) + + assert not res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) + assert res.value.body == "def concept hello world" + + def test_i_can_detect_concepts_too_long(self): + context = self.get_context() + source = "a very very long concept that cannot be an unique one" + res = ExactConceptParser().parse(context, source) + + assert not res.status + assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_TOO_LONG) + assert res.value.body == "a very very long concept that cannot be an unique one" + + def test_i_can_detect_concept_from_tokens(self): + context = self.get_context() + concept = get_concept("hello world", []) + context.sheerka.add_in_cache(concept) + + source = "hello world" + results = ExactConceptParser().parse(context, list(Tokenizer(source))) + + assert len(results) == 1 + assert results[0].status + assert results[0].value.value == concept diff --git a/tests/parsers/test_MultipleConceptsParser.py b/tests/parsers/test_MultipleConceptsParser.py new file mode 100644 index 0000000..87325dc --- /dev/null +++ b/tests/parsers/test_MultipleConceptsParser.py @@ -0,0 +1,215 @@ +import pytest + +from core.builtin_concepts import ParserResultConcept, BuiltinConcepts +from core.concept import Concept +from core.tokenizer import Tokenizer, TokenKind, Token +from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, cnode, utnode, scnode, SourceCodeNode +from parsers.MultipleConceptsParser import MultipleConceptsParser +from parsers.PythonParser import PythonNode + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +def get_return_value(context, grammar, expression): + parser = ConceptLexerParser() + parser.initialize(context, grammar) + + ret_val = parser.parse(context, expression) + assert not ret_val.status + return ret_val + + +class TestMultipleConceptsParser(TestUsingMemoryBasedSheerka): + + def init(self, concepts, grammar, expression): + context = self.get_context() + for c in concepts: + context.sheerka.create_new_concept(context, c) + return_value = get_return_value(context, grammar, expression) + + return context, return_value + + def test_not_interested_if_not_parser_result(self): + context = self.get_context() + text = "not parser result" + + res = MultipleConceptsParser().parse(context, text) + assert res is None + + def test_not_interested_if_not_from_concept_lexer_parser(self): + context = self.get_context() + text = ParserResultConcept(parser="not concept lexer", value="some value") + + res = MultipleConceptsParser().parse(context, text) + assert res is None + + def test_i_can_parse_exact_concepts(self): + foo = Concept("foo", body="'foo'") + bar = Concept("bar", body="'bar'") + baz = Concept("baz", body="'baz'") + grammar = {} + context, return_value = self.init([foo, bar, baz], grammar, "bar foo baz") + + parser = MultipleConceptsParser() + ret_val = parser.parse(context, return_value.body) + + assert ret_val.status + assert ret_val.who == parser.name + assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) + assert ret_val.value.value == [ + ConceptNode(bar, 0, 0, source="bar"), + ConceptNode(foo, 2, 2, source="foo"), + ConceptNode(baz, 4, 4, source="baz")] + assert ret_val.value.source == "bar foo baz" + + def test_i_can_parse_when_ending_with_bnf(self): + foo = Concept("foo", body="'foo'") + bar = Concept("bar", body="'bar'") + grammar = {foo: Sequence("foo1", "foo2", "foo3")} + context, return_value = self.init([foo, bar], grammar, "bar foo1 foo2 foo3") + + parser = MultipleConceptsParser() + ret_val = parser.parse(context, return_value.body) + + assert ret_val.status + assert ret_val.who == parser.name + assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) + assert ret_val.value.value == [cnode("bar", 0, 0, "bar"), cnode("foo", 2, 6, "foo1 foo2 foo3")] + assert ret_val.value.source == "bar foo1 foo2 foo3" + + def test_i_can_parse_when_starting_with_bnf(self): + foo = Concept("foo", body="'foo'") + bar = Concept("bar", body="'bar'") + grammar = {foo: Sequence("foo1", "foo2", "foo3")} + context, return_value = self.init([foo, bar], grammar, "foo1 foo2 foo3 bar") + + parser = MultipleConceptsParser() + ret_val = parser.parse(context, return_value.body) + + assert ret_val.status + assert ret_val.who == parser.name + assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) + assert ret_val.value.value == [cnode("foo", 0, 4, "foo1 foo2 foo3"), cnode("bar", 6, 6, "bar")] + assert ret_val.value.source == "foo1 foo2 foo3 bar" + + def test_i_can_parse_when_concept_are_mixed(self): + foo = Concept("foo") + bar = Concept("bar") + baz = Concept("baz") + grammar = {foo: Sequence("foo1", "foo2", "foo3")} + context, return_value = self.init([foo, bar, baz], grammar, "baz foo1 foo2 foo3 bar") + + parser = MultipleConceptsParser() + ret_val = parser.parse(context, return_value.body) + + assert ret_val.status + assert ret_val.who == parser.name + assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) + assert ret_val.value.value == [ + cnode("baz", 0, 0, "baz"), + cnode("foo", 2, 6, "foo1 foo2 foo3"), + cnode("bar", 8, 8, "bar")] + assert ret_val.value.source == "baz foo1 foo2 foo3 bar" + + def test_i_can_parse_when_multiple_concepts_are_matching(self): + foo = Concept("foo") + bar = Concept("bar", body="bar1") + baz = Concept("bar", body="bar2") + grammar = {foo: "foo"} + context, return_value = self.init([foo, bar, baz], grammar, "foo bar") + + parser = MultipleConceptsParser() + ret_val = parser.parse(context, return_value.body) + + assert len(ret_val) == 2 + assert ret_val[0].status + assert ret_val[0].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")] + assert ret_val[0].value.source == "foo bar" + assert ret_val[0].value.value[1].concept.metadata.body == "bar1" + + assert ret_val[1].status + assert ret_val[1].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")] + assert ret_val[1].value.source == "foo bar" + assert ret_val[1].value.value[1].concept.metadata.body == "bar2" + + def test_i_can_parse_when_source_code(self): + foo = Concept("foo") + grammar = {foo: "foo"} + context, return_value = self.init([foo], grammar, "1 foo") + + parser = MultipleConceptsParser() + ret_val = parser.parse(context, return_value.body) + wrapper = ret_val.value + value = ret_val.value.value + + assert ret_val.status + assert ret_val.who == parser.name + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert wrapper.source == "1 foo" + assert value == [ + scnode(0, 1, "1 "), + cnode("foo", 2, 2, "foo")] + + def test_i_cannot_parse_when_unrecognized_token(self): + twenty_two = Concept("twenty two") + one = Concept("one") + grammar = {twenty_two: Sequence("twenty", "two")} + context, return_value = self.init([twenty_two, one], grammar, "twenty two + one") + + parser = MultipleConceptsParser() + ret_val = parser.parse(context, return_value.body) + + assert not ret_val.status + assert ret_val.who == parser.name + assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) + assert ret_val.value.value == [ + cnode("twenty two", 0, 2, "twenty two"), + utnode(3, 5, " + "), + cnode("one", 6, 6, "one") + ] + assert ret_val.value.source == "twenty two + one" + + def test_i_cannot_parse_when_unknown_concepts(self): + twenty_two = Concept("twenty two") + one = Concept("one") + grammar = {twenty_two: Sequence("twenty", "two")} + context, return_value = self.init([twenty_two, one], grammar, "twenty two plus one") + + parser = MultipleConceptsParser() + ret_val = parser.parse(context, return_value.body) + + assert not ret_val.status + assert ret_val.who == parser.name + assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) + assert ret_val.value.value == [ + cnode("twenty two", 0, 2, "twenty two"), + utnode(3, 5, " plus "), + cnode("one", 6, 6, "one") + ] + assert ret_val.value.source == "twenty two plus one" + + @pytest.mark.parametrize("text, expected_source, expected_end", [ + ("True", "True", 0), + ("1 == 1", "1 == 1", 5), + ("1!xdf", "1", 0), + ("1", "1", 0), + ]) + def test_i_can_get_source_code_node(self, text, expected_source, expected_end): + tokens = list(Tokenizer(text))[:-1] # strip trailing EOF + + start_index = 5 # a random number different of zero + res = MultipleConceptsParser().get_source_code_node(self.get_context(), start_index, tokens) + + assert isinstance(res, SourceCodeNode) + assert isinstance(res.node, PythonNode) + assert res.source == expected_source + assert res.start == start_index + assert res.end == start_index + expected_end + + def test_i_cannot_parse_null_text(self): + res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, []) + assert res is None + + eof = Token(TokenKind.EOF, "", 0, 0, 0) + res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, [eof]) + assert res is None diff --git a/tests/parsers/test_PythonParser.py b/tests/parsers/test_PythonParser.py new file mode 100644 index 0000000..226c1c5 --- /dev/null +++ b/tests/parsers/test_PythonParser.py @@ -0,0 +1,77 @@ +import ast +import pytest +from core.builtin_concepts import ParserResultConcept +from core.tokenizer import Tokenizer, LexerError +from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestPythonParser(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("text, expected", [ + ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))), + ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))), + ]) + def test_i_can_parse_a_simple_expression(self, text, expected): + parser = PythonParser() + res = parser.parse(self.get_context(), text) + + assert res.status + assert res.who == parser.name + assert isinstance(res.value, ParserResultConcept) + assert res.value.value == expected + + @pytest.mark.parametrize("text, expected", [ + ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))), + ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))), + ]) + def test_i_can_parse_from_tokens(self, text, expected): + parser = PythonParser() + tokens = list(Tokenizer(text)) + res = parser.parse(self.get_context(), tokens) + + assert res.status + assert res.who == parser.name + assert isinstance(res.value, ParserResultConcept) + assert res.value.value == expected + + @pytest.mark.parametrize("text", [ + "1+", + "'name", + "foo = 'name" + ]) + def test_i_can_detect_error(self, text): + parser = PythonParser() + res = parser.parse(self.get_context(), text) + + assert not res.status + assert res.who == parser.name + assert isinstance(res.value, ParserResultConcept) + assert isinstance(res.value.value[0], PythonErrorNode) + assert isinstance(res.value.value[0].exception, SyntaxError) + + @pytest.mark.parametrize("text, error_msg, error_text", [ + ("c::", "Concept name not found", ""), + ("c:: + 1", "Concept name not found", ""), + ]) + def test_i_can_detect_lexer_errors(self, text, error_msg, error_text): + parser = PythonParser() + res = parser.parse(self.get_context(), text) + + assert not res.status + assert isinstance(res.body, ParserResultConcept) + assert isinstance(res.body.body[0], LexerError) + assert res.body.body[0].message == error_msg + assert res.body.body[0].text == error_text + + def test_i_can_parse_a_concept(self): + text = "c:concept_name: + 1" + + parser = PythonParser() + res = parser.parse(self.get_context(), text) + + assert res + assert res.value.value == PythonNode( + "c:concept_name: + 1", + ast.parse("__C__USE_CONCEPT__concept_name__C__+1", mode="eval")) diff --git a/tests/parsers/test_PythonWithConceptsParser.py b/tests/parsers/test_PythonWithConceptsParser.py new file mode 100644 index 0000000..2ff54c5 --- /dev/null +++ b/tests/parsers/test_PythonWithConceptsParser.py @@ -0,0 +1,138 @@ +import ast + +import pytest + +from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept +from core.concept import Concept +from core.tokenizer import Token, TokenKind, Tokenizer +from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode +from parsers.MultipleConceptsParser import MultipleConceptsParser +from parsers.PythonParser import PythonNode, PythonErrorNode +from parsers.PythonWithConceptsParser import PythonWithConceptsParser + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + +multiple_concepts_parser = MultipleConceptsParser() + + +def ret_val(*args): + result = [] + index = 0 + for item in args: + if isinstance(item, Concept): + tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)] + result.append(ConceptNode(item, index, index, tokens, item.name)) + index += 1 + else: + tokens = list(Tokenizer(item)) + result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens)) + index += len(tokens) + + return ReturnValueConcept("who", False, ParserResultConcept(parser=multiple_concepts_parser, value=result)) + + +def to_str_ast(expression): + return PythonNode.get_dump(ast.parse(expression, mode="eval")) + + +class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("text, interested", [ + ("not parser result", False), + (ParserResultConcept(parser="not multiple_concepts_parser"), False), + (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True), + ]) + def test_not_interested(self, text, interested): + context = self.get_context() + + res = PythonWithConceptsParser().parse(context, text) + if interested: + assert res is not None + else: + assert res is None + + def test_i_can_parse_concepts_and_python(self): + context = self.get_context() + foo = Concept("foo") + input_return_value = ret_val(foo, " + 1") + + parser = PythonWithConceptsParser() + result = parser.parse(context, input_return_value.body) + wrapper = result.value + return_value = result.value.value + + assert result.status + assert result.who == parser.name + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert wrapper.source == "foo + 1" + assert isinstance(return_value, PythonNode) + assert return_value.source == "foo + 1" + assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1") + assert return_value.concepts["__C__foo__C__"] == foo + + def test_i_can_parse_concepts_and_python_when_concept_is_known(self): + context = self.get_context() + foo = Concept("foo") + foo = context.sheerka.create_new_concept(context, foo).body.body + input_return_value = ret_val(foo, " + 1") + + parser = PythonWithConceptsParser() + result = parser.parse(context, input_return_value.body) + wrapper = result.value + return_value = result.value.value + + assert result.status + assert result.who == parser.name + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert wrapper.source == "foo + 1" + assert isinstance(return_value, PythonNode) + assert return_value.source == "foo + 1" + assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__1001__C__ + 1") + assert return_value.concepts["__C__foo__1001__C__"] == foo + + def test_i_can_parse_when_concept_name_has_invalid_characters(self): + context = self.get_context() + foo = Concept("foo et > (,") + foo = context.sheerka.create_new_concept(context, foo).body.body + input_return_value = ret_val(foo, " + 1") + + parser = PythonWithConceptsParser() + result = parser.parse(context, input_return_value.body) + return_value = result.value.value + + assert result.status + assert return_value.concepts["__C__foo0et000000__1001__C__"] == foo + + def test_python_ids_mappings_are_correct_when_concepts_with_the_same_name(self): + context = self.get_context() + foo1 = Concept("foo") + foo2 = Concept("foo") + foo3 = context.sheerka.create_new_concept(context, Concept("foo", body="foo3")).body.body + foo4 = context.sheerka.create_new_concept(context, Concept("foo", body="foo4")).body.body + + input_return_value = ret_val(foo1, "+", foo2, "+", foo3, "+", foo4) + + parser = PythonWithConceptsParser() + result = parser.parse(context, input_return_value.body) + return_value = result.value.value + + assert result.status + assert return_value.concepts["__C__foo__C__"] == foo1 + assert return_value.concepts["__C__foo_1__C__"] == foo2 + assert return_value.concepts["__C__foo__1001__C__"] == foo3 + assert return_value.concepts["__C__foo__1002__C__"] == foo4 + + def test_i_cannot_parse_if_syntax_error(self): + context = self.get_context() + foo = Concept("foo") + foo = context.sheerka.create_new_concept(context, foo).body.body + input_return_value = ret_val(foo, " + ") + + parser = PythonWithConceptsParser() + result = parser.parse(context, input_return_value.body) + wrapper = result.value + return_value = result.value.value + + assert not result.status + assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) + assert isinstance(return_value[0], PythonErrorNode) diff --git a/tests/sdp/__init__.py b/tests/sdp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_sheerkaDataProvider.py b/tests/sdp/test_sheerkaDataProvider.py similarity index 99% rename from tests/test_sheerkaDataProvider.py rename to tests/sdp/test_sheerkaDataProvider.py index 69e6b7c..1cb5b72 100644 --- a/tests/test_sheerkaDataProvider.py +++ b/tests/sdp/test_sheerkaDataProvider.py @@ -12,7 +12,7 @@ import json from sdp.sheerkaSerializer import JsonSerializer, Serializer, PickleSerializer import core.utils -tests_root = path.abspath("../build/tests") +tests_root = path.abspath("../../build/tests") evt_digest = "3a571cb6034ef6fc8d7fe91948d0d29728eed74de02bac7968b0e9facca2c2d7" @@ -821,17 +821,17 @@ def test_i_can_set_using_reference(root): entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"), use_ref=True) state = sdp.load_state(sdp.get_snapshot()) - assert state.data == {"entry": {"2": '##REF##:95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268'}} + assert state.data == {"entry": {"2": '##REF##:43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9'}} assert entry == "entry" assert key == "2" assert sdp.io.exists(sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, - "95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268")) + "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9")) # sanity check, make sure that I can load back loaded = sdp.get(entry, key) assert loaded == ObjWithKey(2, "foo") - assert getattr(loaded, Serializer.ORIGIN) == "95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268" + assert getattr(loaded, Serializer.ORIGIN) == "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9" @pytest.mark.parametrize("root", [ @@ -1430,7 +1430,7 @@ def test_i_can_modify_a_ref(root): state = sdp.load_state(sdp.get_snapshot()) assert state.data == {"entry": { "key1": ObjWithKey("key1", "foo"), - "key2": "##REF##:d70b0247311645ed18d275337cbcf79ad186d995236cdc8ad4fcfc708085bd3d"}} + "key2": "##REF##:041d3cca905b51bc2c66251e73e56b836aae7b9435ee3d7eb05d44bb67ff575e"}} assert entry == "entry" assert key == "key2" diff --git a/tests/test_sheerkaSerializer.py b/tests/sdp/test_sheerkaSerializer.py similarity index 92% rename from tests/test_sheerkaSerializer.py rename to tests/sdp/test_sheerkaSerializer.py index 2a91e02..95e5c3c 100644 --- a/tests/test_sheerkaSerializer.py +++ b/tests/sdp/test_sheerkaSerializer.py @@ -37,7 +37,7 @@ def test_i_can_serialize_an_event(): def test_i_can_serialize_an_object(): obj = Obj("10", "value") serializer = Serializer() - serializer.register(JsonSerializer("tests.test_sheerkaSerializer.Obj")) + serializer.register(JsonSerializer("tests.sdp.test_sheerkaSerializer.Obj")) context = SerializerContext("kodjo", "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b") stream = serializer.serialize(obj, context) @@ -51,7 +51,7 @@ def test_i_can_serialize_an_object(): @pytest.mark.parametrize("obj, expected", [ - (Obj("10", "value"), "tests.test_sheerkaSerializer.Obj") + (Obj("10", "value"), "tests.sdp.test_sheerkaSerializer.Obj") ]) def test_get_full_qualified_name(obj, expected): assert expected == core.utils.get_full_qualified_name(obj) diff --git a/tests/test_AddConceptEvaluator.py b/tests/test_AddConceptEvaluator.py deleted file mode 100644 index 77ff197..0000000 --- a/tests/test_AddConceptEvaluator.py +++ /dev/null @@ -1,194 +0,0 @@ -import ast - -import pytest - -from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts -from core.concept import VARIABLE_PREFIX, ConceptParts, Concept -from core.sheerka import Sheerka, ExecutionContext -from core.tokenizer import Tokenizer -from evaluators.AddConceptEvaluator import AddConceptEvaluator -from parsers.BaseParser import BaseParser -from parsers.ConceptLexerParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression -from parsers.BnfParser import BnfParser -from parsers.DefaultParser import DefConceptNode, NameNode -from parsers.PythonParser import PythonNode, PythonParser -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None): - concept = DefConceptNode([], name=NameNode(list(Tokenizer(name)))) - - if body: - concept.body = get_concept_part(body) - if where: - concept.where = get_concept_part(where) - if pre: - concept.pre = get_concept_part(pre) - if post: - concept.post = get_concept_part(post) - if definition: - concept.definition = definition - - return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=concept)) - - -def get_concept_part(part): - if isinstance(part, str): - node = PythonNode(part, ast.parse(part, mode="eval")) - return ReturnValueConcept( - who="parsers.Default", - status=True, - value=ParserResultConcept( - source=part, - parser=PythonParser(), - value=node)) - - if isinstance(part, PythonNode): - return ReturnValueConcept( - who="parsers.Default", - status=True, - value=ParserResultConcept( - source=part.source, - parser=PythonParser(), - value=part)) - - if isinstance(part, ReturnValueConcept): - return part - - -def get_concept_definition(source, parsing_expression): - return ReturnValueConcept( - who="Parsers:RegexParser", - status=True, - value=ParserResultConcept( - source=source, - parser=BnfParser(), - value=parsing_expression - ) - ) - - -@pytest.mark.parametrize("ret_val, expected", [ - (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))), True), - (ReturnValueConcept(BaseParser.PREFIX + "some_name", False, ParserResultConcept(value=DefConceptNode([]))), False), - (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not a ParserResultConcept"), False), - (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept()), False), -]) -def test_i_can_match(ret_val, expected): - context = get_context() - assert AddConceptEvaluator().matches(context, ret_val) == expected - - -def test_that_the_source_is_correctly_set(): - context = get_context() - def_concept_return_value = get_def_concept( - name="hello a", - definition=get_concept_definition("hello a", Sequence(StrMatch("hello"), StrMatch("a"))), - where="isinstance(a, str )", - pre="a is not None", - body="print('hello' + a)") - - evaluated = AddConceptEvaluator().eval(context, def_concept_return_value) - - assert evaluated.status - assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT) - - created_concept = evaluated.body.body - assert created_concept.metadata.name == "hello a" - assert created_concept.metadata.where == "isinstance(a, str )" - assert created_concept.metadata.pre == "a is not None" - assert created_concept.metadata.post is None - assert created_concept.metadata.body == "print('hello' + a)" - assert created_concept.metadata.definition == "hello a" - - -# def test_that_the_ast_is_correctly_initialized(): -# """ -# When I parse the definition of a concept, I evaluate the metadata (like the body) -# I wanted to keep in cache these evaluation for further utilisation but I have -# a serialization issue. -# So I had to comment concept.add_codes(def_concept_node.get_asts()) around line 85 -# So this test is now irrelevant -# :return: -# """ -# context = get_context() -# def_concept_return_value = get_concept( -# name="hello a", -# definition=get_concept_definition("hello a", Sequence(StrMatch("hello"), StrMatch("a"))), -# where="isinstance(a, str )", -# pre="a is not None", -# body="print('hello' + a)") -# -# evaluated = AddConceptEvaluator().eval(context, def_concept_return_value) -# -# assert evaluated.status -# assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT) -# -# created_concept = evaluated.body.body -# -# assert ConceptParts.WHERE in created_concept.compiled -# assert ConceptParts.PRE in created_concept.compiled -# assert ConceptParts.BODY in created_concept.compiled -# assert ConceptParts.POST not in created_concept.compiled - - -def test_that_the_new_concept_is_correctly_saved_in_db(): - context = get_context() - def_concept_return_value = get_def_concept( - name="hello a", - definition=get_concept_definition("hello a", Sequence(StrMatch("hello"), StrMatch("a"))), - where="isinstance(a, str )", - pre="a is not None", - body="print('hello' + a)") - - # sanity. Make sure that the concept does not already exist - from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0") - assert context.sheerka.isinstance(from_db, BuiltinConcepts.UNKNOWN_CONCEPT) - - AddConceptEvaluator().eval(context, def_concept_return_value) - context.sheerka.concepts_cache = {} # reset cache - from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0") - - assert from_db.metadata.key == f"hello {VARIABLE_PREFIX}0" - assert from_db.metadata.name == "hello a" - assert from_db.metadata.where == "isinstance(a, str )" - assert from_db.metadata.pre == "a is not None" - assert from_db.metadata.post is None - assert from_db.metadata.body == "print('hello' + a)" - assert from_db.metadata.definition == "hello a" - assert len(from_db.metadata.props) == 1 - assert from_db.metadata.props[0] == ("a", None) - assert "a" in from_db.props - - assert from_db.compiled == {} # ast is not saved in db - - -def test_i_can_get_props_from_python_node(): - ret_val = get_concept_part("isinstance(a, str)") - context = get_context() - - assert AddConceptEvaluator.get_props(context.sheerka, ret_val, ["a"]) == ["a"] - - -def test_i_can_get_props_from_another_concept(): - concept = Concept("hello").def_prop("a").def_prop("b") - ret_val = ReturnValueConcept(who="some_parser", - status=True, - value=ParserResultConcept(value=concept)) - - assert AddConceptEvaluator.get_props(get_context(), ret_val, []) == ["a", "b"] - - -def test_i_can_get_props_from_definition(): - parsing_expression = Sequence(ConceptExpression('mult'), ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add")))) - ret_val = get_concept_definition("mult (('+'|'-') add)?", parsing_expression) - - assert AddConceptEvaluator.get_props(get_context(), ret_val, []) == ["add", "mult"] - - diff --git a/tests/test_AddConceptInSetEvaluator.py b/tests/test_AddConceptInSetEvaluator.py deleted file mode 100644 index 98b7068..0000000 --- a/tests/test_AddConceptInSetEvaluator.py +++ /dev/null @@ -1,112 +0,0 @@ -import pytest - -from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from core.tokenizer import Tokenizer -from evaluators.AddConceptInSetEvaluator import AddConceptInSetEvaluator -from parsers.DefaultParser import IsaConceptNode, NameNode -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def get_ret_val(concept_name, concept_set_name): - n1 = NameNode(list(Tokenizer(concept_name))) - n2 = NameNode(list(Tokenizer(concept_set_name))) - - return ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([], n1, n2))) - - -@pytest.mark.parametrize("ret_val, expected", [ - (ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([]))), True), - (ReturnValueConcept("some_name", False, ParserResultConcept(value=IsaConceptNode([]))), False), - (ReturnValueConcept("some_name", True, "not a ParserResultConcept"), False), - (ReturnValueConcept("some_name", True, ParserResultConcept()), False), -]) -def test_i_can_match(ret_val, expected): - context = get_context() - assert AddConceptInSetEvaluator().matches(context, ret_val) == expected - - -def test_i_cannot_add_if_the_concept_does_not_exists(): - context = get_context() - - ret_val = get_ret_val("foo", "bar") - res = AddConceptInSetEvaluator().eval(context, ret_val) - - assert not res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) - assert res.value.body == "foo" - - -def test_i_cannot_add_if_the_set_does_not_exists(): - context = get_context() - foo = Concept("foo") - context.sheerka.set_id_if_needed(foo, False) - context.sheerka.add_in_cache(foo) - - ret_val = get_ret_val("foo", "bar") - res = AddConceptInSetEvaluator().eval(context, ret_val) - - assert not res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) - assert res.value.body == "bar" - - -def test_i_can_add_concept_to_a_set_of_concept(): - context = get_context() - foo = Concept("foo") - context.sheerka.set_id_if_needed(foo, False) - context.sheerka.add_in_cache(foo) - - bar = Concept("bar") - context.sheerka.set_id_if_needed(bar, False) - context.sheerka.add_in_cache(bar) - - ret_val = get_ret_val("foo", "bar") - res = AddConceptInSetEvaluator().eval(context, ret_val) - - assert res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS) - - -def test_i_can_add_concept_with_a_body_to_a_set_of_concept(): - context = get_context() - foo = Concept("foo", body="1") - context.sheerka.set_id_if_needed(foo, False) - context.sheerka.add_in_cache(foo) - - bar = Concept("bar") - context.sheerka.set_id_if_needed(bar, False) - context.sheerka.add_in_cache(bar) - - ret_val = get_ret_val("foo", "bar") - res = AddConceptInSetEvaluator().eval(context, ret_val) - - assert res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS) - - -def test_i_cannot_add_the_same_concept_twice(): - context = get_context() - foo = Concept("foo") - context.sheerka.set_id_if_needed(foo, False) - context.sheerka.add_in_cache(foo) - - bar = Concept("bar") - context.sheerka.set_id_if_needed(bar, False) - context.sheerka.add_in_cache(bar) - - ret_val = get_ret_val("foo", "bar") - AddConceptInSetEvaluator().eval(context, ret_val) - res = AddConceptInSetEvaluator().eval(context, ret_val) - - assert not res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_IN_SET) - assert res.value.concept == foo - assert res.value.concept_set == bar diff --git a/tests/test_BnfParser.py b/tests/test_BnfParser.py deleted file mode 100644 index f1708aa..0000000 --- a/tests/test_BnfParser.py +++ /dev/null @@ -1,185 +0,0 @@ -import pytest - -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from core.tokenizer import Tokenizer, TokenKind, LexerError -from parsers.BaseParser import UnexpectedTokenErrorNode -from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError -from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \ - ConceptLexerParser, ConceptNode, ConceptExpression, cnode -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - - return ExecutionContext("sheerka", Event(), sheerka) - - -class ClassWithName(): - def __init__(self, name): - self.name = name - - -@pytest.mark.parametrize("expression, expected", [ - ("'str'", StrMatch("str")), - ("1", StrMatch("1")), - (" 1", StrMatch("1")), - (",", StrMatch(",")), - ("'foo'?", Optional(StrMatch("foo"))), - ("'foo'*", ZeroOrMore(StrMatch("foo"))), - ("'foo'+", OneOrMore(StrMatch("foo"))), - ("1 | 2 | 3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))), - ("1|2|3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))), - ("1'|' 2 '|' 3", Sequence(StrMatch("1"), StrMatch("|"), StrMatch("2"), StrMatch("|"), StrMatch("3"))), - ("1 2 'foo'", Sequence(StrMatch("1"), StrMatch("2"), StrMatch("foo"))), - ("1 2 | 3 4+", OrderedChoice( - Sequence(StrMatch("1"), StrMatch("2")), - Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))), - ("1 (2 | 3) 4+", Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))), - ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))), - ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))), - ("1 *", Sequence(StrMatch("1"), StrMatch("*"))), - ("1 ?", Sequence(StrMatch("1"), StrMatch("?"))), - ("1 +", Sequence(StrMatch("1"), StrMatch("+"))), - ("(1|*) +", Sequence(OrderedChoice(StrMatch("1"), StrMatch("*")), StrMatch("+"))), - ("1, :&", Sequence(StrMatch("1"), StrMatch(","), StrMatch(":"), StrMatch("&"))), - ("(1 )", StrMatch("1")), - ("'str'=var", StrMatch("str", rule_name="var")), - ("'foo'?=var", Optional(StrMatch("foo"), rule_name="var")), - ("('foo'?)=var", Optional(StrMatch("foo"), rule_name="var")), - ("'foo'*=var", ZeroOrMore(StrMatch("foo"), rule_name="var")), - ("('foo'*)=var", ZeroOrMore(StrMatch("foo"), rule_name="var")), - ("'foo'+=var", OneOrMore(StrMatch("foo"), rule_name="var")), - ("('foo'+)=var", OneOrMore(StrMatch("foo"), rule_name="var")), - ("'foo'=var?", Optional(StrMatch("foo", rule_name="var"))), - ("('foo'=var)?", Optional(StrMatch("foo", rule_name="var"))), - ("'foo'=var*", ZeroOrMore(StrMatch("foo", rule_name="var"))), - ("('foo'=var)*", ZeroOrMore(StrMatch("foo", rule_name="var"))), - ("'foo'=var+", OneOrMore(StrMatch("foo", rule_name="var"))), - ("('foo'=var)+", OneOrMore(StrMatch("foo", rule_name="var"))), - ("(1 | 2 | 3)=var", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"), rule_name="var")), - ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")), - ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")), - ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))), -]) -def test_i_can_parse_regex(expression, expected): - parser = BnfParser() - res = parser.parse(get_context(), Tokenizer(expression)) - - assert not parser.has_error - assert res.status - assert res.value.value == expected - assert res.value.source == expression - - -@pytest.mark.parametrize("expression, expected", [ - ("foo", Concept("foo").init_key()), - ("foo*", ZeroOrMore(Concept("foo").init_key())), - ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))), - ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))), - ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())), - ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())), -]) -def test_i_can_parse_regex_with_concept(expression, expected): - foo = Concept("foo") - bar = Concept("bar") - var = Concept("var") - context = get_context() - - for c in (foo, bar, var): - context.sheerka.add_in_cache(c) - parser = BnfParser() - res = parser.parse(context, Tokenizer(expression)) - - assert not parser.has_error - assert res.status - assert res.value.value == expected - assert res.value.source == expression - - -def test_i_can_parse_regex_with_concept_when_the_concept_is_still_under_definition(): - expression = "foo" - expected = ConceptExpression("foo") - - context = get_context() - context.obj = ClassWithName("foo") - - parser = BnfParser() - res = parser.parse(context, Tokenizer(expression)) - - assert not parser.has_error - assert res.status - assert res.value.value == expected - assert res.value.source == expression - - -@pytest.mark.parametrize("expression, error", [ - ("1 ", UnexpectedEndOfFileError()), - ("1|", UnexpectedEndOfFileError()), - ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token()'", [TokenKind.RPAR])), - ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token()'", [TokenKind.IDENTIFIER])), - ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6)) -]) -def test_i_can_detect_errors(expression, error): - parser = BnfParser() - res = parser.parse(get_context(), Tokenizer(expression)) - ret_value = res.value.value - assert parser.has_error - assert not res.status - assert ret_value[0] == error - - -def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text(): - foo = Concept(name="foo") - bar = Concept(name="bar") - context = get_context() - context.sheerka.add_in_cache(foo) - context.sheerka.add_in_cache(bar) - - regex_parser = BnfParser() - foo_definition = regex_parser.parse(context, "'twenty' | 'thirty'").value.value - bar_definition = regex_parser.parse(context, "foo ('one' | 'two')").value.value - - concepts = {bar: bar_definition, foo: foo_definition} - concept_parser = ConceptLexerParser() - concept_parser.initialize(context, concepts) - - res = concept_parser.parse(context, "twenty two") - assert res.status - assert res.value.body == [cnode("bar", 0, 2, "twenty two")] - - res = concept_parser.parse(context, "thirty one") - assert res.status - assert res.value.body == [cnode("bar", 0, 2, "thirty one")] - - res = concept_parser.parse(context, "twenty") - assert res.status - assert res.value.body == [cnode("foo", 0, 0, "twenty")] - - -def test_i_cannot_parse_when_too_many_concepts(): - foo1 = Concept(name="foo", body="1") - foo2 = Concept(name="foo", body="2") - context = get_context() - context.sheerka.cache_by_key["foo"] = [foo1, foo2] - - regex_parser = BnfParser() - res = regex_parser.parse(context, "foo") - - assert not res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT) - assert res.value.body == ('key', 'foo') - - -def test_i_cannot_parse_when_unknown_concept(): - context = get_context() - - regex_parser = BnfParser() - res = regex_parser.parse(get_context(), "foo") - - assert not res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) - assert res.value.body == ('key', 'foo') diff --git a/tests/test_ConceptEvaluator.py b/tests/test_ConceptEvaluator.py deleted file mode 100644 index bf609c8..0000000 --- a/tests/test_ConceptEvaluator.py +++ /dev/null @@ -1,139 +0,0 @@ -import pytest - -from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts -from core.concept import Concept, ConceptParts -from core.sheerka import Sheerka, ExecutionContext -from evaluators.ConceptEvaluator import ConceptEvaluator -from parsers.ExactConceptParser import ExactConceptParser -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def get_return_value(concept, source=None): - return ReturnValueConcept( - "some_name", - True, - ParserResultConcept(parser=ExactConceptParser(), - source=source or concept.name, - value=concept, - try_parsed=concept)) - - -@pytest.mark.parametrize("ret_val, expected", [ - (ReturnValueConcept("some_name", True, ParserResultConcept(value=Concept())), True), - (ReturnValueConcept("some_name", False, ParserResultConcept(value=Concept())), False), - (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept")), False), - (ReturnValueConcept("some_name", True, Concept()), False), -]) -def test_i_can_match(ret_val, expected): - context = get_context() - assert ConceptEvaluator().matches(context, ret_val) == expected - - -def test_i_can_evaluate_concept(): - context = get_context() - concept = Concept(name="foo", - where="1", - pre="2", - post="3").def_prop("a", "4").def_prop("b", "5") - - evaluator = ConceptEvaluator() - item = get_return_value(concept) - result = evaluator.eval(context, item) - - assert result.who == evaluator.name - assert result.status - assert result.value.name == "foo" - assert result.value.get_metadata_value(ConceptParts.WHERE) == 1 - assert result.value.get_metadata_value(ConceptParts.PRE) == 2 - assert result.value.get_metadata_value(ConceptParts.POST) == 3 - assert result.value.get_prop("a") == 4 - assert result.value.get_prop("b") == 5 - assert result.value.key == "foo" - assert result.parents == [item] - - -def test_body_is_returned_when_defined_and_requested(): - context = get_context() - concept = Concept(name="foo", - body="'I have a value'", - where="1", - pre="2", - post="3").set_prop("a", "4").set_prop("b", "5") - - evaluator = ConceptEvaluator(return_body=True) - item = get_return_value(concept) - result = evaluator.eval(context, item) - - assert result.who == evaluator.name - assert result.status - assert result.value == "I have a value" - assert result.parents == [item] - - -def test_body_is_not_returned_if_not_requested(): - context = get_context() - concept = Concept(name="foo", - body="'I have a value'", - where="1", - pre="2", - post="3").set_prop("a", "4").set_prop("b", "5") - - evaluator = ConceptEvaluator(return_body=False) # which is the default behaviour - item = get_return_value(concept) - result = evaluator.eval(context, item) - - assert result.who == evaluator.name - assert result.status - assert result.value == concept - assert result.parents == [item] - - -def test_i_can_eval_if_with_the_same_name_is_defined_in_the_context(): - # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'") - # ConceptEvaluator will be called to resolve 'a' while we know that 'a' refers to the string 'property_a' - - context = get_context() - context.obj = Concept("other").set_prop("foo", "'some_other_value'") - concept = Concept(name="foo") - - item = get_return_value(concept) - result = ConceptEvaluator().eval(context, item) - - assert result.status - assert result.value == "'some_other_value'" - - -def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(): - context = get_context() - context.sheerka.add_in_cache(Concept(name="one").init_key()) - concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b") - .def_prop("a", "one") - .def_prop("b", "two").init_key()) - - evaluator = ConceptEvaluator() - item = get_return_value(concept_plus) - result = evaluator.eval(context, item) - - assert not result.status - assert context.sheerka.isinstance(result.value, BuiltinConcepts.CONCEPT_EVAL_ERROR) - assert result.value.property_name == "b" - assert context.sheerka.isinstance(result.value.error, BuiltinConcepts.TOO_MANY_ERRORS) - assert result.value.concept == concept_plus - - -def test_that_error_concepts_are_transformed_into_errors_only_if_the_key_is_different(): - context = get_context() - - error_concept = context.sheerka.new(BuiltinConcepts.ERROR) - item = get_return_value(error_concept) - result = ConceptEvaluator().eval(context, item) - - assert not context.sheerka.is_success(error_concept) # it's indeed an error - assert result.status - assert result.value == error_concept diff --git a/tests/test_ConceptLexerParser.py b/tests/test_ConceptLexerParser.py deleted file mode 100644 index 2499546..0000000 --- a/tests/test_ConceptLexerParser.py +++ /dev/null @@ -1,1199 +0,0 @@ -import pytest -import core.utils -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts, DoNotResolve -from core.sheerka import Sheerka, ExecutionContext -from core.tokenizer import Tokenizer, TokenKind, Token -from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \ - ParsingExpressionVisitor, TerminalNode, NonTerminalNode, LexerNode, ConceptExpression, ZeroOrMore, OneOrMore, \ - UnrecognizedTokensNode, cnode, short_cnode -from sdp.sheerkaDataProvider import Event - - -class ConceptVisitor(ParsingExpressionVisitor): - def __init__(self): - self.concepts = set() - - def visit_ConceptExpression(self, node): - self.concepts.add(node.concept) - - -def u(parsing_expression, start, end, children=None): - """ - u stands for underlying - :param parsing_expression: - :param start: - :param end: - :param children: - :return: - """ - if isinstance(parsing_expression, str): - parsing_expression = StrMatch(parsing_expression) - - if isinstance(parsing_expression, StrMatch): - return TerminalNode(parsing_expression, start, end, parsing_expression.to_match) - - return NonTerminalNode(parsing_expression, start, end, [], children) - - -def evaluated(concept): - c = Concept(name=concept.name, body=concept.name) - - -def t(text): - if text.startswith("'") or text.startswith('"'): - return Token(TokenKind.STRING, text, 0, 0, 0) - - if text.startswith(" "): - return Token(TokenKind.WHITESPACE, text, 0, 0, 0) - - return Token(TokenKind.IDENTIFIER, text, 0, 0, 0) - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - - return ExecutionContext("sheerka", Event(), sheerka) - - -def get_expected(concept, text=None): - c = Concept(name=concept.name) - c.compiled[ConceptParts.BODY] = DoNotResolve(text or concept.name) - c.init_key() - c.metadata.id = concept.id - return c - - -def cbody(concept): - """cbody stands for compiled body""" - if not ConceptParts.BODY in concept.compiled: - return None - return concept.compiled[ConceptParts.BODY] - - -def cprop(concept, prop_name): - """cbody stands for compiled property""" - return concept.compiled[prop_name] - - -def init(concepts, grammar): - context = get_context() - for c in concepts: - context.sheerka.add_in_cache(c) - - parser = ConceptLexerParser() - parser.initialize(context, grammar) - - return context, parser - - -def execute(concepts, grammar, text): - context, parser = init(concepts, grammar) - - res = parser.parse(context, text) - wrapper = res.value - return_value = res.value.value - - return context, res, wrapper, return_value - - -@pytest.mark.parametrize("match, text", [ - ("foo", "foo"), - ("'foo'", "'foo'"), - ("1", "1"), - ("3.14", "3.14"), - ("+", "+"), - (StrMatch("foo"), "foo"), - (StrMatch("'foo'"), "'foo'"), - (StrMatch("1"), "1"), - (StrMatch("3.14"), "3.14"), - (StrMatch("+"), "+"), -]) -def test_i_can_match_simple_tokens(match, text): - foo = Concept(name="foo") - grammar = {foo: match} - - context, res, wrapper, return_value = execute([foo], grammar, text) - - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [ConceptNode(get_expected(foo, text), 0, 0, source=text, underlying=u(match, 0, 0))] - - -def test_i_can_match_multiple_concepts_in_one_input(): - one = Concept(name="one") - two = Concept(name="two") - grammar = {one: "one", two: "two"} - - context, res, wrapper, return_value = execute([one, two], grammar, "one two one") - - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [ - ConceptNode(get_expected(one), 0, 0, source="one", underlying=u("one", 0, 0)), - ConceptNode(get_expected(two), 2, 2, source="two", underlying=u("two", 2, 2)), - ConceptNode(get_expected(one), 4, 4, source="one", underlying=u("one", 4, 4)), - ] - - -def test_i_can_match_sequence(): - foo = Concept(name="foo") - grammar = {foo: Sequence("one", "two", "three")} - - context, res, wrapper, return_value = execute([foo], grammar, "one two three") - - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [ - ConceptNode( - get_expected(foo, "one two three"), - 0, - 4, - source="one two three", - underlying=u(grammar[foo], 0, 4, [ - u("one", 0, 0), - u("two", 2, 2), - u("three", 4, 4)]))] - - -def test_i_always_choose_the_longest_match(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = {bar: Sequence("one", "two"), foo: Sequence("one", "two", "three")} - - context, res, wrapper, return_value = execute([foo, bar], grammar, "one two three") - - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [cnode("foo", 0, 4, "one two three")] - - -def test_i_can_match_several_sequences(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = {bar: Sequence("one", "two"), foo: Sequence("one", "two", "three")} - - context, res, wrapper, return_value = execute([foo, bar], grammar, "one two three one two") - - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [ - cnode("foo", 0, 4, "one two three"), - cnode("bar", 6, 8, "one two"), - ] - - -def test_i_can_match_ordered_choice(): - foo = Concept(name="foo") - grammar = {foo: OrderedChoice("one", "two")} - context, parser = init([foo], grammar) - - res1 = parser.parse(context, "one") - assert res1.status - assert context.sheerka.isinstance(res1.value, BuiltinConcepts.PARSER_RESULT) - assert res1.value.body == [cnode("foo", 0, 0, "one")] - assert res1.value.body[0].underlying == u(grammar[foo], 0, 0, [u("one", 0, 0)]) - - res2 = parser.parse(context, "two") - assert res2.status - assert context.sheerka.isinstance(res2.value, BuiltinConcepts.PARSER_RESULT) - assert res2.value.body == [cnode("foo", 0, 0, "two")] - assert res2.value.body[0].underlying == u(grammar[foo], 0, 0, [u("two", 0, 0)]) - - res3 = parser.parse(context, "three") - assert not res3.status - assert context.sheerka.isinstance(res3.value, BuiltinConcepts.PARSER_RESULT) - assert res3.value.value == [ - UnrecognizedTokensNode(0, 0, [t("three")]) - ] - - -def test_i_cannot_match_ordered_choice_with_empty_alternative(): - foo = Concept(name="foo") - grammar = {foo: Sequence(OrderedChoice("one", ""), "two")} - - context, res, wrapper, return_value = execute([foo], grammar, "ok") - - assert not res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [ - UnrecognizedTokensNode(0, 0, [t("ok")]) - ] - - -def test_i_can_mix_sequences_and_ordered_choices(): - foo = Concept(name="foo") - grammar = {foo: Sequence(OrderedChoice("twenty", "thirty"), "one", "ok")} - - context, parser = init([foo], grammar) - - res1 = parser.parse(context, "twenty one ok") - assert res1.status - assert context.sheerka.isinstance(res1.value, BuiltinConcepts.PARSER_RESULT) - assert res1.value.body == [ConceptNode(get_expected(foo, "twenty one ok"), 0, 4, source="twenty one ok", - underlying=u(grammar[foo], 0, 4, [ - u(OrderedChoice("twenty", "thirty"), 0, 0, [u("twenty", 0, 0)]), - u("one", 2, 2), - u("ok", 4, 4)]))] - - res2 = parser.parse(context, "thirty one ok") - assert res2.status - assert context.sheerka.isinstance(res2.value, BuiltinConcepts.PARSER_RESULT) - assert res2.value.body == [ConceptNode(get_expected(foo, "thirty one ok"), 0, 4, source="thirty one ok", - underlying=u(grammar[foo], 0, 4, [ - u(OrderedChoice("twenty", "thirty"), 0, 0, [u("thirty", 0, 0)]), - u("one", 2, 2), - u("ok", 4, 4)]))] - - res3 = parser.parse(context, "twenty one") - assert not res3.status - assert context.sheerka.isinstance(res2.value, BuiltinConcepts.PARSER_RESULT) - assert res3.value.value == [ - UnrecognizedTokensNode(0, 2, [t("twenty"), t(" "), t("one")]) - ] - - -def test_i_can_mix_ordered_choices_and_sequences(): - foo = Concept(name="foo") - grammar = {foo: OrderedChoice(Sequence("twenty", "thirty"), "one")} - - context, parser = init([foo], grammar) - - res = parser.parse(context, "twenty thirty") - assert res.status - assert res.value.value == [cnode("foo", 0, 2, "twenty thirty")] - - res = parser.parse(context, "one") - assert res.status - assert res.value.value == [cnode("foo", 0, 0, "one")] - - -def test_i_cannot_parse_empty_optional(): - foo = Concept(name="foo") - grammar = {foo: Optional("one")} - context, parser = init([foo], grammar) - - res = parser.parse(context, "") - return_value = res.value - - assert not res.status - assert context.sheerka.isinstance(return_value, BuiltinConcepts.IS_EMPTY) - - -def test_i_can_parse_optional(): - foo = Concept(name="foo") - grammar = {foo: Optional("one")} - - context, res, wrapper, return_value = execute([foo], grammar, "one") - - assert res.status - assert return_value == [ConceptNode(get_expected(foo, "one"), 0, 0, source="one", - underlying=u(grammar[foo], 0, 0, [u("one", 0, 0)]))] - - -def test_i_can_parse_sequence_starting_with_optional(): - foo = Concept(name="foo") - grammar = {foo: Sequence(Optional("twenty"), "one")} - context, parser = init([foo], grammar) - - res = parser.parse(context, "twenty one") - assert res.status - assert res.value.body == [ConceptNode( - get_expected(foo, "twenty one"), 0, 2, - source="twenty one", - underlying=u(grammar[foo], 0, 2, - [ - u(Optional("twenty"), 0, 0, [u("twenty", 0, 0)]), - u("one", 2, 2)] - ))] - - res = parser.parse(context, "one") - assert res.status - assert res.value.body == [ConceptNode(get_expected(foo, "one"), 0, 0, source="one", - underlying=u(grammar[foo], 0, 0, [u("one", 0, 0)]))] - - -def test_i_can_parse_sequence_ending_with_optional(): - foo = Concept(name="foo") - grammar = {foo: Sequence("one", "two", Optional("three"))} - - context, parser = init([foo], grammar) - - res = parser.parse(context, "one two three") - assert res.status - assert res.value.body == [cnode("foo", 0, 4, "one two three")] - - res = parser.parse(context, "one two") - assert res.status - assert res.value.body == [cnode("foo", 0, 2, "one two")] - - -def test_i_can_parse_sequence_with_optional_in_between(): - foo = Concept(name="foo") - - grammar = {foo: Sequence("one", Optional("two"), "three")} - - context, parser = init([foo], grammar) - - res = parser.parse(context, "one two three") - assert res.status - assert res.value.body == [cnode("foo", 0, 4, "one two three")] - - res = parser.parse(context, "one three") - assert res.status - assert res.value.body == [cnode("foo", 0, 2, "one three")] - - -def test_i_cannot_parse_wrong_input_with_optional(): - foo = Concept(name="foo") - grammar = {foo: Optional("one")} - - context, res, wrapper, return_value = execute([foo], grammar, "two") - - assert not res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [ - UnrecognizedTokensNode(0, 0, [t("two")]) - ] - - -def test_i_can_use_reference(): - # when there are multiple matches for the same input - # Do I need to create a choice concept ? - # No, create a return value for every possible graph - - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = {foo: Sequence("one", "two"), bar: foo} - context, parser = init([foo, bar], grammar) - res = parser.parse(context, "one two") - - assert len(res) == 2 - - assert res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [cnode("foo", 0, 2, "one two")] - concept_found_1 = res[0].value.body[0].concept - assert cbody(concept_found_1) == DoNotResolve("one two") - - assert res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [cnode("bar", 0, 2, "one two")] - concept_found_2 = res[1].value.body[0].concept - # the body and the prop['foo'] are the same concept 'foo' - assert cbody(concept_found_2) == get_expected(foo, "one two") - assert id(cprop(concept_found_2, "foo")) == id(cbody(concept_found_2)) - - -def test_i_can_use_a_reference_with_a_body(): - """ - Same test than before (test_i_can_use_reference()) - but this time, the concept 'foo' already has a body. - :return: - """ - - foo = Concept(name="foo", body="'foo'") - bar = Concept(name="bar") - grammar = {foo: Sequence("one", "two"), bar: foo} - context, parser = init([foo, bar], grammar) - res = parser.parse(context, "one two") - - assert len(res) == 2 - - assert res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [cnode("foo", 0, 2, "one two")] - concept_found_1 = res[0].value.body[0].concept - assert concept_found_1.metadata.body == "'foo'" - assert cbody(concept_found_1) is None - - assert res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [cnode("bar", 0, 2, "one two")] - concept_found_2 = res[1].value.body[0].concept - assert cbody(concept_found_2) == foo - # the body and the prop['foo'] are the same concept 'foo' - assert id(cprop(concept_found_2, "foo")) == id(cbody(concept_found_2)) - - -def test_i_can_use_context_reference_with_multiple_levels(): - """ - Same than previous one, but with reference of reference - :return: - """ - - foo = Concept(name="foo") - bar = Concept(name="bar") - baz = Concept(name="baz") - grammar = {foo: Sequence("one", "two"), bar: foo, baz: bar} - context, parser = init([foo, bar, baz], grammar) - - res = parser.parse(context, "one two") - assert len(res) == 3 - - assert res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [cnode("foo", 0, 2, "one two")] - concept_found_1 = res[0].value.body[0].concept - assert cbody(concept_found_1) == DoNotResolve("one two") - - assert res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [cnode("bar", 0, 2, "one two")] - concept_found_2 = res[1].value.body[0].concept - assert cbody(concept_found_2) == get_expected(foo, "one two") - assert id(cprop(concept_found_2, "foo")) == id(cbody(concept_found_2)) - - assert res[2].status - assert context.sheerka.isinstance(res[2].value, BuiltinConcepts.PARSER_RESULT) - assert res[2].value.body == [cnode("baz", 0, 2, "one two")] - concept_found_3 = res[2].value.body[0].concept - expected_foo = get_expected(foo, "one two") - assert cbody(concept_found_3) == get_expected(bar, expected_foo) - assert cprop(concept_found_3, "foo") == expected_foo - assert id(cprop(concept_found_3, "bar")) == id(cbody(concept_found_3)) - - -def test_order_is_not_important_when_using_references(): - """ - Same test than test_i_can_use_reference(), - but this time, 'bar' is declared before 'foo' - So the order of the result is different - :return: - """ - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = {bar: foo, foo: Sequence("one", "two")} - context, parser = init([foo, bar], grammar) - - res = parser.parse(context, "one two") - assert len(res) == 2 - assert res[0].value.body == [cnode("bar", 0, 2, "one two")] - assert res[1].value.body == [cnode("foo", 0, 2, "one two")] - - -def test_i_can_parse_when_reference(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = {bar: Sequence(foo, OrderedChoice("one", "two")), foo: OrderedChoice("twenty", "thirty")} - context, parser = init([foo, bar], grammar) - - res = parser.parse(context, "twenty two") - assert res.status - assert res.value.body == [cnode("bar", 0, 2, "twenty two")] - concept_found = res.value.body[0].concept - assert cbody(concept_found) == DoNotResolve("twenty two") - assert cprop(concept_found, "foo") == get_expected(foo, "twenty") - - res = parser.parse(context, "thirty one") - assert res.status - assert res.value.body == [cnode("bar", 0, 2, "thirty one")] - concept_found = res.value.body[0].concept - assert cbody(concept_found) == DoNotResolve("thirty one") - assert cprop(concept_found, "foo") == get_expected(foo, "thirty") - - res = parser.parse(context, "twenty") - assert res.status - assert res.value.body == [cnode("foo", 0, 0, "twenty")] - concept_found = res.value.body[0].concept - assert cbody(concept_found) == DoNotResolve("twenty") - - -def test_i_can_parse_when_reference_has_a_body(): - foo = Concept(name="foo", body="'one'") - bar = Concept(name="bar") - grammar = {bar: Sequence(foo, OrderedChoice("one", "two")), foo: OrderedChoice("twenty", "thirty")} - context, parser = init([foo, bar], grammar) - - res = parser.parse(context, "twenty two") - assert res.status - assert res.value.body == [cnode("bar", 0, 2, "twenty two")] - concept_found = res.value.body[0].concept - assert cbody(concept_found) == DoNotResolve("twenty two") - assert cprop(concept_found, "foo") == foo - - res = parser.parse(context, "twenty") - assert res.status - assert res.value.body == [cnode("foo", 0, 0, "twenty")] - concept_found = res.value.body[0].concept - assert concept_found.metadata.body == "'one'" - - -def test_i_can_parse_multiple_results(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = { - bar: Sequence("one", "two"), - foo: Sequence("one", OrderedChoice("two", "three")) - } - context, parser = init([foo, bar], grammar) - - res = parser.parse(context, "one two") - assert len(res) == 2 - assert res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [cnode("bar", 0, 2, "one two")] - concept_found_0 = res[0].value.body[0].concept - assert cbody(concept_found_0) == DoNotResolve("one two") - assert len(concept_found_0.props) == 0 - - assert res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [cnode("foo", 0, 2, "one two")] - concept_found_1 = res[1].value.body[0].concept - assert cbody(concept_found_1) == DoNotResolve("one two") - assert len(concept_found_1.props) == 0 - - -def test_i_can_parse_multiple_results_times_two(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = { - bar: Sequence("one", "two"), - foo: Sequence("one", OrderedChoice("two", "three")) - } - context, parser = init([foo, bar], grammar) - - res = parser.parse(context, "one two one two") - assert len(res) == 4 - assert res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [short_cnode("bar", "one two"), short_cnode("bar", "one two")] - - assert res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [short_cnode("foo", "one two"), short_cnode("bar", "one two")] - - assert res[2].status - assert context.sheerka.isinstance(res[2].value, BuiltinConcepts.PARSER_RESULT) - assert res[2].value.body == [short_cnode("bar", "one two"), short_cnode("foo", "one two")] - - assert res[3].status - assert context.sheerka.isinstance(res[3].value, BuiltinConcepts.PARSER_RESULT) - assert res[3].value.body == [short_cnode("foo", "one two"), short_cnode("foo", "one two")] - - -def test_i_can_parse_multiple_results_when_reference(): - """ - TODO : There should no be two answer, has the one with bar is totally useless - Note that bar = Sequence(foo, OrderedChoice("one", "two")) does not match - - :return: - """ - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = { - bar: Sequence(foo, Optional(OrderedChoice("one", "two"))), - foo: OrderedChoice("twenty", "thirty") - } - context, parser = init([foo, bar], grammar) - - res = parser.parse(context, "twenty") - assert len(res) == 2 - assert res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [cnode("bar", 0, 0, "twenty")] - - assert res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [cnode("foo", 0, 0, "twenty")] - - -def test_i_can_parse_concept_reference_that_is_not_in_grammar(): - one = Concept(name="one") - two = Concept(name="two") - foo = Concept(name="foo") - grammar = {foo: Sequence("twenty", OrderedChoice(one, two))} - context, parser = init([one, two, foo], grammar) - - res = parser.parse(context, "twenty two") - assert res.status - assert res.value.body == [cnode("foo", 0, 2, "twenty two")] - concept_found = res.value.body[0].concept - assert cbody(concept_found) == DoNotResolve("twenty two") - assert cprop(concept_found, "two") == get_expected(two, "two") - - res = parser.parse(context, "twenty one") - assert res.status - assert res.value.body == [cnode("foo", 0, 2, "twenty one")] - - -def test_i_can_parse_concept_reference_that_is_group(): - """ - if one is number, then number is a 'group' - a group can be found under the sdp entry 'all_' - """ - - context = get_context() - one = Concept(name="one") - two = Concept(name="two") - number = Concept(name="number") - foo = Concept(name="foo") - for c in [one, two, number, foo]: - context.sheerka.set_id_if_needed(c, False) - context.sheerka.add_in_cache(c) - - context.sheerka.add_concept_to_set(context, one, number) - context.sheerka.add_concept_to_set(context, two, number) - - grammar = {foo: Sequence("twenty", number)} - - parser = ConceptLexerParser() - parser.initialize(context, grammar) - - res = parser.parse(context, "twenty two") - assert res.status - assert res.value.body == [cnode("foo", 0, 2, "twenty two")] - concept_found = res.value.body[0].concept - assert cbody(concept_found) == DoNotResolve("twenty two") - assert cprop(concept_found, "two") == get_expected(two, "two") - assert cprop(concept_found, "number") == get_expected(number, get_expected(two, "two")) - - res = parser.parse(context, "twenty one") - assert res.status - assert res.value.body == [cnode("foo", 0, 2, "twenty one")] - concept_found = res.value.body[0].concept - assert cbody(concept_found) == DoNotResolve("twenty one") - assert cprop(concept_found, "one") == get_expected(one, "one") - assert cprop(concept_found, "number") == get_expected(number, get_expected(one, "one")) - - -def test_i_can_parse_zero_or_more(): - foo = Concept(name="foo") - grammar = {foo: ZeroOrMore("one")} - - context, res, wrapper, return_value = execute([foo], grammar, "one one") - - assert res.status - assert return_value == [cnode("foo", 0, 2, "one one")] - assert return_value[0].underlying == u(grammar[foo], 0, 2, [u("one", 0, 0), u("one", 2, 2)]) - - concept_found = return_value[0].concept - assert cbody(concept_found) == DoNotResolve("one one") - - -def test_i_can_parse_sequence_and_zero_or_more(): - foo = Concept(name="foo") - grammar = {foo: Sequence(ZeroOrMore("one"), "two")} - context, parser = init([foo], grammar) - - res = parser.parse(context, "one one two") - assert res.status - assert res.value.value == [cnode("foo", 0, 4, "one one two")] - - res = parser.parse(context, "two") - assert res.status - assert res.value.value == [cnode("foo", 0, 0, "two")] - - -def test_i_cannot_parse_zero_and_more_when_wrong_entry(): - foo = Concept(name="foo") - grammar = {foo: ZeroOrMore("one")} - context, parser = init([foo], grammar) - - parser = ConceptLexerParser() - parser.initialize(context, grammar) - - res = parser.parse(context, "one two") - assert not res.status - assert res.value.value == [ - cnode("foo", 0, 0, "one"), - UnrecognizedTokensNode(1, 2, [t(" "), t("two")]) - ] - - res = parser.parse(context, "two") - assert not res.status - assert res.value.value == [ - UnrecognizedTokensNode(0, 0, [t("two")]) - ] - - -def test_i_can_parse_zero_and_more_with_separator(): - foo = Concept(name="foo") - grammar = {foo: ZeroOrMore("one", sep=",")} - - context, res, wrapper, return_value = execute([foo], grammar, "one, one , one") - - assert res.status - assert return_value == [cnode("foo", 0, 7, "one, one , one")] - - -def test_that_zero_and_more_is_greedy(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = {foo: ZeroOrMore("one"), bar: "one"} - - context, res, wrapper, return_value = execute([foo], grammar, "one one one") - - assert res.status - assert return_value == [cnode("foo", 0, 4, "one one one")] - - -def test_i_can_parse_one_and_more(): - foo = Concept(name="foo") - grammar = {foo: OneOrMore("one")} - - context, res, wrapper, return_value = execute([foo], grammar, "one one") - - assert res.status - assert return_value == [cnode("foo", 0, 2, "one one")] - assert return_value[0].underlying == u(grammar[foo], 0, 2, [ - u("one", 0, 0), - u("one", 2, 2)]) - - -def test_i_can_parse_sequence_and_one_or_more(): - foo = Concept(name="foo") - grammar = {foo: Sequence(OneOrMore("one"), "two")} - context, parser = init([foo], grammar) - - res = parser.parse(context, "one one two") - assert res.status - assert res.value.value == [cnode("foo", 0, 4, "one one two")] - - res = parser.parse(context, "two") - assert not res.status - assert res.value.value == [ - UnrecognizedTokensNode(0, 0, [t("two")]) - ] - - -def test_i_can_parse_one_and_more_with_separator(): - foo = Concept(name="foo") - grammar = {foo: OneOrMore("one", sep=",")} - - context, res, wrapper, return_value = execute([foo], grammar, "one, one , one") - - assert res.status - assert return_value == [cnode("foo", 0, 7, "one, one , one")] - assert return_value[0].underlying == u(grammar[foo], 0, 7, [ - u("one", 0, 0), - u("one", 3, 3), - u("one", 7, 7)]) - - -def test_that_one_and_more_is_greedy(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = {foo: OneOrMore("one"), bar: "one"} - - context, res, wrapper, return_value = execute([foo], grammar, "one one one") - - assert res.status - assert return_value == [cnode("foo", 0, 4, "one one one")] - - -def test_i_can_detect_infinite_recursion(): - foo = Concept(name="foo") - bar = Concept(name="bar") - - grammar = { - bar: foo, - foo: bar - } - parser = ConceptLexerParser() - parser.initialize(get_context(), grammar) - - assert bar not in parser.concepts_grammars - assert foo not in parser.concepts_grammars - - -def test_i_can_detect_indirect_infinite_recursion_with_ordered_choice(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = { - bar: foo, - foo: OrderedChoice(bar, "foo") - } - - parser = ConceptLexerParser() - parser.initialize(get_context(), grammar) - - assert foo not in parser.concepts_grammars # removed because of the infinite recursion - assert bar not in parser.concepts_grammars # removed because of the infinite recursion - - # the other way around is possible - grammar = { - bar: foo, - foo: OrderedChoice("foo", bar) - } - context, parser = init([foo, bar], grammar) - - assert foo in parser.concepts_grammars - assert bar in parser.concepts_grammars - - res = parser.parse(context, "foo") - assert len(res) == 2 - assert res[0].status - assert res[0].value.body == [cnode("bar", 0, 0, "foo")] - assert res[1].status - assert res[1].value.body == [cnode("foo", 0, 0, "foo")] - - -def test_i_can_detect_indirect_infinite_recursion_with_sequence(): - foo = Concept(name="foo") - bar = Concept(name="bar") - - grammar = { - bar: foo, - foo: Sequence("one", bar, "two") - } - parser = ConceptLexerParser() - parser.initialize(get_context(), grammar) - - assert foo not in parser.concepts_grammars # removed because of the infinite recursion - assert bar not in parser.concepts_grammars # removed because of the infinite recursion - - -def test_i_can_detect_indirect_infinite_recursion_with_sequence_or_ordered_choice(): - foo = Concept(name="foo") - bar = Concept(name="bar") - - grammar = { - bar: foo, - foo: Sequence("one", OrderedChoice(bar, "other"), "two") - } - parser = ConceptLexerParser() - parser.initialize(get_context(), grammar) - - assert foo not in parser.concepts_grammars # removed because of the infinite recursion - assert bar not in parser.concepts_grammars # removed because of the infinite recursion - - -def test_infinite_recursion_does_not_fail_if_a_concept_is_missing(): - foo = Concept(name="foo") - bar = Concept(name="bar") - - grammar = { - foo: bar - } - parser = ConceptLexerParser() - parser.initialize(get_context(), grammar) - - assert foo in parser.concepts_grammars - - -def test_i_can_detect_indirect_infinite_recursion_with_optional(): - # TODO infinite recursion with optional - pass - - -def test_i_can_detect_indirect_infinite_recursion_with_zero_and_more(): - # TODO infinite recursion with optional - pass - - -def test_i_can_detect_indirect_infinite_recursion_with_one_and_more(): - # TODO infinite recursion with optional - pass - - -def test_i_can_visit_parsing_expression(): - mult = Concept(name="mult") - add = Concept(name="add") - - visitor = ConceptVisitor() - visitor.visit(Sequence(mult, Optional(Sequence("+", add)))) - - assert sorted(list(visitor.concepts)) == ["add", "mult"] - - -def test_i_can_initialize_rule_names(): - context = get_context() - foo = Concept(name="foo") - bar = Concept(name="bar") - - grammar = {foo: Sequence("one", "two"), bar: foo} - parser = ConceptLexerParser() - ret = parser.initialize(context, grammar) - return_value = ret.body - - assert return_value[foo].rule_name == "" - assert return_value[bar].rule_name == "foo" - - -@pytest.mark.parametrize("text, end_position", [ - ("foo", 0), - ("foo bar", 2), - ("foo bar ", 3), - (" foo bar ", 4) -]) -def test_cannot_parser_unknown_concepts(text, end_position): - context, res, wrapper, return_value = execute([], {}, text) - tokens = list(Tokenizer(text))[:-1] - - assert not res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [UnrecognizedTokensNode(0, end_position, tokens)] - - -def test_i_cannot_parse_when_part_of_the_input_is_unrecognized(): - one = Concept(name="one") - two = Concept(name="two") - grammar = {one: "one", two: "two"} - - context, res, wrapper, return_value = execute([one, two], grammar, "one two three") - - assert not res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [ - ConceptNode(get_expected(one, "one"), 0, 0, source="one", underlying=u("one", 0, 0)), - ConceptNode(get_expected(two, "two"), 2, 2, source="two", underlying=u("two", 2, 2)), - UnrecognizedTokensNode(3, 4, [t(" "), t("three")]) - ] - - -def test_i_cannot_parse_when_wrong_sequence(): - foo = Concept(name="foo") - grammar = {foo: Sequence("one", "two", "three")} - - context, res, wrapper, return_value = execute([foo], grammar, "one two three one") - - assert not res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [ - short_cnode("foo", "one two three"), - UnrecognizedTokensNode(5, 6, [t(" "), t("one")]) - ] - - -def test_i_cannot_parse_when_sequence_cannot_match_because_of_end_of_file(): - foo = Concept(name="foo") - grammar = {foo: Sequence("one", "two", "three")} - - context, res, wrapper, return_value = execute([foo], grammar, "one two") - - assert not res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [ - UnrecognizedTokensNode(0, 2, [t("one"), t(" "), t("two")]) - ] - - -def test_i_cannot_parse_multiple_results_when_unknown_tokens_at_the_end(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = { - bar: Sequence("one", "two"), - foo: Sequence("one", OrderedChoice("two", "three")) - } - context, parser = init([foo, bar], grammar) - - res = parser.parse(context, "one two four five") - - assert len(res) == 2 - assert not res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [ - cnode("bar", 0, 2, "one two"), - UnrecognizedTokensNode(3, 6, [t(" "), t("four"), t(" "), t("five")]) - ] - - assert not res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [ - cnode("foo", 0, 2, "one two"), - UnrecognizedTokensNode(3, 6, [t(" "), t("four"), t(" "), t("five")]) - ] - - -def test_i_cannot_parse_multiple_results_when_beginning_by_unknown_tokens(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = { - bar: Sequence("one", "two"), - foo: Sequence("one", OrderedChoice("two", "three")) - } - context, parser = init([foo, bar], grammar) - - res = parser.parse(context, "four five one two") - - assert len(res) == 2 - assert not res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [ - UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]), - cnode("bar", 4, 6, "one two"), - ] - - assert not res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [ - UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]), - cnode("foo", 4, 6, "one two"), - ] - - -def test_i_cannot_parse_multiple_results_when_surrounded_by_unknown_tokens(): - foo = Concept(name="foo") - bar = Concept(name="bar") - grammar = { - bar: Sequence("one", "two"), - foo: Sequence("one", OrderedChoice("two", "three")) - } - context, parser = init([foo, bar], grammar) - - res = parser.parse(context, "four five one two six seven") - assert len(res) == 2 - assert not res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [ - UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]), - cnode("bar", 4, 6, "one two"), - UnrecognizedTokensNode(7, 10, [t(" "), t("six"), t(" "), t("seven")]), - ] - - assert not res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [ - UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]), - cnode("foo", 4, 6, "one two"), - UnrecognizedTokensNode(7, 10, [t(" "), t("six"), t(" "), t("seven")]), - ] - - -def test_i_cannot_parse_multiple_results_when_unknown_tokens_in_the_middle(): - context = get_context() - foo = Concept(name="foo") - bar = Concept(name="bar") - baz = Concept(name="baz") - grammar = { - bar: Sequence("one", "two"), - foo: Sequence("one", OrderedChoice("two", "three")), - baz: StrMatch("six"), - } - context, parser = init([foo, bar, baz], grammar) - - res = parser.parse(context, "one two four five six") - assert len(res) == 2 - assert not res[0].status - assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT) - assert res[0].value.body == [ - cnode("bar", 0, 2, "one two"), - UnrecognizedTokensNode(3, 7, [t(" "), t("four"), t(" "), t("five"), t(" ")]), - cnode("baz", 8, 8, "six"), - ] - - assert not res[1].status - assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT) - assert res[1].value.body == [ - cnode("foo", 0, 2, "one two"), - UnrecognizedTokensNode(3, 7, [t(" "), t("four"), t(" "), t("five"), t(" ")]), - cnode("baz", 8, 8, "six"), - ] - - -def test_i_can_get_the_inner_concept_when_possible(): - foo = Concept(name="foo") - one = Concept(name="one") - grammar = {foo: Sequence(Optional(ZeroOrMore(one)), ZeroOrMore("one"))} - - context, res, wrapper, return_value = execute([foo, one], grammar, "one") - - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [cnode("foo", 0, 0, "one")] - concept_found = return_value[0].concept - assert cbody(concept_found) == get_expected(one, "one") - assert id(cprop(concept_found, "one")) == id(cbody(concept_found)) - - -def test_i_can_get_the_inner_concept_when_possible_with_rule_name(): - foo = Concept(name="foo") - one = Concept(name="one") - grammar = {foo: Sequence( - Optional(ZeroOrMore(one, rule_name="zero"), rule_name="opt"), - ZeroOrMore("one"), rule_name="seq")} - - context, res, wrapper, return_value = execute([foo, one], grammar, "one") - - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [cnode("foo", 0, 0, "one")] - concept_found = return_value[0].concept - assert cbody(concept_found) == get_expected(one, "one") - assert id(cprop(concept_found, "one")) == id(cbody(concept_found)) - assert id(cprop(concept_found, "zero")) == id(cbody(concept_found)) - assert id(cprop(concept_found, "opt")) == id(cbody(concept_found)) - assert id(cprop(concept_found, "seq")) == id(cbody(concept_found)) - - -def test_i_get_multiple_props_when_zero_or_more(): - foo = Concept(name="foo") - one = Concept(name="one") - grammar = {foo: ZeroOrMore(one)} - - context, res, wrapper, return_value = execute([foo, one], grammar, "one one one") - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [cnode("foo", 0, 4, "one one one")] - concept_found = return_value[0].concept - assert cbody(concept_found) == DoNotResolve("one one one") - assert len(concept_found.compiled["one"]) == 3 - assert cprop(concept_found, "one")[0] == get_expected(one) - assert cprop(concept_found, "one")[1] == get_expected(one) - assert cprop(concept_found, "one")[2] == get_expected(one) - assert id(cprop(concept_found, "one")[0]) != id(cprop(concept_found, "one")[1]) - assert id(cprop(concept_found, "one")[1]) != id(cprop(concept_found, "one")[2]) - assert id(cprop(concept_found, "one")[2]) != id(cprop(concept_found, "one")[0]) - - -def test_i_get_multiple_props_when_zero_or_more_and_different_values(): - foo = Concept(name="foo") - one = Concept(name="one") - grammar = {foo: ZeroOrMore(Sequence(one, "ok", rule_name="seq")), one: OrderedChoice("one", "un", "uno")} - - context, res, wrapper, return_value = execute([foo, one], grammar, "one ok un ok uno ok") - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert return_value == [short_cnode("foo", "one ok un ok uno ok")] - concept_found = return_value[0].concept - assert cprop(concept_found, "one")[0] == get_expected(one, "one") - assert cprop(concept_found, "one")[1] == get_expected(one, "un") - assert cprop(concept_found, "one")[2] == get_expected(one, "uno") - assert cprop(concept_found, "seq")[0] == DoNotResolve("one ok") - assert cprop(concept_found, "seq")[1] == DoNotResolve("un ok") - assert cprop(concept_found, "seq")[2] == DoNotResolve("uno ok") - - -# -# def test_i_can_parse_basic_arithmetic_operations_and_resolve_properties(): -# context = get_context() -# add = Concept(name="add") -# mult = Concept(name="mult") -# atom = Concept(name="atom") -# -# grammar = { -# add: Sequence(mult, Optional(Sequence(OrderedChoice('+', '-', rule_name="sign"), add))), -# mult: Sequence(atom, Optional(Sequence(OrderedChoice('*', '/'), mult))), -# atom: OrderedChoice(OrderedChoice('1', '2', '3'), Sequence('(', add, ')')), -# } -# -# parser = ConceptLexerParser() -# parser.register(grammar) -# -# # res = parser.parse(context, "1") -# # assert len(res) == 3 # add, mult, atom -# # -# # res = parser.parse(context, "1 * 2") -# # assert len(res) == 2 # add and mult -# # -# # res = parser.parse(context, "1 + 2") -# # assert res.status -# # assert return_value == [ConceptNode(add, 0, 4, source="1 + 2")] -# -# res = parser.parse(context, "1 * 2 + 3") -# assert res.status -# assert return_value == [ConceptNode(add, 0, 4, source="1 + 2 + 3")] - -def test_i_can_register_concepts_with_the_same_name(): - # TODO : concepts are registered by name, - # what when two concepts have the same name ? - pass - - -def test_i_can_parse_very_very_long_input(): - # TODO: In the current implementation, all the tokens are loaded in memory - # It's clearly not the good approach - pass diff --git a/tests/test_ConceptsWithConceptsParser.py b/tests/test_ConceptsWithConceptsParser.py deleted file mode 100644 index b04f913..0000000 --- a/tests/test_ConceptsWithConceptsParser.py +++ /dev/null @@ -1,204 +0,0 @@ -import ast - -import pytest - -from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from core.tokenizer import Token, TokenKind, Tokenizer -from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode -from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser -from parsers.MultipleConceptsParser import MultipleConceptsParser -from parsers.PythonParser import PythonNode -from sdp.sheerkaDataProvider import Event - -multiple_concepts_parser = MultipleConceptsParser() - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def get_ret_from(*args): - result = [] - index = 0 - source = "" - for item in args: - if isinstance(item, Concept): - tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)] - result.append(ConceptNode(item, index, index, tokens, item.name)) - index += 1 - source += item.name - elif isinstance(item, PythonNode): - tokens = list(Tokenizer(item.source))[:-1] # strip trailing EOF - result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source)) - index += len(tokens) - source += item.source - else: - tokens = list(Tokenizer(item))[:-1] # strip trailing EOF - result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens)) - index += len(tokens) - source += item - - return ReturnValueConcept( - "who", - False, - ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source)) - - -def init(concepts, inputs): - context = get_context() - for concept in concepts: - context.sheerka.create_new_concept(context, concept) - - return context, get_ret_from(*inputs) - - -def execute(concepts, inputs): - context, input_return_values = init(concepts, inputs) - - parser = ConceptsWithConceptsParser() - result = parser.parse(context, input_return_values.body) - - wrapper = result.body - return_value = result.body.body - - return context, parser, result, wrapper, return_value - - -@pytest.mark.parametrize("text, interested", [ - ("not parser result", False), - (ParserResultConcept(parser="not multiple_concepts_parser"), False), - (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True), -]) -def test_not_interested(text, interested): - context = get_context() - - res = ConceptsWithConceptsParser().parse(context, text) - if interested: - assert res is not None - else: - assert res is None - - -def test_i_can_parse_composition_of_concepts(): - foo = Concept("foo") - bar = Concept("bar") - plus = Concept("a plus b").def_prop("a").def_prop("b") - - context, parser, result, wrapper, return_value = execute([foo, bar, plus], [foo, " plus ", bar]) - - assert result.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert result.who == wrapper.parser.name - assert wrapper.source == "foo plus bar" - assert context.sheerka.isinstance(return_value, plus) - - assert return_value.compiled["a"] == foo - assert return_value.compiled["b"] == bar - - # sanity check, I can evaluate the result - evaluated = context.sheerka.evaluate_concept(context, return_value) - assert evaluated.key == return_value.key - assert evaluated.get_prop("a") == foo.init_key() - assert evaluated.get_prop("b") == bar.init_key() - - -def test_i_can_parse_when_composition_of_source_code(): - plus = Concept("a plus b", body="a + b").def_prop("a").def_prop("b") - left = PythonNode("1+1", ast.parse("1+1", mode="eval")) - right = PythonNode("2+2", ast.parse("2+2", mode="eval")) - context, parser, result, wrapper, return_value = execute([plus], [left, " plus ", right]) - - assert result.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert result.who == wrapper.parser.name - assert wrapper.source == "1+1 plus 2+2" - assert context.sheerka.isinstance(return_value, plus) - - left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left) - right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right) - assert return_value.compiled["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)] - assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)] - - # sanity check, I can evaluate the result - evaluated = context.sheerka.evaluate_concept(context, return_value) - assert evaluated.key == return_value.key - assert evaluated.get_prop("a") == 2 - assert evaluated.get_prop("b") == 4 - assert evaluated.body == 6 - - -def test_i_can_parse_when_mix_of_concept_and_code(): - plus = Concept("a plus b").def_prop("a").def_prop("b") - code = PythonNode("1+1", ast.parse("1+1", mode="eval")) - foo = Concept("foo") - context, parser, result, wrapper, return_value = execute([plus, foo], [foo, " plus ", code]) - - assert result.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert result.who == wrapper.parser.name - assert wrapper.source == "foo plus 1+1" - assert context.sheerka.isinstance(return_value, plus) - - code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code) - assert return_value.compiled["a"] == foo - assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)] - - # sanity check, I can evaluate the result - evaluated = context.sheerka.evaluate_concept(context, return_value) - assert evaluated.key == return_value.key - assert evaluated.get_prop("a") == foo.init_key() - assert evaluated.get_prop("b") == 2 - - -def test_i_can_parse_when_multiple_concepts_are_recognized(): - foo = Concept("foo") - bar = Concept("bar") - plus_1 = Concept("a plus b", body="body1").def_prop("a").def_prop("b") - plus_2 = Concept("a plus b", body="body2").def_prop("a").def_prop("b") - - context, input_return_values = init([foo, bar, plus_1, plus_2], [foo, " plus ", bar]) - parser = ConceptsWithConceptsParser() - result = parser.parse(context, input_return_values.body) - - assert len(result) == 2 - - res = result[0] - wrapper = res.value - return_value = res.value.value - assert res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert res.who == wrapper.parser.name - assert wrapper.source == "foo plus bar" - assert context.sheerka.isinstance(return_value, plus_1) - assert return_value.compiled["a"] == foo - assert return_value.compiled["b"] == bar - - res = result[1] - wrapper = res.value - return_value = res.value.value - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert res.who == wrapper.parser.name - assert wrapper.source == "foo plus bar" - assert context.sheerka.isinstance(return_value, plus_2) - assert return_value.compiled["a"] == foo - assert return_value.compiled["b"] == bar - - -def test_i_cannot_parse_when_unknown_concept(): - foo = Concept("foo") - bar = Concept("bar") - - context, input_return_values = init([foo, bar], [foo, " plus ", bar]) - parser = ConceptsWithConceptsParser() - result = parser.parse(context, input_return_values.body) - wrapper = result.body - return_value = result.body.body - - assert not result.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME) - assert result.who == parser.name - assert return_value == input_return_values.body.body diff --git a/tests/test_DefaultParser.py b/tests/test_DefaultParser.py deleted file mode 100644 index 1cc07f9..0000000 --- a/tests/test_DefaultParser.py +++ /dev/null @@ -1,368 +0,0 @@ -import pytest -import ast - -from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptExpression -from parsers.PythonParser import PythonParser, PythonNode -from core.tokenizer import Keywords, Tokenizer, LexerError -from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode -from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode -from parsers.BnfParser import BnfParser - -from sdp.sheerkaDataProvider import Event - - -def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None): - def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name)))) - - if body: - def_concept.body = get_concept_part(body) - if where: - def_concept.where = get_concept_part(where) - if pre: - def_concept.pre = get_concept_part(pre) - if post: - def_concept.post = get_concept_part(post) - if definition: - def_concept.definition = ReturnValueConcept( - "parsers.Bnf", - True, - definition) - - return def_concept - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def get_concept_part(part): - if isinstance(part, str): - node = PythonNode(part, ast.parse(part, mode="eval")) - return ReturnValueConcept( - who="parsers.Default", - status=True, - value=ParserResultConcept( - source=part, - parser=PythonParser(), - value=node)) - - if isinstance(part, PythonNode): - return ReturnValueConcept( - who="parsers.Default", - status=True, - value=ParserResultConcept( - source=part.source, - parser=PythonParser(), - value=part)) - - if isinstance(part, ReturnValueConcept): - return part - - -@pytest.mark.parametrize("text, expected", [ - ("def concept hello", get_def_concept(name="hello")), - ("def concept hello ", get_def_concept(name="hello")), - ("def concept a + b", get_def_concept(name="a + b")), - ("def concept a+b", get_def_concept(name="a + b")), - ("def concept 'a+b'+c", get_def_concept(name="'a+b' + c")), - ("def concept 'as if'", get_def_concept(name="'as if'")), - ("def concept 'as' if", get_def_concept(name="'as if'")), - ("def concept hello as 'hello'", get_def_concept(name="hello", body="'hello'")), - ("def concept hello as 1", get_def_concept(name="hello", body="1")), - ("def concept hello as 1 + 1", get_def_concept(name="hello", body="1 + 1")), -]) -def test_i_can_parse_def_concept(text, expected): - parser = DefaultParser() - res = parser.parse(get_context(), text) - node = res.value.value - - assert res.status - assert res.who == parser.name - assert res.value.source == text - assert isinstance(res.value, ParserResultConcept) - assert node == expected - - -def test_i_can_parse_complex_def_concept_statement(): - text = """def concept a plus b -where a,b -pre isinstance(a, int) and isinstance(b, float) -post isinstance(res, int) -as res = a + b -""" - parser = DefaultParser() - res = parser.parse(get_context(), text) - return_value = res.value - expected_concept = get_def_concept( - name="a plus b", - where="a,b", - pre="isinstance(a, int) and isinstance(b, float)", - post="isinstance(res, int)", - body=PythonNode("res = a + b", ast.parse("res = a + b", mode="exec")) - ) - - assert res.status - assert isinstance(return_value, ParserResultConcept) - assert return_value.value == expected_concept - - -def test_i_can_have_mutilines_declarations(): - text = """ -def concept add one to a as -def func(x): - return x+1 -func(a) - """ - - expected_concept = get_def_concept( - name="add one to a ", - body=PythonNode( - "def func(x):\n return x+1\nfunc(a)", - ast.parse("def func(x):\n return x+1\nfunc(a)", mode="exec")) - ) - - parser = DefaultParser() - res = parser.parse(get_context(), text) - return_value = res.value - - assert res.status - assert isinstance(return_value, ParserResultConcept) - assert return_value.value == expected_concept - - -def test_i_can_use_colon_to_use_indentation(): - text = """ -def concept add one to a as: - def func(x): - return x+1 - func(a) -""" - - expected_concept = get_def_concept( - name="add one to a ", - body=PythonNode( - "def func(x):\n return x+1\nfunc(a)", - ast.parse("def func(x):\n return x+1\nfunc(a)", mode="exec")) - ) - - parser = DefaultParser() - res = parser.parse(get_context(), text) - return_value = res.value - - assert res.status - assert isinstance(return_value, ParserResultConcept) - assert return_value.value == expected_concept - - -def test_indentation_is_mandatory_after_a_colon(): - text = """ -def concept add one to a as: -def func(x): - return x+1 -func(a) -""" - - parser = DefaultParser() - res = parser.parse(get_context(), text) - return_value = res.value - - assert not res.status - assert isinstance(return_value, ParserResultConcept) - assert isinstance(return_value.value[0], SyntaxErrorNode) - assert return_value.value[0].message == "Indentation not found." - - -def test_indentation_is_not_allowed_if_the_colon_is_missing(): - text = """ -def concept add one to a as - def func(x): - return x+1 - func(a) - """ - context = get_context() - sheerka = context.sheerka - - parser = DefaultParser() - res = parser.parse(context, text) - return_value = res.value - - assert not res.status - assert context.sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS) - - -def test_name_is_mandatory(): - text = "def concept as 'hello'" - - parser = DefaultParser() - res = parser.parse(get_context(), text) - return_value = res.value - - assert not res.status - assert isinstance(return_value, ParserResultConcept) - assert isinstance(return_value.value[0], SyntaxErrorNode) - assert return_value.value[0].message == "Name is mandatory" - - -def test_concept_keyword_is_mandatory_but_the_concept_is_recognized(): - text = "def hello as a where b pre c post d" - - expected_concept = get_def_concept(name="hello", body="a", where="b", pre="c", post="d") - parser = DefaultParser() - res = parser.parse(get_context(), text) - return_value = res.value - - assert not res.status - assert isinstance(return_value, ParserResultConcept) - assert isinstance(return_value.value[0], UnexpectedTokenErrorNode) - assert return_value.value[0].message == "Syntax error." - assert return_value.value[0].expected_tokens == [Keywords.CONCEPT] - assert return_value.try_parsed == expected_concept - - -@pytest.mark.parametrize("text", [ - "def concept hello where 1+", - "def concept hello pre 1+", - "def concept hello post 1+", - "def concept hello as 1+" -]) -def test_i_can_detect_error_in_declaration(text): - context = get_context() - sheerka = context.sheerka - - parser = DefaultParser() - res = parser.parse(context, text) - return_value = res.value - - assert not res.status - assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS) - - -def test_new_line_is_not_allowed_in_the_name(): - text = "def concept hello \n my friend as 'hello'" - - parser = DefaultParser() - res = parser.parse(get_context(), text) - return_value = res.value - - assert not res.status - assert return_value.value == [SyntaxErrorNode([], "Newline are not allowed in name.")] - - -def test_i_can_parse_def_concept_from_regex(): - context = get_context() - a_concept = Concept("a_concept") - context.sheerka.add_in_cache(a_concept) - - text = "def concept name from bnf a_concept | 'a_string' as __definition[0]" - parser = DefaultParser() - res = parser.parse(context, text) - node = res.value.value - definition = OrderedChoice(a_concept, StrMatch("a_string")) - parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition) - expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result) - - assert res.status - assert res.who == parser.name - assert res.value.source == text - assert isinstance(res.value, ParserResultConcept) - assert node == expected - - -def test_i_can_parse_def_concept_where_bnf_references_itself(): - context = get_context() - a_concept = Concept("a_concept") - context.sheerka.add_in_cache(a_concept) - - text = "def concept name from bnf 'a' + name?" - parser = DefaultParser() - parser.parse(context, text) - - assert not parser.has_error - - -def test_i_can_detect_empty_bnf_declaration(): - text = "def concept name from bnf as __definition[0]" - - parser = DefaultParser() - res = parser.parse(get_context(), text) - - assert not res.status - assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration") - - -def test_i_can_detect_not_for_me(): - text = "hello world" - context = get_context() - parser = DefaultParser() - res = parser.parse(context, text) - - assert not res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME) - assert isinstance(res.value.body[0], CannotHandleErrorNode) - - -def test_i_can_parse_is_a(): - parser = DefaultParser() - text = "the name of my 'concept' isa the name of the set" - res = parser.parse(get_context(), text) - expected = IsaConceptNode([], - concept=NameNode(list(Tokenizer("the name of my 'concept'"))), - set=NameNode(list(Tokenizer("the name of the set")))) - - assert res.status - assert res.who == parser.name - assert res.value.source == text - assert isinstance(res.value, ParserResultConcept) - assert res.value.value == expected - - -@pytest.mark.parametrize("text", [ - "concept", - "isa number", - "name isa", - "def", - "def concept_name" -]) -def test_i_cannot_parse_invalid_entries(text): - parser = DefaultParser() - res = parser.parse(get_context(), text) - - assert not res.status - assert isinstance(res.body, ParserResultConcept) - assert isinstance(res.body.body[0], UnexpectedTokenErrorNode) - - -@pytest.mark.parametrize("text, error_msg, error_text", [ - ("'name", "Missing Trailing quote", "'name"), - ("foo isa 'name", "Missing Trailing quote", "'name"), - ("def concept 'name", "Missing Trailing quote", "'name"), - ("def concept name as 'body", "Missing Trailing quote", "'body"), - ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"), - ("def concept c::", "Concept name not found", ""), -]) -def test_i_cannot_parse_when_tokenizer_fails(text, error_msg, error_text): - parser = DefaultParser() - res = parser.parse(get_context(), text) - - assert not res.status - assert isinstance(res.body, ParserResultConcept) - assert isinstance(res.body.body[0], LexerError) - assert res.body.body[0].message == error_msg - assert res.body.body[0].text == error_text - - -def test_i_cannot_parse_bnf_definition_referencing_unknown_concept(): - context = get_context() - text = "def concept name from bnf unknown" - - parser = DefaultParser() - res = parser.parse(context, text) - - assert not res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) - assert res.value.body == ("key", "unknown") diff --git a/tests/test_EvalEvaluator.py b/tests/test_EvalEvaluator.py deleted file mode 100644 index d21276e..0000000 --- a/tests/test_EvalEvaluator.py +++ /dev/null @@ -1,80 +0,0 @@ -import pytest - -from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from evaluators.EvalEvaluator import EvalEvaluator -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def r(value, status=True): - return ReturnValueConcept("some_name", status, value) - - -eval_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.CONCEPT_EVAL_REQUESTED)) - - -def test_i_can_match_and_eval(): - context = get_context() - - to_eval1 = ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init()) - to_eval2 = ReturnValueConcept("some_name", True, Concept(name="3", body="also to eval").auto_init()) - - return_values = [ - ReturnValueConcept("some_name", True, "not to eval"), - ReturnValueConcept("some_name", True, Concept(name="not to eval")), - ReturnValueConcept("some_name", False, Concept(name="1", body="'not to eval'")), - to_eval1, - to_eval2, - eval_requested - ] - - evaluator = EvalEvaluator() - assert evaluator.matches(context, return_values) - - evaluated = evaluator.eval(context, return_values) - assert len(evaluated) == 2 - assert evaluated[0].value == to_eval1.body.body - assert evaluated[0].parents == [to_eval1, eval_requested] - - assert evaluated[1].value == to_eval2.body.body - assert evaluated[1].parents == [to_eval2, eval_requested] - - -@pytest.mark.parametrize("return_values, expected", [ - ([r(Concept("foo", body="bar")), eval_requested], True), - ([r(Concept("status is false", body="bar"), False), eval_requested], True), - ([r("string_value"), eval_requested], True), - ([r(Concept("no body")), eval_requested], True), - ([r(Concept("eval requested missing", body="bar"))], False), -]) -def test_i_cannot_match_if_eval_request_is_not_present(return_values, expected): - context = get_context() - assert EvalEvaluator().matches(context, return_values) == expected - - -def test_concept_eval_requested_is_reduced_when_nothing_to_reduce(): - context = get_context() - - return_values = [ - ReturnValueConcept("some_name", True, "not to eval"), - ReturnValueConcept("some_name", True, Concept(name="not to eval")), - ReturnValueConcept("some_name", False, Concept(name="1", body="not to eval")), - eval_requested - ] - - evaluator = EvalEvaluator() - assert evaluator.matches(context, return_values) - - evaluated = evaluator.eval(context, return_values) - assert evaluated == ReturnValueConcept( - "evaluators.Eval", - False, - context.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_REQUESTED)) - assert evaluated.parents == [eval_requested] diff --git a/tests/test_ExactConceptParser.py b/tests/test_ExactConceptParser.py deleted file mode 100644 index b36ad8d..0000000 --- a/tests/test_ExactConceptParser.py +++ /dev/null @@ -1,151 +0,0 @@ -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, Property -from core.sheerka import Sheerka, ExecutionContext -from core.tokenizer import Tokenizer -from parsers.ExactConceptParser import ExactConceptParser -from sdp.sheerkaDataProvider import Event - - -def metadata_prop(concept, prop_name): - for name, value in concept.metadata.props: - if name == prop_name: - return value - - return None - - -def test_i_can_compute_combinations(): - parser = ExactConceptParser() - res = parser.combinations(["foo", "bar", "baz"]) - - assert res == {('foo', 'bar', 'baz'), - ('__var__0', 'bar', 'baz'), - ('foo', '__var__0', 'baz'), - ('foo', 'bar', '__var__0'), - ('__var__0', '__var__1', 'baz'), - ('__var__0', 'bar', '__var__1'), - ('foo', '__var__0', '__var__1'), - ('__var__0', '__var__1', '__var__2')} - - -def test_i_can_compute_combinations_with_duplicates(): - parser = ExactConceptParser() - res = parser.combinations(["foo", "bar", "foo"]) - - assert res == {('foo', 'bar', 'foo'), - ('__var__0', 'bar', '__var__0'), - ('foo', '__var__0', 'foo'), - ('__var__0', '__var__1', '__var__0'), - ('__var__1', '__var__0', '__var__1')} - # TODO: the last tuple is not possible, so the algo can be improved - - -def test_i_can_recognize_a_simple_concept(): - context = get_context() - concept = get_concept("hello world", []) - context.sheerka.add_in_cache(concept) - - source = "hello world" - results = ExactConceptParser().parse(context, source) - - assert len(results) == 1 - assert results[0].status - assert results[0].value.value == concept - - -def test_i_can_recognize_concepts_defined_several_times(): - context = get_context() - context.sheerka.add_in_cache(get_concept("hello world", [])) - context.sheerka.add_in_cache(get_concept("hello a", ["a"])) - - source = "hello world" - results = ExactConceptParser().parse(context, source) - - assert len(results) == 2 - results = sorted(results, key=lambda x: x.value.value.name) # because of the usage of sets - - assert results[0].status - assert results[0].value.value.name == "hello a" - assert metadata_prop(results[0].value.value, "a") == "world" - - assert results[1].status - assert results[1].value.value.name == "hello world" - - -def test_i_can_recognize_a_concept_with_variables(): - context = get_context() - concept = get_concept("a + b", ["a", "b"]) - context.sheerka.add_in_cache(concept) - source = "10 + 5" - results = ExactConceptParser().parse(context, source) - - assert len(results) == 1 - assert results[0].status - concept_found = results[0].value.value - assert concept_found.key == concept.key - assert metadata_prop(concept_found, "a") == "10" - assert metadata_prop(concept_found, "b") == "5" - - -def test_i_can_recognize_a_concept_with_duplicate_variables(): - context = get_context() - concept = get_concept("a + b + a", ["a", "b"]) - context.sheerka.cache_by_key[concept.key] = concept - source = "10 + 5 + 10" - results = ExactConceptParser().parse(context, source) - - assert len(results) == 1 - assert results[0].status - concept_found = results[0].value.value - assert concept_found.key == concept.key - assert metadata_prop(concept_found, "a") == "10" - assert metadata_prop(concept_found, "b") == "5" - - -def test_i_can_manage_unknown_concept(): - context = get_context() - source = "def concept hello world" # this is not a concept by itself - res = ExactConceptParser().parse(context, source) - - assert not res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT) - assert res.value.body == "def concept hello world" - - -def test_i_can_detect_concepts_too_long(): - context = get_context() - source = "a very very long concept that cannot be an unique one" - res = ExactConceptParser().parse(context, source) - - assert not res.status - assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_TOO_LONG) - assert res.value.body == "a very very long concept that cannot be an unique one" - - -def test_i_can_detect_concept_from_tokens(): - context = get_context() - concept = get_concept("hello world", []) - context.sheerka.add_in_cache(concept) - - source = "hello world" - results = ExactConceptParser().parse(context, list(Tokenizer(source))) - - assert len(results) == 1 - assert results[0].status - assert results[0].value.value == concept - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - - return ExecutionContext("sheerka", Event(), sheerka) - - -def get_concept(name, variables): - c = Concept(name=name) - if variables: - for v in variables: - c.def_prop(v) - c.init_key() - return c diff --git a/tests/test_LexerNodeEvaluator.py b/tests/test_LexerNodeEvaluator.py deleted file mode 100644 index eceda82..0000000 --- a/tests/test_LexerNodeEvaluator.py +++ /dev/null @@ -1,115 +0,0 @@ -import ast - -import pytest - -from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts -from core.concept import Concept, ConceptParts, DoNotResolve -from core.sheerka import Sheerka, ExecutionContext -from evaluators.LexerNodeEvaluator import LexerNodeEvaluator -from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, StrMatch, UnrecognizedTokensNode, SourceCodeNode -from parsers.PythonParser import PythonNode -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def from_parsing(context, grammar, expression): - parser = ConceptLexerParser() - parser.initialize(context, grammar) - - ret_val = parser.parse(context, expression) - assert ret_val.status - return ret_val - - -def from_fragments(*fragments): - nodes = [] - for fragment in fragments: - if isinstance(fragment, str): - node = PythonNode(fragment, ast.parse(fragment.strip(), mode="eval")) - nodes.append(SourceCodeNode(node, 0, 0, [], fragment)) - else: - nodes.append(ConceptNode(fragment, 0, 0, [], fragment.name)) - - return ReturnValueConcept("somme_name", True, ParserResultConcept(value=nodes)) - - -def init(concept, grammar, text): - context = get_context() - if isinstance(concept, list): - for c in concept: - context.sheerka.add_in_cache(c) - else: - context.sheerka.add_in_cache(concept) - ret_val = from_parsing(context, grammar, text) - node = ret_val.value.value[0] - - return context, node - - -@pytest.mark.parametrize("ret_val, expected", [ - (ReturnValueConcept("some_name", True, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), True), - (ReturnValueConcept("some_name", True, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), True), - (ReturnValueConcept("some_name", True, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), True), - (ReturnValueConcept("some_name", True, ParserResultConcept(value=SourceCodeNode(0, 0, []))), True), - (ReturnValueConcept("some_name", True, ParserResultConcept(value=[UnrecognizedTokensNode(0, 0, [])])), False), - (ReturnValueConcept("some_name", True, ParserResultConcept(value=UnrecognizedTokensNode(0, 0, []))), False), - (ReturnValueConcept("some_name", False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), False), - (ReturnValueConcept("some_name", False, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), False), - (ReturnValueConcept("some_name", False, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), False), - (ReturnValueConcept("some_name", False, ParserResultConcept(value=SourceCodeNode(0, 0, []))), False), - (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept node")), False), - (ReturnValueConcept("some_name", True, ParserResultConcept(value=["Not a concept node"])), False), - (ReturnValueConcept("some_name", True, [ConceptNode(Concept(), 0, 0)]), False), - (ReturnValueConcept("some_name", True, ConceptNode(Concept(), 0, 0)), False), -]) -def test_i_can_match(ret_val, expected): - context = get_context() - assert LexerNodeEvaluator().matches(context, ret_val) == expected - - -def test_concept_is_returned_when_only_one_in_the_list(): - foo = Concept("foo") - context = get_context() - context.sheerka.add_in_cache(foo) - - ret_val = from_parsing(context, {foo: StrMatch("foo")}, "foo") - - evaluator = LexerNodeEvaluator() - result = evaluator.eval(context, ret_val) - wrapper = result.body - return_value = result.body.body - - assert result.who == evaluator.name - assert result.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert wrapper.parser == evaluator - assert wrapper.source == "foo" - assert return_value == Concept("foo").init_key() - assert return_value.compiled[ConceptParts.BODY] == DoNotResolve("foo") - assert result.parents == [ret_val] - - -def test_concept_python_node_is_returned_when_source_code(): - context = get_context() - foo = Concept("foo") - ret_val = from_fragments(foo, " + 1") - - evaluator = LexerNodeEvaluator() - result = evaluator.eval(context, ret_val) - wrapper = result.body - return_value = result.body.body - - assert result.who == evaluator.name - assert result.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert wrapper.parser == evaluator - assert wrapper.source == "foo + 1" - - assert return_value == PythonNode('foo + 1', ast.parse("__C__foo__C__ + 1", mode="eval")) - assert return_value.concepts == {"__C__foo__C__": foo} - assert result.parents == [ret_val] diff --git a/tests/test_MultipleConceptsParser.py b/tests/test_MultipleConceptsParser.py deleted file mode 100644 index 36ecd87..0000000 --- a/tests/test_MultipleConceptsParser.py +++ /dev/null @@ -1,231 +0,0 @@ -import pytest - -from core.builtin_concepts import ParserResultConcept, BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from core.tokenizer import Tokenizer, TokenKind, Token -from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, cnode, utnode, scnode, SourceCodeNode -from parsers.MultipleConceptsParser import MultipleConceptsParser -from parsers.PythonParser import PythonNode -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def get_return_value(context, grammar, expression): - parser = ConceptLexerParser() - parser.initialize(context, grammar) - - ret_val = parser.parse(context, expression) - assert not ret_val.status - return ret_val - - -def init(concepts, grammar, expression): - context = get_context() - for c in concepts: - context.sheerka.create_new_concept(context, c) - return_value = get_return_value(context, grammar, expression) - - return context, return_value - - -def test_not_interested_if_not_parser_result(): - context = get_context() - text = "not parser result" - - res = MultipleConceptsParser().parse(context, text) - assert res is None - - -def test_not_interested_if_not_from_concept_lexer_parser(): - context = get_context() - text = ParserResultConcept(parser="not concept lexer", value="some value") - - res = MultipleConceptsParser().parse(context, text) - assert res is None - - -def test_i_can_parse_exact_concepts(): - foo = Concept("foo", body="'foo'") - bar = Concept("bar", body="'bar'") - baz = Concept("baz", body="'baz'") - grammar = {} - context, return_value = init([foo, bar, baz], grammar, "bar foo baz") - - parser = MultipleConceptsParser() - ret_val = parser.parse(context, return_value.body) - - assert ret_val.status - assert ret_val.who == parser.name - assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) - assert ret_val.value.value == [ - ConceptNode(bar, 0, 0, source="bar"), - ConceptNode(foo, 2, 2, source="foo"), - ConceptNode(baz, 4, 4, source="baz")] - assert ret_val.value.source == "bar foo baz" - - -def test_i_can_parse_when_ending_with_bnf(): - foo = Concept("foo", body="'foo'") - bar = Concept("bar", body="'bar'") - grammar = {foo: Sequence("foo1", "foo2", "foo3")} - context, return_value = init([foo, bar], grammar, "bar foo1 foo2 foo3") - - parser = MultipleConceptsParser() - ret_val = parser.parse(context, return_value.body) - - assert ret_val.status - assert ret_val.who == parser.name - assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) - assert ret_val.value.value == [cnode("bar", 0, 0, "bar"), cnode("foo", 2, 6, "foo1 foo2 foo3")] - assert ret_val.value.source == "bar foo1 foo2 foo3" - - -def test_i_can_parse_when_starting_with_bnf(): - foo = Concept("foo", body="'foo'") - bar = Concept("bar", body="'bar'") - grammar = {foo: Sequence("foo1", "foo2", "foo3")} - context, return_value = init([foo, bar], grammar, "foo1 foo2 foo3 bar") - - parser = MultipleConceptsParser() - ret_val = parser.parse(context, return_value.body) - - assert ret_val.status - assert ret_val.who == parser.name - assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) - assert ret_val.value.value == [cnode("foo", 0, 4, "foo1 foo2 foo3"), cnode("bar", 6, 6, "bar")] - assert ret_val.value.source == "foo1 foo2 foo3 bar" - - -def test_i_can_parse_when_concept_are_mixed(): - foo = Concept("foo") - bar = Concept("bar") - baz = Concept("baz") - grammar = {foo: Sequence("foo1", "foo2", "foo3")} - context, return_value = init([foo, bar, baz], grammar, "baz foo1 foo2 foo3 bar") - - parser = MultipleConceptsParser() - ret_val = parser.parse(context, return_value.body) - - assert ret_val.status - assert ret_val.who == parser.name - assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) - assert ret_val.value.value == [ - cnode("baz", 0, 0, "baz"), - cnode("foo", 2, 6, "foo1 foo2 foo3"), - cnode("bar", 8, 8, "bar")] - assert ret_val.value.source == "baz foo1 foo2 foo3 bar" - - -def test_i_can_parse_when_multiple_concepts_are_matching(): - foo = Concept("foo") - bar = Concept("bar", body="bar1") - baz = Concept("bar", body="bar2") - grammar = {foo: "foo"} - context, return_value = init([foo, bar, baz], grammar, "foo bar") - - parser = MultipleConceptsParser() - ret_val = parser.parse(context, return_value.body) - - assert len(ret_val) == 2 - assert ret_val[0].status - assert ret_val[0].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")] - assert ret_val[0].value.source == "foo bar" - assert ret_val[0].value.value[1].concept.metadata.body == "bar1" - - assert ret_val[1].status - assert ret_val[1].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")] - assert ret_val[1].value.source == "foo bar" - assert ret_val[1].value.value[1].concept.metadata.body == "bar2" - - -def test_i_can_parse_when_source_code(): - foo = Concept("foo") - grammar = {foo: "foo"} - context, return_value = init([foo], grammar, "1 foo") - - parser = MultipleConceptsParser() - ret_val = parser.parse(context, return_value.body) - wrapper = ret_val.value - value = ret_val.value.value - - assert ret_val.status - assert ret_val.who == parser.name - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert wrapper.source == "1 foo" - assert value == [ - scnode(0, 1, "1 "), - cnode("foo", 2, 2, "foo")] - - -def test_i_cannot_parse_when_unrecognized_token(): - twenty_two = Concept("twenty two") - one = Concept("one") - grammar = {twenty_two: Sequence("twenty", "two")} - context, return_value = init([twenty_two, one], grammar, "twenty two + one") - - parser = MultipleConceptsParser() - ret_val = parser.parse(context, return_value.body) - - assert not ret_val.status - assert ret_val.who == parser.name - assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) - assert ret_val.value.value == [ - cnode("twenty two", 0, 2, "twenty two"), - utnode(3, 5, " + "), - cnode("one", 6, 6, "one") - ] - assert ret_val.value.source == "twenty two + one" - - -def test_i_cannot_parse_when_unknown_concepts(): - twenty_two = Concept("twenty two") - one = Concept("one") - grammar = {twenty_two: Sequence("twenty", "two")} - context, return_value = init([twenty_two, one], grammar, "twenty two plus one") - - parser = MultipleConceptsParser() - ret_val = parser.parse(context, return_value.body) - - assert not ret_val.status - assert ret_val.who == parser.name - assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT) - assert ret_val.value.value == [ - cnode("twenty two", 0, 2, "twenty two"), - utnode(3, 5, " plus "), - cnode("one", 6, 6, "one") - ] - assert ret_val.value.source == "twenty two plus one" - - -@pytest.mark.parametrize("text, expected_source, expected_end", [ - ("True", "True", 0), - ("1 == 1", "1 == 1", 5), - ("1!xdf", "1", 0), - ("1", "1", 0), -]) -def test_i_can_get_source_code_node(text, expected_source, expected_end): - tokens = list(Tokenizer(text))[:-1] # strip trailing EOF - - start_index = 5 # a random number different of zero - res = MultipleConceptsParser().get_source_code_node(get_context(), start_index, tokens) - - assert isinstance(res, SourceCodeNode) - assert isinstance(res.node, PythonNode) - assert res.source == expected_source - assert res.start == start_index - assert res.end == start_index + expected_end - - -def test_i_cannot_parse_null_text(): - res = MultipleConceptsParser().get_source_code_node(get_context(), 0, []) - assert res is None - - eof = Token(TokenKind.EOF, "", 0, 0, 0) - res = MultipleConceptsParser().get_source_code_node(get_context(), 0, [eof]) - assert res is None diff --git a/tests/test_MultipleSameSuccessEvaluator.py b/tests/test_MultipleSameSuccessEvaluator.py deleted file mode 100644 index 69348e6..0000000 --- a/tests/test_MultipleSameSuccessEvaluator.py +++ /dev/null @@ -1,216 +0,0 @@ -from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from evaluators.BaseEvaluator import BaseEvaluator -from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator -from parsers.BaseParser import BaseParser -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def test_i_can_match_and_eval(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - evaluator = MultipleSameSuccessEvaluator() - assert evaluator.matches(context, return_values) - - evaluated = evaluator.eval(context, return_values) - assert evaluated.status - assert evaluated.value == Concept(name="1", body="value").auto_init() # the first concept is returned - - -def test_i_can_match_and_eval_when_no_body(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - evaluator = MultipleSameSuccessEvaluator() - assert evaluator.matches(context, return_values) - - evaluated = evaluator.eval(context, return_values) - assert evaluated.status - assert evaluated.value == Concept(name="1").auto_init() - - -def test_i_can_match_and_eval_when_value_is_not_a_concept(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - evaluator = MultipleSameSuccessEvaluator() - assert evaluator.matches(context, return_values) - - evaluated = evaluator.eval(context, return_values) - assert evaluated.status - assert evaluated.value == "value" - - -def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_concept_evaluator_first(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), - ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"), - ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"), - ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="2", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "Concept", True, Concept(name="1", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="3", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - evaluator = MultipleSameSuccessEvaluator() - assert evaluator.matches(context, return_values) - - evaluated = evaluator.eval(context, return_values) - assert evaluated.status - assert evaluated.value == Concept(name="1", body="value").auto_init() # the concept is returned, not the value - - -def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_python_evaluator_first(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), - ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"), - ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"), - ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="2", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="1", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="3", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - evaluator = MultipleSameSuccessEvaluator() - assert evaluator.matches(context, return_values) - - evaluated = evaluator.eval(context, return_values) - assert evaluated.status - assert evaluated.value == Concept(name="1", body="value").auto_init() # the concept is returned, not the value - - -def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value2").auto_init()), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - evaluator = MultipleSameSuccessEvaluator() - assert evaluator.matches(context, return_values) - assert evaluator.eval(context, return_values) is None - - -def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail_when_no_body(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2").auto_init()), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - evaluator = MultipleSameSuccessEvaluator() - assert evaluator.matches(context, return_values) - assert evaluator.eval(context, return_values) is None - - -def test_i_can_match_if_no_parser(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - assert MultipleSameSuccessEvaluator().matches(context, return_values) - - -def test_i_cannot_match_if_not_reduced_requested(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), - ReturnValueConcept("some_name", True, Concept(name="2", body="value")), - # ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - assert not MultipleSameSuccessEvaluator().matches(context, return_values) - - -def test_i_cannot_match_if_only_one_successful_evaluator(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"), - # ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), - ReturnValueConcept("some_name", True, Concept(name="2", body="value")), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - assert not MultipleSameSuccessEvaluator().matches(context, return_values) - - -def test_i_cannot_match_if_at_least_one_parser_is_successful(): - context = get_context() - sheerka = context.sheerka - - return_values = [ - ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"), - ReturnValueConcept(BaseParser.PREFIX + "some_name2", True, "Not relevant"), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()), - ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()), - ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED)) - ] - - assert not MultipleSameSuccessEvaluator().matches(context, return_values) - diff --git a/tests/test_OneErrorEvaluator.py b/tests/test_OneErrorEvaluator.py deleted file mode 100644 index 955a4f1..0000000 --- a/tests/test_OneErrorEvaluator.py +++ /dev/null @@ -1,80 +0,0 @@ -import pytest - -from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from evaluators.OneErrorEvaluator import OneErrorEvaluator -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def r(value, status=True): - return ReturnValueConcept(value, status, value) - - -reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED)) - - -@pytest.mark.parametrize("return_values, expected", [ - ([r("evaluators.one error", False), reduce_requested], True), - ([r("evaluators.one error", False), r("failed", False), r("failed", False), reduce_requested], True), - ([r("evaluators.error", False), r("not a parser in success"), reduce_requested], True), - ([r("evaluators.no reduce required", False), r("failed", False), r("failed", False)], False), - ([r("evaluators.no reduce required", False)], False), - ([r("evaluators.error", False), r("evaluators.success"), reduce_requested], False), - ([r("evaluators.error", False), r("parsers.success"), reduce_requested], False), - ([r("evaluators.success"), r("not an evaluator in error", False), reduce_requested], False), - ([r("evaluators.error", False), r("evaluators.another error", False), reduce_requested], False), -]) -def test_i_can_match(return_values, expected): - context = get_context() - assert OneErrorEvaluator().matches(context, return_values) == expected - - -def test_i_can_eval(): - context = get_context() - - return_values = [ - r("evaluators.one error", False), - r("parsers.failed", False), - r("parsers.failed", False), - reduce_requested - ] - - evaluator = OneErrorEvaluator() - evaluator.matches(context, return_values) - res = evaluator.eval(context, return_values) - - assert not res.status - assert res.body == "evaluators.one error" - assert len(res.parents) == 4 - - -def test_unwanted_return_values_are_not_eaten(): - context = get_context() - - a_successful_concept = r("successful concept") - a_concept_in_error = r("concept in error", False) - return_values = [ - r("evaluators.one error", False), - r("parsers.failed", False), - r("parsers.failed", False), - a_successful_concept, - a_concept_in_error, - reduce_requested - ] - - evaluator = OneErrorEvaluator() - evaluator.matches(context, return_values) - res = evaluator.eval(context, return_values) - - assert not res.status - assert res.body == "evaluators.one error" - assert len(res.parents) == 4 - - assert a_successful_concept not in res.parents diff --git a/tests/test_OneSuccessEvaluator.py b/tests/test_OneSuccessEvaluator.py deleted file mode 100644 index e8d9720..0000000 --- a/tests/test_OneSuccessEvaluator.py +++ /dev/null @@ -1,79 +0,0 @@ -import pytest - -from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from evaluators.OneSuccessEvaluator import OneSuccessEvaluator -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def r(value, status=True): - return ReturnValueConcept(value, status, value) - - -reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED)) - - -@pytest.mark.parametrize("return_values, expected", [ - ([r("evaluators.one success"), reduce_requested], True), - ([r("evaluators.one success"), r("failed", False), r("failed", False), reduce_requested], True), - ([r("evaluators.no reduce required"), r("failed", False), r("failed", False)], False), - ([r("evaluators.no reduce required")], False), - ([r("evaluators.failed", False), r("failed", False), r("failed", False), reduce_requested], False), - ([r("evaluators.failed", False), r("not evaluator success"), reduce_requested], False), - ([r("evaluators.success"), r("evaluators.another success"), reduce_requested], False), - ([r("evaluators.one success"), r("other success"), r("failed", False), reduce_requested], True), - ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False), -]) -def test_i_can_match(return_values, expected): - context = get_context() - assert OneSuccessEvaluator().matches(context, return_values) == expected - - -def test_i_can_eval(): - context = get_context() - - return_values = [ - r("evaluators.one success"), - r("failed", False), - r("failed", False), - reduce_requested - ] - - evaluator = OneSuccessEvaluator() - matches = evaluator.matches(context, return_values) - res = evaluator.eval(context, return_values) - - assert matches - assert res.status - assert res.body == "evaluators.one success" - assert len(res.parents) == 4 - - -def test_i_do_not_eat_the_other_success(): - context = get_context() - - not_a_parser_success = r("other success") - return_values = [ - r("evaluators.one success"), - not_a_parser_success, - r("failed", False), - reduce_requested - ] - - evaluator = OneSuccessEvaluator() - matches = evaluator.matches(context, return_values) - res = evaluator.eval(context, return_values) - - assert matches - assert res.status - assert res.body == "evaluators.one success" - assert len(res.parents) == 3 - - assert not_a_parser_success not in res.parents diff --git a/tests/test_PrepareEvalEvaluator.py b/tests/test_PrepareEvalEvaluator.py deleted file mode 100644 index fbffbd3..0000000 --- a/tests/test_PrepareEvalEvaluator.py +++ /dev/null @@ -1,55 +0,0 @@ -import pytest - -from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from evaluators.PrepareEvalEvaluator import PrepareEvalEvaluator -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -r = ReturnValueConcept - - -@pytest.mark.parametrize("ret_val, expected", [ - (r("name", True, UserInputConcept("eval 1 + 1")), True), - (r("name", True, UserInputConcept(" eval 1 + 1")), True), - (r("name", True, UserInputConcept("eval")), False), - (r("name", True, UserInputConcept("1+1")), False), - (r("name", True, UserInputConcept("")), False), - (r("name", True, UserInputConcept("eval ")), False), - (r("name", True, UserInputConcept([])), False), - (r("name", True, Concept("foo")), False), - (r("name", True, "not a concept"), False), - (r("name", False, UserInputConcept("eval 1 + 1")), False), - (r("name", False, UserInputConcept(" eval 1 + 1")), False), -]) -def test_i_can_match(ret_val, expected): - context = get_context() - assert PrepareEvalEvaluator().matches(context, ret_val) == expected - - -@pytest.mark.parametrize("ret_val, expected", [ - (r("name", True, UserInputConcept("eval 1 + 1")), "1 + 1"), - (r("name", True, UserInputConcept(" eval 1 + 1")), "1 + 1"), -]) -def test_i_can_eval(ret_val, expected): - context = get_context() - sheerka = context.sheerka - - prepare_evaluator = PrepareEvalEvaluator() - prepare_evaluator.matches(context, ret_val) - res = prepare_evaluator.eval(context, ret_val) - - assert len(res) == 2 - assert res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.USER_INPUT) - assert res[0].body.body == expected - - assert res[1].status - assert sheerka.isinstance(res[1].body, BuiltinConcepts.CONCEPT_EVAL_REQUESTED) diff --git a/tests/test_PythonEvaluator.py b/tests/test_PythonEvaluator.py deleted file mode 100644 index da68168..0000000 --- a/tests/test_PythonEvaluator.py +++ /dev/null @@ -1,151 +0,0 @@ -import pytest - -from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts -from core.sheerka import Sheerka, ExecutionContext -from core.concept import Concept -from evaluators.PythonEvaluator import PythonEvaluator -from parsers.PythonParser import PythonNode, PythonParser -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def get_context_name(context): - return context.name - - -@pytest.mark.parametrize("ret_val, expected", [ - (ReturnValueConcept("some_name", True, ParserResultConcept(value=PythonNode("", None))), True), - (ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False), - (ReturnValueConcept("some_name", False, "not relevant"), False), - (ReturnValueConcept("some_name", True, Concept()), False) -]) -def test_i_can_match(ret_val, expected): - context = get_context() - assert PythonEvaluator().matches(context, ret_val) == expected - - -@pytest.mark.parametrize("text, expected", [ - ("1 + 1", 2), - ("sheerka.test()", "I have access to Sheerka !"), - ("a=10\na", 10), -]) -def test_i_can_eval(text, expected): - context = get_context() - parsed = PythonParser().parse(context, text) - - evaluated = PythonEvaluator().eval(context, parsed) - - assert evaluated.status - assert evaluated.value == expected - - -@pytest.mark.parametrize("concept", [ - Concept("foo"), - Concept("foo", body="2"), - Concept("foo").set_prop("prop", "'a'"), - Concept("foo", body="bar") -]) -def test_i_cannot_eval_simple_concept(concept): - context = get_context() - context.sheerka.add_in_cache(Concept("foo")) - - parsed = PythonParser().parse(context, "foo") - evaluated = PythonEvaluator().eval(context, parsed) - - assert not evaluated.status - assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.NOT_FOR_ME) - - -def test_i_can_eval_expression_with_that_references_concepts(): - """ - I can test modules with variables - :return: - """ - context = get_context() - context.sheerka.add_in_cache(Concept("foo", body="1")) - - parsed = PythonParser().parse(context, "foo + 2") - evaluated = PythonEvaluator().eval(context, parsed) - - assert evaluated.status - assert evaluated.value == 3 - - -def test_i_can_eval_module_with_that_references_concepts(): - """ - I can test modules with variables - :return: - """ - context = get_context() - context.sheerka.add_in_cache(Concept("foo")) - - parsed = PythonParser().parse(context, "def a(b):\n return b\na(c:foo:)") - evaluated = PythonEvaluator().eval(context, parsed) - - assert evaluated.status - assert evaluated.value == Concept("foo").init_key() - - -def test_i_can_eval_module_with_that_references_concepts_with_body(): - """ - I can test modules with variables - :return: - """ - context = get_context() - context.sheerka.add_in_cache(Concept("foo", body="2")) - - parsed = PythonParser().parse(context, "def a(b):\n return b\na(foo)") - evaluated = PythonEvaluator().eval(context, parsed) - - assert evaluated.status - assert evaluated.value == 2 - - -def test_i_can_eval_concept_token(): - context = get_context() - context.sheerka.add_in_cache(Concept("foo", body="2")) - - parsed = PythonParser().parse(context, "get_context_name(c:foo:)") - python_evaluator = PythonEvaluator() - python_evaluator.locals["get_context_name"] = get_context_name - evaluated = python_evaluator.eval(context, parsed) - - assert evaluated.status - assert evaluated.value == "foo" - - # sanity, does not work otherwise - parsed = PythonParser().parse(context, "get_context_name(foo)") - python_evaluator = PythonEvaluator() - python_evaluator.locals["get_context_name"] = get_context_name - evaluated = python_evaluator.eval(context, parsed) - - assert not evaluated.status - assert evaluated.body.body.args[0] == "'int' object has no attribute 'name'" - - -@pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [ - ("__C__key__C__", "key", None, False), - ("__C__key__id__C__", "key", "id", False), - ("__C__USE_CONCEPT__key__id__C__", "key", "id", True), - ("__C__USE_CONCEPT__key__id__C__", "key", "id", True), -]) -def test_i_can_resolve_name(text, concept_key, concept_id, use_concept): - context = get_context() - assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept) - - -@pytest.mark.parametrize("text", [ - "__C__", - "__C__key", - "__C__key____", - "__C____", - "__C__USE_CONCEPT__", -]) -def test_i_cannot_resolve_name(text): - context = get_context() - assert PythonEvaluator().resolve_name(context, text) is None diff --git a/tests/test_PythonParser.py b/tests/test_PythonParser.py deleted file mode 100644 index 979f89a..0000000 --- a/tests/test_PythonParser.py +++ /dev/null @@ -1,87 +0,0 @@ -import ast - -import pytest - -from core.builtin_concepts import ParserResultConcept -from core.sheerka import Sheerka, ExecutionContext -from core.tokenizer import Tokenizer, LexerError -from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -@pytest.mark.parametrize("text, expected", [ - ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))), - ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))), -]) -def test_i_can_parse_a_simple_expression(text, expected): - parser = PythonParser() - res = parser.parse(get_context(), text) - - assert res.status - assert res.who == parser.name - assert isinstance(res.value, ParserResultConcept) - assert res.value.value == expected - - -@pytest.mark.parametrize("text, expected", [ - ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))), - ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))), -]) -def test_i_can_parse_from_tokens(text, expected): - parser = PythonParser() - tokens = list(Tokenizer(text)) - res = parser.parse(get_context(), tokens) - - assert res.status - assert res.who == parser.name - assert isinstance(res.value, ParserResultConcept) - assert res.value.value == expected - - -@pytest.mark.parametrize("text", [ - "1+", - "'name", - "foo = 'name" -]) -def test_i_can_detect_error(text): - parser = PythonParser() - res = parser.parse(get_context(), text) - - assert not res.status - assert res.who == parser.name - assert isinstance(res.value, ParserResultConcept) - assert isinstance(res.value.value[0], PythonErrorNode) - assert isinstance(res.value.value[0].exception, SyntaxError) - - -@pytest.mark.parametrize("text, error_msg, error_text", [ - ("c::", "Concept name not found", ""), - ("c:: + 1", "Concept name not found", ""), -]) -def test_i_can_detect_lexer_errors(text, error_msg, error_text): - parser = PythonParser() - res = parser.parse(get_context(), text) - - assert not res.status - assert isinstance(res.body, ParserResultConcept) - assert isinstance(res.body.body[0], LexerError) - assert res.body.body[0].message == error_msg - assert res.body.body[0].text == error_text - - -def test_i_can_parse_a_concept(): - text = "c:concept_name: + 1" - - parser = PythonParser() - res = parser.parse(get_context(), text) - - assert res - assert res.value.value == PythonNode( - "c:concept_name: + 1", - ast.parse("__C__USE_CONCEPT__concept_name__C__+1", mode="eval")) diff --git a/tests/test_PythonWithConceptsParser.py b/tests/test_PythonWithConceptsParser.py deleted file mode 100644 index 5422118..0000000 --- a/tests/test_PythonWithConceptsParser.py +++ /dev/null @@ -1,147 +0,0 @@ -import ast - -import pytest - -from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from core.tokenizer import Token, TokenKind, Tokenizer -from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode -from parsers.MultipleConceptsParser import MultipleConceptsParser -from parsers.PythonParser import PythonNode, PythonErrorNode -from parsers.PythonWithConceptsParser import PythonWithConceptsParser -from sdp.sheerkaDataProvider import Event - -multiple_concepts_parser = MultipleConceptsParser() - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def get_ret_from(*args): - result = [] - index = 0 - for item in args: - if isinstance(item, Concept): - tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)] - result.append(ConceptNode(item, index, index, tokens, item.name)) - index += 1 - else: - tokens = list(Tokenizer(item)) - result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens)) - index += len(tokens) - - return ReturnValueConcept("who", False, ParserResultConcept(parser=multiple_concepts_parser, value=result)) - - -def to_str_ast(expression): - return PythonNode.get_dump(ast.parse(expression, mode="eval")) - - -@pytest.mark.parametrize("text, interested", [ - ("not parser result", False), - (ParserResultConcept(parser="not multiple_concepts_parser"), False), - (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True), -]) -def test_not_interested(text, interested): - context = get_context() - - res = PythonWithConceptsParser().parse(context, text) - if interested: - assert res is not None - else: - assert res is None - - -def test_i_can_parse_concepts_and_python(): - context = get_context() - foo = Concept("foo") - input_return_value = get_ret_from(foo, " + 1") - - parser = PythonWithConceptsParser() - result = parser.parse(context, input_return_value.body) - wrapper = result.value - return_value = result.value.value - - assert result.status - assert result.who == parser.name - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert wrapper.source == "foo + 1" - assert isinstance(return_value, PythonNode) - assert return_value.source == "foo + 1" - assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1") - assert return_value.concepts["__C__foo__C__"] == foo - - -def test_i_can_parse_concepts_and_python_when_concept_is_known(): - context = get_context() - foo = Concept("foo") - foo = context.sheerka.create_new_concept(context, foo).body.body - input_return_value = get_ret_from(foo, " + 1") - - parser = PythonWithConceptsParser() - result = parser.parse(context, input_return_value.body) - wrapper = result.value - return_value = result.value.value - - assert result.status - assert result.who == parser.name - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert wrapper.source == "foo + 1" - assert isinstance(return_value, PythonNode) - assert return_value.source == "foo + 1" - assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__1001__C__ + 1") - assert return_value.concepts["__C__foo__1001__C__"] == foo - - -def test_i_can_parse_when_concept_name_has_invalid_characters(): - context = get_context() - foo = Concept("foo et > (,") - foo = context.sheerka.create_new_concept(context, foo).body.body - input_return_value = get_ret_from(foo, " + 1") - - parser = PythonWithConceptsParser() - result = parser.parse(context, input_return_value.body) - return_value = result.value.value - - assert result.status - assert return_value.concepts["__C__foo0et000000__1001__C__"] == foo - - -def test_python_ids_mappings_are_correct_when_concepts_with_the_same_name(): - context = get_context() - foo1 = Concept("foo") - foo2 = Concept("foo") - foo3 = context.sheerka.create_new_concept(context, Concept("foo", body="foo3")).body.body - foo4 = context.sheerka.create_new_concept(context, Concept("foo", body="foo4")).body.body - - input_return_value = get_ret_from(foo1, "+", foo2, "+", foo3, "+", foo4) - - parser = PythonWithConceptsParser() - result = parser.parse(context, input_return_value.body) - return_value = result.value.value - - assert result.status - assert return_value.concepts["__C__foo__C__"] == foo1 - assert return_value.concepts["__C__foo_1__C__"] == foo2 - assert return_value.concepts["__C__foo__1001__C__"] == foo3 - assert return_value.concepts["__C__foo__1002__C__"] == foo4 - - -def test_i_cannot_parse_if_syntax_error(): - context = get_context() - foo = Concept("foo") - foo = context.sheerka.create_new_concept(context, foo).body.body - input_return_value = get_ret_from(foo, " + ") - - parser = PythonWithConceptsParser() - result = parser.parse(context, input_return_value.body) - wrapper = result.value - return_value = result.value.value - - assert not result.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - assert isinstance(return_value[0], PythonErrorNode) diff --git a/tests/test_TooManySucessEvaluator.py b/tests/test_TooManySucessEvaluator.py deleted file mode 100644 index 8e2879f..0000000 --- a/tests/test_TooManySucessEvaluator.py +++ /dev/null @@ -1,112 +0,0 @@ -import pytest - -from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from evaluators.TooManySuccessEvaluator import TooManySuccessEvaluator -from sdp.sheerkaDataProvider import Event - - -def get_context(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - return ExecutionContext("test", Event(), sheerka) - - -def r(name, status=True, value=None): - value = value or name - return ReturnValueConcept(name, status, value) - - -reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED)) - - -@pytest.mark.parametrize("return_values, expected", [ - ([r("evaluators.success1"), r("evaluators.success2"), reduce_requested], True), - ([r("evaluators.success1"), r("evaluators.success2"), r("other"), reduce_requested], True), - ([r("evaluators.success1"), r("evaluators.success2"), r("other", False), reduce_requested], True), - ([r("evaluators.a", value=Concept("c1", body="1")), - r("evaluators.a", value=Concept("c2", body="1")), - r("other"), - reduce_requested], True), - ([r("evaluators.a", value=Concept("c1", body="1")), - r("evaluators.a", value=Concept("c2", body="1")), - r("parsers.other", False), - reduce_requested], True), - ([r("evaluators.a", value=Concept("c1", body="1")), - r("evaluators.a", value=Concept("c2", body="1")), - r("parsers.other"), - reduce_requested], False), - ([r("evaluators.success1"), reduce_requested], False), - ([reduce_requested], False), - ([r("evaluators.success1"), r("evaluators.success2")], False), -]) -def test_i_can_match(return_values, expected): - context = get_context() - assert TooManySuccessEvaluator().matches(context, return_values) == expected - - -def test_i_can_eval(): - context = get_context() - - value1 = r("evaluators.a", value=Concept("c1", body="1")) - value2 = r("evaluators.a", value=Concept("c2", body="2")) - return_values = [ - value1, - value2, - r("other", False), - reduce_requested - ] - - evaluator = TooManySuccessEvaluator() - matches = evaluator.matches(context, return_values) - res = evaluator.eval(context, return_values) - - assert matches - assert not res.status - assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS) - assert res.body.body == [value1, value2] - assert len(res.parents) == 4 - - -def test_i_can_eval_when_same_success(): - context = get_context() - - return_values = [ - r("evaluators.a", value=Concept("c1", body="1").auto_init()), - r("evaluators.a", value=Concept("c2", body="1").auto_init()), - r("other", False), - reduce_requested - ] - - evaluator = TooManySuccessEvaluator() - matches = evaluator.matches(context, return_values) - res = evaluator.eval(context, return_values) - - assert matches - assert res is None - - -def test_other_success_are_not_reduced(): - context = get_context() - - value1 = r("evaluators.a", value=Concept("c1", body="1").auto_init()) - value2 = r("evaluators.a", value=Concept("c2", body="2").auto_init()) - other_success = r("other") - return_values = [ - value1, - value2, - other_success, - reduce_requested - ] - - evaluator = TooManySuccessEvaluator() - matches = evaluator.matches(context, return_values) - res = evaluator.eval(context, return_values) - - assert matches - assert not res.status - assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS) - assert res.body.body == [value1, value2] - assert len(res.parents) == 3 - assert other_success not in res.parents diff --git a/tests/test_builtin_helpers.py b/tests/test_builtin_helpers.py deleted file mode 100644 index 6fb0ef8..0000000 --- a/tests/test_builtin_helpers.py +++ /dev/null @@ -1,171 +0,0 @@ -import ast - -import pytest - -from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts -from core.sheerka import Sheerka, ExecutionContext -import core.builtin_helpers -from sdp.sheerkaDataProvider import Event - - -def test_i_can_use_expect_one_when_empty(): - sheerka = get_sheerka() - - res = core.builtin_helpers.expect_one(get_context(sheerka), []) - assert not res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY) - - -def test_i_can_use_expect_one_when_too_many_success(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", True, "value1"), - ReturnValueConcept("who", True, "value2"), - ] - res = core.builtin_helpers.expect_one(get_context(sheerka), items) - assert not res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS) - assert res.value.body == items - assert res.parents == items - - -def test_i_can_use_expect_one_when_same_success(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", True, "value"), - ReturnValueConcept("who", True, "value"), - ] - res = core.builtin_helpers.expect_one(get_context(sheerka), items) - assert res.status - assert res.value == items[0].value - assert res.parents == items - - -def test_i_can_use_expect_when_only_errors_1(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", False, sheerka.new(BuiltinConcepts.ERROR)), - ] - res = core.builtin_helpers.expect_one(get_context(sheerka), items) - assert not res.status - assert res.value, items[0] - assert res.parents == items - - -def test_i_can_use_expect_when_only_errors_2(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", False, None), - ReturnValueConcept("who", False, None), - ] - res = core.builtin_helpers.expect_one(get_context(sheerka), items) - assert not res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS) - assert res.value.body == items - assert res.parents == items - - -def test_i_can_use_expect_one_when_one_success_1(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", True, None), - ] - res = core.builtin_helpers.expect_one(get_context(sheerka), items) - assert res.status - assert res.body == items[0].body - assert res.parents == items - - -def test_i_can_use_expect_one_when_one_success_2(): - sheerka = get_sheerka() - - items = [ - ReturnValueConcept("who", False, None), - ReturnValueConcept("who", True, None), - ReturnValueConcept("who", False, None), - ] - res = core.builtin_helpers.expect_one(get_context(sheerka), items) - assert res.status - assert res.body == items[1].body - assert res.parents == items - - -def test_i_can_use_expect_one_when_not_a_list_true(): - sheerka = get_sheerka() - - item = ReturnValueConcept("who", True, None) - res = core.builtin_helpers.expect_one(get_context(sheerka), item) - assert res.status - assert res == item - - -def test_i_can_use_expect_one_when_not_a_list_false(): - sheerka = get_sheerka() - - item = ReturnValueConcept("who", False, None) - res = core.builtin_helpers.expect_one(get_context(sheerka), item) - - assert not res.status - assert res == item - - -@pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [ - ("a == 1", [], [], []), - ("a == 1", ["a"], [], ["a == 1"]), - ("a == 1", [], ["a"], []), - ("predicate(a)", [], [], []), - ("predicate(a)", ["a"], [], ["predicate(a)"]), - ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]), - ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]), - ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]), - ("predicate(a, b)", ["a"], ["b"], []), - ("a + b == 1", [], [], []), - ("a + b == 1", ["a"], [], ["a + b == 1"]), - ("a + b == 1", ["a"], ["b"], []), - ("a + b == 1", ["b"], [], ["a + b == 1"]), - ("a + b == 1", ["a", "b"], [], ["a + b == 1"]), - ("a == 1 and b == 2", [], [], []), - ("a == 1 and b == 2", ["a"], [], ["a == 1"]), - ("a == 1 and b == 2", ["b"], [], ["b == 2"]), - ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]), - ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]), - ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]), - ("not(a == 1)", [], [], []), - ("not(a == 1)", ["a"], [], ["not(a==1)"]), - ("a == 1 or b == 2", [], [], []), - ("a == 1 or b == 2", ["a"], [], ["a == 1"]), - ("a == 1 or b == 2", ["b"], [], ["b == 2"]), - ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]), - ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]), -]) -def test_i_can_extract_predicates(expression, vars_to_include, vars_to_exclude, expected_expr): - sheerka = get_sheerka() - expected = [ast.parse(expr, mode="eval") for expr in expected_expr] - - actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude) - assert len(actual) == len(expected) - for i in range(len(actual)): - assert dump_ast(actual[i]) == dump_ast(expected[i]) - - -def get_sheerka(): - sheerka = Sheerka(skip_builtins_in_db=True) - sheerka.initialize("mem://") - - return sheerka - - -def get_context(sheerka): - return ExecutionContext("test", Event(), sheerka) - - -def dump_ast(node): - dump = ast.dump(node) - for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]: - dump = dump.replace(to_remove, "") - return dump diff --git a/tests/test_sheerka.py b/tests/test_sheerka.py deleted file mode 100644 index 3977367..0000000 --- a/tests/test_sheerka.py +++ /dev/null @@ -1,955 +0,0 @@ -import pytest -import os -from os import path -import shutil - -from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept, ConceptAlreadyInSet, \ - ParserResultConcept -from core.concept import Concept, PROPERTIES_TO_SERIALIZE, Property, ConceptParts, DoNotResolve, simplec -from core.sheerka import Sheerka, ExecutionContext -from parsers.PythonParser import PythonNode -from sdp.sheerkaDataProvider import SheerkaDataProvider, Event - -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - - os.chdir(current_pwd) - - -def get_sheerka(use_dict=True, skip_builtins_in_db=True): - root = "mem://" if use_dict else root_folder - sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db) - sheerka.initialize(root) - - return sheerka - - -def get_context(sheerka): - return ExecutionContext("test", Event(), sheerka) - - -def get_default_concept(): - concept = Concept( - name="a + b", - where="isinstance(a, int) and isinstance(b, int)", - pre="isinstance(a, int) and isinstance(b, int)", - post="isinstance(res, int)", - body="def func(x,y):\n return x+y\nfunc(a,b)", - desc="specific description") - concept.def_prop("a", "value1") - concept.def_prop("b", "value2") - - return concept - - -class ConceptWithGetValue(Concept): - def get_value(self): - return self.get_prop("my_prop") - - -def test_root_folder_is_created_after_initialization(): - return_value = Sheerka().initialize(root_folder) - assert return_value.status, "initialisation should be successful" - assert os.path.exists(root_folder), "init folder should be created" - - -def test_i_can_list_builtin_concepts(): - sheerka = get_sheerka() - builtins = list(sheerka.get_builtins_classes_as_dict()) - - assert str(BuiltinConcepts.ERROR) in builtins - assert str(BuiltinConcepts.RETURN_VALUE) in builtins - - -def test_builtin_concepts_are_initialized(): - sheerka = get_sheerka(skip_builtins_in_db=False) - assert len(sheerka.cache_by_key) == len(BuiltinConcepts) - for concept_name in BuiltinConcepts: - assert str(concept_name) in sheerka.cache_by_key - assert sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, str(concept_name)) is not None - - for key, concept_class in sheerka.get_builtins_classes_as_dict().items(): - assert isinstance(sheerka.cache_by_key[key], concept_class) - - -def test_builtin_concepts_can_be_updated(): - sheerka = get_sheerka(False, False) - loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA) - loaded_sheerka.metadata.desc = "I have a description" - sheerka.sdp.modify("Test", sheerka.CONCEPTS_ENTRY, loaded_sheerka.key, loaded_sheerka) - - sheerka = get_sheerka(False, False) - loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA) - - assert loaded_sheerka.metadata.desc == "I have a description" - - -def test_i_can_add_a_concept(): - sheerka = get_sheerka() - concept = get_default_concept() - - res = sheerka.create_new_concept(get_context(sheerka), concept) - - assert res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT) - - concept_found = res.value.body - for prop in PROPERTIES_TO_SERIALIZE: - assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop) - - assert concept_found.key == "__var__0 + __var__1" - assert concept_found.id == "1001" - - assert concept.key in sheerka.cache_by_key - assert concept.id in sheerka.cache_by_id - assert sheerka.sdp.io.exists( - sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest())) - - -def test_i_cannot_add_the_same_concept_twice(): - """ - Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp - :return: - """ - sheerka = get_sheerka() - concept = get_default_concept() - - sheerka.create_new_concept(get_context(sheerka), concept) - res = sheerka.create_new_concept(get_context(sheerka), concept) - - assert not res.status - assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED) - assert res.value.body == concept - - -def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string(): - """ - Checks that a concept can be found its name - even when there are variables in the name (ex 'hello + a' or 'a + b' ) - :return: - """ - sheerka = get_sheerka() - for key in sheerka.get_builtins_classes_as_dict(): - assert sheerka.get(key) is not None - assert sheerka.get(str(key)) is not None - - -def test_i_can_get_a_newly_created_concept(): - sheerka = get_sheerka() - concept = get_default_concept() - - sheerka.create_new_concept(get_context(sheerka), concept) - - from_cache = sheerka.get(concept.key) - assert from_cache is not None - assert from_cache == concept - - from_cache = sheerka.get_by_id(concept.id) - assert from_cache is not None - assert from_cache == concept - - -def test_i_first_look_in_local_cache(): - sheerka = get_sheerka() - concept = get_default_concept() - - sheerka.create_new_concept(get_context(sheerka), concept) - sheerka.cache_by_key[concept.key].pre = "I have modified the concept in cache" - - from_cache = sheerka.get(concept.key) - assert from_cache is not None - assert from_cache.key == concept.key - assert from_cache.pre == "I have modified the concept in cache" - - -def test_i_can_get_a_known_concept_when_not_in_cache(): - """ - When not in cache, uses sdp - :return: - """ - sheerka = get_sheerka() - concept = get_default_concept() - sheerka.create_new_concept(get_context(sheerka), concept) - - sheerka.cache_by_key = {} # reset the cache - loaded = sheerka.get(concept.key) - - assert loaded is not None - assert loaded == concept - - # I can also get it by its id - loaded = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, concept.id) - assert loaded is not None - assert loaded == concept - - -def test_i_can_get_a_concept_by_its_id(): - sheerka = get_sheerka() - concept = get_default_concept() - sheerka.create_new_concept(get_context(sheerka), concept) - - sheerka.cache_by_key = {} # reset the cache - loaded = sheerka.get_by_id(concept.id) - - assert loaded is not None - assert loaded == concept - - -def test_i_can_get_list_of_concept_when_same_key_when_no_cache(): - sheerka = get_sheerka() - concept1 = get_default_concept() - concept2 = get_default_concept() - concept2.metadata.body = "a+b" - - res1 = sheerka.create_new_concept(get_context(sheerka), concept1) - res2 = sheerka.create_new_concept(get_context(sheerka), concept2) - - assert res1.value.body.key == res2.value.body.key # same key - - sheerka.cache_by_key = {} # reset the cache - - result = sheerka.get(concept1.key) - assert len(result) == 2 - assert result[0] == concept1 - assert result[1] == concept2 - - -def test_i_can_get_list_of_concept_when_same_key_when_cache(): - sheerka = get_sheerka() - concept1 = get_default_concept() - concept2 = get_default_concept() - concept2.metadata.body = "a+b" - - res1 = sheerka.create_new_concept(get_context(sheerka), concept1) - res2 = sheerka.create_new_concept(get_context(sheerka), concept2) - - assert res1.value.body.key == res2.value.body.key # same key - - # sheerka.cache_by_key = {} # Do not reset the cache - - result = sheerka.get(concept1.key) - assert len(result) == 2 - assert result[0] == concept1 - assert result[1] == concept2 - - -def test_i_can_get_the_correct_concept_using_the_id_when_same_key_when_no_cache(): - sheerka = get_sheerka() - concept1 = get_default_concept() - concept2 = get_default_concept() - concept2.metadata.body = "a+b" - - res1 = sheerka.create_new_concept(get_context(sheerka), concept1) - res2 = sheerka.create_new_concept(get_context(sheerka), concept2) - - assert res1.value.body.key == res2.value.body.key # same key - - result = sheerka.get(concept1.key, res2.body.body.id) - assert result.name == "a + b" - assert result.metadata.body == "a+b" - - -def test_i_can_get_the_correct_concept_using_the_id__when_same_key_when_cache(): - sheerka = get_sheerka() - concept1 = get_default_concept() - concept2 = get_default_concept() - concept2.metadata.body = "a+b" - - res1 = sheerka.create_new_concept(get_context(sheerka), concept1) - res2 = sheerka.create_new_concept(get_context(sheerka), concept2) - - assert res1.value.body.key == res2.value.body.key # same key - - result = sheerka.get(concept1.key, res2.body.body.id) - assert result.name == "a + b" - assert result.metadata.body == "a+b" - - -def test_i_cannot_get_the_correct_concept_id_the_id_is_wrong(): - sheerka = get_sheerka() - concept1 = get_default_concept() - concept2 = get_default_concept() - concept2.metadata.body = "a+b" - - res1 = sheerka.create_new_concept(get_context(sheerka), concept1) - res2 = sheerka.create_new_concept(get_context(sheerka), concept2) - - assert res1.value.body.key == res2.value.body.key # same key - - result = sheerka.get(concept1.key, "wrong id") - assert sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT) - - -def test_i_cannot_get_when_key_is_none(): - sheerka = get_sheerka() - - res = sheerka.get(None) - assert sheerka.isinstance(res, BuiltinConcepts.ERROR) - assert res.body == "Concept key is undefined." - - -def test_unknown_concept_is_return_when_the_concept_key_is_not_found(): - sheerka = get_sheerka() - - loaded = sheerka.get("key_that_does_not_exist") - - assert loaded is not None - assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT) - assert loaded.body == ("key", "key_that_does_not_exist") - assert loaded.metadata.is_evaluated - - -def test_unknown_concept_is_return_when_the_concept_id_is_not_found(): - sheerka = get_sheerka() - - loaded = sheerka.get_by_id("id_that_does_not_exist") - - assert loaded is not None - assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT) - assert loaded.body == ("id", "id_that_does_not_exist") - assert loaded.metadata.is_evaluated - - -def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(): - sheerka = get_sheerka() - ret = sheerka.new(BuiltinConcepts.RETURN_VALUE, who="who", status="status", value="value", message="message") - - assert isinstance(ret, ReturnValueConcept) - assert ret.key == str(BuiltinConcepts.RETURN_VALUE) - assert ret.who == "who" - assert ret.status == "status" - assert ret.value == "value" - assert ret.message == "message" - - -def test_i_can_instantiate_a_builtin_concept_when_no_specific_class(): - sheerka = get_sheerka() - ret = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body="fake_concept") - - assert isinstance(ret, Concept) - assert ret.key == str(BuiltinConcepts.UNKNOWN_CONCEPT) - assert ret.body == "fake_concept" - - -def test_i_can_instantiate_a_concept(): - sheerka = get_sheerka() - concept = get_default_concept() - sheerka.create_new_concept(get_context(sheerka), concept) - - new = sheerka.new(concept.key, a=10, b="value") - - assert sheerka.isinstance(new, concept) - for prop in PROPERTIES_TO_SERIALIZE: - assert getattr(new.metadata, prop) == getattr(concept.metadata, prop) - - assert new.props["a"].value == 10 - assert new.props["b"].value == "value" - - -def test_i_can_instantiate_with_the_name_and_the_id(): - sheerka = get_sheerka() - sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo1")) - sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo2")) - - concepts = sheerka.new("foo") - assert len(concepts) == 2 - - foo1 = sheerka.new(("foo", "1001")) - assert foo1.metadata.body == "foo1" - - foo2 = sheerka.new(("foo", "1002")) - assert foo2.metadata.body == "foo2" - - -def test_instances_are_different_when_asking_for_new(): - sheerka = get_sheerka() - concept = get_default_concept() - sheerka.create_new_concept(get_context(sheerka), concept) - - new1 = sheerka.new(concept.key, a=10, b="value") - new2 = sheerka.new(concept.key, a=10, b="value") - - assert new1 == new2 - assert id(new1) != id(new2) - - -def test_i_get_the_same_instance_when_is_unique_is_true(): - sheerka = get_sheerka() - concept = Concept(name="unique", is_unique=True) - sheerka.create_new_concept(get_context(sheerka), concept) - - new1 = sheerka.new(concept.key, a=10, b="value") - new2 = sheerka.new(concept.key, a=10, b="value") - - assert new1 == new2 - assert id(new1) == id(new2) - - -def test_i_cannot_instantiate_an_unknown_concept(): - sheerka = get_sheerka() - - new = sheerka.new("fake_concept") - - assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT) - assert new.body == ('key', 'fake_concept') - - -def test_i_cannot_instantiate_with_invalid_id(): - sheerka = get_sheerka() - sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo1")) - sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo2")) - - new = sheerka.new(("foo", "invalid_id")) - - assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT) - assert new.body == [('key', 'foo'), ('id', 'invalid_id')] - - -def test_i_cannot_instantiate_with_invalid_key(): - sheerka = get_sheerka() - sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo1")) - sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo2")) - - new = sheerka.new(("invalid_key", "1001")) - - assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT) - assert new.body == [('key', 'invalid_key'), ('id', '1001')] - - -def test_concept_id_is_irrelevant_when_only_one_concept(): - sheerka = get_sheerka() - sheerka.create_new_concept(get_context(sheerka), Concept("foo", body="foo1")) - - new = sheerka.new(("foo", "invalid_id")) - - assert sheerka.isinstance(new, "foo") - assert new.metadata.body == "foo1" - - -def test_i_cannot_instantiate_when_properties_are_not_recognized(): - sheerka = get_sheerka() - concept = get_default_concept() - sheerka.create_new_concept(get_context(sheerka), concept) - - new = sheerka.new(concept.key, a=10, c="value") - - assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_PROPERTY) - assert new.property_name == "c" - assert sheerka.isinstance(new.concept, concept) - - -@pytest.mark.parametrize("concept, reduce_simple_list, expected", [ - (None, False, None), - (3.14, False, 3.14), - ("foo", False, "foo"), - (True, False, True), - (Concept("name", body="foo"), False, "foo"), - (Concept("name"), False, Concept("name")), - (ConceptWithGetValue("name").set_prop("my_prop", "my_value"), False, "my_value"), - (ReturnValueConcept(value="return_value"), False, "return_value"), - (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"), - (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"), - (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]), - (Concept("name", body=["foo"]), True, "foo"), - (Concept("name", body=Concept("foo")), False, Concept("foo")), - (Concept("name", body=Concept("foo", body="value")), False, "value"), - (Concept("name", body=Concept("foo", body=Concept("bar", body="value"))), False, "value"), - (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"), -]) -def test_i_can_get_value(concept, reduce_simple_list, expected): - sheerka = get_sheerka() - - # I use auto_init() instead of evaluate_concept() to be quicker - c = concept - while isinstance(c, Concept): - c.auto_init() - c = c.body - - assert sheerka.value(concept, reduce_simple_list) == expected - - -def test_list_of_concept_is_sorted_by_id(): - sheerka = get_sheerka(False, False) - concepts = sheerka.concepts() - - assert concepts[0].id < concepts[-1].id - - -@pytest.mark.parametrize("body, expected", [ - # (None, None), - # ("", ""), - ("1", 1), - ("1+1", 2), - ("'one'", "one"), - ("'one' + 'two'", "onetwo"), - ("True", True), - ("1 > 2", False), -]) -def test_i_can_evaluate_a_concept_with_simple_body(body, expected): - sheerka = get_sheerka() - - concept = Concept("foo", body=body) - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.key - assert evaluated.body == expected - assert evaluated.metadata.body == body - assert evaluated.metadata.pre is None - assert evaluated.metadata.post is None - assert evaluated.metadata.where is None - assert evaluated.props == {} - assert evaluated.metadata.is_evaluated - assert len(evaluated.values) == 0 if body is None else 1 - - -@pytest.mark.parametrize("expr, expected", [ - ("", ""), - ("1", 1), - ("1+1", 2), - ("'one'", "one"), - ("'one' + 'two'", "onetwo"), - ("True", True), - ("1 > 2", False), -]) -def test_i_can_evaluate_the_other_metadata(expr, expected): - """ - I only test WHERE, it's the same for the others - :param expr: - :param expected: - :return: - """ - - sheerka = get_sheerka() - - concept = Concept("foo", where=expr) - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.key - assert evaluated.metadata.body is None - assert evaluated.metadata.pre is None - assert evaluated.metadata.post is None - assert evaluated.metadata.where == expr - assert evaluated.get_metadata_value(ConceptParts.WHERE) == expected - assert evaluated.props == {} - assert evaluated.metadata.is_evaluated - assert len(evaluated.values) == 0 if expr is None else 1 - - -@pytest.mark.parametrize("expr, expected", [ - (None, None), - ("", ""), - ("1", 1), - ("1+1", 2), - ("'one'", "one"), - ("'one' + 'two'", "onetwo"), - ("True", True), - ("1 > 2", False), -]) -def test_i_can_evaluate_a_concept_with_prop(expr, expected): - sheerka = get_sheerka() - - concept = Concept("foo").def_prop("a", expr) - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.key - assert evaluated.metadata.pre is None - assert evaluated.metadata.pre is None - assert evaluated.metadata.post is None - assert evaluated.metadata.where is None - assert evaluated.props == {"a": Property("a", expected)} - assert evaluated.metadata.is_evaluated - - -def test_i_can_evaluate_metadata_using_do_not_resolve(): - sheerka = get_sheerka() - concept = Concept("foo") - concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve") - - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.body == "do not resolve" - assert evaluated.metadata.is_evaluated - - -def test_i_can_evaluate_property_using_do_not_resolve(): - sheerka = get_sheerka() - concept = Concept("foo").def_prop("a") - concept.compiled["a"] = DoNotResolve("do not resolve") - - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.get_prop("a") == "do not resolve" - assert evaluated.metadata.is_evaluated - - -def test_original_value_is_overridden_when_using_do_no_resolve(): - sheerka = get_sheerka() - concept = Concept("foo", body="original value").set_prop("a", "original value") - concept.compiled["a"] = DoNotResolve("do not resolve") - concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve") - - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.body == "do not resolve" - assert evaluated.get_prop("a") == "do not resolve" - assert evaluated.metadata.is_evaluated - - -def test_props_are_evaluated_before_body(): - sheerka = get_sheerka() - - concept = Concept("foo", body="a+1").def_prop("a", "10").init_key() - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.key - assert evaluated.body == 11 - - -def test_i_can_evaluate_when_another_concept_is_referenced(): - sheerka = get_sheerka() - concept_a = Concept("a") - sheerka.add_in_cache(concept_a) - - concept = Concept("foo", body="a").init_key() - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated == simplec("foo", simplec("a", None)) - assert id(evaluated.body) != id(concept_a) - assert evaluated.metadata.is_evaluated - assert evaluated.body.metadata.is_evaluated - - -def test_i_can_evaluate_when_the_referenced_concept_has_a_body(): - sheerka = get_sheerka() - concept_a = Concept("a", body="1") - sheerka.add_in_cache(concept_a) - - concept = Concept("foo", body="a") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.key - assert evaluated.body == simplec("a", 1) - assert not concept_a.metadata.is_evaluated - assert evaluated.metadata.is_evaluated - - -def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(): - sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="a", body="'a'")) - sheerka.add_in_cache(Concept(name="b", body="a")) - sheerka.add_in_cache(Concept(name="c", body="b")) - concept_d = sheerka.add_in_cache(Concept(name="d", body="c")) - - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d) - - assert evaluated.key == concept_d.key - expected = simplec("c", simplec("b", simplec("a", "a"))) - assert evaluated.body == expected - assert sheerka.value(evaluated) == 'a' - assert evaluated.metadata.is_evaluated - - -def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(): - sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="a")) - sheerka.add_in_cache(Concept(name="b", body="a")) - sheerka.add_in_cache(Concept(name="c", body="b")) - concept_d = sheerka.add_in_cache(Concept(name="d", body="c")) - - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d) - - assert evaluated.key == concept_d.key - expected = simplec("c", simplec("b", simplec("a", None))) - assert evaluated.body == expected - assert sheerka.value(evaluated) == Concept(name="a").init_key() - assert evaluated.metadata.is_evaluated - - -def test_i_can_evaluate_concept_when_properties_reference_others_concepts(): - sheerka = get_sheerka() - concept_a = sheerka.add_in_cache(Concept(name="a").init_key()) - - concept = Concept("foo", body="a").def_prop("a", "a").init_key() - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - # first prop a is evaluated to concept_a - # then body is evaluated to prop a -> concept_a - assert evaluated.key == concept.key - assert evaluated.body == concept_a - - -def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2(): - """ - Same test, - but the name of the property and the name of the concept are different - :return: - """ - sheerka = get_sheerka() - concept_a = sheerka.add_in_cache(Concept(name="a")) - - concept = Concept("foo", body="concept_a").def_prop("concept_a", "a") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.key - assert evaluated.body == concept_a - - -def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body(): - sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="a", body="1")) - sheerka.add_in_cache(Concept(name="b", body="2")) - - concept = Concept("foo", body="propA + propB").def_prop("propA", "a").def_prop("propB", "b") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.key - assert evaluated.body == 3 - - -def test_i_can_evaluate_concept_when_properties_is_a_concept(): - sheerka = get_sheerka() - concept_a = sheerka.add_in_cache(Concept(name="a", body="'a'").init_key()) - - concept = Concept("foo").def_prop("a") - concept.compiled["a"] = concept_a - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.key - assert evaluated.get_prop("a") == simplec("a", "a") - - -def test_i_can_evaluate_when_property_asts_is_a_list(): - sheerka = get_sheerka() - foo = Concept("foo", body="1") - - concept = Concept("to_eval").def_prop("prop") - concept.compiled["prop"] = [foo, DoNotResolve("1")] - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - props = evaluated.get_prop("prop") - assert len(props) == 2 - assert props[0] == simplec("foo", 1) - assert props[1] == "1" - - -def test_i_can_evaluate_when_compiled_is_set_up_with_return_value(): - sheerka = get_sheerka() - - python_node = PythonNode("1 +1 ") - parser_result = ParserResultConcept(parser="who", value=python_node) - - concept = Concept("to_eval").def_prop("prop") - concept.compiled["prop"] = [ReturnValueConcept("who", True, parser_result)] - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - assert evaluated.get_prop("prop") == 2 - - # also works when only one return value - concept = Concept("to_eval").def_prop("prop") - concept.compiled["prop"] = ReturnValueConcept("who", True, parser_result) - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - assert evaluated.get_prop("prop") == 2 - - -def test_i_can_reference_sheerka(): - sheerka = get_sheerka() - - concept = Concept("foo", body="sheerka.test()").init_key() - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.key - assert evaluated.body == sheerka.test() - - -def test_properties_values_takes_precedence_over_the_outside_world(): - sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="a", body="'concept_a'")) - sheerka.add_in_cache(Concept(name="b", body="'concept_b'")) - - concept = Concept("foo", body="a") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - assert evaluated.key == concept.key - assert evaluated.body == simplec("a", "concept_a") # this test was already done - - # so check this one. - concept = Concept("foo", body="a").def_prop("a", "'property_a'") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - assert evaluated.key == concept.key - assert evaluated.body == 'property_a' - - # or this one. - concept = Concept("foo", body="a").def_prop("a", "b") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - assert evaluated.key == concept.key - assert evaluated.body == simplec(name="b", body="concept_b") - - -def test_properties_values_takes_precedence(): - sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="a", body="'concept_a'")) - sheerka.add_in_cache(Concept(name="b", body="'concept_b'")) - - concept = Concept("foo", body="a + b").def_prop("a", "'prop_a'") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - assert evaluated.key == concept.key - assert evaluated.body == 'prop_aconcept_b' - - -def test_i_can_reference_sub_property_of_a_property(): - sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="concept_a").def_prop("subProp", "'sub_a'")) - - concept = Concept("foo", body="a.props['subProp'].value").def_prop("a", "concept_a") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - assert evaluated == simplec(concept.key, "sub_a") - - -def test_i_cannot_evaluate_concept_if_property_is_in_error(): - sheerka = get_sheerka() - - concept = Concept(name="concept_a").def_prop("subProp", "undef_concept") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) - - -def test_key_is_initialized_by_evaluation(): - sheerka = get_sheerka() - - concept = Concept("foo") - evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) - - assert evaluated.key == concept.init_key().key - - -def test_builtin_error_concept_are_errors(): - # only test a random one, it will be the same for the others - sheerka = get_sheerka() - assert not sheerka.is_success(sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS)) - - -def test_i_can_add_concept_to_set(): - sheerka = get_sheerka(False, False) - - foo = Concept("foo") - sheerka.set_id_if_needed(foo, False) - - all_foos = Concept("all_foos") - sheerka.set_id_if_needed(all_foos, False) - - context = get_context(sheerka) - res = sheerka.add_concept_to_set(context, foo, all_foos) - - assert res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) - - all_entries = get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False) - assert len(all_entries) == 1 - assert foo.id in all_entries - - -def test_i_can_add_several_concepts_to_set(): - sheerka = get_sheerka(False, False) - - foo1 = Concept("foo1") - sheerka.set_id_if_needed(foo1, False) - - foo2 = Concept("foo1") - sheerka.set_id_if_needed(foo2, False) - - all_foos = Concept("all_foos") - sheerka.set_id_if_needed(all_foos, False) - - context = get_context(sheerka) - sheerka.add_concept_to_set(context, foo1, all_foos) - res = sheerka.add_concept_to_set(context, foo2, all_foos) - - assert res.status - assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) - - all_entries = get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False) - assert len(all_entries) == 2 - assert foo1.id in all_entries - assert foo2.id in all_entries - - -def test_i_cannot_add_the_same_concept_twice_in_a_set(): - sheerka = get_sheerka() - - foo = Concept("foo") - sheerka.set_id_if_needed(foo, False) - - all_foos = Concept("all_foos") - sheerka.set_id_if_needed(all_foos, False) - - context = get_context(sheerka) - sheerka.add_concept_to_set(context, foo, all_foos) - res = sheerka.add_concept_to_set(context, foo, all_foos) - - assert not res.status - assert res.body == ConceptAlreadyInSet(foo, all_foos) - - all_entries = sheerka.sdp.get("All_" + all_foos.id, None, False) - assert len(all_entries) == 1 - assert foo.id in all_entries - - -def test_i_get_elements_from_a_set(): - sheerka = get_sheerka(False, False) - - one = Concept("one") - two = Concept("two") - three = Concept("three") - number = Concept("number") - - for c in [one, two, three, number]: - sheerka.set_id_if_needed(c, False) - sheerka.add_in_cache(c) - - context = get_context(sheerka) - for c in [one, two, three]: - sheerka.add_concept_to_set(context, c, number) - - elements = sheerka.get_set_elements(number) - - assert set(elements) == set([one, two, three]) - - -def test_i_cannot_get_elements_if_not_a_set(): - sheerka = get_sheerka(False, False) - one = Concept("one") - sheerka.set_id_if_needed(one, False) - sheerka.add_in_cache(one) - - error = sheerka.get_set_elements(one) - - assert sheerka.isinstance(error, BuiltinConcepts.NOT_A_SET) - assert error.body == one - - -def test_isa_and_isa_group(): - sheerka = get_sheerka() - - group = Concept("group").init_key() - group.metadata.id = "1001" - assert not sheerka.isagroup(group) - - foo = Concept("foo").init_key() - foo.metadata.id = "1002" - assert not sheerka.isa(foo, group) - - context = get_context(sheerka) - sheerka.add_concept_to_set(context, foo, group) - assert sheerka.isagroup(group) - assert sheerka.isa(foo, group) diff --git a/tests/test_sheerka_call_evaluators.py b/tests/test_sheerka_call_evaluators.py deleted file mode 100644 index da705bd..0000000 --- a/tests/test_sheerka_call_evaluators.py +++ /dev/null @@ -1,399 +0,0 @@ -# Make sure that the evaluators works as expected -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept -from core.sheerka import Sheerka, ExecutionContext -from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator -from sdp.sheerkaDataProvider import Event - - -def get_sheerka(): - sheerka = Sheerka() - sheerka.initialize("mem://") - return sheerka - - -def get_context(sheerka): - return ExecutionContext("test", Event(), sheerka) - - -def get_ret_val(sheerka, concept, who="who"): - concept.init_key() - if concept.key not in sheerka.cache_by_key: - sheerka.cache_by_key[concept.key] = concept - return sheerka.ret(who, True, sheerka.new(concept.key)) - - -class Out: - debug_out = [] - - def out(self, method, name, context, return_value): - name = name[len(BaseEvaluator.PREFIX):] - if isinstance(return_value, list): - target = [str(r.body.key) for r in return_value] - else: - target = str(return_value.body.key) - step = str(context.step) - text = f"{step} [{context.iteration}] " - text += f"{name} - {method} - target={target}" - self.debug_out.append(text) - - def out_all(self, method, name, context, return_values): - name = name[len(BaseEvaluator.PREFIX):] - target = [str(r.body.key) for r in return_values] - step = str(context.step) - text = f"{step} [{context.iteration}] " - text += f"{name} - {method} - target={target}" - self.debug_out.append(text) - - -class OneReturnValueEvaluatorForTestingPurpose(OneReturnValueEvaluator, Out): - def __init__(self, name, steps, priority): - super().__init__(name, steps, priority) - - def matches(self, context, return_value): - self.out("matches", self.name, context, return_value) - return True - - def eval(self, context, return_value): - self.out("eval", self.name, context, return_value) - - -class AllReturnValueEvaluatorForTestingPurpose(AllReturnValuesEvaluator, Out): - def __init__(self, name, steps, priority): - super().__init__(name, steps, priority) - - def matches(self, context, return_values): - self.out("matches", self.name, context, return_values) - return True - - def eval(self, context, return_values): - self.out("eval", self.name, context, return_values) - - -class EvaluatorOneWithPriority(OneReturnValueEvaluatorForTestingPurpose): - def __init__(self, name, priority): - super().__init__(name, [BuiltinConcepts.EVALUATION], priority) - - -class EvaluatorOneWithPriority10(EvaluatorOneWithPriority): - def __init__(self): - super().__init__("priority10", 10) - - -class EvaluatorOneWithPriority15(EvaluatorOneWithPriority): - def __init__(self): - super().__init__("priority15", 15) - - -class EvaluatorOneWithPriority20(EvaluatorOneWithPriority): - def __init__(self): - super().__init__("priority20", 20) - - -class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose): - def __init__(self, name, priority): - super().__init__(name, [BuiltinConcepts.EVALUATION], priority) - - -class EvaluatorAllWithPriority10(EvaluatorAllWithPriority): - def __init__(self): - super().__init__("all_priority10", 10) - - -class EvaluatorAllWithPriority15(EvaluatorAllWithPriority): - def __init__(self): - super().__init__("all_priority15", 15) - - -class EvaluatorAllWithPriority20(EvaluatorAllWithPriority): - def __init__(self): - super().__init__("all_priority20", 20) - - -class EvaluatorOneModifyFoo(EvaluatorOneWithPriority): - def __init__(self): - super().__init__("modifyFoo", 10) - - def matches(self, context, return_value): - super().matches(context, return_value) - return context.sheerka.isinstance(return_value.body, "foo") - - def eval(self, context, return_value): - super().eval(context, return_value) - return get_ret_val(context.sheerka, Concept("bar")) - - -class EvaluatorOneModifyBar(EvaluatorOneWithPriority): - def __init__(self): - super().__init__("modifyBar", 10) - - def matches(self, context, return_value): - super().matches(context, return_value) - return context.sheerka.isinstance(return_value.body, "bar") - - def eval(self, context, return_value): - super().eval(context, return_value) - return get_ret_val(context.sheerka, Concept("baz")) - - -class EvaluatorOnePreEvaluation(OneReturnValueEvaluatorForTestingPurpose): - def __init__(self): - super().__init__("preEval", [BuiltinConcepts.BEFORE_EVALUATION], 10) - - -class EvaluatorOneMultiSteps(OneReturnValueEvaluatorForTestingPurpose): - def __init__(self): - super().__init__("multiStep", [BuiltinConcepts.EVALUATION, BuiltinConcepts.BEFORE_EVALUATION], 10) - - -class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority): - def __init__(self): - super().__init__("all_reduce_foobar", 10) - - def matches(self, context, return_values): - super().matches(context, return_values) - keys = [c.body.key for c in return_values] - return "foo" in keys and "bar" in keys - - def eval(self, context, return_values): - super().eval(context, return_values) - ret = get_ret_val(context.sheerka, context.sheerka.new(BuiltinConcepts.SUCCESS)) - ret.parents = return_values - return ret - - -class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority): - - def __init__(self): - super().__init__("suppress", 100) - - def matches(self, context, return_values): - super().matches(context, return_values) - return True - - def eval(self, context, return_values): - super().eval(context, return_values) - foo = None - for ret in return_values: - if ret.body.name == "foo": - foo = ret - - if foo: - return context.sheerka.ret(self.name, False, Concept("does not matter"), parents=[foo]) - else: - return None - - -def test_that_return_values_is_unchanged_when_no_evaluator(): - sheerka = get_sheerka() - sheerka.evaluators = [] - - entries = get_ret_val(sheerka, Concept("foo")) - return_values = sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) - - assert return_values == [entries] - - -def test_i_can_use_a_list_as_input(): - sheerka = get_sheerka() - sheerka.evaluators = [] - - entries = [get_ret_val(sheerka, Concept("foo"))] - return_values = sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) - - assert return_values == entries - - -def test_step_concept_is_removed_after_processing_if_not_reduced(): - """ - No evaluator - """ - sheerka = get_sheerka() - sheerka.evaluators = [] - - entry = get_ret_val(sheerka, Concept("foo")) - return_values = sheerka.execute(get_context(sheerka), entry, [BuiltinConcepts.EVALUATION]) - - assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values] - - -def test_step_concept_is_removed_after_processing_if_not_reduced_2(): - """ - This time the entry is modified by an evaluator, - nevertheless, step concept is removed - """ - sheerka = get_sheerka() - sheerka.evaluators = [EvaluatorOneModifyFoo] - - entry = get_ret_val(sheerka, Concept("foo")) - return_values = sheerka.execute(get_context(sheerka), entry, [BuiltinConcepts.EVALUATION]) - - assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values] - - -def test_that_higher_priority_evaluators_are_evaluated_first(): - sheerka = get_sheerka() - sheerka.evaluators = [ - EvaluatorAllWithPriority10, - EvaluatorOneWithPriority20, - EvaluatorAllWithPriority15, - EvaluatorOneWithPriority10, - EvaluatorOneWithPriority15, - EvaluatorAllWithPriority20, ] - - entries = [get_ret_val(sheerka, Concept("foo"))] - Out.debug_out = [] - sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) - - assert Out.debug_out == [ - '__EVALUATION [0] priority20 - matches - target=foo', - '__EVALUATION [0] priority20 - eval - target=foo', - '__EVALUATION [0] priority20 - matches - target=__EVALUATION', - '__EVALUATION [0] priority20 - eval - target=__EVALUATION', - "__EVALUATION [0] all_priority20 - matches - target=['foo', '__EVALUATION']", - "__EVALUATION [0] all_priority20 - eval - target=['foo', '__EVALUATION']", - "__EVALUATION [0] all_priority15 - matches - target=['foo', '__EVALUATION']", - "__EVALUATION [0] all_priority15 - eval - target=['foo', '__EVALUATION']", - '__EVALUATION [0] priority15 - matches - target=foo', - '__EVALUATION [0] priority15 - eval - target=foo', - '__EVALUATION [0] priority15 - matches - target=__EVALUATION', - '__EVALUATION [0] priority15 - eval - target=__EVALUATION', - "__EVALUATION [0] all_priority10 - matches - target=['foo', '__EVALUATION']", - "__EVALUATION [0] all_priority10 - eval - target=['foo', '__EVALUATION']", - '__EVALUATION [0] priority10 - matches - target=foo', - '__EVALUATION [0] priority10 - eval - target=foo', - '__EVALUATION [0] priority10 - matches - target=__EVALUATION', - '__EVALUATION [0] priority10 - eval - target=__EVALUATION' - ] - - -def test_that_predicate_is_checked_before_evaluation_for_one_return(): - sheerka = get_sheerka() - sheerka.evaluators = [EvaluatorOneModifyFoo] - - entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("baz"))] - Out.debug_out = [] - sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) - - assert Out.debug_out == [ - '__EVALUATION [0] modifyFoo - matches - target=foo', - '__EVALUATION [0] modifyFoo - eval - target=foo', - '__EVALUATION [0] modifyFoo - matches - target=baz', - '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION', - '__EVALUATION [1] modifyFoo - matches - target=bar', - '__EVALUATION [1] modifyFoo - matches - target=baz', - '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION' - ] - - -def test_that_predicate_is_checked_before_evaluation_for_all_return(): - sheerka = get_sheerka() - sheerka.evaluators = [EvaluatorAllReduceFooBar] - - entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("bar"))] - Out.debug_out = [] - sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) - assert Out.debug_out == [ - "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', 'bar', '__EVALUATION']", - "__EVALUATION [0] all_reduce_foobar - eval - target=['foo', 'bar', '__EVALUATION']", - "__EVALUATION [1] all_reduce_foobar - matches - target=['__SUCCESS']" - ] - - entries = [get_ret_val(sheerka, Concept("foo"))] - Out.debug_out = [] - sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) - assert Out.debug_out == [ - "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', '__EVALUATION']" - ] - - -def test_evaluation_continue_until_no_more_modification(): - sheerka = get_sheerka() - sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar] - - entries = [get_ret_val(sheerka, Concept("foo")), get_ret_val(sheerka, Concept("baz"))] - Out.debug_out = [] - sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) - - assert Out.debug_out == [ - '__EVALUATION [0] modifyFoo - matches - target=foo', - '__EVALUATION [0] modifyFoo - eval - target=foo', - '__EVALUATION [0] modifyFoo - matches - target=baz', - '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION', - '__EVALUATION [0] modifyBar - matches - target=foo', - '__EVALUATION [0] modifyBar - matches - target=baz', - '__EVALUATION [0] modifyBar - matches - target=__EVALUATION', - '__EVALUATION [1] modifyFoo - matches - target=bar', - '__EVALUATION [1] modifyFoo - matches - target=baz', - '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION', - '__EVALUATION [1] modifyBar - matches - target=bar', - '__EVALUATION [1] modifyBar - eval - target=bar', - '__EVALUATION [1] modifyBar - matches - target=baz', - '__EVALUATION [1] modifyBar - matches - target=__EVALUATION', - '__EVALUATION [2] modifyFoo - matches - target=baz', - '__EVALUATION [2] modifyFoo - matches - target=baz', - '__EVALUATION [2] modifyFoo - matches - target=__EVALUATION', - '__EVALUATION [2] modifyBar - matches - target=baz', - '__EVALUATION [2] modifyBar - matches - target=baz', - '__EVALUATION [2] modifyBar - matches - target=__EVALUATION' - ] - - -def test_evaluation_steps_are_respected(): - sheerka = get_sheerka() - sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation] - - entries = [get_ret_val(sheerka, Concept("foo"))] - Out.debug_out = [] - sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION]) - - assert Out.debug_out == [ - '__BEFORE_EVALUATION [0] preEval - matches - target=foo', - '__BEFORE_EVALUATION [0] preEval - eval - target=foo', - '__BEFORE_EVALUATION [0] preEval - matches - target=__BEFORE_EVALUATION', - '__BEFORE_EVALUATION [0] preEval - eval - target=__BEFORE_EVALUATION'] - - -def test_evaluation_multi_steps_are_respected(): - sheerka = get_sheerka() - sheerka.evaluators = [EvaluatorOneMultiSteps] - - entries = [get_ret_val(sheerka, Concept("foo"))] - Out.debug_out = [] - sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION]) - - assert Out.debug_out == [ - '__BEFORE_EVALUATION [0] multiStep - matches - target=foo', - '__BEFORE_EVALUATION [0] multiStep - eval - target=foo', - '__BEFORE_EVALUATION [0] multiStep - matches - target=__BEFORE_EVALUATION', - '__BEFORE_EVALUATION [0] multiStep - eval - target=__BEFORE_EVALUATION', - '__EVALUATION [0] multiStep - matches - target=foo', - '__EVALUATION [0] multiStep - eval - target=foo', - '__EVALUATION [0] multiStep - matches - target=__EVALUATION', - '__EVALUATION [0] multiStep - eval - target=__EVALUATION' - ] - - -def test_evaluators_can_be_pre_processed(): - sheerka = get_sheerka() - sheerka.evaluators = [EvaluatorOneModifyFoo] - - entries = [get_ret_val(sheerka, Concept("foo"))] - - # disable evaluator - context = get_context(sheerka) - context.add_preprocess(EvaluatorOneModifyFoo().name, enabled=False) # disabled for this exec context - Out.debug_out = [] - sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION]) - assert Out.debug_out == [] - - # other contextes are not impacted - Out.debug_out = [] - sheerka.execute(get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) - assert Out.debug_out == [ - '__EVALUATION [0] modifyFoo - matches - target=foo', - '__EVALUATION [0] modifyFoo - eval - target=foo', - '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION', - '__EVALUATION [1] modifyFoo - matches - target=bar', - '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION' - ] diff --git a/tests/test_sheerka_call_parsers.py b/tests/test_sheerka_call_parsers.py deleted file mode 100644 index 70a47da..0000000 --- a/tests/test_sheerka_call_parsers.py +++ /dev/null @@ -1,380 +0,0 @@ -from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept -from core.sheerka import Sheerka, ExecutionContext -from parsers.BaseParser import BaseParser -from sdp.sheerkaDataProvider import Event - - -def get_sheerka(): - sheerka = Sheerka() - sheerka.initialize("mem://") - return sheerka - - -def get_context(sheerka): - return ExecutionContext("test", Event(), sheerka) - - -def get_ret_val(text, who="who"): - return ReturnValueConcept(who, True, UserInputConcept(text, "user_name")) - - -class BaseTestParser(BaseParser): - debug_out = [] - - def __init__(self, name, priority, status=None, parser_result=None): - super().__init__(name, priority) - self.status = status - self.parser_result = parser_result - - @staticmethod - def _get_name(name): - return name[8:] if name.startswith("parsers.") else name - - @staticmethod - def _get_source(text_): - return text_ if isinstance(text_, str) else text_.body - - def _out(self, name, priority, status, source): - debug = f"name={name}" - debug += f", priority={priority}" - debug += f", status={status}" - debug += f", source={source}" - self.debug_out.append(debug) - - def parse(self, context, text): - self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) - value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) - parser_result = ParserResultConcept(parser=self, value=value) - return ReturnValueConcept(self, self.status, self.parser_result or parser_result) - - -class Enabled90FalseParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Enabled90False", 90, False) - - -class Enabled80FalseParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Enabled80False", 80, False) - - -class Enabled80MultipleFalseParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Enabled80MultipleFalse", 80, False) - - def parse(self, context, text): - self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) - value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1" - value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2" - return [ - ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)), - ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)), - ] - - -class Enabled80MultipleTrueParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Enabled80MultipleTrue", 80) - - def parse(self, context, text): - self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) - value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1" - value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2" - return [ - ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)), - ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)), - ] - - -class Enabled70FalseParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Enabled70False", 70, False, "Not a ParserResult") - - -class Enabled50TrueParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Enabled50True", 50, True) - - def parse(self, context, text): - source = self._get_source(text) - status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world" - self._out(self._get_name(self.name), self.priority, status, source) - - value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) - return_value = ParserResultConcept(parser=self, value=value) - return ReturnValueConcept(self, status, return_value) - - -class Enabled50bisTrueParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Enabled50BisTrue", 50, True) - - -class Enabled50FalseParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Enabled50False", 50, False) - - -class Enabled10TrueParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Enabled10True", 10, True) - - -class DisabledParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("Disabled", 90, True) - self.enabled = False - - -class NoneParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("None", 90, True, None) - - def parse(self, context, text): - self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) - return None - - -class ListOfNoneParser(BaseTestParser): - def __init__(self, **kwargs): - super().__init__("ListOfNone", 90, True, None) - - def parse(self, context, text): - self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text)) - return [None, None] - - -def test_disabled_parsers_are_not_executed(): - sheerka = get_sheerka() - sheerka.parsers = { - "Enabled": Enabled10TrueParser, - "Disabled": DisabledParser - } - - user_input = [get_ret_val("hello world")] - BaseTestParser.debug_out = [] - sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - - assert BaseTestParser.debug_out == ['name=Enabled10True, priority=10, status=True, source=hello world'] - - -def test_parser_are_executed_by_priority(): - sheerka = get_sheerka() - sheerka.parsers = { - "Enabled90False": Enabled90FalseParser, - "Enabled80False": Enabled80FalseParser, - "Enabled50True": Enabled50TrueParser, - } - - user_input = [get_ret_val("hello world")] - BaseTestParser.debug_out = [] - sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - - assert BaseTestParser.debug_out == [ - 'name=Enabled90False, priority=90, status=False, source=hello world', - 'name=Enabled80False, priority=80, status=False, source=hello world', - 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', - 'name=Enabled50True, priority=50, status=False, source=hello world', - 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', - 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', - 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', - ] - - -def test_parsing_stop_at_the_first_success(): - sheerka = get_sheerka() - sheerka.parsers = { - "Enabled80False": Enabled80FalseParser, - "Enabled50bisTrue": Enabled50bisTrueParser, - "Enabled10True": Enabled10TrueParser, - } - - user_input = [get_ret_val("hello world")] - BaseTestParser.debug_out = [] - sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - - assert BaseTestParser.debug_out == [ - 'name=Enabled80False, priority=80, status=False, source=hello world', - 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', - ] - - -def test_parsing_stop_at_the_first_success_2(): - """ - Same test than before, but Enabled50True takes more time to find a match - :return: - """ - sheerka = get_sheerka() - sheerka.parsers = { - "Enabled90False": Enabled90FalseParser, - "Enabled80False": Enabled80FalseParser, - "Enabled50True": Enabled50TrueParser, - "Enabled10True": Enabled10TrueParser, - } - - user_input = [get_ret_val("hello world")] - BaseTestParser.debug_out = [] - sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - - assert BaseTestParser.debug_out == [ - 'name=Enabled90False, priority=90, status=False, source=hello world', - 'name=Enabled80False, priority=80, status=False, source=hello world', - 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', - 'name=Enabled50True, priority=50, status=False, source=hello world', - 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', - 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', - 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', - ] - - -def test_all_parsers_of_a_given_priority_are_executed(): - """ - Make sure that all parsers with priority 50 are executed - :return: - """ - sheerka = get_sheerka() - sheerka.parsers = { - "Enabled90False": Enabled90FalseParser, - "Enabled80False": Enabled80FalseParser, - "Enabled50True": Enabled50TrueParser, - "Enabled50bisTrue": Enabled50bisTrueParser, - "Enabled50False": Enabled50FalseParser, - "Enabled10True": Enabled10TrueParser, - } - - user_input = [get_ret_val("hello world")] - BaseTestParser.debug_out = [] - sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - - assert BaseTestParser.debug_out == [ - 'name=Enabled90False, priority=90, status=False, source=hello world', - 'name=Enabled80False, priority=80, status=False, source=hello world', - 'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world', - 'name=Enabled50True, priority=50, status=False, source=hello world', - 'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world', - 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', - 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', - 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', - 'name=Enabled50False, priority=50, status=False, source=hello world', - 'name=Enabled50False, priority=50, status=False, source=Enabled90False:hello world', - 'name=Enabled50False, priority=50, status=False, source=Enabled80False:hello world', - 'name=Enabled50False, priority=50, status=False, source=Enabled80False:Enabled90False:hello world', - ] - - -def test_a_parser_has_access_to_the_output_of_its_predecessors(): - sheerka = get_sheerka() - sheerka.parsers = { - "Enabled90False": Enabled90FalseParser, - "Enabled80False": Enabled80FalseParser, - "Enabled50True": Enabled50TrueParser, - } - - user_input = [get_ret_val("hello world")] - - res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] - assert res_as_tuple == [ - ('Enabled90False', False, 'Enabled90False:hello world'), - ('Enabled80False', False, 'Enabled80False:hello world'), - ('Enabled80False', False, 'Enabled80False:Enabled90False:hello world'), - ('Enabled50True', False, 'Enabled50True:hello world'), - ('Enabled50True', False, 'Enabled50True:Enabled90False:hello world'), - ('Enabled50True', False, 'Enabled50True:Enabled80False:hello world'), - ('Enabled50True', True, 'Enabled50True:Enabled80False:Enabled90False:hello world'), - ] - - -def test_none_return_values_are_discarded(): - sheerka = get_sheerka() - sheerka.parsers = { - "NoneParser": NoneParser, - "ListOfNone": ListOfNoneParser, - } - - user_input = [get_ret_val("hello world")] - - BaseTestParser.debug_out = [] - res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - assert res == [] - assert BaseTestParser.debug_out == [ - 'name=None, priority=90, status=True, source=hello world', - 'name=ListOfNone, priority=90, status=True, source=hello world' - ] - - -def test_following_priorities_can_only_see_parser_result_return_values(): - """ - Normally, lower priority parsers can see the result of the higher priority parsers - This is true only if the higher priority parser return a ParserResultConcept - :return: - """ - - sheerka = get_sheerka() - sheerka.parsers = { - "Enabled80False": Enabled80FalseParser, - "Enabled70False": Enabled70FalseParser, - "Enabled50True": Enabled50TrueParser, - } - - user_input = [get_ret_val("hello world")] - BaseTestParser.debug_out = [] - sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - - assert BaseTestParser.debug_out == [ - 'name=Enabled80False, priority=80, status=False, source=hello world', - 'name=Enabled70False, priority=70, status=False, source=hello world', - 'name=Enabled70False, priority=70, status=False, source=Enabled80False:hello world', - 'name=Enabled50True, priority=50, status=False, source=hello world', - 'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world', - ] - - -def test_i_can_manage_parser_with_multiple_results(): - sheerka = get_sheerka() - sheerka.parsers = { - "Enabled80MultipleFalse": Enabled80MultipleFalseParser, - "Enabled50True": Enabled50TrueParser, - } - - user_input = [get_ret_val("hello world")] - BaseTestParser.debug_out = [] - res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - - assert BaseTestParser.debug_out == [ - 'name=Enabled80MultipleFalse, priority=80, status=False, source=hello world', - 'name=Enabled50True, priority=50, status=False, source=hello world', - 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_1', - 'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_2', - ] - - res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] - assert res_as_tuple == [ - ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_1'), - ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_2'), - ('Enabled50True', False, 'Enabled50True:hello world'), - ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_1'), - ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'), - ] - - -def test_i_can_manage_parser_with_multiple_results_and_a_sucess(): - sheerka = get_sheerka() - sheerka.parsers = { - "Enabled80MultipleTrue": Enabled80MultipleTrueParser, - "Enabled50True": Enabled50TrueParser, - } - - user_input = [get_ret_val("hello world")] - BaseTestParser.debug_out = [] - res = sheerka.execute(get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - - assert BaseTestParser.debug_out == [ - 'name=Enabled80MultipleTrue, priority=80, status=None, source=hello world', - ] - - res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] - assert res_as_tuple == [ - ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'), - ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'), - ] diff --git a/tests/test_sheerka_non_reg.py b/tests/test_sheerka_non_reg.py deleted file mode 100644 index 4a88a08..0000000 --- a/tests/test_sheerka_non_reg.py +++ /dev/null @@ -1,696 +0,0 @@ -import os -import shutil -from os import path - -import pytest - -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, PROPERTIES_TO_SERIALIZE, Property, simplec -from core.sheerka import Sheerka, ExecutionContext -from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator -from parsers.ConceptLexerParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression -from sdp.sheerkaDataProvider import SheerkaDataProvider, Event - -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - - os.chdir(current_pwd) - - -def get_sheerka(use_dict=True, skip_builtins_in_db=True): - root = "mem://" if use_dict else root_folder - sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db) - sheerka.initialize(root) - - return sheerka - - -def get_context(sheerka): - return ExecutionContext("test", Event(), sheerka) - - -def get_default_concept(): - concept = Concept( - name="a + b", - where="isinstance(a, int) and isinstance(b, int)", - pre="isinstance(a, int) and isinstance(b, int)", - post="isinstance(res, int)", - body="def func(x,y):\n return x+y\nfunc(a,b)", - desc="specific description") - concept.def_prop("a", "value1") - concept.def_prop("b", "value2") - - return concept - - -@pytest.mark.parametrize("text, expected", [ - ("1 + 1", 2), - ("sheerka.test()", 'I have access to Sheerka !') -]) -def test_i_can_eval_python_expressions_with_no_variable(text, expected): - sheerka = get_sheerka() - - res = sheerka.evaluate_user_input(text) - - assert len(res) == 1 - assert res[0].status - assert res[0].value == expected - - -def test_i_can_eval_concept_with_python_body(): - sheerka = get_sheerka() - concept = Concept(name="one", body="1") - sheerka.add_in_cache(concept) - - text = "one" - res = sheerka.evaluate_user_input(text) - assert len(res) == 1 - assert res[0].status - assert res[0].value == simplec("one", 1) # by default, the concept is returned - - -def test_i_can_eval_concept_with_concept_body(): - sheerka = get_sheerka() - concept_one = Concept(name="one") - concept_un = Concept(name="un", body="one") - sheerka.add_in_cache(concept_one) - sheerka.add_in_cache(concept_un) - - res = sheerka.evaluate_user_input("un") - return_value = res[0].value - assert len(res) == 1 - assert res[0].status - assert return_value == simplec("un", simplec("one", None)) - - -def test_i_can_eval_concept_with_no_body(): - sheerka = get_sheerka() - concept = Concept(name="one") - sheerka.add_in_cache(concept) - - text = "one" - res = sheerka.evaluate_user_input(text) - assert len(res) == 1 - assert res[0].status - assert res[0].value == concept - assert id(res[0].value) != id(concept) - - -def test_is_unique_property_is_used_when_evaluating(): - sheerka = get_sheerka() - concept = Concept(name="one", is_unique=True) - sheerka.add_in_cache(concept) - - text = "one" - res = sheerka.evaluate_user_input(text) - assert len(res) == 1 - assert res[0].status - assert res[0].value == concept - assert id(res[0].value) == id(concept) - - -def test_i_can_eval_def_concept_request(): - text = """ -def concept a + b -where isinstance(a, int) and isinstance(b, int) -pre isinstance(a, int) and isinstance(b, int) -post isinstance(res, int) -as: - def func(x,y): - return x+y - func(a,b) - """ - - expected = get_default_concept() - expected.metadata.id = "1001" - expected.metadata.desc = None - expected.metadata.props = [("a", None), ("b", None)] - expected.init_key() - - sheerka = get_sheerka() - res = sheerka.evaluate_user_input(text) - - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT) - - concept_saved = res[0].value.body - - for prop in PROPERTIES_TO_SERIALIZE: - assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop) - - assert concept_saved.key in sheerka.cache_by_key - assert concept_saved.id in sheerka.cache_by_id - assert sheerka.sdp.io.exists( - sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) - - -def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept(): - """ - In this test, we test that the properties of 'concept a xx b' (which are 'a' and 'b') - are correctly detected, thanks to the source code 'a plus b' in its body - :return: - """ - sheerka = get_sheerka() - - # concept 'a plus b' is known - concept_a_plus_b = Concept(name="a plus b").def_prop("a").def_prop("b") - sheerka.add_in_cache(concept_a_plus_b) - - res = sheerka.evaluate_user_input("def concept a xx b as a plus b") - expected = Concept(name="a xx b", body="a plus b").def_prop("a").def_prop("b").init_key() - expected.metadata.id = "1001" - - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT) - - concept_saved = res[0].value.body - - for prop in PROPERTIES_TO_SERIALIZE: - assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop) - - assert concept_saved.key in sheerka.cache_by_key - assert concept_saved.id in sheerka.cache_by_id - assert sheerka.sdp.io.exists( - sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) - - -def test_i_cannot_eval_the_same_def_concept_twice(): - text = """ -def concept a + b -where isinstance(a, int) and isinstance(b, int) -pre isinstance(a, int) and isinstance(b, int) -post isinstance(res, int) -as: - def func(x,y): - return x+y - func(a,b) - """ - - sheerka = get_sheerka() - sheerka.evaluate_user_input(text) - res = sheerka.evaluate_user_input(text) - - assert len(res) == 1 - assert not res[0].status - assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED) - - -@pytest.mark.parametrize("text", [ - "", - " ", - "\n", -]) -def test_i_can_eval_a_empty_input(text): - sheerka = get_sheerka() - - res = sheerka.evaluate_user_input(text) - - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].value, BuiltinConcepts.NOP) - - -def test_i_can_eval_concept_with_variable(): - sheerka = get_sheerka() - concept_hello = Concept(name="hello a").def_prop("a") - concept_foo = Concept(name="foo") - sheerka.add_in_cache(concept_hello) - sheerka.add_in_cache(concept_foo) - - res = sheerka.evaluate_user_input("hello foo") - return_value = res[0].value - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(return_value, concept_hello) - assert return_value.props["a"].value == concept_foo - - -def test_i_can_eval_concept_with_variable_and_python_as_body(): - sheerka = get_sheerka() - hello_a = sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a")) - sheerka.add_in_cache(Concept(name="foo", body="'foo'")) - - res = sheerka.evaluate_user_input("hello foo") - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].value, hello_a) - assert res[0].value.body == "hello foo" - assert res[0].value.metadata.is_evaluated - assert res[0].value.props["a"].value == simplec("foo", "foo") - assert res[0].value.props["a"].value.metadata.is_evaluated - - -def test_i_can_eval_duplicate_concepts_with_same_value(): - sheerka = get_sheerka() - - sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a")) - sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'")) - sheerka.add_in_cache(Concept(name="foo", body="'foo'")) - - res = sheerka.evaluate_user_input("hello foo") - assert len(res) == 1 - assert res[0].status - assert res[0].value.body == "hello foo" - assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME) - - -def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different(): - sheerka = get_sheerka() - - sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a")) - sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'")) - sheerka.add_in_cache(Concept(name="foo", body="'another value'")) - - res = sheerka.evaluate_user_input("hello foo") - assert len(res) == 1 - assert not res[0].status - assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS) - - concepts = res[0].value.body - assert len(concepts) == 2 - sorted_values = sorted(concepts, key=lambda x: x.value.body) - assert sorted_values[0].value.body == "hello another value" - assert sorted_values[1].value.body == "hello foo" - - -def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(): - sheerka = get_sheerka() - context = get_context(sheerka) - - sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").def_prop("a")) - sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").def_prop("b")) - - res = sheerka.evaluate_user_input("hello 'foo'") - assert len(res) == 1 - assert res[0].status - assert res[0].value.body == "hello foo" # I don't know yet the one to choose - assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME) - - -def test_i_can_create_concepts_with_python_code_as_body(): - sheerka = get_sheerka() - context = get_context(sheerka) - - sheerka.create_new_concept(context, Concept(name="concepts", body="sheerka.concepts()")) - res = sheerka.evaluate_user_input("concepts") - - assert len(res) == 1 - assert res[0].status - assert isinstance(res[0].value.body, list) - - -def test_i_can_create_concept_with_bnf_definition(): - sheerka = get_sheerka(False, False) - a = Concept("a") - sheerka.add_in_cache(a) - sheerka.concepts_definition_cache = {a: OrderedChoice("one", "two")} - - res = sheerka.evaluate_user_input("def concept plus from bnf a ('plus' plus)?") - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT) - - saved_concept = sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, "plus") - assert saved_concept.key == "plus" - assert saved_concept.metadata.definition == "a ('plus' plus)?" - assert "a" in saved_concept.props - assert "plus" in saved_concept.props - - saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY) - expected_bnf = Sequence( - a, Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus")))) - assert saved_definitions[saved_concept] == expected_bnf - - new_concept = res[0].value.body - assert new_concept.metadata.name == "plus" - assert new_concept.metadata.definition == "a ('plus' plus)?" - assert new_concept.bnf == expected_bnf - assert "a" in new_concept.props - assert "plus" in new_concept.props - - -def test_i_can_eval_bnf_definitions(): - sheerka = get_sheerka() - concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body - - res = sheerka.evaluate_user_input("one") - - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].value, concept_a) - - -def test_i_can_eval_bnf_definitions_with_variables(): - sheerka = get_sheerka() - concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body - concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body - - res = sheerka.evaluate_user_input("one three") - - assert len(res) == 1 - assert res[0].status - return_value = res[0].value - - assert sheerka.isinstance(return_value, concept_b) - assert return_value.body == "one three" - assert return_value.metadata.is_evaluated - - assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one").init_key()) - assert return_value.props["a"].value.metadata.is_evaluated - - -def test_i_can_eval_bnf_definitions_from_separate_instances(): - """ - Same test then before, - but make sure that the BNF are correctly persisted and loaded - """ - sheerka = get_sheerka(False) - concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' 'two'")[0].body.body - - res = get_sheerka(False).evaluate_user_input("one two") - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].value, concept_a) - - # add another bnf definition - concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body - - res = get_sheerka(False).evaluate_user_input("one two") # previous one still works - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].value, concept_a) - - res = get_sheerka(False).evaluate_user_input("one two three") # new one works - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].value, concept_b) - assert res[0].value.body == "one two three" - assert res[0].value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key()) - - -@pytest.mark.parametrize("desc, definitions", [ - ("Simple form", [ - "def concept one as 1", - "def concept two as 2", - "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit" - ]), - ("When twenty is a concept", [ - "def concept one as 1", - "def concept two as 2", - "def concept twenty as 20", - "def concept twenties from bnf twenty (one|two)=unit as twenty + unit" - ]), - ("When digit is a concept", [ - "def concept one as 1", - "def concept two as 2", - "def concept twenty as 20", - "def concept digit from bnf one|two", - "def concept twenties from bnf twenty digit as twenty + digit" - ]), - ("When using isa and concept twenty", [ - "def concept one as 1", - "def concept two as 2", - "def concept number", - "one isa number", - "two isa number", - "def concept twenties from bnf 'twenty' number as 20 + number" - ]), - ("When using isa and concept twenty", [ - "def concept one as 1", - "def concept two as 2", - "def concept twenty as 20", - "def concept number", - "one isa number", - "two isa number", - "def concept twenties from bnf twenty number as 20 + number" - ]), -]) -def test_i_can_mix_concept_with_python_to_define_numbers(desc, definitions): - sheerka = get_sheerka() - - for definition in definitions: - sheerka.evaluate_user_input(definition) - - res = sheerka.evaluate_user_input("twenty one") - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].body, "twenties") - assert res[0].body.body == 21 - - res = sheerka.evaluate_user_input("twenty one + 1") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 22 - - res = sheerka.evaluate_user_input("twenty one + one") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 22 - - res = sheerka.evaluate_user_input("twenty one + twenty two") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 43 - - res = sheerka.evaluate_user_input("1 + twenty one") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 22 - - res = sheerka.evaluate_user_input("1 + 1 + twenty one") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 23 - - -def test_i_can_mix_bnf_and_isa(): - """ - if 'one' isa 'number, twenty number should be recognized - :return: - """ - sheerka = get_sheerka() - sheerka.evaluate_user_input("def concept one as 1") - sheerka.evaluate_user_input("def concept two as 2") - sheerka.evaluate_user_input("def concept number") - sheerka.evaluate_user_input("one isa number") - sheerka.evaluate_user_input("two isa number") - sheerka.evaluate_user_input("def concept twenties from bnf 'twenty' number as 20 + number") - - res = sheerka.evaluate_user_input("twenty one") - assert len(res) == 1 - assert res[0].status - assert res[0].body == simplec("twenties", 21) - - res = sheerka.evaluate_user_input("twenty one + 1") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 22 - - res = sheerka.evaluate_user_input("twenty one + one") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 22 - - res = sheerka.evaluate_user_input("twenty one + twenty two") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 43 - - res = sheerka.evaluate_user_input("1 + twenty one") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 22 - - res = sheerka.evaluate_user_input("1 + 1 + twenty one") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 23 - - -def test_i_can_mix_concept_of_concept(): - sheerka = get_sheerka() - - definitions = [ - "def concept one as 1", - "def concept two as 2", - "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit", - "def concept a plus b as a + b" - ] - - for definition in definitions: - sheerka.evaluate_user_input(definition) - - res = sheerka.evaluate_user_input("1 plus 2") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 3 - - res = sheerka.evaluate_user_input("1 plus one") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 2 - - res = sheerka.evaluate_user_input("1 + 1 plus 1") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 3 - - res = sheerka.evaluate_user_input("1 plus twenty one") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 22 - - res = sheerka.evaluate_user_input("one plus 1") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 2 - - res = sheerka.evaluate_user_input("one plus two") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 3 - - res = sheerka.evaluate_user_input("one plus twenty one") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 22 - - res = sheerka.evaluate_user_input("twenty one plus 1") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 22 - - res = sheerka.evaluate_user_input("twenty one plus one") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 22 - - res = sheerka.evaluate_user_input("twenty one plus twenty two") - assert len(res) == 1 - assert res[0].status - assert res[0].body.body == 43 - - -# def test_i_can_evaluate_concept_of_concept_when_multiple_choices(): -# sheerka = get_sheerka() -# -# definitions = [ -# "def concept little a where a", -# "def concept blue a where a", -# "def concept little blue a where a", -# "def concept house" -# ] -# -# for definition in definitions: -# sheerka.evaluate_user_input(definition) -# -# ### CAUTION #### -# # this test cannot work !! -# # it is just to hint the result that I would like to achieve -# -# res = sheerka.evaluate_user_input("little blue house") -# assert len(res) == 2 -# assert res[0].status -# assert res[0].body == "little(blue(house))" -# -# assert res[1].status -# assert res[1].body == "little blue(house)" - - -def test_i_can_say_that_a_concept_isa_another_concept(): - sheerka = get_sheerka() - sheerka.evaluate_user_input("def concept foo") - sheerka.evaluate_user_input("def concept bar") - - res = sheerka.evaluate_user_input("foo isa bar") - assert len(res) == 1 - assert res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) - - -def test_eval_does_not_break_valid_result(): - sheerka = get_sheerka() - sheerka.evaluate_user_input("def concept one as 1") - sheerka.evaluate_user_input("def concept two as 2") - - res = sheerka.evaluate_user_input("one + two") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 3 - - res = sheerka.evaluate_user_input("eval one + two") - assert len(res) == 1 - assert res[0].status - assert res[0].body == 3 - - -@pytest.mark.parametrize("text", [ - "'hello", - '"foo" + "string', - "c::", - "c:foo\nbar:", - "c:foo", - "def concept 'name", - "def concept name from bnf 'name" -]) -def test_i_can_manage_tokenizer_error(text): - sheerka = get_sheerka() - sheerka.add_in_cache(Concept("foo")) - - res = sheerka.evaluate_user_input(text) - - assert len(res) > 1 - for r in [r for r in res if r.who.startswith("parsers.")]: - assert not r.status - - -def test_i_can_recognize_concept_from_string(): - sheerka = get_sheerka() - sheerka.add_in_cache(Concept("one", body="1")) - - res = sheerka.evaluate_user_input("'one'") - - assert len(res) == 1 - assert res[0].status - assert res[0].body == "one" - - res = sheerka.evaluate_user_input("eval 'one'") - - assert len(res) == 1 - assert res[0].status - assert res[0].body == "one" - - -@pytest.mark.parametrize("expression", [ - "def concept twenties from bnf 'twenty' (one | two)=unit as 20 + unit", - "def concept twenties from bnf 'twenty' (one | two)=unit as twenty + unit", - "def concept twenties from bnf twenty (one | two)=unit as 20 + unit", - "def concept twenties from bnf twenty (one | two)=unit as twenty + unit", -]) -def test_i_can_evaluate_bnf_concepts(expression): - sheerka = get_sheerka() - - sheerka.evaluate_user_input("def concept one as 1") - sheerka.evaluate_user_input("def concept two as 2") - sheerka.evaluate_user_input("def concept twenty as 20") - sheerka.evaluate_user_input(expression) - res = sheerka.evaluate_user_input("eval twenty one") - - assert len(res) == 1 - assert res[0].status - assert res[0].body == 21 diff --git a/tests/test_sheerka_transform.py b/tests/test_sheerka_transform.py deleted file mode 100644 index 1837aab..0000000 --- a/tests/test_sheerka_transform.py +++ /dev/null @@ -1,336 +0,0 @@ -from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts -from core.sheerka import Sheerka, ExecutionContext, ExecutionContextIdManager -from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY -from sdp.sheerkaDataProvider import Event - - -def get_sheerka(): - sheerka = Sheerka() - sheerka.initialize("mem://") - return sheerka - - -def get_context(sheerka): - return ExecutionContext("test", Event(), sheerka) - - -def test_i_can_transform_an_unknown_concept(): - sheerka = get_sheerka() - - foo = Concept("foo", body="body") - concept_with_sub = Concept("concept_with_sub", body=foo) - - concept = Concept( - name="concept_name", - is_builtin=True, - is_unique=True, - key="concept_key", - body=concept_with_sub, - where=[foo, 1, "1", True, 1.0], - pre=foo, - post=None, # will not appear - definition="it is a definition", - definition_type="def type", - desc="this this the desc" - ).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub") - - # add values and props - concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init() - concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0] - concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init() - concept.values[ConceptParts.POST] = "a value for POST" - concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub) - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(concept) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 0, - 'meta.name': 'concept_name', - 'meta.key': 'concept_key', - 'meta.is_builtin': True, - 'meta.is_unique': True, - 'meta.definition': 'it is a definition', - 'meta.definition_type': 'def type', - 'meta.desc': 'this this the desc', - 'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 1, - 'meta.body': 'body', - 'meta.name': 'foo'}, 1, '1', True, 1.0], - 'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, - 'meta.body': { - '__type__': SheerkaTransformType.Concept, - '__id__': 2, - 'meta.name': 'concept_with_sub', - 'meta.body': { - '__type__': SheerkaTransformType.Reference, - '__id__': 1}}, - 'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']], - 'pre': {'__type__': SheerkaTransformType.Concept, - '__id__': 4, - 'meta.body': 'body', - 'meta.name': 'foo', - 'body': 'body'}, - 'post': "a value for POST", - 'body': {'__type__': SheerkaTransformType.Concept, - '__id__': 3, - 'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}, - 'meta.name': 'concept_with_sub', - 'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}}, - 'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0], - 'props': [('a', 10), - ('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}), - ('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})], - } - - -def test_i_can_transform_unknown_concept_with_almost_same_value(): - sheerka = get_sheerka() - concept = Concept("foo") - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(concept) - - assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'} - - -def test_i_can_transform_known_concept_when_the_values_are_the_same(): - """ - Values are the same means that we are serializing a concept which has kept all its default values - There is not diff between the concept to serialize and the one which was registered with create_new_concept() - We serialize only the id of the concept - :return: - """ - sheerka = get_sheerka() - - concept = Concept( - name="concept_name", - is_builtin=True, - is_unique=False, - key="concept_key", - body="body definition", - where="where definition", - pre="pre definition", - post="post definition", - definition="it is a definition", - definition_type="def type", - desc="this this the desc" - ).def_prop("a").def_prop("b") - sheerka.create_new_concept(get_context(sheerka), concept) - - new_concept = sheerka.new(concept.key) - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(new_concept) - - assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"} - - -def test_i_can_transform_known_concept_when_the_values_are_different(): - """ - Values are the different means the concept was modified. - It's different from the one which was registered with create_new_concept() - We serialize only the differences - :return: - """ - sheerka = get_sheerka() - - concept = Concept( - name="concept_name", - is_builtin=True, - is_unique=False, - key="concept_key", - body="body definition", - where="where definition", - pre="pre definition", - post="post definition", - definition="it is a definition", - definition_type="def type", - desc="this this the desc" - ).def_prop("a").def_prop("b") - sheerka.create_new_concept(get_context(sheerka), concept) - - new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre") - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(new_concept) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 0, - "id": "1001", - 'pre': 'another pre', - "body": "another", - 'props': [('a', 10)] - } - - -def test_i_can_transform_concept_with_circular_reference(): - sheerka = get_sheerka() - foo = Concept("foo") - bar = Concept("bar", body=foo) - foo.metadata.body = bar - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(foo) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 0, - 'meta.name': 'foo', - 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 1, - 'meta.name': 'bar', - 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, - OBJ_ID_KEY: 0}, - }, - } - - -def test_i_can_transform_concept_with_circular_reference_2(): - sheerka = get_sheerka() - foo = Concept("foo") - bar = Concept("foo", body=foo) - foo.metadata.body = bar - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(foo) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 0, - 'meta.name': 'foo', - 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept, - OBJ_ID_KEY: 1, - 'meta.name': 'foo', - 'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, - OBJ_ID_KEY: 0}, - }, - } - - -def test_i_can_transform_the_unknown_concept(): - sheerka = get_sheerka() - - unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT) - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(unknown) - - assert len(to_dict) == 3 - assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept - assert to_dict[OBJ_ID_KEY] == 0 - assert "id" in to_dict - - -def test_i_can_transform_simple_execution_context(): - sheerka = get_sheerka() - ExecutionContextIdManager.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(context) - - assert to_dict == { - OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext, - OBJ_ID_KEY: 0, - '_parent': None, - '_id': 0, - '_tab': '', - '_bag': {}, - '_start': 0, - '_stop': 0, - 'who': 'requester', - 'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'}, - 'desc': 'this is the desc', - 'children': [], - 'preprocess': None, - 'inputs': {}, - 'values': {}, - 'obj': None, - 'concepts': {} - } - - -def test_i_can_transform_list(): - sheerka = get_sheerka() - ExecutionContextIdManager.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict([context]) - - assert len(to_dict) == 1 - assert isinstance(to_dict, list) - assert to_dict[0]["who"] == "requester" - assert to_dict[0]["desc"] == "this is the desc" - - -def test_i_can_transform_set(): - sheerka = get_sheerka() - ExecutionContextIdManager.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict({context}) - - assert len(to_dict) == 1 - assert isinstance(to_dict, list) - assert to_dict[0]["who"] == "requester" - assert to_dict[0]["desc"] == "this is the desc" - - -def test_i_can_transform_dict(): - sheerka = get_sheerka() - ExecutionContextIdManager.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key() - sheerka.create_new_concept(get_context(sheerka), known_concept) - unknown_concept = Concept("bar") - known = sheerka.new("foo") - - bag = { - "context": context, - "known_concept": known_concept, - "unknown_concept": unknown_concept, - "True": True, - "Number": 1.1, - "String": "a string value", - "None": None, - unknown_concept: "hello", - known: "world" - } - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(bag) - - assert isinstance(to_dict, dict) - assert to_dict['Number'] == 1.1 - assert to_dict['String'] == 'a string value' - assert to_dict['True'] - assert to_dict['None'] is None - assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext - assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept - assert to_dict["known_concept"]["id"] == '1001' - assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept - assert to_dict["(None)bar"] == "hello" - assert to_dict["(1001)foo"] == "world" - - -def test_i_can_transform_when_circular_references(): - sheerka = get_sheerka() - ExecutionContextIdManager.ids = {} - context = ExecutionContext("requester", Event(), sheerka, 'this is the desc') - context.push("another requester", "another desc") - - st = SheerkaTransform(sheerka) - to_dict = st.to_dict(context) - - assert isinstance(to_dict, dict) - assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext - assert len(to_dict["children"]) == 1 - assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext - assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference - assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0 - assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference - assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1