diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 3184f60..6f06d4c 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -1,199 +1,8 @@ +from core.builtin_concepts_ids import BuiltinConcepts from core.concept import Concept, ConceptParts from core.error import ErrorObj -class BuiltinConcepts: - """ - List of builtin concepts that do no need any specific implementation - Please note that the value of the enum is informal. It is not used in the system - For example, the concept 'NODE' DOES NOT have the key, the id or whatever 200 - The key if the name of the concept - The id is a sequential number given just before the concept is saved in sdp - - The values of the enum is not used the code - """ - SHEERKA = "__SHEERKA" - - # processing instructions during sheerka.execute() or sheerka.evaluate_concept() - # The instructions may alter how the actions work - DEBUG = "__DEBUG" # activate all debug information - EVAL_BODY_REQUESTED = "__EVAL_BODY_REQUESTED" # to evaluate the body - EVAL_WHERE_REQUESTED = "__EVAL_WHERE_REQUESTED" # to evaluate the where clause - RETURN_BODY_REQUESTED = "__RETURN_BODY_REQUESTED" # returns the body of the concept instead of the concept itself - REDUCE_REQUESTED = "__REDUCE_REQUESTED" # remove meaningless error when possible - EVAL_UNTIL_SUCCESS_REQUESTED = "__EVAL_UNTIL_SUCCESS_REQUESTED" # PythonEvaluator tries combination until True is found - EVAL_QUESTION_REQUESTED = "__EVAL_QUESTION_REQUESTED" # the user input must be treated as question - - # possible actions during sheerka.execute() or sheerka.evaluate_rules() - INIT_SHEERKA = "__INIT_SHEERKA" # - PROCESS_INPUT = "__PROCESS_INPUT" # Processing user input or other input - PROCESSING = "__PROCESSING" # Processing user input or other input - BEFORE_PARSING = "__BEFORE_PARSING" # activated before evaluation by the parsers - PARSING = "__PARSING" # activated during the parsing. It contains the text to parse - AFTER_PARSING = "__AFTER_PARSING" # after parsing - BEFORE_EVALUATION = "__BEFORE_EVALUATION" # before evaluation - EVALUATION = "__EVALUATION" # activated when the parsing process seems to be finished - AFTER_EVALUATION = "__AFTER_EVALUATION" # activated when the parsing process seems to be finished - BEFORE_RENDERING = "__BEFORE_RENDERING" # activate before the output is rendered - RENDERING = "__RENDERING" # rendering the response from sheerka - AFTER_RENDERING = "__AFTER_RENDERING" # rendering the response from sheerka - EVALUATE_SOURCE = "__EVALUATE_SOURCE" # - EVALUATE_CONCEPT = "__EVALUATE_CONCEPT" # a concept will be evaluated - EVALUATING_CONCEPT = "__EVALUATING_CONCEPT" # a concept will be evaluated - EVALUATING_ATTRIBUTE = "__EVALUATING_ATTRIBUTE" # - VALIDATE_CONCEPT = "__VALIDATE_CONCEPT" - VALIDATING_CONCEPT = "__VALIDATING_CONCEPT" - INIT_COMPILED = "__INIT_COMPILED" - INIT_BNF = "__INIT_BNF" - MANAGE_INFINITE_RECURSION = "__MANAGE_INFINITE_RECURSION" - PARSE_CODE = "__PARSE_CODE" - EXEC_CODE = "__EXEC_CODE" # to use when executing Python or other language compiled code - TESTING = "__TESTING" - EVALUATOR_PRE_PROCESS = "__EVALUATOR_PRE_PROCESS" # used modify / tweak behaviour of evaluators - EVALUATING_RULES = "__EVALUATING_RULES" - - # builtin attributes - ISA = "__ISA" # when a concept is an instance of another one - HASA = "__HASA" # when a concept has/owns another concept - AUTO_EVAL = "__AUTO_EVAL" # when the concept must be auto evaluated - - # object - USER_INPUT = "__USER_INPUT" # represent an input from an user - SUCCESS = "__SUCCESS" - ERROR = "__ERROR" - UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT" # the request concept is not recognized - CANNOT_RESOLVE_CONCEPT = "__CANNOT_RESOLVE_CONCEPT" # when too many concepts with the same name - RETURN_VALUE = "__RETURN_VALUE" # a value is returned - CONCEPT_TOO_LONG = "__CONCEPT_TOO_LONG" # concept cannot be processed by exactConcept parser - NEW_CONCEPT = "__NEW_CONCEPT" # when a new concept is added - UNKNOWN_PROPERTY = "__UNKNOWN_PROPERTY" # when requesting for a unknown property - PARSER_RESULT = "__PARSER_RESULT" - TOO_MANY_SUCCESS = "__TOO_MANY_SUCCESS" # when expecting a limited number of successful return value - TOO_MANY_ERRORS = "__TOO_MANY_ERRORS" # when expecting a limited number of successful return value - ONLY_SUCCESSFUL = "__ONLY_SUCCESSFUL" # filter the result, only keep successful ones - MULTIPLE_ERRORS = "__MULTIPLE_ERRORS" # filter the result, only keep evaluator in error - NOT_FOR_ME = "__NOT_FOR_ME" # a parser recognize that the entry is not meant for it - IS_EMPTY = "__IS_EMPTY" # when a set is empty - NO_RESULT = "__NO_RESULT" # no return value returned - INVALID_RETURN_VALUE = "__INVALID_RETURN_VALUE" # the return value of an evaluator is not correct - CONCEPT_ALREADY_DEFINED = "__CONCEPT_ALREADY_DEFINED" # when you try to add the same object twice (a concept or whatever) - PROPERTY_ALREADY_DEFINED = "__PROPERTY_ALREADY_DEFINED" # When you try to add the same element in a property - NOP = "__NOP" # no operation concept. Does nothing - CONCEPT_EVAL_ERROR = "__CONCEPT_EVAL_ERROR" # cannot evaluate a property or metadata of a concept - ENUMERATION = "__ENUMERATION" # represents a list or a set - LIST = "__LIST" # represents a list - FILTERED = "__FILTERED" # represents the result of a filtering - CONCEPT_ALREADY_IN_SET = "__CONCEPT_ALREADY_IN_SET" - NOT_A_SET = "__NOT_A_SET" # the concept has no entry in sets - CONDITION_FAILED = "__CONDITION_FAILED" # failed to validate where clause during evaluation - CHICKEN_AND_EGG = "__CHICKEN_AND_EGG" # infinite recursion when declaring concept - EXPLANATION = "__EXPLANATION" - PRECEDENCE = "__PRECEDENCE" # use to set priority among concepts when parsing - ASSOCIATIVITY = "__ASSOCIATIVITY" # use to set priority among concepts when parsing - NOT_FOUND = "__NOT_FOUND" # when the wanted resource is not found - FORMAT_INSTRUCTIONS = "__FORMAT_INSTRUCTIONS" # to express how to print the concept - NOT_IMPLEMENTED = "__NOT_IMPLEMENTED" # instead of raise an error - PYTHON_SECURITY_ERROR = "__PYTHON_SECURITY_ERROR" # when trying to execute statement when only expression is allowed - INVALID_LESSER_OPERATION = "__INVALID_LESSER_OPERATION" - INVALID_GREATEST_OPERATION = "__INVALID_GREATEST_OPERATION" - NEW_RULE = "__NEW_RULE" - UNKNOWN_RULE = "__UNKNOWN_RULE" - - NODE = "__NODE" - GENERIC_NODE = "__GENERIC_NODE" - IDENTIFIER_NODE = "__IDENTIFIER_NODE" - - # formatting - TO_LIST = "__TO_LIST" - TO_DICT = "__TO_DICT" - TO_MULTI = "__TO_MULTI" # used when there are multiple output to display - - -AllBuiltinConcepts = [v for n, v in BuiltinConcepts.__dict__.items() if not n.startswith("__")] - -BuiltinUnique = [ - BuiltinConcepts.EVAL_BODY_REQUESTED, - BuiltinConcepts.EVAL_WHERE_REQUESTED, - BuiltinConcepts.RETURN_BODY_REQUESTED, - BuiltinConcepts.REDUCE_REQUESTED, - BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED, - BuiltinConcepts.EVAL_QUESTION_REQUESTED, - - BuiltinConcepts.INIT_SHEERKA, - BuiltinConcepts.PROCESS_INPUT, - BuiltinConcepts.PROCESSING, - BuiltinConcepts.BEFORE_PARSING, - BuiltinConcepts.PARSING, - BuiltinConcepts.AFTER_PARSING, - BuiltinConcepts.BEFORE_EVALUATION, - BuiltinConcepts.EVALUATION, - BuiltinConcepts.AFTER_EVALUATION, - BuiltinConcepts.BEFORE_RENDERING, - BuiltinConcepts.RENDERING, - BuiltinConcepts.AFTER_RENDERING, - BuiltinConcepts.EVALUATE_CONCEPT, - BuiltinConcepts.EVALUATING_CONCEPT, - BuiltinConcepts.EVALUATING_ATTRIBUTE, - BuiltinConcepts.VALIDATE_CONCEPT, - BuiltinConcepts.VALIDATING_CONCEPT, - BuiltinConcepts.INIT_COMPILED, - BuiltinConcepts.INIT_BNF, - BuiltinConcepts.MANAGE_INFINITE_RECURSION, - BuiltinConcepts.PARSE_CODE, - BuiltinConcepts.EXEC_CODE, - BuiltinConcepts.TESTING, - - BuiltinConcepts.ISA, - BuiltinConcepts.AUTO_EVAL, - - BuiltinConcepts.INVALID_LESSER_OPERATION, - BuiltinConcepts.INVALID_GREATEST_OPERATION, -] - -BuiltinErrors = [ - BuiltinConcepts.ERROR, - BuiltinConcepts.UNKNOWN_CONCEPT, - BuiltinConcepts.CANNOT_RESOLVE_CONCEPT, - BuiltinConcepts.CONCEPT_TOO_LONG, - BuiltinConcepts.UNKNOWN_PROPERTY, - BuiltinConcepts.TOO_MANY_SUCCESS, - BuiltinConcepts.TOO_MANY_ERRORS, - BuiltinConcepts.MULTIPLE_ERRORS, - BuiltinConcepts.INVALID_RETURN_VALUE, - BuiltinConcepts.CONCEPT_ALREADY_DEFINED, - BuiltinConcepts.PROPERTY_ALREADY_DEFINED, - BuiltinConcepts.CONCEPT_EVAL_ERROR, - BuiltinConcepts.CONCEPT_ALREADY_IN_SET, - BuiltinConcepts.NOT_A_SET, - BuiltinConcepts.CONDITION_FAILED, - BuiltinConcepts.CHICKEN_AND_EGG, - BuiltinConcepts.NOT_FOUND, - BuiltinConcepts.INVALID_LESSER_OPERATION, - BuiltinConcepts.INVALID_GREATEST_OPERATION, -] - -BuiltinContainers = [ - BuiltinConcepts.PARSER_RESULT, - BuiltinConcepts.ONLY_SUCCESSFUL, - BuiltinConcepts.FILTERED, - BuiltinConcepts.EXPLANATION, - BuiltinConcepts.TO_LIST, - BuiltinConcepts.TO_DICT, - BuiltinConcepts.TO_MULTI, -] - -BuiltinOutConcepts = [ - BuiltinConcepts.TO_LIST, - BuiltinConcepts.TO_DICT, - BuiltinConcepts.TO_MULTI, -] - -""" -Some concepts have a specific implementation -It's mainly to ease the usage -""" - - class UserInputConcept(Concept): ALL_ATTRIBUTES = ["text", "user_name"] diff --git a/src/core/builtin_concepts_ids.py b/src/core/builtin_concepts_ids.py new file mode 100644 index 0000000..637789d --- /dev/null +++ b/src/core/builtin_concepts_ids.py @@ -0,0 +1,191 @@ +class BuiltinConcepts: + """ + List of builtin concepts that do no need any specific implementation + Please note that the value of the enum is informal. It is not used in the system + For example, the concept 'NODE' DOES NOT have the key, the id or whatever 200 + The key if the name of the concept + The id is a sequential number given just before the concept is saved in sdp + + The values of the enum is not used the code + """ + SHEERKA = "__SHEERKA" + + # processing instructions during sheerka.execute() or sheerka.evaluate_concept() + # The instructions may alter how the actions work + DEBUG = "__DEBUG" # activate all debug information + EVAL_BODY_REQUESTED = "__EVAL_BODY_REQUESTED" # to evaluate the body + EVAL_WHERE_REQUESTED = "__EVAL_WHERE_REQUESTED" # to evaluate the where clause + RETURN_BODY_REQUESTED = "__RETURN_BODY_REQUESTED" # returns the body of the concept instead of the concept itself + REDUCE_REQUESTED = "__REDUCE_REQUESTED" # remove meaningless error when possible + EVAL_UNTIL_SUCCESS_REQUESTED = "__EVAL_UNTIL_SUCCESS_REQUESTED" # PythonEvaluator tries combination until True is found + EVAL_QUESTION_REQUESTED = "__EVAL_QUESTION_REQUESTED" # the user input must be treated as question + + # possible actions during sheerka.execute() or sheerka.evaluate_rules() + INIT_SHEERKA = "__INIT_SHEERKA" # + PROCESS_INPUT = "__PROCESS_INPUT" # Processing user input or other input + PROCESSING = "__PROCESSING" # Processing user input or other input + BEFORE_PARSING = "__BEFORE_PARSING" # activated before evaluation by the parsers + PARSING = "__PARSING" # activated during the parsing. It contains the text to parse + AFTER_PARSING = "__AFTER_PARSING" # after parsing + BEFORE_EVALUATION = "__BEFORE_EVALUATION" # before evaluation + EVALUATION = "__EVALUATION" # activated when the parsing process seems to be finished + AFTER_EVALUATION = "__AFTER_EVALUATION" # activated when the parsing process seems to be finished + BEFORE_RENDERING = "__BEFORE_RENDERING" # activate before the output is rendered + RENDERING = "__RENDERING" # rendering the response from sheerka + AFTER_RENDERING = "__AFTER_RENDERING" # rendering the response from sheerka + EVALUATE_SOURCE = "__EVALUATE_SOURCE" # + EVALUATE_CONCEPT = "__EVALUATE_CONCEPT" # a concept will be evaluated + EVALUATING_CONCEPT = "__EVALUATING_CONCEPT" # a concept will be evaluated + EVALUATING_ATTRIBUTE = "__EVALUATING_ATTRIBUTE" # + VALIDATE_CONCEPT = "__VALIDATE_CONCEPT" + VALIDATING_CONCEPT = "__VALIDATING_CONCEPT" + INIT_COMPILED = "__INIT_COMPILED" + INIT_BNF = "__INIT_BNF" + MANAGE_INFINITE_RECURSION = "__MANAGE_INFINITE_RECURSION" + PARSE_CODE = "__PARSE_CODE" + EXEC_CODE = "__EXEC_CODE" # to use when executing Python or other language compiled code + TESTING = "__TESTING" + EVALUATOR_PRE_PROCESS = "__EVALUATOR_PRE_PROCESS" # used modify / tweak behaviour of evaluators + EVALUATING_RULES = "__EVALUATING_RULES" + + # builtin attributes + ISA = "__ISA" # when a concept is an instance of another one + HASA = "__HASA" # when a concept has/owns another concept + AUTO_EVAL = "__AUTO_EVAL" # when the concept must be auto evaluated + + # object + USER_INPUT = "__USER_INPUT" # represent an input from an user + SUCCESS = "__SUCCESS" + ERROR = "__ERROR" + UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT" # the request concept is not recognized + CANNOT_RESOLVE_CONCEPT = "__CANNOT_RESOLVE_CONCEPT" # when too many concepts with the same name + RETURN_VALUE = "__RETURN_VALUE" # a value is returned + CONCEPT_TOO_LONG = "__CONCEPT_TOO_LONG" # concept cannot be processed by exactConcept parser + NEW_CONCEPT = "__NEW_CONCEPT" # when a new concept is added + UNKNOWN_PROPERTY = "__UNKNOWN_PROPERTY" # when requesting for a unknown property + PARSER_RESULT = "__PARSER_RESULT" + TOO_MANY_SUCCESS = "__TOO_MANY_SUCCESS" # when expecting a limited number of successful return value + TOO_MANY_ERRORS = "__TOO_MANY_ERRORS" # when expecting a limited number of successful return value + ONLY_SUCCESSFUL = "__ONLY_SUCCESSFUL" # filter the result, only keep successful ones + MULTIPLE_ERRORS = "__MULTIPLE_ERRORS" # filter the result, only keep evaluator in error + NOT_FOR_ME = "__NOT_FOR_ME" # a parser recognize that the entry is not meant for it + IS_EMPTY = "__IS_EMPTY" # when a set is empty + NO_RESULT = "__NO_RESULT" # no return value returned + INVALID_RETURN_VALUE = "__INVALID_RETURN_VALUE" # the return value of an evaluator is not correct + CONCEPT_ALREADY_DEFINED = "__CONCEPT_ALREADY_DEFINED" # when you try to add the same object twice (a concept or whatever) + PROPERTY_ALREADY_DEFINED = "__PROPERTY_ALREADY_DEFINED" # When you try to add the same element in a property + NOP = "__NOP" # no operation concept. Does nothing + CONCEPT_EVAL_ERROR = "__CONCEPT_EVAL_ERROR" # cannot evaluate a property or metadata of a concept + ENUMERATION = "__ENUMERATION" # represents a list or a set + LIST = "__LIST" # represents a list + FILTERED = "__FILTERED" # represents the result of a filtering + CONCEPT_ALREADY_IN_SET = "__CONCEPT_ALREADY_IN_SET" + NOT_A_SET = "__NOT_A_SET" # the concept has no entry in sets + CONDITION_FAILED = "__CONDITION_FAILED" # failed to validate where clause during evaluation + CHICKEN_AND_EGG = "__CHICKEN_AND_EGG" # infinite recursion when declaring concept + EXPLANATION = "__EXPLANATION" + PRECEDENCE = "__PRECEDENCE" # use to set priority among concepts when parsing + ASSOCIATIVITY = "__ASSOCIATIVITY" # use to set priority among concepts when parsing + NOT_FOUND = "__NOT_FOUND" # when the wanted resource is not found + FORMAT_INSTRUCTIONS = "__FORMAT_INSTRUCTIONS" # to express how to print the concept + NOT_IMPLEMENTED = "__NOT_IMPLEMENTED" # instead of raise an error + PYTHON_SECURITY_ERROR = "__PYTHON_SECURITY_ERROR" # when trying to execute statement when only expression is allowed + INVALID_LESSER_OPERATION = "__INVALID_LESSER_OPERATION" + INVALID_GREATEST_OPERATION = "__INVALID_GREATEST_OPERATION" + NEW_RULE = "__NEW_RULE" + UNKNOWN_RULE = "__UNKNOWN_RULE" + + NODE = "__NODE" + GENERIC_NODE = "__GENERIC_NODE" + IDENTIFIER_NODE = "__IDENTIFIER_NODE" + + # formatting + TO_LIST = "__TO_LIST" + TO_DICT = "__TO_DICT" + TO_MULTI = "__TO_MULTI" # used when there are multiple output to display + + +AllBuiltinConcepts = [v for n, v in BuiltinConcepts.__dict__.items() if not n.startswith("__")] + +BuiltinUnique = [ + BuiltinConcepts.EVAL_BODY_REQUESTED, + BuiltinConcepts.EVAL_WHERE_REQUESTED, + BuiltinConcepts.RETURN_BODY_REQUESTED, + BuiltinConcepts.REDUCE_REQUESTED, + BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED, + BuiltinConcepts.EVAL_QUESTION_REQUESTED, + + BuiltinConcepts.INIT_SHEERKA, + BuiltinConcepts.PROCESS_INPUT, + BuiltinConcepts.PROCESSING, + BuiltinConcepts.BEFORE_PARSING, + BuiltinConcepts.PARSING, + BuiltinConcepts.AFTER_PARSING, + BuiltinConcepts.BEFORE_EVALUATION, + BuiltinConcepts.EVALUATION, + BuiltinConcepts.AFTER_EVALUATION, + BuiltinConcepts.BEFORE_RENDERING, + BuiltinConcepts.RENDERING, + BuiltinConcepts.AFTER_RENDERING, + BuiltinConcepts.EVALUATE_CONCEPT, + BuiltinConcepts.EVALUATING_CONCEPT, + BuiltinConcepts.EVALUATING_ATTRIBUTE, + BuiltinConcepts.VALIDATE_CONCEPT, + BuiltinConcepts.VALIDATING_CONCEPT, + BuiltinConcepts.INIT_COMPILED, + BuiltinConcepts.INIT_BNF, + BuiltinConcepts.MANAGE_INFINITE_RECURSION, + BuiltinConcepts.PARSE_CODE, + BuiltinConcepts.EXEC_CODE, + BuiltinConcepts.TESTING, + + BuiltinConcepts.ISA, + BuiltinConcepts.AUTO_EVAL, + + BuiltinConcepts.INVALID_LESSER_OPERATION, + BuiltinConcepts.INVALID_GREATEST_OPERATION, +] + +BuiltinErrors = [ + BuiltinConcepts.ERROR, + BuiltinConcepts.UNKNOWN_CONCEPT, + BuiltinConcepts.CANNOT_RESOLVE_CONCEPT, + BuiltinConcepts.CONCEPT_TOO_LONG, + BuiltinConcepts.UNKNOWN_PROPERTY, + BuiltinConcepts.TOO_MANY_SUCCESS, + BuiltinConcepts.TOO_MANY_ERRORS, + BuiltinConcepts.MULTIPLE_ERRORS, + BuiltinConcepts.INVALID_RETURN_VALUE, + BuiltinConcepts.CONCEPT_ALREADY_DEFINED, + BuiltinConcepts.PROPERTY_ALREADY_DEFINED, + BuiltinConcepts.CONCEPT_EVAL_ERROR, + BuiltinConcepts.CONCEPT_ALREADY_IN_SET, + BuiltinConcepts.NOT_A_SET, + BuiltinConcepts.CONDITION_FAILED, + BuiltinConcepts.CHICKEN_AND_EGG, + BuiltinConcepts.NOT_FOUND, + BuiltinConcepts.INVALID_LESSER_OPERATION, + BuiltinConcepts.INVALID_GREATEST_OPERATION, +] + +BuiltinContainers = [ + BuiltinConcepts.PARSER_RESULT, + BuiltinConcepts.ONLY_SUCCESSFUL, + BuiltinConcepts.FILTERED, + BuiltinConcepts.EXPLANATION, + BuiltinConcepts.TO_LIST, + BuiltinConcepts.TO_DICT, + BuiltinConcepts.TO_MULTI, +] + +BuiltinOutConcepts = [ + BuiltinConcepts.TO_LIST, + BuiltinConcepts.TO_DICT, + BuiltinConcepts.TO_MULTI, +] + +# BuiltinDynamicProps means that the attributes of the concept can vary for one instance to another +# So do not put them in cache +BuiltinDynamicAttrs = [ + BuiltinConcepts.EVALUATOR_PRE_PROCESS +] diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index 9695a25..aa44863 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -304,7 +304,7 @@ def parse_unrecognized(context, source, parsers, who=None, prop=None, filter_fun with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context: # disable all parsers but the requested ones if parsers != "all": - sub_context.preprocess_parsers = [BaseParser.PREFIX + parser for parser in parsers] + sub_context.preprocess_parsers = parsers # sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False) # for parser in parsers: # sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True) diff --git a/src/core/concept.py b/src/core/concept.py index ab5e637..635761b 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import Union import core.utils +from core.builtin_concepts_ids import BuiltinDynamicAttrs from core.tokenizer import Tokenizer, TokenKind PROPERTIES_FOR_DIGEST = ("name", "key", @@ -86,13 +87,14 @@ def get_concept_attrs(concept): pass all_attributes = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"] - if concept.id: + if concept.id and concept.key not in BuiltinDynamicAttrs: ALL_ATTRIBUTES[concept.id] = all_attributes return all_attributes def freeze_concept_attrs(concept): - ALL_ATTRIBUTES[concept.id] = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"] + if concept.key not in BuiltinDynamicAttrs: + ALL_ATTRIBUTES[concept.id] = [k for k in concept.__dict__ if k[0] != "_" and k[0] != "#"] class Concept: diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index d5801eb..586e810 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -8,7 +8,8 @@ from cache.Cache import Cache from cache.CacheManager import CacheManager from cache.DictionaryCache import DictionaryCache from cache.IncCache import IncCache -from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, UnknownConcept +from core.builtin_concepts import ErrorConcept, ReturnValueConcept, UnknownConcept +from core.builtin_concepts_ids import BuiltinErrors, BuiltinConcepts from core.concept import Concept, ConceptParts, NotInit, get_concept_attrs from core.error import ErrorObj from core.global_symbols import EVENT_USER_INPUT_EVALUATED diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index 1bf93e8..adde154 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -2,7 +2,7 @@ import sys import time from os import path -from core.builtin_concepts import BuiltinConcepts, BuiltinContainers +from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers from core.builtin_helpers import ensure_concept from core.concept import Concept from core.sheerka.services.sheerka_service import BaseService diff --git a/src/core/sheerka/services/SheerkaConceptManager.py b/src/core/sheerka/services/SheerkaConceptManager.py index 9639ecf..1cfbd16 100644 --- a/src/core/sheerka/services/SheerkaConceptManager.py +++ b/src/core/sheerka/services/SheerkaConceptManager.py @@ -5,7 +5,8 @@ from cache.Cache import Cache from cache.CacheManager import ConceptNotFound from cache.ListIfNeededCache import ListIfNeededCache from cache.SetCache import SetCache -from core.builtin_concepts import BuiltinConcepts, ErrorConcept, AllBuiltinConcepts, BuiltinUnique +from core.builtin_concepts import ErrorConcept +from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, BuiltinUnique from core.builtin_helpers import ensure_concept from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, NotInit, \ ConceptMetadata diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index 1f72109..76500bf 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -9,6 +9,7 @@ from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.sheerka_service import BaseService from core.tokenizer import Tokenizer from core.utils import unstr_concept +from parsers.BaseNodeParser import ConceptNode from parsers.ExpressionParser import ExpressionParser, TrueifyVisitor CONCEPT_EVALUATION_STEPS = [ @@ -17,6 +18,11 @@ CONCEPT_EVALUATION_STEPS = [ BuiltinConcepts.AFTER_EVALUATION] +@dataclass +class ChickenAndEggException(Exception): + error: Concept + + @dataclass class WhereClauseDef: concept: Concept # concept on which the where clause is applied @@ -148,6 +154,27 @@ class SheerkaEvaluateConcept(BaseService): else: return None + def get_recursive_definitions(self, concept, return_values): + """ + Returns the name of the parsers that will resolve to a recursive evaluation + :param concept: + :param return_values: + :return: + """ + if concept.name in concept.variables(): + # There is a variable with the same name as the concept + # During evaluation, inner variables take precedence other concepts + # So there won't be any cyclic reference, the variable will be picked + return + for parser in [r.body for r in return_values if + r.status and self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]: + parsed = parser.body if isinstance(parser.body, list) else [parser.body] + for parsed_item in parsed: + if isinstance(parsed_item, Concept) and parsed_item.id == concept.id: + yield parser.parser + elif isinstance(parsed_item, ConceptNode) and parsed_item.concept.id == concept.id: + yield parser.parser + def apply_where_clause(self, context, where_clause_def, return_values): """ Apply intermediate where clause when evaluating concept variables @@ -225,14 +252,57 @@ class SheerkaEvaluateConcept(BaseService): """ def is_only_successful(r): + """ + + :param r: return_value + :return: + """ return context.sheerka.isinstance(r, BuiltinConcepts.RETURN_VALUE) and \ context.sheerka.isinstance(r.body, BuiltinConcepts.ONLY_SUCCESSFUL) def parse_token_concept(s): + """ + + :param s: source + :return: + """ if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None): return self.sheerka.fast_resolve(identifier) return None + def get_return_value(current_context, c, s, p): + """ + + :param current_context: + :param c: concept + :param s: source + :param p: part of the concept being parsed + :return: + """ + while True: + return_value = parse_unrecognized(current_context, + s, + parsers="all", + prop=p, + filter_func=only_successful) + + if not return_value.status: + if current_context.preprocess: + raise ChickenAndEggException(self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c})) + else: + raise Exception(f"Failed to build '{s}'. But it doesn't seems to be recursion") + + return_value = return_value.body.body if is_only_successful(return_value) else [return_value] + recursive_parsers = list(self.get_recursive_definitions(c, return_value)) + + if len(recursive_parsers) == 0: + return return_value + + desc = f"Removing parsers {recursive_parsers}" + current_context = current_context.push(context.action, context.action_context, desc=desc) + for recursive_parser in recursive_parsers: + current_context.add_preprocess(recursive_parser.name, enabled=False) + for part_key in AllConceptParts: if part_key in concept.get_compiled(): continue @@ -253,12 +323,7 @@ class SheerkaEvaluateConcept(BaseService): concept.get_compiled()[part_key] = concept_found else: # ...or a list of ReturnValueConcept to resolve - res = parse_unrecognized(context, - source, - parsers="all", - prop=part_key, - filter_func=only_successful) - concept.get_compiled()[part_key] = res.body.body if is_only_successful(res) else res + concept.get_compiled()[part_key] = get_return_value(context, concept, source, part_key) for var_name, default_value in concept.get_metadata().variables: if var_name in concept.get_compiled(): @@ -279,12 +344,7 @@ class SheerkaEvaluateConcept(BaseService): concept.get_compiled()[var_name] = concept_found else: # ...or a list of ReturnValueConcept to resolve - res = parse_unrecognized(context, - default_value, - parsers="all", - prop=var_name, - filter_func=only_successful) - concept.get_compiled()[var_name] = res.body.body if is_only_successful(res) else res + concept.get_compiled()[var_name] = get_return_value(context, concept, default_value, var_name) # Updates the cache of concepts when possible # This piece of code is not used, a the compile part is removed by sheerka.new_from_template() @@ -469,7 +529,10 @@ class SheerkaEvaluateConcept(BaseService): if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)): sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - self.initialize_concept_asts(sub_context, concept) + try: + self.initialize_concept_asts(sub_context, concept) + except ChickenAndEggException as ex: + return ex.error # to make sure of the order, it don't use ConceptParts.get_parts() # variables must be evaluated first, body must be evaluated before where @@ -525,7 +588,8 @@ class SheerkaEvaluateConcept(BaseService): # validate PRE and WHERE condition if part_key in (ConceptParts.PRE, ConceptParts.WHERE) and not self.sheerka.objvalue(resolved): return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, - body=getattr(concept.get_metadata(), concept_part_value(metadata_to_eval)), + body=getattr(concept.get_metadata(), + concept_part_value(metadata_to_eval)), concept=concept, prop=part_key) diff --git a/src/core/sheerka/services/SheerkaExecute.py b/src/core/sheerka/services/SheerkaExecute.py index ee4de60..0e5e2b8 100644 --- a/src/core/sheerka/services/SheerkaExecute.py +++ b/src/core/sheerka/services/SheerkaExecute.py @@ -169,16 +169,34 @@ class SheerkaExecute(BaseService): self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20) self.instantiated_evaluators = None self.evaluators_by_name = None - self.grouped_evaluators_cache = {} # key=step, value=tuple(evaluators for this step, sorted priorities) + + self.instantiated_parsers = None + self.parsers_by_name = None self.old_values = [] + # cache for all preregistered evaluator combination + # the key is the concatenation of the step and the name of evaluators in the group + # ex : BEFORE_EVALUATION|Python|Sya|Bnf + # The value is a tuple, + # The first entry is the grouped evaluators + # ex : {60 : [PythonEvaluator(), SyaEvaluator()], 50: [BnfEvaluator()]} + # The second entry are the sorted priorities ex [60, 50] + self.grouped_evaluators_cache = {} + + # cache for preregistered parsers + # Same construction than the evaluators + # Except 1 : the key does not have a step component. It is simple the list of parsers' names + # Except 2 : we store the type of the parser, not its instance + self.grouped_parsers_cache = {} + def initialize(self): self.sheerka.bind_service_method(self.execute, True) self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False) - self.reset_evaluators() + self.reset_registered_evaluators() + self.reset_registered_parsers() - def reset_evaluators(self): + def reset_registered_evaluators(self): # instantiate evaluators, once for all, only keep when it's enabled self.instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators] self.instantiated_evaluators = [e for e in self.instantiated_evaluators if e.enabled] @@ -186,44 +204,49 @@ class SheerkaExecute(BaseService): # get default evaluators by process step for process_step in EVALUATOR_STEPS: - self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped_evaluators( + self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped( [e for e in self.instantiated_evaluators if process_step in e.steps]) - # @staticmethod - # def get_grouped_evaluators(instantiated_evaluators, process_step): - # """ - # For a given list of evaluators and a given process step - # Computes - # * the evaluators eligible for this step - # * the list of sorted priorities for theses evaluators - # :param instantiated_evaluators: - # :param process_step: - # :return: - # """ - # grouped = {} - # for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]: - # grouped.setdefault(evaluator.priority, []).append(evaluator) - # - # sorted_groups = sorted(grouped.keys(), reverse=True) - # return grouped, sorted_groups + def reset_registered_parsers(self): + """ + Browse all parsers and only keep those which are enabled + :return: + """ + self.instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()] + self.instantiated_parsers = [p for p in self.instantiated_parsers if p.enabled] + self.parsers_by_name = {p.short_name: p for p in self.instantiated_parsers} + + self.grouped_parsers_cache["__default"] = self.get_grouped(self.instantiated_parsers, use_classes=True) @staticmethod - def get_grouped_evaluators(evaluators): + def get_grouped(evaluators, use_classes=False): """ For a given list of evaluators, group them by priorities sort the priorities :param evaluators: + :param use_classes: if True, store the class (the type) of the evaluator, not its instance :return: tuple({priority: List of evaluators with this priority}, list of sorted priorities) """ grouped = {} for evaluator in evaluators: - grouped.setdefault(evaluator.priority, []).append(evaluator) + grouped.setdefault(evaluator.priority, []).append(type(evaluator) if use_classes else evaluator) sorted_groups = sorted(grouped.keys(), reverse=True) return grouped, sorted_groups def preprocess(self, items, preprocess_definitions): + """ + Modifies the attributes of item + :param items: either a parser or an evaluator + :param preprocess_definitions: how to modify List of BuiltinConcepts.EVALUATOR_PRE_PROCESS + preprocess.get_value("preprocess_name") gives the name of the property to alter + preprocess.values() gives the alterations + ex: + Sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS, preprocess_name="parser_name", enabled=False) + Will disable parser 'parser_name' + :return: + """ for preprocess in preprocess_definitions: for item in items: if self.matches(item.name, preprocess.get_value("preprocess_name")): @@ -234,51 +257,9 @@ class SheerkaExecute(BaseService): self.old_values.append((item, var_name, getattr(item, var_name))) setattr(item, var_name, value) - def preprocess_old(self, context, parsers_or_evaluators, mode): - if mode == "parsers": - if not context.preprocess and not context.preprocess_parsers: - return parsers_or_evaluators - items = context.preprocess_parsers - elif mode == "evaluators": - if not context.preprocess and not context.preprocess_evaluators: - return parsers_or_evaluators - items = context.preprocess_evaluators - else: - raise ValueError(mode) - - if not hasattr(parsers_or_evaluators, "__iter__"): - single_one = True - parsers_or_evaluators = [parsers_or_evaluators] - else: - single_one = False - - if items: - res = [] - for item in items: - for e in parsers_or_evaluators: - if item == e.name: - res.append(e) - break - else: - raise ValueError(f"{item} not found.") - parsers_or_evaluators = res - - if context.preprocess: - for preprocess in context.preprocess: - for e in parsers_or_evaluators: - if self.matches(e.name, preprocess.get_value("name")): - for var_name in preprocess.values: - if var_name == "name": - continue - if hasattr(e, var_name): - self.old_values.append((e, var_name, getattr(e, var_name))) - setattr(e, var_name, preprocess.get_value(var_name)) - - return parsers_or_evaluators[0] if single_one else parsers_or_evaluators - def get_evaluators(self, context, process_step): """ - Returns the list of evaluators to use for a specific test + Returns the list of evaluators to use for a specific context need :param context: :param process_step: :return: @@ -287,30 +268,64 @@ class SheerkaExecute(BaseService): if not context.preprocess_evaluators and not context.preprocess: return self.grouped_evaluators_cache[f"{process_step}|__default"] - # First case, only use a subset of evaluators - if context.preprocess_evaluators and not context.preprocess: - key = str(process_step) + "|" + "|".join(context.preprocess_evaluators) + # Other case, only use a subset of evaluators + selected = context.preprocess_evaluators + if selected and not context.preprocess: + key = str(process_step) + "|" + "|".join(selected) try: return self.grouped_evaluators_cache[key] except KeyError: - evaluators = [self.evaluators_by_name[e] for e in context.preprocess_evaluators] - grouped = self.get_grouped_evaluators(evaluators) + evaluators = [self.evaluators_by_name[e] for e in selected if e in self.evaluators_by_name] + evaluators = [e for e in evaluators if process_step in e.steps] # check the process step + grouped = self.get_grouped(evaluators) self.grouped_evaluators_cache[key] = grouped return grouped - # final case, evaluators attributes are modified by the context - # So first, get the modified evaluators - evaluators = [self.evaluators_by_name[e] for e in - context.preprocess_evaluators] if context.preprocess_evaluators else self.instantiated_evaluators + # Final case, evaluators attributes are modified by the context + evaluators = [self.evaluators_by_name[e] for e in selected if + e in self.evaluators_by_name] if selected else self.instantiated_evaluators + evaluators = [e for e in evaluators if process_step in e.steps] # check the process step self.preprocess(evaluators, context.preprocess) evaluators = [e for e in evaluators if e.enabled] # make sure they are still enabled - key = str(process_step) + "|" + "|".join([e.name for e in evaluators if e.enabled]) - try: - return self.grouped_evaluators_cache[key] - except KeyError: - grouped = self.get_grouped_evaluators(evaluators) - self.grouped_evaluators_cache[key] = grouped - return grouped + return self.get_grouped(evaluators) + + def get_parsers(self, context): + """ + We cannot use a single instance of a parser shared among executions as it's common to have a parser + calling another parser or even calling itself + So the cache holds the parser classes or sorted priorities + :param context: + :return: + """ + + def get_instances(from_cache): + grouped_instances = {priority: [p(sheerka=self.sheerka) for p in parsers_classes] + for priority, parsers_classes in from_cache[0].items()} + return grouped_instances, from_cache[1] + + # Normal case, use all registered parsers + if not context.preprocess_parsers and not context.preprocess: + return get_instances(self.grouped_parsers_cache["__default"]) + + # Other case, only use a subset of parsers + # This case is heavily used by lexer node parsers, thru parse_unrecognized + if context.preprocess_parsers and not context.preprocess: + key = "|".join(context.preprocess_parsers) + try: + return get_instances(self.grouped_parsers_cache[key]) + except KeyError: + parsers = [self.parsers_by_name[p] for p in context.preprocess_parsers if p in self.parsers_by_name] + self.grouped_parsers_cache[key] = self.get_grouped(parsers, use_classes=True) + return get_instances(self.grouped_parsers_cache[key]) + + # final case, parsers attributes are modified by the context + # This a the case when we want to disable a specific parser, or change the order of priority + parsers = [self.parsers_by_name[p] for p in context.preprocess_parsers if p in self.parsers_by_name] \ + if context.preprocess_parsers else self.instantiated_parsers + self.preprocess(parsers, context.preprocess) + parsers = [p for p in parsers if p.enabled] # only keep those that are still enabled + groups, sorted_priorities = self.get_grouped(parsers, use_classes=True) + return get_instances((groups, sorted_priorities)) def get_parser_input(self, text, tokens=None): """ @@ -367,14 +382,7 @@ class SheerkaExecute(BaseService): # keep track of the originals user inputs, as they need to be removed at the end user_inputs = to_process[:] - # group the parsers by priorities - instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()] - instantiated_parsers = self.preprocess_old(context, instantiated_parsers, "parsers") - - grouped_parsers = {} - for parser in [p for p in instantiated_parsers if p.enabled]: - grouped_parsers.setdefault(parser.priority, []).append(parser) - sorted_priorities = sorted(grouped_parsers.keys(), reverse=True) + grouped_parsers, sorted_priorities = self.get_parsers(context) stop_processing = False for priority in sorted_priorities: @@ -427,6 +435,7 @@ class SheerkaExecute(BaseService): break # Do not try the other priorities if a match is found result = core.utils.remove_list_from_list(result, user_inputs) + return result def call_evaluators(self, context, return_values, process_step): @@ -470,7 +479,7 @@ class SheerkaExecute(BaseService): # init the evaluator is possible # KSI. 20201102 : Evaluators are now instantiated at startup, - # Can we move this section into reset_evaluators() + # Can we move this section into reset_registered_evaluators() if hasattr(evaluator, "init_evaluator") and not evaluator.is_initialized: evaluator.init_evaluator(sub_context, original_items) diff --git a/src/evaluators/BaseEvaluator.py b/src/evaluators/BaseEvaluator.py index 792f483..94e7dbb 100644 --- a/src/evaluators/BaseEvaluator.py +++ b/src/evaluators/BaseEvaluator.py @@ -1,5 +1,4 @@ from core.sheerka.Sheerka import ExecutionContext -from core.sheerka_logger import get_logger class BaseEvaluator: @@ -14,7 +13,7 @@ class BaseEvaluator: # self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__) # self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__) - self.name = self.PREFIX + name + self.name = BaseEvaluator.get_name(name) self.short_name = name self.steps = steps self.priority = priority @@ -23,9 +22,25 @@ class BaseEvaluator: def __repr__(self): return f"{self.name} ({self.priority})" + def __eq__(self, other): + if not isinstance(other, BaseEvaluator): + return False + + return self.name == other.name and \ + self.priority == other.priority and \ + self.steps == other.steps and \ + self.enabled == other.enabled + + def __hash__(self): + return hash((self.name, self.priority, self.steps, self.enabled)) + def reset(self): pass + @staticmethod + def get_name(name): + return BaseEvaluator.PREFIX + name + class OneReturnValueEvaluator(BaseEvaluator): """ diff --git a/src/evaluators/MultipleOutEvaluator.py b/src/evaluators/MultipleOutEvaluator.py index ef9875e..195ed09 100644 --- a/src/evaluators/MultipleOutEvaluator.py +++ b/src/evaluators/MultipleOutEvaluator.py @@ -1,4 +1,4 @@ -from core.builtin_concepts import BuiltinConcepts, BuiltinOutConcepts +from core.builtin_concepts_ids import BuiltinConcepts, BuiltinOutConcepts from evaluators.BaseEvaluator import AllReturnValuesEvaluator from parsers.BaseParser import BaseParser diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index a793982..8cfb0fc 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -824,14 +824,6 @@ class BaseNodeParser(BaseParser): else: self.concepts_by_first_keyword = None - # self.token = None - # self.pos = -1 - # self.tokens = None - # - # self.context: ExecutionContext = None - # self.text = None - # self.sheerka = None - def init_from_concepts(self, context, concepts, **kwargs): """ Initialize the parser with a list of concepts diff --git a/src/parsers/BaseParser.py b/src/parsers/BaseParser.py index b672cb6..b1df588 100644 --- a/src/parsers/BaseParser.py +++ b/src/parsers/BaseParser.py @@ -94,7 +94,8 @@ class BaseParser: # self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__) # self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__) - self.name = self.PREFIX + name + self.name = BaseParser.get_name(name) + self.short_name = name self.priority = priority self.enabled = enabled @@ -298,6 +299,10 @@ class BaseParser: return list_a + @staticmethod + def get_name(name): + return BaseParser.PREFIX + name + class BaseTokenizerIterParser(BaseParser): diff --git a/src/parsers/BnfDefinitionParser.py b/src/parsers/BnfDefinitionParser.py index 1c4b57f..83311e2 100644 --- a/src/parsers/BnfDefinitionParser.py +++ b/src/parsers/BnfDefinitionParser.py @@ -48,9 +48,11 @@ class BnfDefinitionParser(BaseParser): return True def reset_parser(self, context, text): + self.error_sink.clear() self.context = context self.sheerka = context.sheerka - + self.source = "" + self.lexer_iter = iter(Tokenizer(text.strip())) if isinstance(text, str) else iter(text) self._current = None self.after_current = None diff --git a/src/parsers/ExactConceptParser.py b/src/parsers/ExactConceptParser.py index 56d086b..f8a5b91 100644 --- a/src/parsers/ExactConceptParser.py +++ b/src/parsers/ExactConceptParser.py @@ -1,10 +1,8 @@ -import logging - import core.builtin_helpers from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts from core.concept import VARIABLE_PREFIX from core.sheerka.services.SheerkaExecute import ParserInput -from core.tokenizer import TokenKind, LexerError +from core.tokenizer import TokenKind from core.utils import str_concept from parsers.BaseParser import BaseParser @@ -31,12 +29,13 @@ class ExactConceptParser(BaseParser): context.log(f"Parsing '{parser_input}'", self.name) sheerka = context.sheerka - try: + if self.reset_parser(context, parser_input): parser_input.reset() words = self.get_words(parser_input) - except LexerError as e: - context.log(f"Error found in tokenizer {e}", self.name) - return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=e)) + else: + error = self.error_sink[0] + context.log(f"Error found in tokenizer {error}", self.name) + return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=error)) if len(words) > (self.max_word_size or self.MAX_WORDS_SIZE): context.log(f"Max words reached. Stopping.", self.name) diff --git a/src/parsers/PythonParser.py b/src/parsers/PythonParser.py index 43b80f0..aaa8bd3 100644 --- a/src/parsers/PythonParser.py +++ b/src/parsers/PythonParser.py @@ -5,7 +5,7 @@ from dataclasses import dataclass import core.utils from core.builtin_concepts import BuiltinConcepts from core.sheerka.services.SheerkaExecute import ParserInput -from core.tokenizer import LexerError, TokenKind +from core.tokenizer import TokenKind from parsers.BaseParser import BaseParser, Node, ErrorNode log = logging.getLogger(__name__) @@ -101,9 +101,11 @@ class PythonParser(BaseParser): Parse Python scripts """ + NAME = "Python" + def __init__(self, **kwargs): - BaseParser.__init__(self, "Python", 50) + BaseParser.__init__(self, PythonParser.NAME, 50) self.source = kwargs.get("source", "") def parse(self, context, parser_input: ParserInput): @@ -119,9 +121,7 @@ class PythonParser(BaseParser): TokenKind.RULE: lambda t: core.utils.encode_concept(t.value, "R") } - try: - parser_input.reset() - + if self.reset_parser(context, parser_input): source_code = parser_input.as_text(python_switcher, tracker) source_code = source_code.strip() @@ -134,14 +134,11 @@ class PythonParser(BaseParser): error_node = PythonErrorNode(parser_input.as_text(), error) self.error_sink.append(error_node) - except LexerError as e: - self.error_sink.append(e) - - # Python parser will refuse input that directly refers to a concept - if isinstance(tree, ast.Expression) and isinstance(tree.body, ast.Name): - if tree.body.id in tracker or context.sheerka.fast_resolve(tree.body.id, return_new=False) is not None: - context.log("It's a simple concept. Not for me.", self.name) - self.error_sink.append(ConceptDetected(tree.body.id)) + # Python parser will refuse input that directly refers to a concept + if isinstance(tree, ast.Expression) and isinstance(tree.body, ast.Name): + if tree.body.id in tracker or context.sheerka.fast_resolve(tree.body.id, return_new=False) is not None: + context.log("It's a simple concept. Not for me.", self.name) + self.error_sink.append(ConceptDetected(tree.body.id)) if self.has_error: ret = sheerka.ret( diff --git a/src/parsers/PythonWithConceptsParser.py b/src/parsers/PythonWithConceptsParser.py index c16b7fb..2a4033b 100644 --- a/src/parsers/PythonWithConceptsParser.py +++ b/src/parsers/PythonWithConceptsParser.py @@ -34,6 +34,7 @@ class PythonWithConceptsParser(BaseParser): yield node def parse(self, context, parser_input): + self.error_sink.clear() nodes = self.get_input_as_lexer_nodes(parser_input, unrecognized_nodes_parser) return self.parse_nodes(context, nodes) diff --git a/src/parsers/RuleParser.py b/src/parsers/RuleParser.py index b0e5567..b63ea39 100644 --- a/src/parsers/RuleParser.py +++ b/src/parsers/RuleParser.py @@ -1,7 +1,7 @@ from core.builtin_concepts import BuiltinConcepts from core.rule import Rule, ACTION_TYPE_DEFERRED from core.sheerka.services.SheerkaExecute import ParserInput -from core.tokenizer import LexerError, TokenKind +from core.tokenizer import TokenKind from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode @@ -40,49 +40,47 @@ class RuleParser(BaseParser): False, sheerka.new(BuiltinConcepts.IS_EMPTY)) - try: - parser_input.reset() + if not self.reset_parser(context, parser_input): + error = self.error_sink[0] + context.log(f"Error found in tokenizer {error}", self.name) + return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=error)) - parser_input.next_token() - if parser_input.token.type != TokenKind.RULE: - return sheerka.ret(self.name, - False, - sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text())) + parser_input.next_token() + if parser_input.token.type != TokenKind.RULE: + return sheerka.ret(self.name, + False, + sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text())) - token = parser_input.token + token = parser_input.token - if parser_input.next_token(): - reason = UnexpectedTokenErrorNode("Only one rule supported", - parser_input.token, - [TokenKind.EOF]) - return sheerka.ret(self.name, - False, - sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text(), reason=reason)) + if parser_input.next_token(): + reason = UnexpectedTokenErrorNode("Only one rule supported", + parser_input.token, + [TokenKind.EOF]) + return sheerka.ret(self.name, + False, + sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text(), reason=reason)) - if token.value[1] is None: - return sheerka.ret(self.name, - False, - sheerka.new(BuiltinConcepts.NOT_IMPLEMENTED)) + if token.value[1] is None: + return sheerka.ret(self.name, + False, + sheerka.new(BuiltinConcepts.NOT_IMPLEMENTED)) - if token.value[1].isdigit(): - rule = sheerka.get_rule_by_id(token.value[1]) - else: - rule = Rule().set_id(token.value[1]) - rule.metadata.action_type = ACTION_TYPE_DEFERRED + if token.value[1].isdigit(): + rule = sheerka.get_rule_by_id(token.value[1]) + else: + rule = Rule().set_id(token.value[1]) + rule.metadata.action_type = ACTION_TYPE_DEFERRED - if sheerka.isinstance(rule, BuiltinConcepts.UNKNOWN_RULE): - return sheerka.ret(self.name, - False, - sheerka.new(BuiltinConcepts.ERROR, - body=[RuleNotFound(token.value)])) - body = sheerka.new(BuiltinConcepts.PARSER_RESULT, - parser=self, - source=parser_input.as_text(), - body=[rule], - try_parsed=[rule]) + if sheerka.isinstance(rule, BuiltinConcepts.UNKNOWN_RULE): + return sheerka.ret(self.name, + False, + sheerka.new(BuiltinConcepts.ERROR, + body=[RuleNotFound(token.value)])) + body = sheerka.new(BuiltinConcepts.PARSER_RESULT, + parser=self, + source=parser_input.as_text(), + body=[rule], + try_parsed=[rule]) - return sheerka.ret(self.name, True, body) - - except LexerError as e: - context.log(f"Error found in tokenizer {e}", self.name) - return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=e)) + return sheerka.ret(self.name, True, body) diff --git a/tests/core/test_ExecutionContext.py b/tests/core/test_ExecutionContext.py index dd8bfc5..4687483 100644 --- a/tests/core/test_ExecutionContext.py +++ b/tests/core/test_ExecutionContext.py @@ -1,4 +1,4 @@ -from core.builtin_concepts import BuiltinConcepts +from core.builtin_concepts_ids import BuiltinConcepts from core.concept import Concept from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka.services.SheerkaMemory import SheerkaMemory diff --git a/tests/core/test_SheerkaAdmin.py b/tests/core/test_SheerkaAdmin.py index 4a62ae8..60cf184 100644 --- a/tests/core/test_SheerkaAdmin.py +++ b/tests/core/test_SheerkaAdmin.py @@ -1,5 +1,3 @@ -from core.builtin_concepts import BuiltinConcepts - from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka diff --git a/tests/core/test_SheerkaEvaluateConcept.py b/tests/core/test_SheerkaEvaluateConcept.py index db18167..457331c 100644 --- a/tests/core/test_SheerkaEvaluateConcept.py +++ b/tests/core/test_SheerkaEvaluateConcept.py @@ -1,12 +1,17 @@ +from dataclasses import dataclass + import pytest from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, CB, NotInit, \ - concept_part_value + concept_part_value, DEFINITION_TYPE_DEF from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept from core.sheerka.services.SheerkaMemory import SheerkaMemory -from parsers.PythonParser import PythonNode +from parsers.BaseParser import BaseParser +from parsers.PythonParser import PythonNode, PythonParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, python_ret_val + class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): @@ -709,7 +714,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(evaluated, BuiltinConcepts.CHICKEN_AND_EGG) assert evaluated.body == {foo, bar, baz, qux} - def test_i_can_detect_auto_recursion(self): + def test_i_can_detect_infinite_recursion(self): sheerka, context, foo = self.init_concepts( Concept("foo", body="foo"), eval_body=True @@ -863,11 +868,11 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): service.initialize_concept_asts(context, concept) assert service.compute_metadata_to_eval(context, concept) == ["#where#"] - concept = Concept("foo", where="where a").def_var("a") + concept = Concept("foo", where="a").def_var("a") service.initialize_concept_asts(context, concept) assert service.compute_metadata_to_eval(context, concept) == ["variables", "#where#"] - concept = Concept("foo", where="where self") + concept = Concept("foo", where="self") service.initialize_concept_asts(context, concept) assert service.compute_metadata_to_eval(context, concept) == ["#body#", "#where#"] @@ -925,6 +930,53 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): res = sheerka.evaluate_concept(context, foo, eval_body=True) assert res.body == "Print return values" + def test_i_can_manage_python_concept_infinite_recursion_when_initializing_ast(self): + sheerka, context, foo = self.init_concepts(Concept("a + b", body="a + b").def_var("a").def_var("b")) + evaluator = SheerkaEvaluateConcept(sheerka) + + evaluator.initialize_concept_asts(context, foo) + res = foo.get_compiled()["#body#"] + + assert len(res) == 1 + assert sheerka.isinstance(res[0], BuiltinConcepts.RETURN_VALUE) + assert res[0].who == BaseParser.get_name(PythonParser.NAME) + + # TODO validate that a rule is created + + def test_can_detect_recursive_definition_with_exact_concept(self): + sheerka, context, foo = self.init_concepts("foo") + evaluator = SheerkaEvaluateConcept(sheerka) + + # 'def concept foo as foo' + return_values = [pr_ret_val(foo, parser="ExactConcept"), python_ret_val("foo")] + + res = evaluator.get_recursive_definitions(foo, return_values) + + assert list(res) == [BaseParser.get_name("ExactConcept")] + + def test_i_can_detect_when_no_recursive_definition(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar") + evaluator = SheerkaEvaluateConcept(sheerka) + + # 'def concept foo as bar' + return_values = [pr_ret_val(bar, parser="ExactConcept"), python_ret_val("foo")] + + res = evaluator.get_recursive_definitions(foo, return_values) + + assert list(res) == [] + + def test_i_can_detect_when_no_recursive_definition2(self): + sheerka, context, q = self.init_concepts( + Concept("q", definition="q ?", definition_type=DEFINITION_TYPE_DEF).def_var("q")) + evaluator = SheerkaEvaluateConcept(sheerka) + + # i dunno how to construct the return value + return_values = [pr_ret_val(q, parser="ExactConcept")] + + res = evaluator.get_recursive_definitions(q, return_values) + + assert list(res) == [] + # I cannot implement value cache for now # def test_values_when_no_variables_are_computed_only_once(self): # sheerka, context, foo = self.init_concepts(Concept("foo", body="10")) diff --git a/tests/core/test_SheerkaSetsManager.py b/tests/core/test_SheerkaSetsManager.py index 8d5d90e..46940b0 100644 --- a/tests/core/test_SheerkaSetsManager.py +++ b/tests/core/test_SheerkaSetsManager.py @@ -204,8 +204,10 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka): service.add_concepts_to_set(context, [one, two, twenty, twenties], number) assert sheerka.isinset(twenties, number) - twenty_one = sheerka.evaluate_user_input("twenty one", "")[0].body - assert sheerka.isinset(twenty_one, number) + res = sheerka.evaluate_user_input("twenty one", "") + assert len(res) == 1 + assert res[0].status + assert sheerka.isinset(res[0].body, number) def test_a_concept_can_be_in_multiple_sets(self): sheerka, context, foo, all_foo, all_bar = self.init_concepts( diff --git a/tests/core/test_sheerka.py b/tests/core/test_sheerka.py index 421dc8b..3af892c 100644 --- a/tests/core/test_sheerka.py +++ b/tests/core/test_sheerka.py @@ -1,7 +1,8 @@ import os import pytest -from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept, AllBuiltinConcepts +from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept +from core.builtin_concepts_ids import AllBuiltinConcepts from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager diff --git a/tests/core/test_sheerka_call_evaluators.py b/tests/core/test_sheerka_call_evaluators.py index b2a2b3e..0e2e712 100644 --- a/tests/core/test_sheerka_call_evaluators.py +++ b/tests/core/test_sheerka_call_evaluators.py @@ -72,8 +72,10 @@ class EvaluatorOneWithPriority15(EvaluatorOneWithPriority): class EvaluatorOneWithPriority20(EvaluatorOneWithPriority): - def __init__(self): + def __init__(self, enabled=None): super().__init__("priority20", 20) + if enabled is not None: + self.enabled = enabled class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose): @@ -87,8 +89,8 @@ class EvaluatorAllWithPriority10(EvaluatorAllWithPriority): class EvaluatorAllWithPriority15(EvaluatorAllWithPriority): - def __init__(self): - super().__init__("all_priority15", 15) + def __init__(self, priority=15): + super().__init__("all_priority15", priority) class EvaluatorAllWithPriority20(EvaluatorAllWithPriority): @@ -238,6 +240,12 @@ class EvaluatorOneDoNotModifyExecutionFlow(EvaluatorOneWithPriority): return return_value +class DisabledEvaluatorOneWithPriority90(EvaluatorOneWithPriority): + def __init__(self): + super().__init__("disabled90", 90) + self.enabled = False + + class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): @classmethod @@ -246,6 +254,127 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): # Ask for a new one TestUsingMemoryBasedSheerka.singleton_instance = None + def test_i_can_get_evaluators_when_context_is_not_altered(self): + sheerka, context = self.init_concepts() + sheerka.evaluators = [ + EvaluatorOneWithPriority20, + EvaluatorAllWithPriority15, + EvaluatorOnePreEvaluation, # wrong step + DisabledEvaluatorOneWithPriority90, # disabled, + ] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_evaluators() + + groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION) + assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]} + assert sorted_priorities == [20, 15] + + def test_i_can_get_selected_evaluators(self): + sheerka, context = self.init_concepts() + sheerka.evaluators = [ + EvaluatorOneWithPriority20, + EvaluatorAllWithPriority15, + EvaluatorOnePreEvaluation, # wrong step + DisabledEvaluatorOneWithPriority90, # disabled + EvaluatorOneWithPriority15, # not selected + ] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_evaluators() + evaluators_names = ["priority20", "all_priority15", "preEval", "disabled90"] + context.preprocess_evaluators = evaluators_names + + groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION) + assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]} + assert sorted_priorities == [20, 15] + + key = BuiltinConcepts.EVALUATION + "|" + "|".join(evaluators_names) + assert key in service.grouped_evaluators_cache + groups, sorted_priorities = service.grouped_evaluators_cache[key] + assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]} + assert sorted_priorities == [20, 15] + + def test_i_can_get_altered_evaluators(self): + sheerka, context = self.init_concepts() + sheerka.evaluators = [ + EvaluatorOneWithPriority20, + EvaluatorAllWithPriority15, + EvaluatorOnePreEvaluation, # wrong step + DisabledEvaluatorOneWithPriority90, # always disabled + EvaluatorOneWithPriority15, # dynamically disabled + ] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_evaluators() + context.add_preprocess(BaseEvaluator.get_name("priority15"), enabled=False) + context.add_preprocess(BaseEvaluator.get_name("all_priority15"), priority=99) + + groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION) + assert groups == {20: [EvaluatorOneWithPriority20()], 99: [EvaluatorAllWithPriority15(99)]} + assert sorted_priorities == [99, 20] + service.undo_preprocess() + + # make sure that the result in not kept in cache + another_context = self.get_context(sheerka) + another_context.add_preprocess(BaseEvaluator.get_name("all_priority15"), priority=50) + + groups, sorted_priorities = service.get_evaluators(another_context, BuiltinConcepts.EVALUATION) + assert groups == {20: [EvaluatorOneWithPriority20()], + 15: [EvaluatorOneWithPriority15()], + 50: [EvaluatorAllWithPriority15(50)]} + assert sorted_priorities == [50, 20, 15] + + def test_i_can_get_altered_and_selected_evaluators(self): + sheerka, context = self.init_concepts() + sheerka.evaluators = [ + EvaluatorOneWithPriority20, + EvaluatorAllWithPriority15, + EvaluatorOnePreEvaluation, # wrong step + DisabledEvaluatorOneWithPriority90, # always disabled + EvaluatorOneWithPriority15, # dynamically disabled + ] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_evaluators() + evaluators_names = ["all_priority15", "preEval", "disabled90", "priority15"] + context.preprocess_evaluators = evaluators_names + context.add_preprocess(BaseEvaluator.get_name("priority15"), enabled=False) + context.add_preprocess(BaseEvaluator.get_name("all_priority15"), priority=99) + context.add_preprocess(BaseEvaluator.get_name("priority20"), priority=98) # not selected + + groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION) + assert groups == {99: [EvaluatorAllWithPriority15(99)]} + assert sorted_priorities == [99] + + key = BuiltinConcepts.EVALUATION + "|" + "|".join(evaluators_names) + assert key not in service.grouped_evaluators_cache + + def test_i_can_revert_back_evaluators_alterations(self): + sheerka, context = self.init_concepts() + sheerka.evaluators = [ + EvaluatorOneWithPriority20, + EvaluatorAllWithPriority15, + ] + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_evaluators() + + context.add_preprocess(BaseEvaluator.get_name("priority20"), enabled=False) + context.add_preprocess(BaseEvaluator.get_name("all_priority15"), priority=99) + + groups, sorted_priorities = service.get_evaluators(context, BuiltinConcepts.EVALUATION) + assert groups == {99: [EvaluatorAllWithPriority15(99)]} + assert sorted_priorities == [99] + + another_context = self.get_context(sheerka) + # the result is taken from the default cache, so the priorities are okay + # But the attributes of the evaluators are not reset + groups, sorted_priorities = service.get_evaluators(another_context, BuiltinConcepts.EVALUATION) + assert groups == {15: [EvaluatorAllWithPriority15(99)], 20: [EvaluatorOneWithPriority20(enabled=False)]} + assert sorted_priorities == [20, 15] + + # let's revert + service.undo_preprocess() + groups, sorted_priorities = service.get_evaluators(another_context, BuiltinConcepts.EVALUATION) + assert groups == {15: [EvaluatorAllWithPriority15()], 20: [EvaluatorOneWithPriority20()]} + assert sorted_priorities == [20, 15] + def test_that_return_values_is_unchanged_when_no_evaluator(self): sheerka = self.get_sheerka() sheerka.evaluators = [] @@ -299,7 +428,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): EvaluatorOneWithPriority15, EvaluatorAllWithPriority20, ] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -324,7 +453,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneModifyFoo] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))] Out.debug_out = [] @@ -342,7 +471,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorAllReduceFooBar] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar"))] Out.debug_out = [] @@ -364,7 +493,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))] Out.debug_out = [] @@ -391,7 +520,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -406,7 +535,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneMultiSteps] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -426,8 +555,9 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): EvaluatorAllWithPriority15, EvaluatorAllWithPriority10, ] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() - context.preprocess_evaluators = [EvaluatorAllWithPriority10().short_name] # it will be the only one to be evaluated + service.reset_registered_evaluators() + context.preprocess_evaluators = [ + EvaluatorAllWithPriority10().short_name] # it will be the only one to be evaluated entries = [self.tretval(sheerka, Concept("foo"))] @@ -448,7 +578,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): EvaluatorOneWithPriority15, EvaluatorOneWithPriority10, ] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() # invert the priorities context.add_preprocess(EvaluatorOneWithPriority20().name, priority=1) @@ -468,17 +598,13 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): '__EVALUATION [0] priority20 - eval - target=foo', ] - # grouped evaluator is in cache - service = sheerka.services[SheerkaExecute.NAME] - assert "__EVALUATION|evaluators.priority20|evaluators.priority15|evaluators.priority10" in service.grouped_evaluators_cache - def test_evaluators_enabled_can_be_tweaked_by_the_context(self): sheerka, context = self.init_concepts() sheerka.evaluators = [EvaluatorOneWithPriority20, EvaluatorOneWithPriority15, EvaluatorOneWithPriority10, ] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() # invert the priorities context.add_preprocess(EvaluatorOneWithPriority20().name, enabled=False) @@ -493,15 +619,11 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): "__EVALUATION [0] priority10 - eval - target=foo", ] - # grouped evaluator is in cache - service = sheerka.services[SheerkaExecute.NAME] - assert "__EVALUATION|evaluators.priority10" in service.grouped_evaluators_cache - def test_evaluators_can_be_pre_processed(self): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneModifyFoo] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] @@ -525,7 +647,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz") sheerka.evaluators = [EvaluatorOneInitializationOnce] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [ self.tretval(sheerka, foo), @@ -548,7 +670,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka, context, foo, bar, baz = self.init_concepts("foo", "bar", "baz") sheerka.evaluators = [EvaluatorOneInitializationMultiple] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [ self.tretval(sheerka, foo), @@ -578,7 +700,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneDoNotModifyExecutionFlow] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -600,7 +722,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorOneModifyFoo] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo"))] Out.debug_out = [] @@ -619,7 +741,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() sheerka.evaluators = [EvaluatorAllSuppressEntries] service = sheerka.services[SheerkaExecute.NAME] - service.reset_evaluators() + service.reset_registered_evaluators() entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar")), diff --git a/tests/core/test_sheerka_call_parsers.py b/tests/core/test_sheerka_call_parsers.py index dfae9b3..0030fac 100644 --- a/tests/core/test_sheerka_call_parsers.py +++ b/tests/core/test_sheerka_call_parsers.py @@ -1,5 +1,5 @@ from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept -from core.sheerka.services.SheerkaExecute import ParserInput +from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute from parsers.BaseParser import BaseParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -142,12 +142,74 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): # Ask for a new one TestUsingMemoryBasedSheerka.singleton_instance = None + def test_i_can_get_parser_when_context_is_not_altered(self): + sheerka, context = self.init_concepts() + sheerka.parsers = { + "Enabled90False": Enabled90FalseParser, + "Enabled80False": Enabled80FalseParser, + } + service = SheerkaExecute(sheerka) + service.reset_registered_parsers() + + groups, sorted_priorities = service.get_parsers(context) + assert groups == {80: [Enabled80FalseParser()], 90: [Enabled90FalseParser()]} + assert sorted_priorities == [90, 80] + + def test_i_can_get_selected_parsers(self): + sheerka, context = self.init_concepts() + sheerka.parsers = { + "Enabled90False": Enabled90FalseParser, + "Enabled80False": Enabled80FalseParser, + "Enabled70False": Enabled70FalseParser, + "Enabled50True": Enabled50TrueParser, + "Disabled": Enabled50TrueParser, # <= this one is disabled. It can't be used + } + service = SheerkaExecute(sheerka) + service.reset_registered_parsers() + parsers_names = ["Enabled50True", "Enabled70False", "Disabled"] + context.preprocess_parsers = parsers_names + + groups, sorted_priorities = service.get_parsers(context) + assert groups == {50: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]} + assert sorted_priorities == [70, 50] # Disabled parser does not appear + + key = "|".join(parsers_names) + assert key in service.grouped_parsers_cache + groups, sorted_priorities = service.grouped_parsers_cache[key] + assert groups == {50: [Enabled50TrueParser], 70: [Enabled70FalseParser]} + assert sorted_priorities == [70, 50] + + def test_i_can_get_altered_parsers(self): + sheerka, context = self.init_concepts() + sheerka.parsers = { + "Enabled90False": Enabled90FalseParser, + "Enabled80False": Enabled80FalseParser, + "Enabled70False": Enabled70FalseParser, + "Enabled50True": Enabled50TrueParser, + "Disabled": Enabled50TrueParser, # <= this one is disabled. It can't be used + } + service = SheerkaExecute(sheerka) + service.reset_registered_parsers() + parsers_names = ["Enabled90False", "Enabled50True", "Enabled70False", "Disabled"] + context.preprocess_parsers = parsers_names + context.add_preprocess(BaseParser.get_name("Enabled90False"), enabled=False) + context.add_preprocess(BaseParser.get_name("Enabled50True"), priority=80) + + groups, sorted_priorities = service.get_parsers(context) + assert groups == {80: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]} + assert sorted_priorities == [80, 70] # Disabled parsers does not appear + + key = "|".join(parsers_names) + assert key not in service.grouped_parsers_cache # not saved in cache + def test_disabled_parsers_are_not_executed(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled": Enabled10TrueParser, "Disabled": DisabledParser } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] @@ -162,6 +224,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] @@ -177,22 +241,25 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): 'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world', ] - # def test_parsing_stop_at_the_first_success(self): - # sheerka = self.get_sheerka() - # sheerka.parsers = { - # "Enabled80False": Enabled80FalseParser, - # "Enabled50bisTrue": Enabled50bisTrueParser, - # "Enabled10True": Enabled10TrueParser, - # } - # - # user_input = [get_ret_val("hello world")] - # BaseTestParser.debug_out = [] - # sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) - # - # assert BaseTestParser.debug_out == [ - # 'name=Enabled80False, priority=80, status=False, source=hello world', - # 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', - # ] + def test_parsing_stop_at_the_first_success(self): + sheerka = self.get_sheerka() + sheerka.parsers = { + "Enabled80False": Enabled80FalseParser, + "Enabled50bisTrue": Enabled50bisTrueParser, + "Enabled10True": Enabled10TrueParser, + } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() + + user_input = [get_ret_val("hello world")] + BaseTestParser.debug_out = [] + sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) + + assert BaseTestParser.debug_out == [ + 'name=Enabled80False, priority=80, status=False, source=hello world', + 'name=Enabled50BisTrue, priority=50, status=True, source=hello world', + 'name=Enabled50BisTrue, priority=50, status=True, source=Enabled80False:hello world', + ] def test_parsing_stop_at_the_first_success_2(self): """ @@ -206,6 +273,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): "Enabled50True": Enabled50TrueParser, "Enabled10True": Enabled10TrueParser, } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] @@ -235,6 +304,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): "Enabled50False": Enabled50FalseParser, "Enabled10True": Enabled10TrueParser, } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] @@ -265,6 +336,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): "Enabled80False": Enabled80FalseParser, "Enabled50True": Enabled50TrueParser, } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() user_input = [get_ret_val("hello world")] @@ -286,6 +359,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): "NoneParser": NoneParser, "ListOfNone": ListOfNoneParser, } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() user_input = [get_ret_val("hello world")] @@ -310,6 +385,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): "Enabled70False": Enabled70FalseParser, "Enabled50True": Enabled50TrueParser, } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] @@ -329,6 +406,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): "Enabled80MultipleFalse": Enabled80MultipleFalseParser, "Enabled50True": Enabled50TrueParser, } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] @@ -356,6 +435,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): "Enabled80MultipleTrue": Enabled80MultipleTrueParser, "Enabled50True": Enabled50TrueParser, } + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() user_input = [get_ret_val("hello world")] BaseTestParser.debug_out = [] diff --git a/tests/evaluators/EvaluatorTestsUtils.py b/tests/evaluators/EvaluatorTestsUtils.py index 7a2c38f..c580ea8 100644 --- a/tests/evaluators/EvaluatorTestsUtils.py +++ b/tests/evaluators/EvaluatorTestsUtils.py @@ -1,7 +1,10 @@ -from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +import ast + +from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts, ParserResultConcept from core.concept import Concept from evaluators.BaseEvaluator import BaseEvaluator from parsers.BaseParser import BaseParser +from parsers.PythonParser import PythonNode reduced_requested = ReturnValueConcept("Sheerka", True, Concept(name=BuiltinConcepts.REDUCE_REQUESTED, key=BuiltinConcepts.REDUCE_REQUESTED)) @@ -12,7 +15,7 @@ def ret_val(value="value", who="who", status=True): def p_ret_val(value="value", parser="parser", status=True): - return ReturnValueConcept(BaseParser.PREFIX + parser, status, value) + return ReturnValueConcept(BaseParser.get_name(parser), status, value) def e_ret_val(value="value", evaluator="evaluator", status=True): @@ -40,6 +43,24 @@ def e_ret_val_new(key, evaluator="evaluator", status=True, **kwargs): return e_ret_val(body, evaluator, status) +def pr_ret_val(value, parser="parser", source=None): + """ + ParserResult ReturnValue + eg: ReturnValue with a ParserResult + :param value: + :param parser: + :param source: + :return: + """ + source = source or (value.name if isinstance(value, Concept) else "source") + parser_result = ParserResultConcept(BaseParser.get_name(parser), source=source, value=value) + return p_ret_val(parser_result, parser) + + +def python_ret_val(source): + python_node = PythonNode(source, ast.parse(source, f"", 'eval')) + return pr_ret_val(python_node, parser="Python", source=source) + def new_concept(key, **kwargs): res = Concept(key=key, name=key, is_builtin=False, is_unique=False) for k, v in kwargs.items(): diff --git a/tests/evaluators/test_ConceptEvaluator.py b/tests/evaluators/test_ConceptEvaluator.py index 4d870c2..d5281e2 100644 --- a/tests/evaluators/test_ConceptEvaluator.py +++ b/tests/evaluators/test_ConceptEvaluator.py @@ -91,12 +91,10 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): assert result.value == "'some_other_value'" def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(self): - context = self.get_context() + sheerka, context, one, concept_plus = self.init_concepts( + "one", + Concept(name="a plus b").def_var("a", "one").def_var("b", "two")) context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - context.sheerka.test_only_add_in_cache(Concept(name="one").init_key()) - concept_plus = context.sheerka.test_only_add_in_cache(Concept(name="a plus b") - .def_var("a", "one") - .def_var("b", "two").init_key()) evaluator = ConceptEvaluator() item = self.pretval(concept_plus) diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index 7b107cd..51d1bee 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -43,11 +43,7 @@ class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka): assert evaluated == simplec("one", 1) def test_i_can_recognize_concept_with_concept_body(self): - sheerka = self.get_sheerka() - concept_one = Concept(name="one") - concept_un = Concept(name="un", body="one") - sheerka.test_only_add_in_cache(concept_one) - sheerka.test_only_add_in_cache(concept_un) + sheerka, context, concept_one, concept_un = self.init_concepts("one", Concept(name="un", body="one")) res = sheerka.evaluate_user_input("un") return_value = res[0].value @@ -202,9 +198,10 @@ as: assert evaluated.get_value("a") == concept_foo def test_i_can_recognize_concept_with_variable_and_python_as_body(self): - sheerka = self.get_sheerka() - hello_a = sheerka.test_only_add_in_cache(Concept(name="hello a", body="'hello ' + a").def_var("a")) - sheerka.test_only_add_in_cache(Concept(name="foo", body="'foo'")) + sheerka, context, hello_a, foo = self.init_concepts( + Concept(name="hello a", body="'hello ' + a").def_var("a"), + Concept(name="foo", body="'foo'") + ) res = sheerka.evaluate_user_input("hello foo") assert len(res) == 1 @@ -1199,6 +1196,18 @@ as: assert res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.TO_MULTI) + def test_i_can_evaluate_pseudo_recursive_definition(self): + init = [ + "def concept a + b as a + b", + ] + sheerka = self.init_scenario(init) + + res = sheerka.evaluate_user_input("eval 1 + 1") + + assert len(res) == 1 + assert res[0].status + assert res[0].body == 2 + class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): def test_i_can_def_several_concepts(self):