diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..92e0f7b --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,16 @@ +pipeline { + agent none + stages { + stage('Build') { + agent { + docker { + image 'python:2-alpine' + } + } + steps { + sh 'make clean' + // stash(name: 'compiled-results', includes: 'sources/*.py*') + } + } + } +} \ No newline at end of file diff --git a/_concepts_lite.txt b/_concepts_lite.txt index 47688a2..715c747 100644 --- a/_concepts_lite.txt +++ b/_concepts_lite.txt @@ -2,4 +2,10 @@ def concept one as 1 def concept two as 2 def concept plus from a plus b as a + b def concept explain as get_results() | filter("id == 0") | recurse(2) -def concept explain last as get_last_results() | filter("id == 0") | recurse(2) \ No newline at end of file +set_isa(c:explain:, __COMMAND) +def concept explain last as get_last_results() | filter("id == 0") | recurse(2) +set_isa(c:explain last:, __COMMAND) +def concept precedence a > precedence b as set_is_greater_than(BuiltinConcepts.PRECEDENCE, a, b) +set_isa(c:precedence a > precedence b:, __COMMAND) +def concept x is a command as set_isa(x, __COMMAND) +set_isa(c:x is a command:, __COMMAND) \ No newline at end of file diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 0286b53..63e77b8 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -15,7 +15,15 @@ class BuiltinConcepts(Enum): """ SHEERKA = "sheerka" - # Execution context actions + # processing instructions during sheerka.execute() + EVAL_BODY_REQUESTED = "eval body" # to evaluate the body + EVAL_WHERE_REQUESTED = "eval where" # to evaluate the where clause + RETURN_VALUE_REQUESTED = "return value" # returns the body of the concept instead of the concept itself + REDUCE_REQUESTED = "reduce" # remove meaningless error when possible + EVAL_UNTIL_SUCCESS_REQUESTED = "eval until success" # PythonEvaluator tries combination until True is found + QUESTION_REQUESTED = "question" # a question is asked + + # possible actions during sheerka.execute() INIT_SHEERKA = "init sheerka" # PROCESS_INPUT = "process input" # Processing user input or other input PROCESSING = "processing input" # Processing user input or other input @@ -30,22 +38,28 @@ class BuiltinConcepts(Enum): AFTER_RENDERING = "after rendering" # rendering the response from sheerka EVALUATE_CONCEPT = "evaluate concept" # a concept will be evaluated EVALUATING_CONCEPT = "evaluating concept" # a concept will be evaluated + EVALUATING_ATTRIBUTE = "evaluating concept attribute" # VALIDATE_CONCEPT = "validate concept" VALIDATING_CONCEPT = "validating concept" INIT_COMPILED = "initializing concept compiled" - INIT_BNF = "ensure bnf" + INIT_BNF = "initialize bnf" MANAGE_INFINITE_RECURSION = "manage infinite recursion" PARSE_CODE = "execute source code" EXEC_CODE = "execute source code" TESTING = "testing" - USER_INPUT = "user input" # represent an input from an user - SUCCESS = "success" - ERROR = "error" + # builtin attributes + ISA = "is a" # when a concept is an instance of another one + COMMAND = "command" # when the concept must be auto evaluated + + # object + USER_INPUT = "user input concept" # represent an input from an user + SUCCESS = "success concept" + ERROR = "error concept" UNKNOWN_CONCEPT = "unknown concept" # the request concept is not recognized CANNOT_RESOLVE_CONCEPT = "cannot resolve concept" # when too many concepts with the same name - RETURN_VALUE = "return value" # a value is returned - CONCEPT_TOO_LONG = "concept too long" # concept cannot be processed by exactConcept parser + RETURN_VALUE = "return value concept" # a value is returned + CONCEPT_TOO_LONG = "concept too long concept" # concept cannot be processed by exactConcept parser NEW_CONCEPT = "new concept" # when a new concept is added UNKNOWN_PROPERTY = "unknown property" # when requesting for a unknown property PARSER_RESULT = "parser result" @@ -64,15 +78,9 @@ class BuiltinConcepts(Enum): FILTERED = "filtered" # represents the result of a filtering CONCEPT_ALREADY_IN_SET = "concept already in set" EVALUATOR_PRE_PROCESS = "evaluator pre process" # used modify / tweak behaviour of evaluators - EVAL_BODY_REQUESTED = "eval body requested" # to evaluate the body - EVAL_WHERE_REQUESTED = "eval where requested" # to evaluate the where clause - CONCEPT_VALUE_REQUESTED = "concept value requested" # returns the body of the concept instead of the concept itself - REDUCE_REQUESTED = "reduce requested" # remove meaningless error when possible - EVAL_SUCCESS_REQUESTED = "Try to find a successful evaluation" # PyhtonEvaluator tries combination until True is found NOT_A_SET = "not a set" # the concept has no entry in sets - WHERE_CLAUSE_FAILED = "where clause failed" # failed to validate where clause during evaluation + CONDITION_FAILED = "where clause failed" # failed to validate where clause during evaluation CHICKEN_AND_EGG = "chicken and egg" # infinite recursion when declaring concept - ISA = "is a" # builtin concept to express that a concept is an instance of another one EXPLANATION = "explanation" PRECEDENCE = "precedence" # use to set priority among concepts when parsing ASSOCIATIVITY = "associativity" # use to set priority among concepts when parsing @@ -80,6 +88,7 @@ class BuiltinConcepts(Enum): NOT_FOUND = "not found" # when the wanted resource is not found FORMAT_INSTRUCTIONS = "format instructions" # to express how to print the concept NOT_IMPLEMENTED = "not implemented" # instead of raise an error + PYTHON_SECURITY_ERROR = "security error" # when trying to execute statement when only expression is allowed NODE = "node" GENERIC_NODE = "generic node" @@ -108,6 +117,16 @@ class BuiltinConcepts(Enum): BuiltinUnique = [ + BuiltinConcepts.EVAL_BODY_REQUESTED, + BuiltinConcepts.EVAL_WHERE_REQUESTED, + BuiltinConcepts.RETURN_VALUE_REQUESTED, + BuiltinConcepts.REDUCE_REQUESTED, + BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED, + BuiltinConcepts.QUESTION_REQUESTED, + + BuiltinConcepts.INIT_SHEERKA, + BuiltinConcepts.PROCESS_INPUT, + BuiltinConcepts.PROCESSING, BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING, @@ -117,10 +136,20 @@ BuiltinUnique = [ BuiltinConcepts.BEFORE_RENDERING, BuiltinConcepts.RENDERING, BuiltinConcepts.AFTER_RENDERING, - BuiltinConcepts.SUCCESS, - BuiltinConcepts.NOP, - BuiltinConcepts.EVAL_BODY_REQUESTED, - BuiltinConcepts.REDUCE_REQUESTED, + BuiltinConcepts.EVALUATE_CONCEPT, + BuiltinConcepts.EVALUATING_CONCEPT, + BuiltinConcepts.EVALUATING_ATTRIBUTE, + BuiltinConcepts.VALIDATE_CONCEPT, + BuiltinConcepts.VALIDATING_CONCEPT, + BuiltinConcepts.INIT_COMPILED, + BuiltinConcepts.INIT_BNF, + BuiltinConcepts.MANAGE_INFINITE_RECURSION, + BuiltinConcepts.PARSE_CODE, + BuiltinConcepts.EXEC_CODE, + BuiltinConcepts.TESTING, + + BuiltinConcepts.ISA, + BuiltinConcepts.COMMAND, ] BuiltinErrors = [str(e) for e in { @@ -137,7 +166,7 @@ BuiltinErrors = [str(e) for e in { BuiltinConcepts.CONCEPT_EVAL_ERROR, BuiltinConcepts.CONCEPT_ALREADY_IN_SET, BuiltinConcepts.NOT_A_SET, - BuiltinConcepts.WHERE_CLAUSE_FAILED, + BuiltinConcepts.CONDITION_FAILED, BuiltinConcepts.CHICKEN_AND_EGG, BuiltinConcepts.NOT_INITIALIZED, BuiltinConcepts.NOT_FOUND @@ -194,7 +223,7 @@ class ReturnValueConcept(Concept): It's the main input for the evaluators """ - def __init__(self, who=None, status=None, value=None, message=None, parents=None): + def __init__(self, who=None, status=None, value=None, message=None, parents=None, concept_id=None): super().__init__(BuiltinConcepts.RETURN_VALUE, True, False, BuiltinConcepts.RETURN_VALUE) self.set_value(ConceptParts.BODY, value) self.set_value("who", who) @@ -202,6 +231,7 @@ class ReturnValueConcept(Concept): self.set_value("message", message) self.set_value("parents", parents) self.metadata.is_evaluated = True + self.metadata.id = concept_id @property def who(self): @@ -429,21 +459,19 @@ class ConceptAlreadyInSet(Concept): return self.get_value("concept_set") -class WhereClauseFailed(Concept): - def __init__(self, concept=None): - super().__init__(BuiltinConcepts.WHERE_CLAUSE_FAILED, +class ConditionFailed(Concept): + def __init__(self, condition=None, concept=None, prop=None): + super().__init__(BuiltinConcepts.CONDITION_FAILED, True, False, - BuiltinConcepts.WHERE_CLAUSE_FAILED) - self.set_value(ConceptParts.BODY, concept) + BuiltinConcepts.CONDITION_FAILED) + self.set_value(ConceptParts.BODY, condition) + self.set_value("concept", concept) + self.set_value("prop", prop) self.metadata.is_evaluated = True def __repr__(self): - return f"WhereClauseFailed(concept={self.concept})" - - @property - def concept(self): - return self.body + return f"ConditionFailed(condition='{self.body}', concept='{self.concept}', prop='{self.prop}')" class NotForMeConcept(Concept): @@ -472,3 +500,18 @@ class ExplanationConcept(Concept): self.set_value("instructions", instructions) # instructions for SheerkaPrint self.set_value(ConceptParts.BODY, execution_result) # list of results self.metadata.is_evaluated = True + + +class PythonSecurityError(Concept): + def __init__(self, prop=None, source_code=None, source=None, line=None, column=None): + super().__init__(BuiltinConcepts.PYTHON_SECURITY_ERROR, + True, + False, + BuiltinConcepts.PYTHON_SECURITY_ERROR) + + self.set_value("prop", prop) # property or variable that was evaluated + self.set_value("source", source) # origin of the source code (eg. file name) + self.set_value("line", line) # line number + self.set_value("column", column) # column number + self.set_value(ConceptParts.BODY, source_code) # code being executed + self.metadata.is_evaluated = True diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index 7f7d931..9b49d42 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -5,7 +5,7 @@ import core.ast.nodes from core.ast.nodes import CallNodeConcept from core.ast.visitors import UnreferencedNamesVisitor from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, NotInit +from core.concept import Concept, NotInit, ConceptParts from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode from parsers.BaseParser import BaseParser, ErrorNode @@ -25,13 +25,10 @@ def is_same_success(context, return_values): if isinstance(ret_val.body, Concept): if not ret_val.body.metadata.is_evaluated: - with context.push(BuiltinConcepts.EVALUATE_CONCEPT, - ret_val.body, - desc=f"Evaluating concept '{ret_val.body}'") as sub_context: - sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - evaluated = context.sheerka.evaluate_concept(sub_context, ret_val.body) - if evaluated.key != ret_val.body.key: - raise Exception("Failed to evaluate evaluate") + evaluated = context.sheerka.evaluate_concept(context, ret_val.body, eval_body=True) + if not context.sheerka.is_success(evaluated): + raise Exception("Failed to evaluate evaluate") + return context.sheerka.objvalue(evaluated) else: return context.sheerka.objvalue(ret_val.body) @@ -173,6 +170,129 @@ def only_successful(context, return_values): sheerka.new(BuiltinConcepts.ONLY_SUCCESSFUL, body=successful_results), parents=return_values) +# +# def remove_ambiguity(context, return_values): +# """ +# When multiple concepts are matching, try to find the one(s) which is/are the more accurate +# :param context: +# :param return_values: +# :return: +# """ +# # example : +# # Concept("x is a y", PRE=QUESTION) +# # Concept("x is a y") +# # Concept("x is a command") +# # with the source 'foo is a command' +# # The first one do not respect the pre condition (assuming that QUESTION is not in context) +# # The second one is Ok, but the third is more specific (as it's a concept that explicitly asks for 'command') +# # The third one will remain +# if not return_values: +# return return_values +# +# sheerka = context.sheerka +# by_number_of_vars = {} # sorted by number of variables +# to_keep = [] # return values values that are not eligible (ex non parser result) +# passed_validation = [] +# +# # sort by number of variables +# # The less variables there are, the more accurate is the concept +# for r in return_values: +# if (r.status and +# sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT) and +# isinstance(r.body.body, Concept)): +# by_number_of_vars.setdefault(len(r.body.body.metadata.variables), []).append(r) +# else: +# to_keep.append(r) +# +# # Check the concepts, starting with the ones that have the less variables +# # Once a concept with a given number of variables is found, we do not process the concepts with an higher +# # number of variables +# for nb_vars in sorted(by_number_of_vars.keys()): +# for r in by_number_of_vars[nb_vars]: +# concept = r.body.body +# if concept.metadata.pre is None or concept.metadata.pre.strip() == "": +# passed_validation.append(r) +# else: +# evaluated = context.sheerka.evaluate_concept(context, concept, metadata=["pre"]) +# if evaluated.key == concept.key and evaluated.get_value(ConceptParts.PRE): +# passed_validation.append(r) +# if len(passed_validation) > 0: +# break +# +# # Nothing was filtered, return the original return values +# if len(passed_validation) + len(to_keep) == len(return_values): +# return return_values # nothing to filter +# +# # All return_values fail, return empty list +# if len(passed_validation) == 0: +# return sheerka.ret( +# context.who, +# True, +# sheerka.new(BuiltinConcepts.FILTERED, +# body=to_keep, +# iterable=return_values, +# predicate="remove_ambiguity(context, iterable)"), +# parents=return_values) +# +# # Final check, we consider that the concepts that match a PRE condition are better than concepts with no condition +# by_condition_complexity = {} +# for r in passed_validation: +# by_condition_complexity.setdefault(get_condition_complexity(r.body.body, "pre"), []).append(r) +# +# return sheerka.ret( +# context.who, +# True, +# sheerka.new(BuiltinConcepts.FILTERED, +# body=to_keep + by_condition_complexity[max(by_condition_complexity.keys())], +# iterable=return_values, +# predicate="remove_ambiguity(context, iterable)"), +# parents=return_values) + + +def resolve_ambiguity(context, concepts): + """ + From the list of concept, elect the one(s) that best suit(s) the context + :param context: + :param concepts: + :return: + """ + + # we first sort by condition complexity. The more complex is the PRE condition, the more likely + # the concept matches the context + by_complexity = {} + for c in concepts: + by_complexity.setdefault(get_condition_complexity(c, "pre"), []).append(c) + + remaining_concepts = [] + for complexity in sorted(by_complexity.keys(), reverse=True): + for c in by_complexity[complexity]: + evaluated = context.sheerka.evaluate_concept(context, c, metadata=["pre"]) + if evaluated.key == c.key: + remaining_concepts.append(c) + + if len(remaining_concepts) > 0: + break # no need to check concept with lower complexity + + if len(remaining_concepts) in (0, 1): + return remaining_concepts # they all failed the pre conditions or one champ is found + + # for concepts with the same condition complexity, we choose the one that has the less number of variables + # We consider that Concept("hello world") is more specific than Concept("hello a").def_var("a") + # when the input is "hello world" + by_number_of_vars = {} + for c in remaining_concepts: + by_number_of_vars.setdefault(len(c.metadata.variables), []).append(c) + + return by_number_of_vars[min(by_number_of_vars.keys())] + + +def get_condition_complexity(concept, concept_part_str): + concept_part_value = getattr(concept.metadata, concept_part_str) + if concept_part_value is None or concept_part_value.strip() == 0: + return 0 + + return 1 # no real computing as of now + def only_parsers_results(context, return_values): """ @@ -197,7 +317,8 @@ def only_parsers_results(context, return_values): sheerka.new(BuiltinConcepts.IS_EMPTY, body=return_values), parents=return_values) - return_values_ok = [item for item in return_values if sheerka.isinstance(item.body, BuiltinConcepts.PARSER_RESULT)] + return_values_ok = [item for item in return_values if + sheerka.isinstance(item.body, BuiltinConcepts.PARSER_RESULT)] # hack because some parsers don't follow the NOT_FOR_ME rule temp_ret_val = [] @@ -291,7 +412,8 @@ def get_lexer_nodes(return_values, start, tokens): continue end = start + len(tokens) - 1 - lexer_nodes.append([SourceCodeNode(ret_val.body.body, start, end, tokens, ret_val.body.source, ret_val)]) + lexer_nodes.append( + [SourceCodeNode(ret_val.body.body, start, end, tokens, ret_val.body.source, ret_val)]) elif ret_val.who == "parsers.ExactConcept": concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body] @@ -333,12 +455,7 @@ def ensure_evaluated(context, concept, eval_body=True): (var[0] not in concept.values or concept.get_value(var[0]) == NotInit): return concept - with context.push(BuiltinConcepts.EVALUATE_CONCEPT, concept, desc=f"Evaluating concept {concept}") as sub_context: - if eval_body: - sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - evaluated = context.sheerka.evaluate_concept(sub_context, concept) - sub_context.add_values(return_values=evaluated) - + evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=eval_body) return evaluated diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index dcb749c..aa6151c 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -1,5 +1,6 @@ import inspect import logging +from dataclasses import dataclass import core.builtin_helpers import core.utils @@ -22,6 +23,15 @@ BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser" EXIT_COMMANDS = ("quit", "exit", "bye") +@dataclass +class SheerkaMethod: + """ + Wrapper to sheerka method, to indicate if it's safe to call + """ + method: object + has_side_effect: bool + + class Sheerka(Concept): """ Main controller for the project @@ -55,6 +65,7 @@ class Sheerka(Concept): self.log.debug("Starting Sheerka.") self.bnp = None # reference to the BaseNodeParser class (to compute first keyword token) + self.return_value_concept_id = None # a concept can be instantiated # ex: File is a concept, but File('foo.txt') is an instance @@ -84,10 +95,10 @@ class Sheerka(Concept): self.save_execution_context = True - self.methods_with_context = {"test_using_context"} + self.methods_with_context = {"test_using_context"} # only the names, the method is defined in sheerka_methods self.sheerka_methods = { - "test": self.test, - "test_using_context": self.test_using_context + "test": SheerkaMethod(self.test, False), + "test_using_context": SheerkaMethod(self.test_using_context, False) } self.sheerka_pipeables = {} @@ -119,10 +130,11 @@ class Sheerka(Concept): def chicken_and_eggs(self): return self.cache_manager.caches[self.CONCEPTS_GRAMMARS_ENTRY].cache - def bind_service_method(self, bound_method, as_name=None): + def bind_service_method(self, bound_method, has_side_effect, as_name=None): """ Bind service method to sheerka instance for ease of use ? :param bound_method: + :param has_side_effect: False if the method is safe :param as_name: :return: """ @@ -132,18 +144,19 @@ class Sheerka(Concept): signature = inspect.signature(bound_method) if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context": self.methods_with_context.add(as_name) - self.sheerka_methods[as_name] = bound_method + self.sheerka_methods[as_name] = SheerkaMethod(bound_method, has_side_effect) setattr(self, as_name, bound_method) - def add_pipeable(self, func_name, function): + def add_pipeable(self, func_name, function, has_side_effect): """ Adds a function that can bu used with pipe '|' :param func_name: :param function: + :param has_side_effect: :return: """ - self.sheerka_pipeables[func_name] = function + self.sheerka_pipeables[func_name] = SheerkaMethod(function, has_side_effect) def initialize(self, root_folder: str = None, save_execution_context=True): """ @@ -292,6 +305,10 @@ class Sheerka(Concept): self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.") self.set_id_if_needed(concept, True) self.cache_manager.add_concept(concept) + + if key == BuiltinConcepts.RETURN_VALUE: + self.return_value_concept_id = concept.id + else: self.init_log.debug(f"Found concept '{from_db}' in db. Updating.") concept.update_from(from_db) @@ -595,6 +612,8 @@ class Sheerka(Concept): :param concept_id: :return: """ + if concept_id is None: + return False return self.cache_manager.has(self.CONCEPTS_BY_ID_ENTRY, concept_id) def has_key(self, concept_key): @@ -652,6 +671,7 @@ class Sheerka(Concept): return self.new_from_template(template, concept_key, **kwargs) def new_from_template(self, template, key, **kwargs): + # core.utils.my_debug(f"Created {template}, {key=}, {kwargs=}") # manage singleton if template.metadata.is_unique: return template @@ -677,7 +697,7 @@ class Sheerka(Concept): return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept) # TODO : add the concept to the list of known concepts (self.instances) - concept.metadata.is_evaluated = True # because we have manually set the variables + concept.metadata.is_evaluated = True # because we have manually set the variables return concept def ret(self, who: str, status: bool, value, message=None, parents=None): @@ -690,13 +710,23 @@ class Sheerka(Concept): :param parents: :return: """ - return self.new( - BuiltinConcepts.RETURN_VALUE, + + # 1 second saved every twenty seconds in unit tests + return ReturnValueConcept( who=who, status=status, value=value, message=message, - parents=parents) + parents=parents, + concept_id=self.return_value_concept_id + ) + # return self.new( + # BuiltinConcepts.RETURN_VALUE, + # who=who, + # status=status, + # value=value, + # message=message, + # parents=parents) def objvalue(self, obj, reduce_simple_list=False): if obj is None: @@ -812,6 +842,10 @@ class Sheerka(Concept): if isinstance(obj, ReturnValueConcept): return obj.status + # other cases ? + # ... + + # manage internal errors if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors: return False diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index 73a91fa..16ddd88 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -14,11 +14,11 @@ class SheerkaAdmin(BaseService): super().__init__(sheerka) def initialize(self): - self.sheerka.bind_service_method(self.caches_names) - self.sheerka.bind_service_method(self.cache) - self.sheerka.bind_service_method(self.restore) - self.sheerka.bind_service_method(self.concepts) - self.sheerka.bind_service_method(self.last_created_concept) + self.sheerka.bind_service_method(self.caches_names, False) + self.sheerka.bind_service_method(self.cache, False) + self.sheerka.bind_service_method(self.restore, True) + self.sheerka.bind_service_method(self.concepts, False) + self.sheerka.bind_service_method(self.last_created_concept, False) def caches_names(self): """ diff --git a/src/core/sheerka/services/SheerkaComparisonManager.py b/src/core/sheerka/services/SheerkaComparisonManager.py index 767d45b..7453671 100644 --- a/src/core/sheerka/services/SheerkaComparisonManager.py +++ b/src/core/sheerka/services/SheerkaComparisonManager.py @@ -90,10 +90,10 @@ class SheerkaComparisonManager(BaseService): cache = Cache() self.sheerka.cache_manager.register_cache(self.RESOLVED_COMPARISON_ENTRY, cache, persist=False) - self.sheerka.bind_service_method(self.set_is_greater_than) - self.sheerka.bind_service_method(self.set_is_less_than) - self.sheerka.bind_service_method(self.get_partition) - self.sheerka.bind_service_method(self.get_concepts_weights) + self.sheerka.bind_service_method(self.set_is_greater_than, True) + self.sheerka.bind_service_method(self.set_is_less_than, True) + self.sheerka.bind_service_method(self.get_partition, False) + self.sheerka.bind_service_method(self.get_concepts_weights, False) def set_is_greater_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"): """ diff --git a/src/core/sheerka/services/SheerkaCreateNewConcept.py b/src/core/sheerka/services/SheerkaCreateNewConcept.py index b1c85a8..30dc820 100644 --- a/src/core/sheerka/services/SheerkaCreateNewConcept.py +++ b/src/core/sheerka/services/SheerkaCreateNewConcept.py @@ -20,7 +20,7 @@ class SheerkaCreateNewConcept(BaseService): self.bnp = core.utils.get_class(BASE_NODE_PARSER_CLASS) # BaseNodeParser def initialize(self): - self.sheerka.bind_service_method(self.create_new_concept) + self.sheerka.bind_service_method(self.create_new_concept, True) def create_new_concept(self, context, concept: Concept): """ diff --git a/src/core/sheerka/services/SheerkaDump.py b/src/core/sheerka/services/SheerkaDump.py index e1a5b76..7b565bc 100644 --- a/src/core/sheerka/services/SheerkaDump.py +++ b/src/core/sheerka/services/SheerkaDump.py @@ -21,8 +21,8 @@ class SheerkaDump(BaseService): super().__init__(sheerka) def initialize(self): - self.sheerka.bind_service_method(self.dump_desc, "desc") - self.sheerka.bind_service_method(self.dump_sdp, "dump_sdp") + self.sheerka.bind_service_method(self.dump_desc, True, "desc") # because concept is evaluated + self.sheerka.bind_service_method(self.dump_sdp, False, "dump_sdp") def dump_desc(self, *concept_names, eval=False): first = True @@ -42,7 +42,7 @@ class SheerkaDump(BaseService): for c in concepts: if eval: - evaluated = self.sheerka.evaluate_concept(context, c) + evaluated = self.sheerka.evaluate_concept(context, c, eval_body=eval) value = evaluated.body if evaluated.key == c.key else evaluated if not first: diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index 5518db1..f50c0ac 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -2,6 +2,7 @@ from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import expect_one, only_successful from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved from core.sheerka.services.sheerka_service import BaseService +from core.tokenizer import Tokenizer from core.utils import unstr_concept CONCEPT_EVALUATION_STEPS = [ @@ -17,16 +18,23 @@ class SheerkaEvaluateConcept(BaseService): super().__init__(sheerka) def initialize(self): - self.sheerka.bind_service_method(self.evaluate_concept) + self.sheerka.bind_service_method(self.evaluate_concept, True) @staticmethod def infinite_recursion_detected(context, concept): + """ + Browse the parents, looking for another evaluation of the same concept + :param context: + :param concept: + :return: + """ if concept is None: return False parent = context.get_parent() while parent is not None: - if parent.who == context.who and parent.obj == concept and parent.obj.compiled == concept.compiled: + if parent.who == context.who and parent.action == BuiltinConcepts.EVALUATING_CONCEPT and \ + parent.obj == concept and parent.obj.compiled == concept.compiled: return True parent = parent.get_parent() @@ -64,7 +72,7 @@ class SheerkaEvaluateConcept(BaseService): parent = context concepts_found = set() while parent and parent.obj: - if parent.who == context.who and parent.desc == context.desc: + if parent.who == context.who and parent.action == BuiltinConcepts.EVALUATING_CONCEPT: body = parent.obj.metadata.body try: return self.sheerka.ret(self.NAME, True, InfiniteRecursionResolved(eval(body))) @@ -180,8 +188,7 @@ class SheerkaEvaluateConcept(BaseService): path = get_path(context, current_prop) desc = f"Evaluating {path} (concept={current_concept})" - context.log(desc, self.NAME) - with context.push(BuiltinConcepts.EVALUATING_CONCEPT, + with context.push(BuiltinConcepts.EVALUATING_ATTRIBUTE, current_prop, desc=desc, obj=current_concept, @@ -191,14 +198,14 @@ class SheerkaEvaluateConcept(BaseService): sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) if expect_success: - sub_context.local_hints.add(BuiltinConcepts.EVAL_SUCCESS_REQUESTED) + sub_context.local_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) # when it's a concept, evaluate it if isinstance(to_resolve, Concept) and \ not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE): evaluated = self.evaluate_concept(sub_context, to_resolve) sub_context.add_values(return_values=evaluated) - if evaluated.key == to_resolve.key: + if evaluated.key == to_resolve.key: # quicker (and dirtier) than sheerka.is_success() return self.apply_ret(evaluated) else: error = evaluated @@ -253,51 +260,78 @@ class SheerkaEvaluateConcept(BaseService): return res - def evaluate_concept(self, context, concept: Concept): + def evaluate_concept(self, context, concept: Concept, eval_body=False, metadata=None): """ Evaluation a concept It means that if the where clause is True, will evaluate the body :param context: :param concept: + :param eval_body: + :param metadata: list of metadata to evaluate ('pre', 'post'...) :return: value of the evaluation or error """ if concept.metadata.is_evaluated: return concept - self.initialize_concept_asts(context, concept) + # I cannot use cache because of concept like 'number'. + # They don't have variables, but their values change every time they are instanciated + # TODO: Need to find a way to cache despite of them + # need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) + # if need_body and len(concept.metadata.variables) == 0 and context.sheerka.has_id(concept.id): + # from_cache = context.sheerka.get_by_id(concept.id) + # if from_cache.metadata.is_evaluated: + # concept.set_value(ConceptParts.BODY, from_cache.body) + # concept.metadata.is_evaluated = True + # return concept - # to make sure of the order, it don't use ConceptParts.get_parts() - # variables must be evaluated first, body must be evaluated before where - all_metadata_to_eval = self.choose_metadata_to_eval(context, concept) + desc = f"Evaluating concept {concept}" + with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc, eval_body=eval_body) as sub_context: - for metadata_to_eval in all_metadata_to_eval: - if metadata_to_eval == "variables": - for var_name in (v for v in concept.variables() if v in concept.compiled): - prop_ast = concept.compiled[var_name] + if eval_body: + # ask for body evaluation + sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - if isinstance(prop_ast, list): - # Do not send the current concept for the properties - resolved = self.resolve_list(context, prop_ast, var_name, None, True, False) - else: - # Do not send the current concept for the properties - resolved = self.resolve(context, prop_ast, var_name, None, True, False) + # auto evaluate commands + if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.COMMAND)): + sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved): - resolved.set_value("concept", concept) # since current concept was not sent - return resolved - else: - concept.set_value(var_name, resolved) - else: - part_key = ConceptParts(metadata_to_eval) + self.initialize_concept_asts(sub_context, concept) - # do not evaluate where when the body is a set - # Indeed, the way that the where clause is expressed is not a valid python or concept code - if part_key == ConceptParts.WHERE and self.sheerka.isaset(context, concept.body): - continue + # to make sure of the order, it don't use ConceptParts.get_parts() + # variables must be evaluated first, body must be evaluated before where + all_metadata_to_eval = metadata or self.compute_metadata_to_eval(sub_context, concept) + + for metadata_to_eval in all_metadata_to_eval: + if metadata_to_eval == "variables": + for var_name in (v for v in concept.variables() if v in concept.compiled): + prop_ast = concept.compiled[var_name] + + if isinstance(prop_ast, list): + # Do not send the current concept for the properties + resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, False) + else: + # Do not send the current concept for the properties + resolved = self.resolve(sub_context, prop_ast, var_name, None, True, False) + + if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved): + resolved.set_value("concept", concept) # since current concept was not sent + return resolved + else: + concept.set_value(var_name, resolved) + else: + part_key = ConceptParts(metadata_to_eval) + + # do not evaluate where when the body is a set + # Indeed, the way that the where clause is expressed is not a valid python or concept code + if part_key == ConceptParts.WHERE and self.sheerka.isaset(sub_context, concept.body): + continue + + if part_key not in concept.compiled or concept.compiled[part_key] is None: + continue - if part_key in concept.compiled and concept.compiled[part_key] is not None: metadata_ast = concept.compiled[part_key] + # if part_key is PRE, POST or WHERE, the concept need to be evaluated # if we want the predicates to be resolved => so force_eval = True # otherwise no need to force @@ -306,91 +340,111 @@ class SheerkaEvaluateConcept(BaseService): # when resolving predicate (where or pre), we need to make sure that PythonEvaluator # will try every possibilities before returning False expect_success = part_key in (ConceptParts.WHERE, ConceptParts.PRE) - resolved = self.resolve(context, + + # resolve + resolved = self.resolve(sub_context, metadata_ast, part_key, concept, force_concept_eval, expect_success) - if isinstance(resolved, Concept) and not context.sheerka.is_success(resolved): + + # 'FATAL' error is detected, let's stop + if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved): return resolved - else: - concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved) - # - # TODO : Validate the PRE condition - # + concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved) - # validate where clause - if ConceptParts.WHERE in concept.values: - where_value = concept.get_value(ConceptParts.WHERE) - if not (where_value is None or self.sheerka.objvalue(where_value)): - return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept) + # validate PRE and WHERE condition + if part_key in (ConceptParts.PRE, ConceptParts.WHERE) and not self.sheerka.objvalue(resolved): + return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, + body=getattr(concept.metadata, metadata_to_eval), + concept=concept, + prop=part_key) - # - # TODO : Validate the POST condition - # + # + # TODO : Validate the POST condition + # - concept.init_key() # only does it if needed - concept.metadata.is_evaluated = "body" in all_metadata_to_eval + concept.init_key() # Necessary for old unit tests. To remove someday - # update the cache for concepts with no variable - if len(concept.metadata.variables) == 0: - self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept) + if "body" in all_metadata_to_eval: + concept.metadata.is_evaluated = True - return concept + # # update the cache for concepts with no variables + # Cannot use cache. See the comment at the beginning of this method + # if len(concept.metadata.variables) == 0: + # self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept) - def choose_metadata_to_eval(self, context, concept): - if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED): - return ["pre", "post", "variables", "body", "where", "ret"] + return concept + + def compute_metadata_to_eval(self, context, concept): + to_eval = [] + + needed, variables, body = self.get_needed_metadata(concept, ConceptParts.PRE, True, True) + to_eval.extend(needed) - metadata = ["pre", "post", "ret"] if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.metadata.need_validation: - needed = self.needed_metadata(concept, ConceptParts.WHERE) - for e in needed: - if e not in metadata: - metadata.append(e) - if "where" not in metadata: - metadata.append("where") + # What are the cases where we do not need a validation ? + # see test_sheerka_non_reg::test_i_can_evaluate_bnf_concept_with_where_clause() + # res = sheerka.evaluate_user_input("foobar") + needed, v, b = self.get_needed_metadata(concept, ConceptParts.WHERE, not variables, not body) + variables |= v + body |= b + to_eval.extend(needed) - return metadata + needed, v, b = self.get_needed_metadata(concept, ConceptParts.RET, not variables, not body) + variables |= v + body |= b + to_eval.extend(needed) - def needed_metadata(self, concept, metadata): + needed, v, b = self.get_needed_metadata(concept, ConceptParts.POST, not variables, not body) + variables |= v + body |= b + to_eval.extend(needed) + + if context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED): + if not variables: + to_eval.append('variables') + + if not body: + to_eval.append("body") + + return to_eval + + @staticmethod + def get_needed_metadata(concept, concept_part, check_vars, check_body): """ - Tries to find out if the evaluation of the body is necessary - It's a very basic approach that will need to be improved + Check if the concept_part has to be evaluated + It also checks if the variables and the body need to be evaluated prior to it :param concept: - :param metadata: + :param concept_part: + :param check_vars: + :param check_body: :return: """ + ret = [] + vars_needed = False + body_needed = False - if metadata not in concept.compiled: - return [] + if concept_part in concept.compiled and concept.compiled[concept_part] is not None: + concept_part_source = getattr(concept.metadata, concept_part.value) - return_values = concept.compiled[metadata] - if not isinstance(return_values, list): - return [] + assert concept_part_source is not None - needed = [] - for return_value in return_values: - if not self.sheerka.isinstance(return_value, BuiltinConcepts.RETURN_VALUE): - continue + tokens = [t.str_value for t in Tokenizer(concept_part_source)] - if not return_value.status: - continue + if check_vars: + for var_name in (v[0] for v in concept.metadata.variables): + if var_name in tokens: + vars_needed = True + ret.append("variables") + break - if not self.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT): - continue + if check_body and "self" in tokens: + body_needed = True + ret.append("body") - if not isinstance(return_value.body.source, str): - continue + ret.append(concept_part.value) - for var_name in (p[0] for p in concept.metadata.variables): - if var_name in return_value.body.source: - needed.append("variables") - break - - if "self" in return_value.body.source: - needed.append("body") - - return needed + return ret, vars_needed, body_needed diff --git a/src/core/sheerka/services/SheerkaExecute.py b/src/core/sheerka/services/SheerkaExecute.py index 56c4a7e..9292788 100644 --- a/src/core/sheerka/services/SheerkaExecute.py +++ b/src/core/sheerka/services/SheerkaExecute.py @@ -144,7 +144,7 @@ class SheerkaExecute(BaseService): self.pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20) def initialize(self): - self.sheerka.bind_service_method(self.execute) + self.sheerka.bind_service_method(self.execute, True) self.sheerka.cache_manager.register_cache(self.PARSERS_INPUTS_ENTRY, self.pi_cache, False) @@ -362,7 +362,8 @@ class SheerkaExecute(BaseService): if not isinstance(results, list): results = [results] for result in results: - evaluated_items.append(result) + if result.body: + evaluated_items.append(result) to_delete.extend(result.parents) sub_context.add_values(return_values=results) else: diff --git a/src/core/sheerka/services/SheerkaFilter.py b/src/core/sheerka/services/SheerkaFilter.py index d9ad59f..5dd4d4a 100644 --- a/src/core/sheerka/services/SheerkaFilter.py +++ b/src/core/sheerka/services/SheerkaFilter.py @@ -96,9 +96,9 @@ class SheerkaFilter(BaseService): for k, v in SheerkaFilter.__dict__.items(): if k.startswith("pipe_"): if isinstance(v, staticmethod): - self.sheerka.add_pipeable(k[5:], v.__func__) + self.sheerka.add_pipeable(k[5:], v.__func__, True) else: - self.sheerka.add_pipeable(k[5:], v.__get__(self, self.__class__)) + self.sheerka.add_pipeable(k[5:], v.__get__(self, self.__class__), True) self.sheerka.cache_manager.register_cache(self.PREDICATES_ENTRY, self.cache, False, False) diff --git a/src/core/sheerka/services/SheerkaHistoryManager.py b/src/core/sheerka/services/SheerkaHistoryManager.py index c140eff..6d1f463 100644 --- a/src/core/sheerka/services/SheerkaHistoryManager.py +++ b/src/core/sheerka/services/SheerkaHistoryManager.py @@ -57,7 +57,7 @@ class SheerkaHistoryManager(BaseService): super().__init__(sheerka) def initialize(self): - self.sheerka.bind_service_method(self.history) + self.sheerka.bind_service_method(self.history, False) def history(self, depth=10, start=0): """ diff --git a/src/core/sheerka/services/SheerkaModifyConcept.py b/src/core/sheerka/services/SheerkaModifyConcept.py index a4c79a7..43f0fba 100644 --- a/src/core/sheerka/services/SheerkaModifyConcept.py +++ b/src/core/sheerka/services/SheerkaModifyConcept.py @@ -10,7 +10,7 @@ class SheerkaModifyConcept(BaseService): super().__init__(sheerka) def initialize(self): - self.sheerka.bind_service_method(self.modify_concept) + self.sheerka.bind_service_method(self.modify_concept, True) def modify_concept(self, context, concept): old_version = self.sheerka.get_by_id(concept.id) diff --git a/src/core/sheerka/services/SheerkaResultManager.py b/src/core/sheerka/services/SheerkaResultManager.py index 22de478..4634405 100644 --- a/src/core/sheerka/services/SheerkaResultManager.py +++ b/src/core/sheerka/services/SheerkaResultManager.py @@ -10,10 +10,10 @@ class SheerkaResultConcept(BaseService): self.page_size = page_size def initialize(self): - self.sheerka.bind_service_method(self.get_results_by_digest) - self.sheerka.bind_service_method(self.get_results_by_command) - self.sheerka.bind_service_method(self.get_last_results) - self.sheerka.bind_service_method(self.get_results) + self.sheerka.bind_service_method(self.get_results_by_digest, True) # digest is recorded + self.sheerka.bind_service_method(self.get_results_by_command, True) # digest is recorded + self.sheerka.bind_service_method(self.get_last_results, True) # digest is recorded + self.sheerka.bind_service_method(self.get_results, False) def get_results_by_digest(self, context, digest, record_digest=True): """ diff --git a/src/core/sheerka/services/SheerkaSetsManager.py b/src/core/sheerka/services/SheerkaSetsManager.py index 5b234b7..06115c4 100644 --- a/src/core/sheerka/services/SheerkaSetsManager.py +++ b/src/core/sheerka/services/SheerkaSetsManager.py @@ -13,7 +13,7 @@ GROUP_PREFIX = 'All_' class SheerkaSetsManager(BaseService): NAME = "SetsManager" CONCEPTS_GROUPS_ENTRY = "SetsManager:Concepts_Groups" - CONCEPTS_IN_GROUPS_ENTRY = "SetsManager:Concepts_In_Groups" # cache for get_set_elements() + CONCEPTS_IN_GROUPS_ENTRY = "SetsManager:Concepts_In_Groups" # cache for get_set_elements() def __init__(self, sheerka): super().__init__(sheerka) @@ -21,12 +21,12 @@ class SheerkaSetsManager(BaseService): self.concepts_in_set = Cache() def initialize(self): - self.sheerka.bind_service_method(self.set_isa) - self.sheerka.bind_service_method(self.get_set_elements) - self.sheerka.bind_service_method(self.add_concept_to_set) - self.sheerka.bind_service_method(self.isinset) - self.sheerka.bind_service_method(self.isa) - self.sheerka.bind_service_method(self.isaset) + self.sheerka.bind_service_method(self.set_isa, True) + self.sheerka.bind_service_method(self.get_set_elements, True) # concepts are evaluated + self.sheerka.bind_service_method(self.add_concept_to_set, True) + self.sheerka.bind_service_method(self.isinset, False) + self.sheerka.bind_service_method(self.isa, False) + self.sheerka.bind_service_method(self.isaset, True) # concept is evaluated, need to change the code self.sheerka.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, self.sets) self.sheerka.cache_manager.register_cache(self.CONCEPTS_IN_GROUPS_ENTRY, self.concepts_in_set, persist=False) @@ -49,6 +49,8 @@ class SheerkaSetsManager(BaseService): False, self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set)) + # KSI 20200709 add the concept, not its 'id' or 'key' + # It will allow conditions handling if concept set has its WHERE or PRE set to something concept.add_prop(BuiltinConcepts.ISA, concept_set) res = self.sheerka.modify_concept(context, concept) @@ -141,7 +143,7 @@ class SheerkaSetsManager(BaseService): if sub_concept.metadata.where: new_condition = self._validate_where_clause(sub_concept) if not new_condition: - return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=sub_concept) + return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=sub_concept) # This methods sucks, but I don't have enough tools (like proper AST manipulation functions) # to do it properly now. It will be enhanced later @@ -206,7 +208,7 @@ class SheerkaSetsManager(BaseService): if not (isinstance(concept, Concept) and concept.id): return False - # KSI 29062020 + # KSI 20200629 # To resolve infinite recursion between group concepts and BNF concepts if concept.metadata.definition_type == DEFINITION_TYPE_BNF: return False diff --git a/src/core/sheerka/services/SheerkaVariableManager.py b/src/core/sheerka/services/SheerkaVariableManager.py index 288b16e..ecd3cb7 100644 --- a/src/core/sheerka/services/SheerkaVariableManager.py +++ b/src/core/sheerka/services/SheerkaVariableManager.py @@ -27,11 +27,11 @@ class SheerkaVariableManager(BaseService): super().__init__(sheerka) def initialize(self): - self.sheerka.bind_service_method(self.record) - self.sheerka.bind_service_method(self.load) - self.sheerka.bind_service_method(self.delete) - self.sheerka.bind_service_method(self.set) - self.sheerka.bind_service_method(self.get) + self.sheerka.bind_service_method(self.record, True) + self.sheerka.bind_service_method(self.load, False) + self.sheerka.bind_service_method(self.delete, True) + self.sheerka.bind_service_method(self.set, True) + self.sheerka.bind_service_method(self.get, False) cache = Cache(default=lambda k: self.sheerka.sdp.get(self.VARIABLES_ENTRY, k)) self.sheerka.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True) diff --git a/src/core/tokenizer.py b/src/core/tokenizer.py index dbdff91..72147ed 100644 --- a/src/core/tokenizer.py +++ b/src/core/tokenizer.py @@ -487,6 +487,11 @@ class Tokenizer: return result, lines_count, column_index def eat_word(self, start): + """ + Word is an alphanum (no space) + :param start: + :return: + """ result = self.text[start] i = start + 1 while i < self.text_len: diff --git a/src/evaluators/AddConceptEvaluator.py b/src/evaluators/AddConceptEvaluator.py index 260b769..8244255 100644 --- a/src/evaluators/AddConceptEvaluator.py +++ b/src/evaluators/AddConceptEvaluator.py @@ -2,7 +2,7 @@ from core.ast.nodes import python_to_concept from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts from core.builtin_helpers import get_names from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF -from core.tokenizer import TokenKind +from core.tokenizer import TokenKind, Tokenizer from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.BaseParser import NotInitializedNode from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor @@ -53,7 +53,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator): sheerka = context.sheerka # validate the node - props_found = set() + variables_found = set() concept = Concept(def_concept_node.name) concept.metadata.definition_type = def_concept_node.definition_type @@ -80,16 +80,16 @@ class AddConceptEvaluator(OneReturnValueEvaluator): # try to find what can be a property for p in self.get_variables(sheerka, part_ret_val, name_to_use): - props_found.add(p) + variables_found.add(p) # add variables by order of appearance when possible for name_part in name_to_use: - if name_part in props_found: + if name_part in variables_found: concept.def_var(name_part, None) # add the remaining properties # They mainly come from BNF definition - for p in props_found: + for p in variables_found: if p not in concept.values: concept.def_var(p, None) @@ -130,24 +130,31 @@ class AddConceptEvaluator(OneReturnValueEvaluator): names = [str(t.value) for t in ret_value.tokens if t.type in ( TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)] variables = filter(lambda x: x in concept_name, names) - return list(variables) + return set(variables) # # Case of python code # if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode): if len(concept_name) > 1: - python_node = ret_value.value.value - as_concept_node = python_to_concept(python_node.ast_) - names = get_names(sheerka, as_concept_node) - variables = filter(lambda x: x in concept_name, names) - return list(variables) + # tokens from ParserResult or source from python node + variables = set() + tokens = ret_value.value.tokens or list(Tokenizer(ret_value.value.value.source)) + tokens = [t.str_value for t in tokens] + for identifier in [i for i in concept_name if str(i).isalnum()]: + if identifier in tokens: + variables.add(identifier) + # python_node = ret_value.value.value + # as_concept_node = python_to_concept(python_node.ast_) + # names = get_names(sheerka, as_concept_node) + # variables = filter(lambda x: x in concept_name, names) + return variables # # case of concept # if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, Concept): - return list(ret_value.value.value.values.keys()) + return set(ret_value.value.value.values.keys()) # # case of BNF @@ -155,6 +162,6 @@ class AddConceptEvaluator(OneReturnValueEvaluator): if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, ParsingExpression): visitor = ConceptOrRuleNameVisitor() visitor.visit(ret_value.value.value) - return sorted(list(visitor.names)) + return set(visitor.names) return [] diff --git a/src/evaluators/ConceptEvaluator.py b/src/evaluators/ConceptEvaluator.py index e25c34f..19c30d0 100644 --- a/src/evaluators/ConceptEvaluator.py +++ b/src/evaluators/ConceptEvaluator.py @@ -25,7 +25,7 @@ class ConceptEvaluator(OneReturnValueEvaluator): # self.evaluate_body = True # # for r in return_values: - # if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.CONCEPT_VALUE_REQUESTED): + # if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.RETURN_VALUE_REQUESTED): # self.evaluate_body = True # break # diff --git a/src/evaluators/EvalEvaluator.py b/src/evaluators/EvalEvaluator.py index efee9c4..1a8276a 100644 --- a/src/evaluators/EvalEvaluator.py +++ b/src/evaluators/EvalEvaluator.py @@ -16,7 +16,7 @@ class EvalEvaluator(AllReturnValuesEvaluator): def matches(self, context, return_values): evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING) is_root = len(evaluation_parents) <= 1 - return context.in_context(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) and is_root + return context.in_context(BuiltinConcepts.RETURN_VALUE_REQUESTED) and is_root def eval(self, context, return_values): sheerka = context.sheerka diff --git a/src/evaluators/PostExecutionEvaluator.py b/src/evaluators/PostExecutionEvaluator.py new file mode 100644 index 0000000..e12a7f7 --- /dev/null +++ b/src/evaluators/PostExecutionEvaluator.py @@ -0,0 +1,34 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept +from evaluators.BaseEvaluator import OneReturnValueEvaluator + + +class PostExecutionEvaluator(OneReturnValueEvaluator): + """ + Last chance to alter the return_value + This evaluator is supposed to be a generic evaluator for all rules that must be executed just before + the aggregations + """ + + NAME = "PostExecution" + + def __init__(self): + super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 90) + + def matches(self, context, return_value): + evaluation_parents = context.get_parents(lambda c: c.action == BuiltinConcepts.PROCESSING) + if len(evaluation_parents) > 1: + return False # It must be executed only when the top level context + + # only support the rule for the COMMANDS + value = return_value.body + return isinstance(value, Concept) and context.sheerka.isa(value, context.sheerka.new(BuiltinConcepts.COMMAND)) + + def eval(self, context, return_value): + # only support the rule for the COMMANDS + body = return_value.body.body + return context.sheerka.ret( + self.name, + True, + body if body != BuiltinConcepts.NOT_INITIALIZED else return_value.body, + parents=[return_value]) diff --git a/src/evaluators/PrepareEvalEvaluator.py b/src/evaluators/PrepareEvalEvaluator.py index 7ad82bd..cbd9f35 100644 --- a/src/evaluators/PrepareEvalEvaluator.py +++ b/src/evaluators/PrepareEvalEvaluator.py @@ -34,6 +34,7 @@ class PrepareEvalEvaluator(OneReturnValueEvaluator): True, sheerka.new(BuiltinConcepts.USER_INPUT, body=self.text[5:], user_name=context.event.user_id)) context.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) + context.global_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) + context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED) return new_text_to_parse diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index 9307648..448a8b9 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -7,7 +7,7 @@ import core.ast.nodes import core.utils from core.ast.visitors import UnreferencedNamesVisitor from core.builtin_concepts import BuiltinConcepts, ParserResultConcept -from core.concept import ConceptParts, Concept +from core.concept import ConceptParts, Concept, NotInit from core.sheerka.services.SheerkaFilter import Pipe from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.PythonParser import PythonNode @@ -73,15 +73,29 @@ class PythonEvaluator(OneReturnValueEvaluator): not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node) return sheerka.ret(self.name, False, not_for_me, parents=[return_value]) + attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE) + if attr_under_eval: + attr_under_eval = attr_under_eval[0] + expression_only = attr_under_eval.action_context != ConceptParts.BODY + + if expression_only and isinstance(node.ast_, ast.Module): + # Module execution is forbidden in where, pre, post and ret concept parts + security_error = sheerka.new(BuiltinConcepts.PYTHON_SECURITY_ERROR, + prop=attr_under_eval.action_context, + body=node.source) + return sheerka.ret(self.name, False, security_error, parents=[return_value]) + else: + expression_only = False + # get globals - my_globals = self.get_globals(context, node) + my_globals = self.get_globals(context, node, expression_only) context.log(f"globals={my_globals}", self.name) all_possible_globals = self.get_all_possible_globals(context, my_globals) concepts_entries = None evaluated = BuiltinConcepts.NOT_INITIALIZED errors = [] - expect_success = BuiltinConcepts.EVAL_SUCCESS_REQUESTED in context.local_hints + expect_success = BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED in context.local_hints for globals_ in all_possible_globals: try: # eval @@ -117,14 +131,27 @@ class PythonEvaluator(OneReturnValueEvaluator): context.log(f"{evaluated=}", self.name) return sheerka.ret(self.name, True, evaluated, parents=[return_value]) - def get_globals(self, context, node): + def get_globals(self, context, node, expression_only): + """ + Creates the global variables for python source code evaluation + :param context: + :param node: + :param expression_only: most of the commands are refused + :return: + """ my_globals = { "Concept": core.concept.Concept, "BuiltinConcepts": core.builtin_concepts.BuiltinConcepts, + "in_context": context.in_context } + if expression_only: + # disable builtin + my_globals["__builtins__"] = None + # has to be the first, to allow override - method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context) + method_from_sheerka = self.update_globals_with_sheerka_methods(my_globals, context, expression_only) + self.update_globals_with_context(my_globals, context) already_know = set(my_globals.keys()) self.update_globals_with_node(my_globals, context, node, already_know) @@ -136,32 +163,54 @@ class PythonEvaluator(OneReturnValueEvaluator): return my_globals @staticmethod - def update_globals_with_sheerka_methods(my_locals, context): + def update_globals_with_sheerka_methods(my_locals, context, expression_only): + # methods_from_sheerka = {} + # + # # Make sure that methods that need the concept are correctly wrapped + # for method_name, method in context.sheerka.sheerka_methods.items(): + # if expression_only and method.has_side_effect: + # continue + # + # if method_name in context.sheerka.methods_with_context: + # methods_from_sheerka[method_name] = inject_context(context)(method.method) + # else: + # methods_from_sheerka[method_name] = method.method methods_from_sheerka = {} - # Make sure that methods that need the concept are correctly wrapped - for method_name, method in context.sheerka.sheerka_methods.items(): - if method_name in context.sheerka.methods_with_context: - methods_from_sheerka[method_name] = inject_context(context)(method) - else: - methods_from_sheerka[method_name] = method - # Add all the methods as a direct access - for method_name, method in methods_from_sheerka.items(): - my_locals[method_name] = method + for method_name, method in context.sheerka.sheerka_methods.items(): + if expression_only and method.has_side_effect: + continue + + if method_name in context.sheerka.methods_with_context: + my_locals[method_name] = inject_context(context)(method.method) + else: + my_locals[method_name] = method.method + methods_from_sheerka[method_name] = my_locals[method_name] # Add pipeable functions for func_name, function in context.sheerka.sheerka_pipeables.items(): - my_locals[func_name] = Pipe(function, context) + if expression_only and function.has_side_effect: + continue + + my_locals[func_name] = Pipe(function.method, context) return methods_from_sheerka # to allow access using prefix "sheerka." def update_globals_with_context(self, my_globals, context): + """ + Update globals with the current object being evaluated (and its variables) + :param my_globals: + :param context: + :return: + """ if context.obj: context.log(f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name) for prop_name in context.obj.variables(): - my_globals[prop_name] = context.obj.get_value(prop_name) + value = context.obj.get_value(prop_name) + if value != NotInit: + my_globals[prop_name] = value my_globals["self"] = context.obj def update_globals_with_node(self, my_globals, context, node, already_known): @@ -202,19 +251,11 @@ class PythonEvaluator(OneReturnValueEvaluator): context.log(f"Concept {name} is already evaluated.", self.name) else: context.log(f"Evaluating '{concept}'", self.name) - with context.push(BuiltinConcepts.EVALUATE_CONCEPT, - concept, - who=self.name, - desc=f"Evaluating '{concept}'", - obj=concept) as sub_context: - sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) - evaluated = context.sheerka.evaluate_concept(sub_context, concept) - sub_context.add_values(return_values=evaluated) - - if evaluated.key != concept.key: - context.log(f"Error while evaluating '{name}'. Skipping.", self.name) - continue - concept = evaluated + evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=True) + if evaluated.key != concept.key: + context.log(f"Error while evaluating '{name}'. Skipping.", self.name) + continue + concept = evaluated my_globals[name] = concept @@ -293,7 +334,7 @@ class PythonEvaluator(OneReturnValueEvaluator): last_ast = copy.deepcopy(code_ast) last_ast.body = code_ast.body[-1:] - exec(compile(init_ast, "", "exec"), {}, my_locals) + exec(compile(init_ast, "", "exec"), my_globals, my_locals) if type(last_ast.body[0]) == ast.Expr: return eval(compile(self.expr_to_expression(last_ast.body[0]), "", "eval"), my_globals, my_locals) else: diff --git a/src/evaluators/ResolveAmbiguityEvaluator.py b/src/evaluators/ResolveAmbiguityEvaluator.py new file mode 100644 index 0000000..d760356 --- /dev/null +++ b/src/evaluators/ResolveAmbiguityEvaluator.py @@ -0,0 +1,60 @@ +from core.builtin_concepts import BuiltinConcepts +from core.builtin_helpers import resolve_ambiguity +from core.concept import Concept +from evaluators.BaseEvaluator import AllReturnValuesEvaluator + + +class ResolveAmbiguityEvaluator(AllReturnValuesEvaluator): + """ + Use when multiple concepts match the same input source + """ + + NAME = "ResolveAmbiguity" + + def __init__(self): + super().__init__(self.NAME, [BuiltinConcepts.AFTER_PARSING], 50) + self.sources = None + + def matches(self, context, return_values): + # first, arrange return_values by sources. + # If they share the same source, that means that there are multiple results for one ParserInput + self.sources = {} + success = False + for ret in [ret for ret in return_values if ret.status]: + source = self.get_source(context, ret) + + if source: + self.sources.setdefault(source, []).append(ret) + + if len(self.sources[source]) > 1: + success = True # at least one source are more than one entry -> let's resolve it + + return success + + def eval(self, context, return_values): + ret = [] + + for ret_vals in self.sources.values(): + parser_results = {id(r.body.body): r.body for r in ret_vals} + resolved = resolve_ambiguity(context, [r.body.body for r in ret_vals]) + if len(resolved) == 0: + ret.append(context.sheerka.ret(self.name, True, [], parents=ret_vals)) + else: + for c in resolved: + ret.append(context.sheerka.ret(self.name, True, parser_results[id(c)], parents=ret_vals)) + + return ret + + @staticmethod + def get_source(context, return_value): + """ + Try to get the source (the ParserInput) of the return_value + We only consider parser result of concepts + :param context: + :param return_value: + :return: + """ + if context.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT) and \ + isinstance(return_value.body.body, Concept): + return return_value.body.source + return None diff --git a/src/evaluators/RetEvaluator.py b/src/evaluators/RetEvaluator.py index 8fd916b..79ca476 100644 --- a/src/evaluators/RetEvaluator.py +++ b/src/evaluators/RetEvaluator.py @@ -6,7 +6,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator class RetEvaluator(OneReturnValueEvaluator): """ - The evaluator transform the a concept, using the ret value + The evaluator transforms the concept, using the RET metadata value """ NAME = "Ret" diff --git a/src/parsers/ExactConceptParser.py b/src/parsers/ExactConceptParser.py index 453bba8..5176a3b 100644 --- a/src/parsers/ExactConceptParser.py +++ b/src/parsers/ExactConceptParser.py @@ -2,7 +2,7 @@ import logging import core.builtin_helpers from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts -from core.concept import VARIABLE_PREFIX +from core.concept import VARIABLE_PREFIX, ConceptParts from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Keywords, TokenKind, LexerError from core.utils import str_concept diff --git a/src/repl/SheerkaPromptCompleter.py b/src/repl/SheerkaPromptCompleter.py index 047358c..4876667 100644 --- a/src/repl/SheerkaPromptCompleter.py +++ b/src/repl/SheerkaPromptCompleter.py @@ -36,18 +36,16 @@ class SheerkaPromptCompleter(Completer): self.params_history_service = self.sheerka.services[SheerkaFunctionsParametersHistory.NAME] self.builtins = [] for name, bound_method in sheerka.sheerka_methods.items(): - self.builtins.append(self.get_completion_desc(name, bound_method, "builtin", ["context"])) + self.builtins.append(self.get_completion_desc(name, bound_method.method, "builtin", ["context"])) self.pipeable_builtins = [] for name, pipeable in self.sheerka.sheerka_pipeables.items(): - self.pipeable_builtins.append(self.get_completion_desc(name, pipeable, "builtin", ["context", "iterable"])) + self.pipeable_builtins.append( + self.get_completion_desc(name, pipeable.method, "builtin", ["context", "iterable"])) self.exit_commands = [CompletionDesc(c, c, "command") for c in EXIT_COMMANDS] - self.globals = self.sheerka.sheerka_methods.copy() - self.globals.update(self.sheerka.sheerka_pipeables) - - def get_locals(self): - return self.sheerka.sheerka_methods + self.globals = {k: v.method for k, v in self.sheerka.sheerka_methods.items()} + self.globals.update({k: v.method for k, v in self.sheerka.sheerka_pipeables.items()}) def get_completions(self, document, complete_event): diff --git a/tests/BaseTest.py b/tests/BaseTest.py index eadd138..2351591 100644 --- a/tests/BaseTest.py +++ b/tests/BaseTest.py @@ -81,7 +81,7 @@ class BaseTest: @staticmethod def retval(obj, who="who", status=True): """ret_val""" - return ReturnValueConcept.ret(who, status, obj) + return ReturnValueConcept(who, status, obj) @staticmethod def tretval(sheerka, obj, who="who"): @@ -93,11 +93,11 @@ class BaseTest: return sheerka.ret(who, True, obj) @staticmethod - def pretval(concept, source=None, parser="parsers.name", who="some_name"): + def pretval(concept, source=None, parser="parsers.name", who="some_name", status=True): """ParserResult ret_val (p stands for ParserResult)""" return ReturnValueConcept( who, - True, + status, ParserResultConcept(parser=parser, source=source or concept.name, value=concept, diff --git a/tests/core/test_SheerkaEvaluateConcept.py b/tests/core/test_SheerkaEvaluateConcept.py index 53b8bc8..a8837ab 100644 --- a/tests/core/test_SheerkaEvaluateConcept.py +++ b/tests/core/test_SheerkaEvaluateConcept.py @@ -1,6 +1,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept from core.concept import Concept, DoNotResolve, ConceptParts, Property, InfiniteRecursionResolved, CB, NotInit +from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept from parsers.PythonParser import PythonNode from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -50,16 +51,16 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): :return: """ - sheerka, context, concept = self.init_concepts(Concept("foo", pre=expr)) + sheerka, context, concept = self.init_concepts(Concept("foo", post=expr)) evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key assert evaluated.metadata.body is None - assert evaluated.metadata.pre == expr - assert evaluated.metadata.post is None + assert evaluated.metadata.post == expr + assert evaluated.metadata.pre is None assert evaluated.metadata.where is None - assert evaluated.get_value(ConceptParts.PRE) == expected + assert evaluated.get_value(ConceptParts.POST) == expected assert evaluated.variables() == {} assert not evaluated.metadata.is_evaluated assert len(evaluated.values) == 0 if expr is None else 1 @@ -353,27 +354,34 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == concept.init_key().key - @pytest.mark.parametrize("where_clause, expected", [ - ("True", True), - ("False", False), - ("self < 10", False), - ("self < 11", True), - ("a < 20", False), - ("a > 19", True), - ("a + self > 20", True), + @pytest.mark.parametrize("where_clause, expected, expected_body", [ + ("True", True, BuiltinConcepts.NOT_INITIALIZED), + ("False", False, BuiltinConcepts.NOT_INITIALIZED), + ("self < 10", False, 10), + ("self < 11", True, 10), + ("a < 20", False, BuiltinConcepts.NOT_INITIALIZED), + ("a > 19", True, BuiltinConcepts.NOT_INITIALIZED), + ("a + self > 20", True, 10), ]) - def test_i_can_evaluate_simple_where(self, where_clause, expected): + def test_i_can_evaluate_simple_where(self, where_clause, expected, expected_body): + # We check that the WHERE condition is correctly evaluated + # We also check that the body is evaluated only when it's mandatory + sheerka, context, concept = self.init_concepts( Concept("foo", body="10", where=where_clause).def_var("a", "20"), ) - evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept) + evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept, eval_body=False) if expected: assert evaluated.key == concept.key + assert concept.body == expected_body else: - assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED) - assert evaluated.body == concept + assert sheerka.isinstance(evaluated, BuiltinConcepts.CONDITION_FAILED) + assert evaluated.body == where_clause + assert evaluated.concept == concept + assert evaluated.prop == ConceptParts.WHERE + assert concept.body == expected_body def test_i_can_evaluate_where_when_using_other_concept(self): sheerka, context, foo_true, foo_false = self.init_concepts( @@ -381,20 +389,20 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): Concept("foo_false", body="False"), ) - concept = Concept("foo", where="foo_true").init_key() + concept = Concept("foo", where="foo_true") evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept) assert evaluated.key == concept.key concept = Concept("foo", where="foo_false") evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept) - assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED) - assert evaluated.body == concept + assert sheerka.isinstance(evaluated, BuiltinConcepts.CONDITION_FAILED) + assert evaluated.body == "foo_false" - concept = Concept("foo", where="foo_false").init_key() + concept = Concept("foo", where="foo_false") evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, False), concept) assert evaluated.key == concept.key - def test_i_can_evaluate_disable_where_clause_evaluation(self): + def test_i_can_enable_disable_where_clause_evaluation(self): sheerka, context, concept = self.init_concepts( Concept("foo", body="10", where="a > 10").def_var("a", None), ) @@ -405,7 +413,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) - def test_i_can_detect_infinite_recursion_with_numeric_constant(self): + def test_i_can_detect_and_resolve_infinite_recursion_with_numeric_constant(self): sheerka, context, one_str, one_digit = self.init_concepts( Concept("one", body="1"), Concept("1", body="one"), @@ -423,7 +431,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == one_str.key assert evaluated.body == InfiniteRecursionResolved(1) - def test_i_can_detect_infinite_recursion_with_boolean_constant(self): + def test_i_can_detect_and_resolve_infinite_recursion_with_boolean_constant(self): sheerka, context, true_str, true_bool = self.init_concepts( Concept("true", body="True"), Concept("True", body="true"), @@ -523,3 +531,147 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, forty_one_thousand) assert evaluated.body == 41000 + + def test_i_can_evaluate_command(self): + sheerka, context, command = self.init_concepts(Concept("command", body="a = 10")) + + evaluated = sheerka.evaluate_concept(context, command) + assert evaluated.key == command.key + assert "a" not in sheerka.locals + + sheerka.set_isa(context, command, sheerka.new(BuiltinConcepts.COMMAND)) + evaluated = sheerka.evaluate_concept(context, sheerka.new("command")) + assert evaluated.key == command.key + assert "a" in sheerka.locals + + @pytest.mark.parametrize("metadata", [ + "where", + "pre", + "post", + "ret" + ]) + def test_i_cannot_evaluate_python_statement_in_where_pre_post_ret(self, metadata, capsys): + sheerka, context, foo = self.init_concepts("foo") + setattr(foo.metadata, metadata, "a=10; print('10')") + foo.metadata.need_validation = True + + evaluated = sheerka.evaluate_concept(context, foo) + captured = capsys.readouterr() + + assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) + error = evaluated.body + assert sheerka.isinstance(error, BuiltinConcepts.PYTHON_SECURITY_ERROR) + assert error.prop.value == metadata + assert error.body == "a=10; print('10')" + assert captured.out == "" + + def test_python_builtin_function_are_forbidden_in_where_pre_post_ret(self, capsys): + # I do the test only for PRE, as it will be the same for the other CounceptPart + sheerka, context, foo, bar = self.init_concepts( + Concept("foo", body="print('10')"), # print will be executed + Concept("bar", pre="print('10')"), # print won't be executed + ) + + evaluated = sheerka.evaluate_concept(context, foo, eval_body=True) + captured = capsys.readouterr() + assert evaluated.key == foo.key + assert captured.out == "10\n" + + evaluated = sheerka.evaluate_concept(context, bar) + captured = capsys.readouterr() + assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) + error = evaluated.body + assert sheerka.isinstance(error, BuiltinConcepts.ERROR) + assert captured.out == "" + + def test_i_can_failed_a_concept_when_pre_clause_is_not_validated(self, capsys): + sheerka, context, concept = self.init_concepts( + Concept("foo", pre="in_context('foo')", body="print('10')"), + ) + + evaluated = sheerka.evaluate_concept(context, concept, eval_body=True) + assert sheerka.isinstance(evaluated, BuiltinConcepts.CONDITION_FAILED) + assert evaluated.body == "in_context('foo')" + assert evaluated.concept == concept + assert evaluated.prop == ConceptParts.PRE + + captured = capsys.readouterr() + assert captured.out == "" + + @pytest.mark.parametrize("concept, expected", [ + (Concept("foo"), []), + (Concept("foo", pre="pre", post="post", ret="ret", where="where"), ["pre", "ret", "post"]), + (Concept("foo", pre="a").def_var("a"), ["variables", "pre"]), + (Concept("foo", pre="self"), ["body", "pre"]), + (Concept("foo", pre="self + a").def_var("a"), ["variables", "body", "pre"]), + (Concept("foo", pre="self + a", ret="ret").def_var("a"), ["variables", "body", "pre", "ret"]), + (Concept("foo", body="body"), []) # only if eval_body_is_set + ]) + def test_i_can_compute_metadata_to_eval(self, concept, expected): + sheerka, context, concept = self.init_concepts(concept) + service = sheerka.services[SheerkaEvaluateConcept.NAME] + + service.initialize_concept_asts(context, concept) + assert service.compute_metadata_to_eval(context, concept) == expected + + def test_i_can_compute_metadata_to_eval_for_where_and_body(self): + sheerka = self.get_sheerka() + service = sheerka.services[SheerkaEvaluateConcept.NAME] + + context = self.get_context(sheerka, eval_where=True) + concept = Concept("foo", where="where") + service.initialize_concept_asts(context, concept) + assert service.compute_metadata_to_eval(context, concept) == ["where"] + + concept = Concept("foo", where="where a").def_var("a") + service.initialize_concept_asts(context, concept) + assert service.compute_metadata_to_eval(context, concept) == ["variables", "where"] + + concept = Concept("foo", where="where self") + service.initialize_concept_asts(context, concept) + assert service.compute_metadata_to_eval(context, concept) == ["body", "where"] + + context = self.get_context(sheerka, eval_body=True) + concept = Concept("foo") + assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"] + + context = self.get_context(sheerka, eval_body=True) + concept = Concept("foo").def_var("a") + assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"] + + context = self.get_context(sheerka, eval_body=True) + concept = Concept("foo", body="body").def_var("a") + assert service.compute_metadata_to_eval(context, concept) == ["variables", "body"] + + @pytest.mark.parametrize("concept, expected", [ + (Concept("foo"), True), + ]) + def test_is_evaluated_is_correctly_set(self, concept, expected): + sheerka, context, concept = self.init_concepts(concept) + + evaluated = sheerka.evaluate_concept(context, concept, eval_body=True) + + assert evaluated.key == concept.key + assert concept.metadata.is_evaluated == expected + + def test_i_only_compute_the_requested_metadata(self): + sheerka, context, concept = self.init_concepts( + Concept("foo", pre="'pre'", post="'post'", ret="'ret'", where="'where'", body="'body'").def_var("a", "'a'") + ) + context.local_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) # to prove that we do not care + context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) # to prove that we do not care + + evaluated = sheerka.evaluate_concept(context, concept, metadata=['pre']) + assert evaluated.values == {"a": Property("a", NotInit), ConceptParts.PRE: Property(ConceptParts.PRE, 'pre')} + + # I cannot implement value cache for now + # def test_values_when_no_variables_are_computed_only_once(self): + # sheerka, context, foo = self.init_concepts(Concept("foo", body="10")) + # + # evaluated = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True) + # assert evaluated.body == 10 + # assert len(evaluated.compiled) > 0 + # + # evaluated_2 = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True) + # assert evaluated_2.body == 10 + # assert len(evaluated_2.compiled) == 0 diff --git a/tests/core/test_SheerkaSetsManager.py b/tests/core/test_SheerkaSetsManager.py index e49a89f..544a6e8 100644 --- a/tests/core/test_SheerkaSetsManager.py +++ b/tests/core/test_SheerkaSetsManager.py @@ -239,8 +239,6 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka): Concept("foo"), Concept("bar"), Concept("baz"), - create_new=True - ) sheerka.set_isa(context, foo, bar) @@ -250,6 +248,18 @@ class TestSheerkaSetsManager(TestUsingMemoryBasedSheerka): assert sheerka.isa(bar, baz) assert sheerka.isa(foo, baz) + def test_i_cannot_manage_isa_transitivity_when_using_body(self): + sheerka, context, one, another_one, number = self.init_concepts( + "one", + Concept("another one", body="one"), + "number" + ) + + sheerka.set_isa(context, one, number) + + assert sheerka.isa(one, number) # sanity + assert not sheerka.isa(another_one, number) # Correct this misbehaviour when BuiltinConcepts.IS is implemented + def test_concept_expression_recurse_id_is_updated(self): sheerka, context, one, number, twenties = self.init_concepts( "one", diff --git a/tests/core/test_builtin_helpers.py b/tests/core/test_builtin_helpers.py index 6ca5123..c3e0a8f 100644 --- a/tests/core/test_builtin_helpers.py +++ b/tests/core/test_builtin_helpers.py @@ -3,6 +3,7 @@ import ast import core.builtin_helpers import pytest from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts +from core.concept import Concept from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -142,3 +143,73 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka): assert len(actual) == len(expected) for i in range(len(actual)): assert self.dump_ast(actual[i]) == self.dump_ast(expected[i]) + + @pytest.mark.parametrize("concepts, expected", [ + ([], []), + ([Concept("foo", pre="True"), Concept("bar")], ["foo"]), + ([Concept("foo").def_var("a"), Concept("bar")], ["bar"]), + ]) + def test_i_can_resolve_ambiguity_when_empty(self, concepts, expected): + context = self.get_context() + res = core.builtin_helpers.resolve_ambiguity(context, concepts) + assert [c.name for c in res] == expected + + # @pytest.mark.parametrize("return_values", [ + # None, + # [] + # ]) + # def test_i_can_resolve_simple_ambiguity_when_no_return_values(self, return_values): + # context = self.get_context() + # + # assert core.builtin_helpers.remove_ambiguity(context, return_values) == return_values + + # def test_resolve_ambiguity_concepts_with_no_variable_take_precedence(self): + # context = self.get_context() + # return_values = [ + # self.pretval(Concept("hello a").def_var("a", "world"), "hello word"), + # self.pretval(Concept("hello world"), "hello word"), + # # self.pretval(Concept("hello world", pre="False"), "hello word"), + # self.retval(Concept("not a parser result")), + # self.retval(Concept("status is false"), status=False), + # self.pretval(Concept("false parser result"), status=False), + # ] + # + # ret = core.builtin_helpers.remove_ambiguity(context, return_values) + # assert ret.status + # assert ret.parents == return_values + # + # filtered = ret.body + # assert context.sheerka.isinstance(ret.body, BuiltinConcepts.FILTERED) + # assert filtered.body == [ + # return_values[2], + # return_values[3], + # return_values[4], + # return_values[1], + # ] + # assert filtered.iterable == return_values + # assert filtered.predicate == "remove_ambiguity(context, iterable)" + # + # def test_resolve_ambiguity_failed_pre_condition_are_discarded(self): + # context = self.get_context() + # return_values = [ + # self.pretval(Concept("hello world"), "hello word"), + # self.pretval(Concept("hello world", pre="False"), "hello word"), + # ] + # + # ret = core.builtin_helpers.remove_ambiguity(context, return_values) + # filtered = ret.body + # assert context.sheerka.isinstance(ret.body, BuiltinConcepts.FILTERED) + # assert filtered.body == [ + # return_values[0], + # ] + # + # def test_resolve_ambiguity_original_return_value_is_returned_when_nothing_to_filter(self): + # context = self.get_context() + # return_values = [ + # self.pretval(Concept("hello a").def_var("a", "world"), "hello word"), + # self.retval(Concept("not a parser result")), + # self.retval(Concept("status is false"), status=False), + # self.pretval(Concept("false parser result"), status=False), + # ] + # + # assert core.builtin_helpers.remove_ambiguity(context, return_values) == return_values diff --git a/tests/core/test_sheerka_call_evaluators.py b/tests/core/test_sheerka_call_evaluators.py index 97150dc..a84a726 100644 --- a/tests/core/test_sheerka_call_evaluators.py +++ b/tests/core/test_sheerka_call_evaluators.py @@ -165,10 +165,13 @@ class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority): return ret -class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority): +class EvaluatorAllReplaceFooEntry(EvaluatorAllWithPriority): + """ + This evaluator replaces the concept 'foo' by another one + """ def __init__(self): - super().__init__("suppress", 100) + super().__init__("replace", 100) def matches(self, context, return_values): super().matches(context, return_values) @@ -187,6 +190,30 @@ class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority): return None +class EvaluatorAllSuppressEntries(EvaluatorAllWithPriority): + """ + This evaluator totally removes 'foo' and 'bar' the entries from the workflow + """ + + def __init__(self): + super().__init__("suppress", 100) + + def matches(self, context, return_values): + super().matches(context, return_values) + to_remove = [r for r in return_values if r.body.name in ("foo", "bar")] + return len(to_remove) > 0 + + def eval(self, context, return_values): + super().eval(context, return_values) + + to_remove = [r for r in return_values if r.body.name in ("foo", "bar")] + return context.sheerka.ret( + self.name, + True, + [], + parents=to_remove) + + class EvaluatorOneDoNotModifyExecutionFlow(EvaluatorOneWithPriority): """ To test that when eval() returns the initial return_value, the execution flow is not modified @@ -485,3 +512,22 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): # check that 'foo' is no longer in res, but 'bar' is added assert res == [self.tretval(sheerka, Concept("bar"))] + + def test_i_can_remove_return_values_from_the_execution_workflow(self): + sheerka = self.get_sheerka() + sheerka.evaluators = [EvaluatorAllSuppressEntries] + + entries = [self.tretval(sheerka, Concept("foo")), + self.tretval(sheerka, Concept("bar")), + self.tretval(sheerka, Concept("baz"))] + Out.debug_out = [] + res = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION]) + + assert Out.debug_out == [ + "__EVALUATION [0] suppress - matches - target=['foo', 'bar', 'baz']", + "__EVALUATION [0] suppress - eval - target=['foo', 'bar', 'baz']", + "__EVALUATION [1] suppress - matches - target=['baz']", + ] + + # check that 'foo' is no longer in res, but 'bar' is added + assert res == [self.tretval(sheerka, Concept("baz"))] diff --git a/tests/evaluators/test_AddConceptEvaluator.py b/tests/evaluators/test_AddConceptEvaluator.py index 9d523e5..031d4c6 100644 --- a/tests/evaluators/test_AddConceptEvaluator.py +++ b/tests/evaluators/test_AddConceptEvaluator.py @@ -19,7 +19,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): @staticmethod def get_concept_part(part): if isinstance(part, str): - node = PythonNode(part, ast.parse(part, mode="eval")) + node = PythonNode(part, ast.parse(part, mode="exec")) return ReturnValueConcept( who="parsers.Default", status=True, @@ -169,11 +169,16 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): assert from_db.compiled == {} # ast is not saved in db - def test_i_can_get_variables_from_python_node_when_long_name(self): - ret_val = self.get_concept_part("isinstance(a, str)") + @pytest.mark.parametrize("expression, name, expected", [ + ("isinstance(a, str)", "a b", {"a"}), + ("a.location=b", "a is in b", {"a", "b"}), + ("a.location=b", "'a' is in b", {"b"}), + ]) + def test_i_can_get_variables_from_python_node_when_long_name(self, expression, name, expected): + ret_val = self.get_concept_part(expression) context = self.get_context() - assert AddConceptEvaluator.get_variables(context.sheerka, ret_val, ["a", "b"]) == ["a"] + assert AddConceptEvaluator.get_variables(context.sheerka, ret_val, name.split()) == expected def test_i_can_get_variables_when_keywords(self): sheerka, context = self.init_concepts() @@ -182,7 +187,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): name_to_use = AddConceptEvaluator.get_name_to_use(def_concept) concept_part = self.get_concept_part("pre") - assert AddConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == ["pre"] + assert AddConceptEvaluator.get_variables(context.sheerka, concept_part, name_to_use) == {"pre"} def test_i_cannot_get_variables_from_python_node_when_name_has_only_one_token(self): ret_val = self.get_concept_part("isinstance(a, str)") @@ -196,14 +201,14 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): status=True, value=ParserResultConcept(value=concept)) - assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == ["a", "b"] + assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"a", "b"} def test_i_can_get_variables_from_definition(self): parsing_expression = Sequence(ConceptExpression('mult'), ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add")))) ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression) - assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == ["add", "mult"] + assert AddConceptEvaluator.get_variables(self.get_sheerka(), ret_val, []) == {"add", "mult"} def test_concept_that_references_itself_is_correctly_created(self): context = self.get_context() diff --git a/tests/evaluators/test_ConceptEvaluator.py b/tests/evaluators/test_ConceptEvaluator.py index 3ed57e3..b1d093b 100644 --- a/tests/evaluators/test_ConceptEvaluator.py +++ b/tests/evaluators/test_ConceptEvaluator.py @@ -1,8 +1,8 @@ import pytest - from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts from core.concept import Concept, ConceptParts from evaluators.ConceptEvaluator import ConceptEvaluator + from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -20,11 +20,12 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): def test_i_can_evaluate_concept(self): context = self.get_context() - context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + context.local_hints.update({BuiltinConcepts.EVAL_BODY_REQUESTED, BuiltinConcepts.EVAL_WHERE_REQUESTED}) concept = Concept(name="foo", where="True", - pre="2", - post="3").def_var("a", "4").def_var("b", "5") + pre="2 > 1", + ret="3", + post="4").def_var("a", "5").def_var("b", "6") evaluator = ConceptEvaluator() item = self.pretval(concept) @@ -34,10 +35,11 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): assert result.status assert result.value.name == "foo" assert result.value.get_value(ConceptParts.WHERE) == True - assert result.value.get_value(ConceptParts.PRE) == 2 - assert result.value.get_value(ConceptParts.POST) == 3 - assert result.value.get_value("a") == 4 - assert result.value.get_value("b") == 5 + assert result.value.get_value(ConceptParts.PRE) == True + assert result.value.get_value(ConceptParts.RET) == 3 + assert result.value.get_value(ConceptParts.POST) == 4 + assert result.value.get_value("a") == 5 + assert result.value.get_value("b") == 6 assert result.value.key == "foo" assert result.parents == [item] @@ -118,4 +120,3 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): assert not context.sheerka.is_success(error_concept) # it's indeed an error assert result.status assert result.value == error_concept - diff --git a/tests/evaluators/test_EvalEvaluator.py b/tests/evaluators/test_EvalEvaluator.py index 75cd9a0..7fce824 100644 --- a/tests/evaluators/test_EvalEvaluator.py +++ b/tests/evaluators/test_EvalEvaluator.py @@ -15,7 +15,7 @@ def retval(obj, who="who", status=True): class TestEvalEvaluator(TestUsingMemoryBasedSheerka): def test_i_can_match_and_eval(self): context = self.get_context() - context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) + context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED) to_eval1 = ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init()) to_eval2 = ReturnValueConcept("some_name", True, Concept(name="3", body="also to eval").auto_init()) @@ -49,12 +49,12 @@ class TestEvalEvaluator(TestUsingMemoryBasedSheerka): context = self.get_context() assert not EvalEvaluator().matches(context, [return_value]) - context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) + context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED) assert EvalEvaluator().matches(context, [return_value]) def test_i_can_match_depending_on_builtin_concept_processing(self): context = self.get_context() - context.global_hints.add(BuiltinConcepts.CONCEPT_VALUE_REQUESTED) + context.global_hints.add(BuiltinConcepts.RETURN_VALUE_REQUESTED) return_values = [ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init())] evaluator = EvalEvaluator() diff --git a/tests/evaluators/test_PrepareEvalEvaluator.py b/tests/evaluators/test_PrepareEvalEvaluator.py index b82d1d2..91903db 100644 --- a/tests/evaluators/test_PrepareEvalEvaluator.py +++ b/tests/evaluators/test_PrepareEvalEvaluator.py @@ -45,4 +45,4 @@ class TestPrepareEvalEvaluator(TestUsingMemoryBasedSheerka): assert res.body.body == expected assert BuiltinConcepts.EVAL_BODY_REQUESTED in context.global_hints - assert BuiltinConcepts.CONCEPT_VALUE_REQUESTED in context.global_hints + assert BuiltinConcepts.RETURN_VALUE_REQUESTED in context.global_hints diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py index cb09a15..f2d9cf8 100644 --- a/tests/evaluators/test_PythonEvaluator.py +++ b/tests/evaluators/test_PythonEvaluator.py @@ -1,6 +1,6 @@ import pytest from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts -from core.concept import Concept, CB +from core.concept import Concept, CB, NotInit from core.sheerka.services.SheerkaExecute import ParserInput from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError from parsers.PythonParser import PythonNode, PythonParser @@ -137,7 +137,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert evaluated.status assert not evaluated.value # the first test is between Concept(foo) and int(2) - context.local_hints.add(BuiltinConcepts.EVAL_SUCCESS_REQUESTED) + context.local_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) evaluated = python_evaluator.eval(context, parsed) assert evaluated.status assert evaluated.value # we test until we compare foo.body and 2 @@ -224,3 +224,18 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert isinstance(error1.error, TypeError) assert error1.error.args[0] == 'can only concatenate str (not "int") to str' assert error1.concepts == {'foo': 'string'} + + def test_i_do_not_include_not_initialized_variables_when_evaluating(self): + sheerka, context, foo = self.init_concepts( + Concept("foo a", pre="a == 'True'").def_var("a", "'True'").def_var("b")) + + foo.set_value("b", "'Initialized!'") + context.obj = foo + + assert foo.get_value("a") == NotInit + assert foo.get_value("b") == "'Initialized!'" + + my_globals = {} + PythonEvaluator().update_globals_with_context(my_globals, context) + + assert my_globals == {"self": foo, "b": "'Initialized!'"} diff --git a/tests/evaluators/test_ResolveAmbiguityEvaluator.py b/tests/evaluators/test_ResolveAmbiguityEvaluator.py new file mode 100644 index 0000000..3d361da --- /dev/null +++ b/tests/evaluators/test_ResolveAmbiguityEvaluator.py @@ -0,0 +1,74 @@ +import pytest +from core.concept import Concept +from evaluators.ResolveAmbiguityEvaluator import ResolveAmbiguityEvaluator + +from tests.BaseTest import BaseTest +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +pretval = BaseTest.pretval + + +class TestResolveAmbiguityEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("return_values, expected", [ + ([pretval(Concept("foo"), source="source"), pretval(Concept("bar"), source="source")], True), + ([pretval(Concept("foo"), source="source"), pretval(Concept("bar"), source="source", status=False)], False), + ([pretval(Concept("foo"), source="source1"), pretval(Concept("bar"), source="source2")], False), + ]) + def test_i_can_match(self, return_values, expected): + context = self.get_context() + assert ResolveAmbiguityEvaluator().matches(context, return_values) == expected + + def test_i_can_manage_when_no_source(self): + context = self.get_context() + return_values = [BaseTest.retval(Concept("foo"))] + + assert not ResolveAmbiguityEvaluator().matches(context, return_values) + + def test_i_can_eval(self): + context = self.get_context() + return_values = [ + self.pretval(Concept("hello a").def_var("a", "world"), "hello word"), + self.pretval(Concept("hello world"), "hello word"), + self.pretval(Concept("hello world", pre="False"), "hello word"), + self.retval(Concept("not a parser result")), + self.retval(Concept("status is false"), status=False), + self.pretval(Concept("false parser result"), status=False), + ] + + evaluator = ResolveAmbiguityEvaluator() + evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert len(res) == 1 + resolved = res[0] + + assert resolved.who == evaluator.name + assert resolved.body == return_values[1].body + assert resolved.parents == [ + return_values[0], + return_values[1], + return_values[2], + ] + + def test_i_can_eval_all_fail(self): + context = self.get_context() + return_values = [ + self.pretval(Concept("hello world", pre="2 < 1"), "hello word"), + self.pretval(Concept("hello world", pre="False"), "hello word"), + ] + + evaluator = ResolveAmbiguityEvaluator() + evaluator.matches(context, return_values) + res = evaluator.eval(context, return_values) + + assert len(res) == 1 + resolved = res[0] + + assert resolved.who == evaluator.name + assert resolved.body == [] + assert resolved.parents == [ + return_values[0], + return_values[1], + ] diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index dd21d53..4e6033c 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -1,7 +1,8 @@ import pytest from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit, CC +from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, NotInit from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator +from evaluators.OneSuccessEvaluator import OneSuccessEvaluator from evaluators.PythonEvaluator import PythonEvalError from parsers.BaseNodeParser import SyaAssociativity from parsers.BnfNodeParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression @@ -216,6 +217,7 @@ as: assert evaluated.get_value("a").metadata.is_evaluated def test_i_can_recognize_duplicate_concepts_with_same_value(self): + # when multiple result, choose the one that is the more specific (that has the less variables) sheerka = self.get_sheerka() self.create_and_add_in_cache_concept(sheerka, Concept(name="hello a", body="'hello ' + a"), variables=["a"]) self.create_and_add_in_cache_concept(sheerka, Concept(name="hello foo", body="'hello foo'")) @@ -224,13 +226,14 @@ as: res = sheerka.evaluate_user_input("hello foo") assert len(res) == 1 assert res[0].status - assert res[0].value.body == "hello foo" - assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME) + assert sheerka.isinstance(res[0].body, "hello foo") + assert res[0].value.body == BuiltinConcepts.NOT_INITIALIZED + assert res[0].who == sheerka.get_evaluator_name(OneSuccessEvaluator.NAME) def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different(self): sheerka = self.get_sheerka() - self.create_and_add_in_cache_concept(sheerka, Concept(name="hello a", body="'hello ' + a"), variables=["a"]) - self.create_and_add_in_cache_concept(sheerka, Concept(name="hello foo", body="'hello foo'")) + self.create_and_add_in_cache_concept(sheerka, Concept(name="hello a", body="'hello ' + a").def_var("a")) + self.create_and_add_in_cache_concept(sheerka, Concept(name="hello b", body="'hello you ' + b").def_var("b")) self.create_and_add_in_cache_concept(sheerka, Concept(name="foo", body="'another value'")) res = sheerka.evaluate_user_input("hello foo") @@ -242,7 +245,7 @@ as: assert len(concepts) == 2 sorted_values = sorted(concepts, key=lambda x: x.value.body) assert sorted_values[0].value.body == "hello another value" - assert sorted_values[1].value.body == "hello foo" + assert sorted_values[1].value.body == "hello you another value" def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(self): sheerka = self.get_sheerka() @@ -692,12 +695,12 @@ as: assert len(res) == 1 assert not res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.MULTIPLE_ERRORS) - assert str(BuiltinConcepts.WHERE_CLAUSE_FAILED) in [error.key for error in sheerka.get_error(res[0].body.body)] + assert str(BuiltinConcepts.CONDITION_FAILED) in [error.key for error in sheerka.get_error(res[0].body.body)] res = sheerka.evaluate_user_input("eval twenty three") assert len(res) == 1 assert not res[0].status - assert str(BuiltinConcepts.WHERE_CLAUSE_FAILED) in [error.key for error in sheerka.get_error(res[0].body.body)] + assert str(BuiltinConcepts.CONDITION_FAILED) in [error.key for error in sheerka.get_error(res[0].body.body)] def test_i_can_manage_some_type_of_infinite_recursion(self): sheerka = self.get_sheerka() @@ -735,12 +738,12 @@ as: res = sheerka.evaluate_user_input("foo baz") assert len(res) == 1 assert not res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.WHERE_CLAUSE_FAILED) + assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED) res = sheerka.evaluate_user_input("eval foo baz") assert len(res) == 1 assert not res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.WHERE_CLAUSE_FAILED) + assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED) res = sheerka.evaluate_user_input("foobar") assert len(res) == 1 @@ -749,7 +752,7 @@ as: res = sheerka.evaluate_user_input("eval foobar") assert len(res) == 1 # error assert not res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.WHERE_CLAUSE_FAILED) + assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED) @pytest.mark.skip("Not ready for that") def test_i_can_manage_missing_variables_from_bnf_parsing(self): @@ -998,6 +1001,20 @@ as: assert res[0].status assert res[0].body == 2 + def test_i_can_evaluate_command(self): + init = [ + "def concept command as 'Executed !'", + "set_isa(c:command:, __COMMAND)", + ] + + # Since command is a __COMMAND, the body is auto evaluated + # and we return the body, not the concept + + sheerka = self.init_scenario(init) + res = sheerka.evaluate_user_input("command") + assert res[0].status + assert res[0].body == "Executed !" + class TestSheerkaNonRegFile(TestUsingFileBasedSheerka): def test_i_can_def_several_concepts(self): diff --git a/tests/parsers/test_ExactConceptParser.py b/tests/parsers/test_ExactConceptParser.py index d4f1b82..fe784c2 100644 --- a/tests/parsers/test_ExactConceptParser.py +++ b/tests/parsers/test_ExactConceptParser.py @@ -1,7 +1,6 @@ from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, CMV from core.sheerka.services.SheerkaExecute import ParserInput -from core.tokenizer import Tokenizer from parsers.ExactConceptParser import ExactConceptParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka diff --git a/tests/sheerkapickle/test_sheerka_handlers.py b/tests/sheerkapickle/test_sheerka_handlers.py index f9e3a65..1fcd321 100644 --- a/tests/sheerkapickle/test_sheerka_handlers.py +++ b/tests/sheerkapickle/test_sheerka_handlers.py @@ -16,6 +16,14 @@ def set_full_serialization(concept): class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): + user_input_id = 0 + return_value_id = 0 + + @classmethod + def setup(cls): + sheerka = cls().get_sheerka() + cls.user_input_id = sheerka.get_by_key("__USER_INPUT").id + cls.return_value_id = sheerka.get_by_key("__RETURN_VALUE").id def test_i_can_encode_decode_unknown_concept_metadata(self): sheerka = self.get_sheerka() @@ -201,7 +209,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): to_string = sheerkapickle.encode(sheerka, user_input) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == user_input - assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "23"], "user_name": "my_user_name", "text": "my_text"}' + assert to_string == f'{{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "{self.user_input_id}"], "user_name": "my_user_name", "text": "my_text"}}' def test_i_can_encode_decode_user_input_when_tokens(self): sheerka = self.get_sheerka() @@ -213,7 +221,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): to_string = sheerkapickle.encode(sheerka, user_input) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="my_user_name") - assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "23"], "user_name": "my_user_name", "text": "{text}"' + '}' + assert to_string == f'{{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "{self.user_input_id}"], "user_name": "my_user_name", "text": "{text}"}}' def test_i_can_encode_decode_return_value(self): sheerka = self.get_sheerka() @@ -223,7 +231,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): to_string = sheerkapickle.encode(sheerka, ret_val) decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ret_val - assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "28"], "who": "who", "status": true, "value": 10}' + assert to_string == f'{{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "{self.return_value_id}"], "who": "who", "status": true, "value": 10}}' def test_i_can_encode_decode_return_value_with_parent(self): sheerka = self.get_sheerka() @@ -236,7 +244,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka): decoded = sheerkapickle.decode(sheerka, to_string) assert decoded == ret_val assert decoded.parents == ret_val.parents - id_str = ', "concept/id": ["__RETURN_VALUE", "28"]' + id_str = f', "concept/id": ["__RETURN_VALUE", "{self.return_value_id}"]' parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]' assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}'