diff --git a/sheerka_backup/default.sb b/sheerka_backup/default.sb index aa7b53b..079d232 100644 --- a/sheerka_backup/default.sb +++ b/sheerka_backup/default.sb @@ -1,6 +1,6 @@ # question push_ontology("english") -def concept q from q ? as question(q) pre is_question() auto_eval True +def concept q from q ? as question(q) auto_eval True set_is_lesser(__PRECEDENCE, q, 'Sya') def concept the x where isinstance(x, Concept) ret memory(x) diff --git a/sheerka_backup/numbers.sb b/sheerka_backup/numbers.sb index dd228dd..826886c 100644 --- a/sheerka_backup/numbers.sb +++ b/sheerka_backup/numbers.sb @@ -93,5 +93,5 @@ set_is_greater_than(__PRECEDENCE, multiplied, minus, 'Sya') set_is_greater_than(__PRECEDENCE, divided, minus, 'Sya') def concept quantity -def concept quantify x from bnf number x as set_attr(x, quantity, number) ret x +def concept quantify x from bnf number x where not isa(x, number) as set_attr(x, quantity, number) ret x def concept how many x pre is_question() as get_attr(memory(x), quantity) \ No newline at end of file diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 2109aef..fdfc1e4 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -15,7 +15,7 @@ class UserInputConcept(Concept): BuiltinConcepts.USER_INPUT, bound_body="text") self.set_value("text", text) self.set_value("user_name", user_name) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"({self.id}){self.name}: '{self.body}'" @@ -33,7 +33,7 @@ class ErrorConcept(Concept, ErrorObj): id=concept_id, bound_body="error") self.set_value("error", error) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"({self.id}){self.name}: {self.body}" @@ -49,7 +49,7 @@ class UnknownConcept(Concept, ErrorObj): False, BuiltinConcepts.UNKNOWN_CONCEPT, bound_body="concept_ref") self.set_value("concept_ref", concept_ref) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"({self.id}){self.name}: {self.body}" @@ -75,7 +75,7 @@ class ReturnValueConcept(Concept): self.set_value("status", status) self.set_value("value", value) self.set_value("parents", parents) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"ReturnValue(who={self.who}, status={self.status}, value={self.value})" @@ -110,7 +110,7 @@ class UnknownPropertyConcept(Concept, ErrorObj): bound_body="property_name") self.set_value("property_name", property_name) self.set_value("concept", concept) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"UnknownProperty(property={self.property_name}, concept={self.concept})" @@ -136,7 +136,7 @@ class ParserResultConcept(Concept): self.set_value("tokens", tokens) self.set_value("value", value) self.set_value("try_parsed", try_parsed) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): text = f"ParserResult(parser={self.parser}" @@ -179,7 +179,7 @@ class RuleEvaluationResultConcept(Concept): id=concept_id, bound_body="rule") self.set_value("rule", rule) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"RuleEvaluationResult(rule={self.rule})" @@ -212,7 +212,7 @@ class InvalidReturnValueConcept(Concept, ErrorObj): bound_body="return_value") self.set_value("return_value", return_value) self.set_value("evaluator", evaluator) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True class ConceptEvalError(Concept, ErrorObj): @@ -227,7 +227,7 @@ class ConceptEvalError(Concept, ErrorObj): self.set_value("error", error) self.set_value("concept", concept) self.set_value("property_name", property_name) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"ConceptEvalError(error={self.error}, concept={self.concept}, property={self.property_name})" @@ -244,16 +244,19 @@ class ListConcept(Concept): BuiltinConcepts.LIST, bound_body="items") self.set_value("items", items or []) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def append(self, obj): self.body.append(obj) class FilteredConcept(Concept): - ALL_ATTRIBUTES = ["filtered", "iterable", "predicate"] + ALL_ATTRIBUTES = ["filtered", "iterable", "predicate", "reason"] + # To explain the reason why it's filtered, you can either + # provide the original list (iterable) and the predicate + # provide the reason (It may be a CONDITION_FAILED concept) - def __init__(self, filtered=None, iterable=None, predicate=None): + def __init__(self, filtered=None, iterable=None, predicate=None, reason=None): Concept.__init__(self, BuiltinConcepts.FILTERED, True, @@ -263,7 +266,8 @@ class FilteredConcept(Concept): self.set_value("filtered", filtered) self.set_value("iterable", iterable) self.set_value("predicate", predicate) - self._metadata.is_evaluated = True + self.set_value("reason", reason) + self._hints.is_evaluated = True class ConceptAlreadyInSet(Concept, ErrorObj): @@ -278,7 +282,7 @@ class ConceptAlreadyInSet(Concept, ErrorObj): bound_body="concept") self.set_value("concept", concept) self.set_value("concept_set", concept_set) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"ConceptAlreadyInSet(concept={self.concept}, concept_set={self.concept_set})" @@ -297,7 +301,7 @@ class PropertyAlreadyDefined(Concept, ErrorObj): self.set_value("property_name", property_name) self.set_value("property_value", property_value) self.set_value("concept", concept) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"PropertyAlreadyDefined(property={self.property_name}, value={self.property_value}, concept={self.concept})" @@ -317,7 +321,7 @@ class ConditionFailed(Concept, ErrorObj): self.set_value("concept", concept) self.set_value("prop", prop) self.set_value("reason", reason) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"ConditionFailed(condition='{self.body}', concept='{self.concept}', prop='{self.prop}')" @@ -335,7 +339,7 @@ class NotForMeConcept(Concept): # Not considered as an error ? bound_body="source") self.set_value("source", source) self.set_value("reason", reason) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): return f"NotForMeConcept(source={self.body}, reason={self.get_value('reason')})" @@ -356,7 +360,7 @@ class ExplanationConcept(Concept): self.set_value("title", title) # a title to the explanation self.set_value("instructions", instructions) # instructions for SheerkaPrint self.set_value("execution_result", execution_result) # list of results - self._metadata.is_evaluated = True + self._hints.is_evaluated = True class PythonSecurityError(Concept, ErrorObj): @@ -375,7 +379,7 @@ class PythonSecurityError(Concept, ErrorObj): self.set_value("line", line) # line number self.set_value("column", column) # column number self.set_value("source_code", source_code) # code being executed - self._metadata.is_evaluated = True + self._hints.is_evaluated = True class NotFoundConcept(Concept, ErrorObj): @@ -407,7 +411,7 @@ class ToListConcept(Concept): self.set_value("recursion_depth", recursion_depth) # recursion depth when showing children self.set_value("recurse_on", recurse_on) # which sub items should we display self.set_value("tab", tab) # customise tab (content and length) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True class NewConceptConcept(Concept): @@ -422,7 +426,7 @@ class NewConceptConcept(Concept): bound_body="concept") self.set_value("concept", concept) - self._metadata.is_evaluated = True + self._hints.is_evaluated = True def __repr__(self): if self.concept: diff --git a/src/core/builtin_concepts_ids.py b/src/core/builtin_concepts_ids.py index 6f5c403..2557091 100644 --- a/src/core/builtin_concepts_ids.py +++ b/src/core/builtin_concepts_ids.py @@ -1,12 +1,9 @@ class BuiltinConcepts: """ List of builtin concepts that do no need any specific implementation - Please note that the value of the enum is informal. It is not used in the system - For example, the concept 'NODE' DOES NOT have the key, the id or whatever 200 - The key if the name of the concept + Please note that the value of the enum is informal. + The key is the name of the concept The id is a sequential number given just before the concept is saved in sdp - - The values of the enum is not used the code """ SHEERKA = "__SHEERKA" @@ -19,6 +16,7 @@ class BuiltinConcepts: REDUCE_REQUESTED = "__REDUCE_REQUESTED" # remove meaningless error when possible EVAL_UNTIL_SUCCESS_REQUESTED = "__EVAL_UNTIL_SUCCESS_REQUESTED" # PythonEvaluator tries combination until True is found EVAL_QUESTION_REQUESTED = "__EVAL_QUESTION_REQUESTED" # the user input must be treated as question + VALIDATION_ONLY_REQUESTED = "__VALIDATION_ONLY_REQUESTED" # Validation mode activated. Never evaluate the body # possible actions during sheerka.execute() or sheerka.evaluate_rules() INIT_SHEERKA = "__INIT_SHEERKA" # @@ -54,7 +52,6 @@ class BuiltinConcepts: ISA = "__ISA" # when a concept is an instance of another one HASA = "__HASA" # when a concept has/owns another concept AUTO_EVAL = "__AUTO_EVAL" # when the concept must be auto evaluated - RECOGNIZED_BY = "__RECOGNIZED_BY" # indicate how a concept was recognized # object USER_INPUT = "__USER_INPUT" # represent an input from an user @@ -102,6 +99,7 @@ class BuiltinConcepts: UNKNOWN_RULE = "__UNKNOWN_RULE" ONTOLOGY_ALREADY_DEFINED = "__ONTOLOGY_ALREADY_DEFINED" ONTOLOGY_REMOVED = "__ONTOLOGY_REMOVED" + METHOD_ACCESS_ERROR = "__METHOD_ACCESS_ERROR" NODE = "__NODE" GENERIC_NODE = "__GENERIC_NODE" @@ -174,7 +172,8 @@ BuiltinErrors = [ BuiltinConcepts.NOT_FOUND, BuiltinConcepts.IS_LESSER_CONSTRAINT_ERROR, BuiltinConcepts.IS_GREATEST_CONSTRAINT_ERROR, - BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED + BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED, + BuiltinConcepts.METHOD_ACCESS_ERROR ] BuiltinContainers = [ diff --git a/src/core/builtin_helpers.py b/src/core/builtin_helpers.py index 2436482..81e2158 100644 --- a/src/core/builtin_helpers.py +++ b/src/core/builtin_helpers.py @@ -5,17 +5,17 @@ from cache.Cache import Cache from core.ast_helpers import ast_to_props from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value -from core.global_symbols import NotInit, NotFound, CURRENT_OBJ +from core.global_symbols import NotInit, NotFound, INIT_AST_PARSERS, DEFAULT_EVALUATORS from core.rule import Rule -from core.tokenizer import Tokenizer +from core.tokenizer import Tokenizer, TokenKind from core.utils import as_bag from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \ - RuleNode, VariableNode + RuleNode, LexerNode from parsers.BaseParser import ParsingError PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING] -EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION, - BuiltinConcepts.AFTER_EVALUATION] +EVAL_ONLY_STEPS = [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION] +EVAL_STEPS = PARSE_STEPS + EVAL_ONLY_STEPS PARSERS = ["EmptyString", "ShortTermMemory", "Sequence", "Bnf", "Sya", "Python"] @@ -48,7 +48,7 @@ def is_same_success(context, return_values): if not ret_val.status: raise Exception("Status is false") - if isinstance(ret_val.body, Concept) and not ret_val.body.get_metadata().is_evaluated: + if isinstance(ret_val.body, Concept) and not ret_val.body.get_hints().is_evaluated: raise Exception("Concept is not evaluated") return context.sheerka.objvalue(ret_val) @@ -223,7 +223,10 @@ def resolve_ambiguity(context, concepts): remaining_concepts.extend(by_complexity[complexity]) else: for c in by_complexity[complexity]: - evaluated = context.sheerka.evaluate_concept(context, c, metadata=[ConceptParts.PRE]) + evaluated = context.sheerka.evaluate_concept(context, c, + eval_body=False, + validation_only=True, + metadata=[ConceptParts.PRE]) if context.sheerka.is_success(evaluated) or evaluated.key == c.key: remaining_concepts.append(c) @@ -316,7 +319,8 @@ def only_parsers_results(context, return_values): def evaluate_from_source(context, source, - evaluators="all", + parsers=INIT_AST_PARSERS, + evaluators=DEFAULT_EVALUATORS, desc=None, eval_body=True, eval_where=True, @@ -327,13 +331,14 @@ def evaluate_from_source(context, :param context: :param source: + :param parsers: :param evaluators: :param desc: :param eval_body: :param eval_where: :param is_question: :param expect_success: - :param stm: short term memories entries + :param stm: short term memories entries AKA current namespace :return: """ @@ -362,12 +367,11 @@ def evaluate_from_source(context, for k, v in stm.items(): sub_context.add_to_short_term_memory(k, v) - # disable all evaluators but the requested ones + if parsers != "all": + sub_context.preprocess_parsers = parsers + if evaluators != "all": - from evaluators.BaseEvaluator import BaseEvaluator - sub_context.add_preprocess(BaseEvaluator.PREFIX + "*", enabled=False) - for evaluator in evaluators: - sub_context.add_preprocess(BaseEvaluator.PREFIX + evaluator, enabled=True) + sub_context.preprocess_evaluators = evaluators user_input = sheerka.ret(context.who, True, sheerka.new(BuiltinConcepts.USER_INPUT, body=source)) ret = sheerka.execute(sub_context, [user_input], EVAL_STEPS) @@ -376,6 +380,52 @@ def evaluate_from_source(context, return ret +def evaluate_return_values(context, + source, + return_values, + evaluators=DEFAULT_EVALUATORS, + desc=None, + eval_body=True, + eval_where=True, + is_question=False, + expect_success=False, + stm=None): + sheerka = context.sheerka + desc = desc or f"Eval '{source}' using return values" + hints_to_reset = { + BuiltinConcepts.EVAL_BODY_REQUESTED, + BuiltinConcepts.EVAL_WHERE_REQUESTED, + BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED, + BuiltinConcepts.EVAL_QUESTION_REQUESTED, + } + with context.push(BuiltinConcepts.EVALUATE_SOURCE, source, desc=desc, reset_hints=hints_to_reset) as sub_context: + if eval_body: + sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + + if eval_where: + sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) + + if expect_success: + sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) + + if is_question: + sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + + if stm: + for k, v in stm.items(): + sub_context.add_to_short_term_memory(k, v) + + if evaluators != "all": + sub_context.preprocess_evaluators = evaluators + + sub_context.add_inputs(return_values=return_values) + res = sheerka.execute(sub_context, return_values.copy(), EVAL_ONLY_STEPS) + one_r = expect_one(context, res) + sub_context.add_values(return_values=one_r) + + return one_r + + def get_lexer_nodes(return_values, start, tokens): """ Transform all elements from return_values into lexer nodes (ConceptNode, UnrecognizedTokensNode, SourceCodeNode...) @@ -386,10 +436,14 @@ def get_lexer_nodes(return_values, start, tokens): :param tokens: :return: list of list (list of concept node sequence) """ + from evaluators.BaseEvaluator import BaseEvaluator lexer_nodes = [] for ret_val in return_values: - if ret_val.who == "parsers.Python": + # To manage AFTER_PARSING evaluators + who = ret_val.parents[0].who if ret_val.who.startswith(BaseEvaluator.PREFIX) else ret_val.who + + if who == "parsers.Python": if ret_val.body.source.strip().isidentifier(): # Discard SourceCodeNode which seems to be a concept name @@ -404,27 +458,29 @@ def get_lexer_nodes(return_values, start, tokens): python_node=ret_val.body.body, return_value=ret_val)]) - elif ret_val.who == "parsers.ExactConcept": + elif who == "parsers.ExactConcept": concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body] end = start + len(tokens) - 1 for concept in concepts: lexer_nodes.append([ConceptNode(concept, start, end, tokens, ret_val.body.source)]) - elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"): - nodes = [node for node in ret_val.body.body] + elif who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"): + nodes = [node.clone() for node in ret_val.body.body] for node in nodes: node.start += start node.end += start if isinstance(node, ConceptNode): for k, v in node.concept.get_compiled().items(): - if hasattr(v, "start"): + if isinstance(v, LexerNode): + v = v.clone() v.start += start v.end += start + node.concept.get_compiled()[k] = v # but append the whole sequence if when it's a sequence lexer_nodes.append(nodes) - elif ret_val.who == "parsers.Rule": + elif who == "parsers.Rule": rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body] end = start + len(tokens) - 1 for rule in rules: @@ -447,9 +503,14 @@ def get_lexer_nodes_using_positions(return_values, positions): :return: """ + from evaluators.BaseEvaluator import BaseEvaluator + lexer_nodes = [] for ret_val, position in zip(return_values, positions): - if ret_val.who in ("parsers.Python", 'parsers.PythonWithConcepts'): + # To manage AFTER_PARSING evaluators + who = ret_val.parents[0].who if ret_val.who.startswith(BaseEvaluator.PREFIX) else ret_val.who + + if who in ("parsers.Python", 'parsers.PythonWithConcepts'): lexer_nodes.append(SourceCodeNode(position.start, position.end, @@ -458,7 +519,7 @@ def get_lexer_nodes_using_positions(return_values, positions): python_node=ret_val.body.body, return_value=ret_val)) - elif ret_val.who == "parsers.ExactConcept": + elif who == "parsers.ExactConcept": concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body] for concept in concepts: lexer_nodes.append(ConceptNode(concept, @@ -467,21 +528,23 @@ def get_lexer_nodes_using_positions(return_values, positions): position.tokens, ret_val.body.source)) - elif ret_val.who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"): - nodes = [node for node in ret_val.body.body] + elif who in ("parsers.Bnf", "parsers.Sya", "parsers.Sequence"): + nodes = [node.clone() for node in ret_val.body.body] for node in nodes: node.start = position.start node.end = position.end if isinstance(node, ConceptNode): for k, v in node.concept.get_compiled().items(): - if hasattr(v, "start"): + if isinstance(v, LexerNode): + v = v.clone() v.start += position.start v.end += position.start + node.concept.get_compiled()[k] = v # but append the whole sequence if when it's a sequence lexer_nodes.extend(nodes) - elif ret_val.who == "parsers.Rule": + elif who == "parsers.Rule": rules = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body] for rule in rules: lexer_nodes.append(RuleNode(rule, @@ -489,7 +552,7 @@ def get_lexer_nodes_using_positions(return_values, positions): position.end, position.tokens, ret_val.body.source)) - elif ret_val.who == "parsers.Function": + elif who == "parsers.Function": node = ret_val.body.body node.start = position.start node.end = position.end @@ -510,8 +573,10 @@ def ensure_evaluated(context, concept, eval_body=True, metadata=None): :param metadata: :return: """ - if concept.get_metadata().is_evaluated: - return concept + if concept.get_hints().is_evaluated: + from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept + return SheerkaEvaluateConcept.apply_ret(concept, + eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)) # do not try to evaluate concept that are not fully initialized if concept.get_metadata().definition_type != DEFINITION_TYPE_BNF: @@ -533,19 +598,6 @@ def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers :param parsers: :return: """ - - # first look into short term memory to see if the unrecognized is not a variable of the current object - if (current_obj := context.sheerka.get_from_short_term_memory(context, CURRENT_OBJ)) is not NotFound: - if isinstance(current_obj, Concept): - source = unrecognized_tokens_node.source - if source in current_obj.get_compiled() or source in current_obj.variables(): - return [[VariableNode(current_obj, - source, - unrecognized_tokens_node.start, - unrecognized_tokens_node.end, - unrecognized_tokens_node.tokens, - unrecognized_tokens_node.source)]] - res = context.sheerka.parse_unrecognized(context, unrecognized_tokens_node.source, parsers) res = only_parsers_results(context, res) @@ -630,7 +682,7 @@ def update_compiled(context, concept, errors, parsers=None): if _get_source(concept.get_compiled(), name) != name: break else: - concept.get_metadata().is_evaluated = True + concept.get_hints().is_evaluated = True def add_to_ret_val(sheerka, context, return_values, concept_key): @@ -665,10 +717,10 @@ def set_is_evaluated(concepts, check_nb_variables=False): if hasattr(concepts, "__iter__"): for c in concepts: if not check_nb_variables or check_nb_variables and len(c.get_metadata().variables) > 0: - c.get_metadata().is_evaluated = True + c.get_hints().is_evaluated = True else: if not check_nb_variables or check_nb_variables and len(concepts.get_metadata().variables) > 0: - concepts.get_metadata().is_evaluated = True + concepts.get_hints().is_evaluated = True def ensure_concept(*concepts): @@ -701,7 +753,7 @@ def ensure_concept_or_rule(*items): raise TypeError(f"'{items}' must be a concept or rule") -def ensure_bnf(context, concept, parser_name="BaseNodeParser", update_bnf_for_cached_concept=True): +def ensure_bnf(context, concept, parser_name="BaseNodeParser"): if concept.get_metadata().definition_type == DEFINITION_TYPE_BNF and not concept.get_bnf(): from parsers.BnfDefinitionParser import BnfDefinitionParser regex_parser = BnfDefinitionParser() @@ -719,8 +771,6 @@ def ensure_bnf(context, concept, parser_name="BaseNodeParser", update_bnf_for_ca raise Exception(bnf_parsing_ret_val.value) concept.set_bnf(bnf_parsing_ret_val.body.body) - if concept.id and update_bnf_for_cached_concept: - context.sheerka.get_by_id(concept.id).set_bnf(concept.get_bnf()) # update bnf in cache expressions_cache = Cache() @@ -828,7 +878,24 @@ def get_possible_variables_from_concept(context, concept): concept_name = [t.str_value for t in Tokenizer(concept.name, yield_eof=False)] names = [v_value or v_name for v_name, v_value in concept.get_metadata().variables if v_name in concept_name] possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names) - return set(possible_vars) + to_keep = set() + for var in possible_vars: + tokens = Tokenizer(var, yield_eof=False) + for t in tokens: + if t.type in (TokenKind.IDENTIFIER, TokenKind.KEYWORD): + to_keep.add(var) + return to_keep + + +def is_only_successful(sheerka, return_value): + """ + + :param sheerka: + :param return_value + :return: + """ + return sheerka.isinstance(return_value, BuiltinConcepts.RETURN_VALUE) and \ + sheerka.isinstance(return_value.body, BuiltinConcepts.ONLY_SUCCESSFUL) class CreateObjectIdentifiers: diff --git a/src/core/concept.py b/src/core/concept.py index 52398b0..d65de01 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -40,6 +40,20 @@ def concept_part_value(c): return c[1:-1] +@dataclass +class ConceptHints: + is_evaluated: bool = False # True is the concept is evaluated by sheerka.eval_concept() + need_validation: bool = False # True if the properties of the concept need to be validated + recognized_by: str = None + use_copy: bool = False + + def copy(self): + return ConceptHints(self.is_evaluated, + self.need_validation, + self.recognized_by, + self.use_copy) + + @dataclass class ConceptMetadata: name: str @@ -57,8 +71,6 @@ class ConceptMetadata: id: str # unique identifier for a concept. The id will never be modified (but the key can) props: dict # hashmap of properties, values variables: list # list of concept variables(tuple), with their default values - is_evaluated: bool = False # True is the concept is evaluated by sheerka.eval_concept() - need_validation = False # True if the properties of the concept need to be validated full_serialization: bool = False # If True, the full object will be serialized, rather than just the diff all_attributes: List[str] = None # list of instance attributes @@ -158,7 +170,7 @@ class Concept: self._bnf = None # parsing expression self._original_definition_hash = None # concept hash before any alteration of the metadata self._format = None # how to print the concept - self._hints = {} # extra processing information to help processing + self._hints = ConceptHints() def __repr__(self): text = f"({self._metadata.id}){self._metadata.name}" @@ -201,7 +213,7 @@ class Concept: return False for name, value in self_values.items(): - if value == self: # not very resilient... + if value == self: # not very resilient... continue if value != other.get_value(name): @@ -253,6 +265,9 @@ class Concept: def get_metadata(self): return self._metadata + def get_hints(self) -> ConceptHints: + return self._hints + def get_compiled(self): return self._compiled @@ -381,13 +396,15 @@ class Concept: setattr(self._metadata, prop, as_dict[prop]) return self - def update_from(self, other, update_value=True): + def update_from(self, other, update_value=True, update_hint=False, update_compiled=False): """ Update self using the properties of another concept This method is to mimic the class to instance pattern 'other' is the class, the template, and 'self' is a new instance :param other: :param update_value: + :param update_hint: + :param update_compiled: :return: """ if other is None: @@ -405,14 +422,23 @@ class Concept: else: setattr(self._metadata, prop, getattr(other.get_metadata(), prop)) - # # update metadata - # self.from_dict(other.to_dict()) - # update values if update_value: for k, v in other.values().items(): self.set_value(k, v) + if update_hint: + self._hints = other.get_hints().copy() + + if update_compiled: + for k, v in other.get_compiled().items(): + if isinstance(v, Concept) and v.get_hints().use_copy: + copied = v.copy() + copied.get_hints().use_copy = False + self._compiled[k] = copied + else: + self._compiled[k] = v + # update bnf definition self._bnf = other.get_bnf() @@ -510,15 +536,6 @@ class Concept: def variables(self): return {k: v for k, v in self.values().items() if not k[0] == "#"} - def set_hint(self, name, value): - self._hints[name] = value - - def get_hint(self, name): - try: - return self._hints[name] - except KeyError: - return None - def auto_init(self): """ Sometimes (for tests purposes) @@ -527,7 +544,7 @@ class Concept: :return: """ - if self._metadata.is_evaluated: + if self.get_hints().is_evaluated: return self for metadata in AllConceptParts: @@ -538,7 +555,7 @@ class Concept: for var, value in self._metadata.variables: self.set_value(var, value) - self._metadata.is_evaluated = True + self.get_hints().is_evaluated = True return self def freeze_definition_hash(self): @@ -596,6 +613,19 @@ class Concept: return self._format.get(key, None) + def copy(self): + """ + A copy is not a clone, as not all the attributes are copied + :return: + """ + if self._metadata.is_unique: + return self + + return Concept().update_from(self, update_hint=True, update_compiled=True) + + def get_concept(self): + return self + @dataclass() class DoNotResolve: diff --git a/src/core/global_symbols.py b/src/core/global_symbols.py index c401799..35d8c9b 100644 --- a/src/core/global_symbols.py +++ b/src/core/global_symbols.py @@ -84,3 +84,16 @@ class SyaAssociativity(Enum): SHEERKA_BACKUP_FOLDER = "SHEERKA_BACKUP_FOLDER" SHEERKA_BACKUP_FILE = "SHEERKA_BACKUP_FILE" + +INIT_AST_PARSERS = ["ExactConcept", + "Rule", + "Sequence", + "Sya", + "Python", + "Bnf", + "UnrecognizedNode", + "PythonWithConcepts"] + +INIT_AST_QUESTION_PARSERS = ["Expression"] + +DEFAULT_EVALUATORS = ["Python", "Concept", "LexerNode", "ValidateConcept", "ResolveAmbiguity"] diff --git a/src/core/sheerka/ExecutionContext.py b/src/core/sheerka/ExecutionContext.py index d7f2614..b694c0c 100644 --- a/src/core/sheerka/ExecutionContext.py +++ b/src/core/sheerka/ExecutionContext.py @@ -74,6 +74,7 @@ class ExecutionContext: self.preprocess_evaluators = None self.preprocess = None self.stm = False # True if the context has short term memory entries + self.possible_variables = None # when concepts must be considered as variables self.private_hints = set() self.protected_hints = set() @@ -191,6 +192,7 @@ class ExecutionContext: new.preprocess_parsers = self.preprocess_parsers new.preprocess_evaluators = self.preprocess_evaluators new.protected_hints.update(protected_hints) + new.possible_variables = self.possible_variables self._children.append(new) @@ -266,6 +268,9 @@ class ExecutionContext: """ return self.sheerka.get_from_short_term_memory(self, key) + def debug_short_term_memory(self): + return self.sheerka.get_all_short_term_memory(self, recursive=True) + def get_concept(self, key): # search in obj if isinstance(self.obj, Concept): diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 66d0230..f03d4c4 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -116,6 +116,7 @@ class Sheerka(Concept): "test_dict": SheerkaMethod(self.test_dict, False), "test_error": SheerkaMethod(self.test_error, False), "is_sheerka": SheerkaMethod(self.is_sheerka, False), + "objvalue": SheerkaMethod(self.objvalue, False), } self.concepts_ids = None @@ -409,7 +410,7 @@ class Sheerka(Concept): """ def add_recognized_by(c, _recognized_by): - c.set_hint(BuiltinConcepts.RECOGNIZED_BY, _recognized_by) + c.get_hints().recognized_by = _recognized_by return c def new_instances(concepts, _recognized_by): @@ -444,8 +445,8 @@ class Sheerka(Concept): if concept[1]: if self.is_known(found := self.get_by_id(concept[1])): instance = self.new_from_template(found, found.key) - instance._metadata.is_evaluated = True - instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_ID) + instance.get_hints().is_evaluated = True + instance.get_hints().recognized_by = RECOGNIZED_BY_ID return instance elif concept[0]: if self.is_known(found := self.get_by_name(concept[0])): @@ -466,7 +467,7 @@ class Sheerka(Concept): def fast_resolve(self, key, return_new=True): def add_recognized_by(c, _recognized_by): - c.set_hint(BuiltinConcepts.RECOGNIZED_BY, _recognized_by) + c.get_hints().recognized_by = _recognized_by return c def new_instances(concepts, _recognized_by): @@ -557,7 +558,7 @@ class Sheerka(Concept): return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept) # TODO : add the concept to the list of known concepts (self.instances) - concept._metadata.is_evaluated = True # because we have manually set the variables + concept.get_hints().is_evaluated = True # because we have manually set the variables return concept def push_ontology(self, context, name, cache_only=False): @@ -720,6 +721,9 @@ class Sheerka(Concept): else: return [_obj] + if self.isinstance(_obj, BuiltinConcepts.FILTERED): + return inner_get_errors(_obj.reason) + if isinstance(_obj, Concept) and _obj.body != NotInit: return inner_get_errors(_obj.body) @@ -831,7 +835,7 @@ class Sheerka(Concept): unknown_concept = UnknownConcept() # don't use new() for prevent circular reference unknown_concept.set_value(ConceptParts.BODY, metadata) - unknown_concept._metadata.is_evaluated = True + unknown_concept.get_hints().is_evaluated = True return unknown_concept @staticmethod diff --git a/src/core/sheerka/services/SheerkaAdmin.py b/src/core/sheerka/services/SheerkaAdmin.py index 00c9059..9f0f7dd 100644 --- a/src/core/sheerka/services/SheerkaAdmin.py +++ b/src/core/sheerka/services/SheerkaAdmin.py @@ -6,6 +6,7 @@ from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers from core.builtin_helpers import ensure_concept_or_rule, ensure_concept from core.concept import Concept from core.global_symbols import SHEERKA_BACKUP_FOLDER +from core.sheerka.services.SheerkaExecute import SheerkaExecute from core.sheerka.services.SheerkaHistoryManager import SheerkaHistoryManager from core.sheerka.services.SheerkaMemory import SheerkaMemory from core.sheerka.services.sheerka_service import BaseService @@ -26,6 +27,8 @@ class SheerkaAdmin(BaseService): self.sheerka.bind_service_method(self.restore, True) self.sheerka.bind_service_method(self.concepts, False) self.sheerka.bind_service_method(self.desc, False) + self.sheerka.bind_service_method(self.desc_evaluators, False) + self.sheerka.bind_service_method(self.desc_parsers, False) self.sheerka.bind_service_method(self.extended_isinstance, False) self.sheerka.bind_service_method(self.is_container, False) self.sheerka.bind_service_method(self.format_rules, False) @@ -35,7 +38,6 @@ class SheerkaAdmin(BaseService): self.sheerka.bind_service_method(self.ontologies, False) self.sheerka.bind_service_method(self.in_memory, False) self.sheerka.bind_service_method(self.admin_history, False, as_name="history") - self.sheerka.bind_service_method(self.admin_history, False, as_name="history") self.sheerka.bind_service_method(self.sdp, False) self.sheerka.bind_service_method(self.atomic_def, False) @@ -165,7 +167,27 @@ class SheerkaAdmin(BaseService): concepts = sorted(self.sheerka.om.list(self.sheerka.CONCEPTS_BY_ID_ENTRY), key=lambda item: int(item.id)) return self.sheerka.new(BuiltinConcepts.TO_LIST, body=concepts) + def desc_evaluators(self): + evaluators = {k: sorted(v[0].items(), reverse=True) + for k, v in self.sheerka.services[SheerkaExecute.NAME].grouped_evaluators_cache.items()} + return self.sheerka.new(BuiltinConcepts.TO_DICT, body=evaluators) + + def desc_parsers(self): + res = {} + for k, v in self.sheerka.services[SheerkaExecute.NAME].grouped_parsers_cache.items(): + parsers = {k1: [p.__name__ for p in v1] for k1, v1 in v[0].items()} + sorted_parsers = sorted(parsers.items(), reverse=True) + res[k] = sorted_parsers + return self.sheerka.new(BuiltinConcepts.TO_DICT, body=res) + def desc(self, *items): + if len(items) == 1 and isinstance(items[0], str): + name = items[0].strip().lower() + if name == "parsers": + return self.desc_parsers() + elif name == "evaluators": + return self.desc_evaluators() + ensure_concept_or_rule(*items) res = [] for item in items: diff --git a/src/core/sheerka/services/SheerkaConceptManager.py b/src/core/sheerka/services/SheerkaConceptManager.py index b80c062..35cf2a7 100644 --- a/src/core/sheerka/services/SheerkaConceptManager.py +++ b/src/core/sheerka/services/SheerkaConceptManager.py @@ -14,7 +14,7 @@ from core.builtin_helpers import ensure_concept, ensure_bnf from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \ VARIABLE_PREFIX from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound, ErrorObj, EVENT_CONCEPT_DELETED, NoFirstToken, \ - EVENT_CONCEPT_MODIFIED + EVENT_CONCEPT_MODIFIED, CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.sheerka_service import BaseService from core.tokenizer import Tokenizer, TokenKind from parsers.BnfNodeParser import RegExDef @@ -132,6 +132,7 @@ class SheerkaConceptManager(BaseService): self.sheerka.bind_service_method(self.get_concepts_by_first_regex, False, visible=False) self.sheerka.bind_service_method(self.get_concepts_bnf_definitions, False, visible=False) self.sheerka.bind_service_method(self.clear_bnf_definition, True, visible=False) + self.sheerka.bind_service_method(self.set_precedence, True) register_concept_cache = self.sheerka.om.register_concept_cache @@ -194,7 +195,7 @@ class SheerkaConceptManager(BaseService): if key in BuiltinUnique: concept.get_metadata().is_unique = True - concept.get_metadata().is_evaluated = True + concept.get_hints().is_evaluated = True from_db = self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, concept.get_metadata().key) if from_db is NotFound: @@ -728,6 +729,39 @@ class SheerkaConceptManager(BaseService): else: self.sheerka.om.clear(self.CONCEPTS_BNF_DEFINITIONS_ENTRY) + def set_precedence(self, context, *concepts): + """ + Set the precedence order when parsing concept with SyaNodeParser + The first concept in the list have the highest priority + :param context: + :param concepts: + :return: + """ + + if len(concepts) < 2: + return self.sheerka.err("Not enough elements") + + as_iterable = iter(concepts) + first = next(as_iterable) + ensure_concept(first) + + try: + while True: + second = next(as_iterable) + ret = self.sheerka.set_is_greater_than(context, + BuiltinConcepts.PRECEDENCE, + first, + second, + CONCEPT_COMPARISON_CONTEXT) + if not ret.status: + return ret + + first = second + except StopIteration: + pass + + return self.sheerka.new(BuiltinConcepts.SUCCESS) + @staticmethod def _name_has_changed(to_add): if to_add is None or "meta" not in to_add: @@ -821,7 +855,7 @@ class SheerkaConceptManager(BaseService): concept.get_metadata().key = None if self._definition_has_changed(to_add) and concept.get_metadata().definition_type == DEFINITION_TYPE_BNF: concept.set_bnf(None) - ensure_bnf(context, concept, update_bnf_for_cached_concept=False) + ensure_bnf(context, concept) concept.init_key() diff --git a/src/core/sheerka/services/SheerkaEvaluateConcept.py b/src/core/sheerka/services/SheerkaEvaluateConcept.py index 0e2c3de..ce1dc47 100644 --- a/src/core/sheerka/services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/services/SheerkaEvaluateConcept.py @@ -1,18 +1,21 @@ from dataclasses import dataclass from core.builtin_concepts import BuiltinConcepts -from core.builtin_helpers import expect_one, only_successful, evaluate_from_source, ensure_concept +from core.builtin_helpers import expect_one, only_successful, ensure_concept, is_only_successful, ensure_bnf from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \ concept_part_value -from core.global_symbols import NotInit, CURRENT_OBJ +from core.global_symbols import NotInit, CURRENT_OBJ, INIT_AST_PARSERS, INIT_AST_QUESTION_PARSERS from core.rule import Rule -from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager -from core.sheerka.services.SheerkaExecute import ParserInput -from core.sheerka.services.sheerka_service import BaseService +from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules +from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute +from core.sheerka.services.SheerkaRuleManager import PythonConditionExprVisitor +from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, ChickenAndEggException from core.tokenizer import Tokenizer from core.utils import unstr_concept from parsers.BaseExpressionParser import TrueifyVisitor from parsers.BaseNodeParser import ConceptNode +from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor +from parsers.ExpressionParser import ExpressionParser from parsers.LogicalOperatorParser import LogicalOperatorParser CONCEPT_EVALUATION_STEPS = [ @@ -21,11 +24,6 @@ CONCEPT_EVALUATION_STEPS = [ BuiltinConcepts.AFTER_EVALUATION] -@dataclass -class ChickenAndEggException(Exception): - error: Concept - - @dataclass class ConceptEvalException(Exception): error: Concept @@ -37,7 +35,7 @@ class WhereClauseDef: clause: str # original where clause trueified: str # modified where clause (where unresolvable variables are removed) prop: str # variable to test - compiled: object # trueified where clause Python compiled + conditions: list # compiled trueified class SheerkaEvaluateConcept(BaseService): @@ -45,6 +43,7 @@ class SheerkaEvaluateConcept(BaseService): def __init__(self, sheerka): super().__init__(sheerka) + self.rule_evaluator = None def initialize(self): self.sheerka.bind_service_method(self.evaluate_concept, True) @@ -52,6 +51,9 @@ class SheerkaEvaluateConcept(BaseService): self.sheerka.bind_service_method(self.call_concept, False, as_name="evaluate_question") self.sheerka.bind_service_method(self.set_auto_eval, True) + def initialize_deferred(self, context, is_first_time): + self.rule_evaluator = self.sheerka.services[SheerkaEvaluateRules.NAME] + @staticmethod def infinite_recursion_detected(context, concept): """ @@ -86,14 +88,20 @@ class SheerkaEvaluateConcept(BaseService): return None @staticmethod - def apply_ret(concept): + def apply_ret(concept, eval_body=True): """ Check if a concept has its RET part defined If True, returns it :param concept: + :param eval_body: Do not return the ret is eval body is not requested :return: """ - return concept.get_value(ConceptParts.RET) if ConceptParts.RET in concept.values else concept + if (eval_body and + ConceptParts.RET in concept.values() and + (ret_value := concept.get_value(ConceptParts.RET)) != NotInit): + return ret_value + else: + return concept @staticmethod def get_needed_metadata(concept, concept_part, check_vars, check_body): @@ -153,20 +161,37 @@ class SheerkaEvaluateConcept(BaseService): to_trueify = [v[0] for v in concept.get_metadata().variables if v[0] != var_name] trueified_where = str(TrueifyVisitor(to_trueify, [var_name]).visit(expr)) - tokens = [t.str_value for t in Tokenizer(trueified_where)] - if var_name in tokens: - compiled = None - try: - compiled = compile(trueified_where, "", "eval") - except Exception: - pass - return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, compiled) - else: + try: + parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(trueified_where) + parsed = ExpressionParser(auto_compile=False).parse(context, parser_input) + python_visitor = PythonConditionExprVisitor(context) + conditions = python_visitor.get_conditions(parsed.body.body) + return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, conditions) + except FailedToCompileError: + # TODO: manage invalid where clause return None - def get_recursive_definitions(self, concept, return_values): + # tokens = [t.str_value for t in Tokenizer(trueified_where)] + # if var_name in tokens: + # compiled = None + # try: + # compiled = compile(trueified_where, "", "eval") + # except Exception: + # pass + # return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, compiled) + # else: + # return None + + @staticmethod + def get_recursive_definitions(context, concept, return_values): """ Returns the name of the parsers that will resolve to a recursive evaluation + For example, when getting ast for Concept("a and b", body="a and b") + Chances are that we will end up with two parsers + - Python parser for 'a and b' + - ExactConcept parser that point to the concept itself + The ExactConcept will be returned (to be removed from the list as it's a cyclic reference to itself) + :param context: :param concept: :param return_values: :return: @@ -176,8 +201,9 @@ class SheerkaEvaluateConcept(BaseService): # During evaluation, inner variables take precedence other concepts # So there won't be any cyclic reference, the variable will be picked return + for parser in [r.body for r in return_values if - r.status and self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]: + r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]: parsed = parser.body if isinstance(parser.body, list) else [parser.body] for parsed_item in parsed: if isinstance(parsed_item, Concept) and parsed_item.id == concept.id: @@ -185,6 +211,85 @@ class SheerkaEvaluateConcept(BaseService): elif isinstance(parsed_item, ConceptNode) and parsed_item.concept.id == concept.id: yield parser.parser + @staticmethod + def get_asts(context, who, source, concept, part_key, possible_variables): + """ + Get the return_value_concept or the concept for a given source + :param context: + :param who: + :param concept: + :param part_key: Concept part (#body#, #pre#, #where#...) being compiled + :param source: string or parser input, it does not matter + :param possible_variables: concepts that must be considered as variables + :return: + """ + + def parse_token_concept(s): + """ + The source is a direct reference / call to another concept + :param s: source + :return: + """ + if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None): + return context.sheerka.fast_resolve(identifier) + return None + + def get_return_value(current_context, c, s, p): + """ + + :param current_context: + :param c: concept + :param s: source + :param p: part of the concept being parsed + :return: + """ + parsers_to_use = INIT_AST_QUESTION_PARSERS if p in [ConceptParts.WHERE, + ConceptParts.PRE] else INIT_AST_PARSERS + while True: + return_value = current_context.sheerka.parse_unrecognized(current_context, + s, + parsers=parsers_to_use, + who=who, + prop=p, + filter_func=only_successful, + possible_variables=possible_variables) + + if not return_value.status: + if current_context.preprocess: + raise ChickenAndEggException(context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c})) + else: + raise FailedToCompileError([return_value.body]) + + return_value = return_value.body.body if is_only_successful(context.sheerka, return_value) else \ + [return_value] + + if c is None: + # No concept provided, we cannot look for recursive definition + return return_value + + recursive_parsers = list(SheerkaEvaluateConcept.get_recursive_definitions(context, c, return_value)) + + if len(recursive_parsers) == 0: + return return_value + + desc = f"Removing parsers {recursive_parsers}" + current_context = current_context.push(context.action, context.action_context, desc=desc) + for recursive_parser in recursive_parsers: + current_context.add_preprocess(recursive_parser.name, enabled=False) + + as_str = source.as_text() if isinstance(source, ParserInput) else source + + if as_str.strip() == "": + return DoNotResolve(as_str) + else: + if concept_found := parse_token_concept(as_str): + # the compiled can be a reference to another concept... + context.log(f"Recognized concept '{concept_found}'", SheerkaEvaluateConcept.NAME) + return concept_found + else: + # ...or a list of ReturnValueConcept to resolve + return get_return_value(context, concept, source, part_key) + def apply_where_clause(self, context, where_clause_def, return_values): """ Apply intermediate where clause when evaluating concept variables @@ -195,26 +300,53 @@ class SheerkaEvaluateConcept(BaseService): """ ret = [] valid_return_values = [r for r in return_values if r.status] + with context.push(BuiltinConcepts.VALIDATING_CONCEPT, + {"concept": where_clause_def.concept, "attr": ConceptParts.WHERE}, + desc=f"Apply where clause on '{where_clause_def.prop}'") as sub_context: + sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) + sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + # sub_context.sheerka.add_many_to_short_term_memory(sub_context, objects) + for r in valid_return_values: - if where_clause_def.compiled: - try: - if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}): - ret.append(r) - except NameError: - ret.append(r) # it cannot be solved unitary, let's give a chance to the global where condition + sub_context.add_to_short_term_memory(where_clause_def.prop, r.body) + res = self.rule_evaluator.evaluate_conditions(sub_context, + where_clause_def.conditions, + {where_clause_def.prop: r.body}) + if len(res) == 0: + # case where missing variables were detected + # This means that the 'where' clause can only be evaluated after all the parts are evaluated + ret.append(r) else: - # it means that the where condition is an expression that needs to be executed - evaluation_res = evaluate_from_source(context, - where_clause_def.trueified, - desc=f"Apply where clause on '{where_clause_def.prop}'", - expect_success=True, - is_question=True, - stm={where_clause_def.prop: r.body}) - one_res = expect_one(context, evaluation_res) + one_res = expect_one(context, res) if one_res.status: value = context.sheerka.objvalue(one_res) if isinstance(value, bool) and value: ret.append(r) + # value = context.sheerka.objvalue(res.body) + # if res.status and isinstance(bool, value) and value: + # ret.append(r) + + # if where_clause_def.compiled: + # try: + # if eval(where_clause_def.compiled, {where_clause_def.prop: self.sheerka.objvalue(r)}): + # ret.append(r) + # except NameError: + # ret.append(r) # it cannot be solved unitary, let's give a chance to the global where condition + # else: + # # it means that the where condition is an expression that needs to be executed + # evaluation_res = evaluate_from_source(context, + # where_clause_def.trueified, + # desc=f"Apply where clause on '{where_clause_def.prop}'", + # expect_success=True, + # is_question=True, + # stm={where_clause_def.prop: r.body}) + # one_res = expect_one(context, evaluation_res) + # if one_res.status: + # value = context.sheerka.objvalue(one_res) + # if isinstance(value, bool) and value: + # ret.append(r) if len(ret) > 0: return ret @@ -261,57 +393,19 @@ class SheerkaEvaluateConcept(BaseService): :return: """ - def is_only_successful(r): - """ + # for BNF concepts, concepts are sometimes considered as variables + # example : + # def concept a from bnf 'bar' | 'baz' + # def concept foo a as 'foo' a where a == 'baz' + # In the second concept (foo) as is a still a concept, but also a variable in the where clause - :param r: return_value - :return: - """ - return context.sheerka.isinstance(r, BuiltinConcepts.RETURN_VALUE) and \ - context.sheerka.isinstance(r.body, BuiltinConcepts.ONLY_SUCCESSFUL) - - def parse_token_concept(s): - """ - - :param s: source - :return: - """ - if s.startswith("c:") and (identifier := unstr_concept(s)) != (None, None): - return self.sheerka.fast_resolve(identifier) - return None - - def get_return_value(current_context, c, s, p): - """ - - :param current_context: - :param c: concept - :param s: source - :param p: part of the concept being parsed - :return: - """ - while True: - return_value = current_context.sheerka.parse_unrecognized(current_context, - s, - parsers="all", - prop=p, - filter_func=only_successful) - - if not return_value.status: - if current_context.preprocess: - raise ChickenAndEggException(self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c})) - else: - raise Exception(f"Failed to build '{s}'. But it doesn't seems to be recursion") - - return_value = return_value.body.body if is_only_successful(return_value) else [return_value] - recursive_parsers = list(self.get_recursive_definitions(c, return_value)) - - if len(recursive_parsers) == 0: - return return_value - - desc = f"Removing parsers {recursive_parsers}" - current_context = current_context.push(context.action, context.action_context, desc=desc) - for recursive_parser in recursive_parsers: - current_context.add_preprocess(recursive_parser.name, enabled=False) + ensure_bnf(context, concept) + if concept.get_bnf(): + visitor = BnfNodeConceptExpressionVisitor() + visitor.visit(concept.get_bnf()) + possible_variables = [c.name if isinstance(c, Concept) else c for c in visitor.references] + else: + possible_variables = None for part_key in AllConceptParts: if part_key in concept.get_compiled(): @@ -324,16 +418,12 @@ class SheerkaEvaluateConcept(BaseService): if not isinstance(source, str): raise Exception("Invalid concept init. metadata must be a string") - if source.strip() == "": - concept.get_compiled()[part_key] = DoNotResolve(source) - else: - if concept_found := parse_token_concept(source): - # the compiled can be a reference to another concept... - context.log(f"Recognized concept '{concept_found}'", self.NAME) - concept.get_compiled()[part_key] = concept_found - else: - # ...or a list of ReturnValueConcept to resolve - concept.get_compiled()[part_key] = get_return_value(context, concept, source, part_key) + concept.get_compiled()[part_key] = self.get_asts(context, + self.NAME, + source, + concept, + part_key, + possible_variables) for var_name, default_value in concept.get_metadata().variables: if var_name in concept.get_compiled(): @@ -345,22 +435,12 @@ class SheerkaEvaluateConcept(BaseService): if not isinstance(default_value, str): raise Exception("Invalid concept init. variable metadata must be a string") - if default_value.strip() == "": - concept.get_compiled()[var_name] = DoNotResolve(default_value) - else: - if concept_found := parse_token_concept(default_value): - # the compiled can be a reference to another concept... - context.log(f"Recognized concept '{concept_found}'", self.NAME) - concept.get_compiled()[var_name] = concept_found - else: - # ...or a list of ReturnValueConcept to resolve - concept.get_compiled()[var_name] = get_return_value(context, concept, default_value, var_name) - - # Updates the cache of concepts when possible - # This piece of code is not used, a the compile part is removed by sheerka.new_from_template() - service = context.sheerka.services[SheerkaConceptManager.NAME] - if service.has_id(concept.id): - self.sheerka.get_by_id(concept.id).set_compiled(concept.get_compiled()) + concept.get_compiled()[var_name] = self.get_asts(context, + self.NAME, + default_value, + concept, + var_name, + possible_variables) def resolve(self, context, @@ -368,6 +448,7 @@ class SheerkaEvaluateConcept(BaseService): current_prop, current_concept, force_evaluation, + forbid_methods_with_side_effect, where_clause_def): """ Resolve a variable or a Concept @@ -375,7 +456,8 @@ class SheerkaEvaluateConcept(BaseService): :param to_resolve: Concept or list of ReturnValueConcept to resolve :param current_prop: current property or ConceptPart :param current_concept: current concept - :param force_evaluation: Force body evaluation + :param force_evaluation: force body evaluation + :param forbid_methods_with_side_effect: Do not call methods with side effect when EVAL_BODY_REQUESTED :param where_clause_def: intermediate where clause for variables :return: """ @@ -412,6 +494,9 @@ class SheerkaEvaluateConcept(BaseService): if force_evaluation: sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + if forbid_methods_with_side_effect: + sub_context.protected_hints.add(BuiltinConcepts.VALIDATION_ONLY_REQUESTED) + if current_prop in (ConceptParts.WHERE, ConceptParts.PRE): sub_context.protected_hints.add(BuiltinConcepts.EVAL_UNTIL_SUCCESS_REQUESTED) @@ -440,7 +525,7 @@ class SheerkaEvaluateConcept(BaseService): value = current_concept.get_value(var[0]) if value != NotInit: sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0])) - use_copy = [r for r in to_resolve] if hasattr(to_resolve, "__iter__") else to_resolve + use_copy = to_resolve.copy() if isinstance(to_resolve, list) else to_resolve r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS) if where_clause_def: @@ -469,6 +554,7 @@ class SheerkaEvaluateConcept(BaseService): current_prop, current_concept, force_evaluation, + forbid_methods_with_side_effect, where_clause_def): """When dealing with a list, there are two possibilities""" # It may be a list of ReturnValueConcept to execute (always the case for metadata) @@ -483,6 +569,7 @@ class SheerkaEvaluateConcept(BaseService): current_prop, current_concept, force_evaluation, + forbid_methods_with_side_effect, where_clause_def) res = [] @@ -499,6 +586,7 @@ class SheerkaEvaluateConcept(BaseService): current_prop, current_concept, force_evaluation, + forbid_methods_with_side_effect, where_clause_def) if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR): return r @@ -506,19 +594,27 @@ class SheerkaEvaluateConcept(BaseService): return res - def evaluate_concept(self, context, concept: Concept, eval_body=False, metadata=None): + def evaluate_concept(self, context, concept: Concept, eval_body=False, validation_only=False, metadata=None): """ Evaluation a concept ie : resolve its body :param context: :param concept: :param eval_body: + :param validation_only: When set, the body is never evaluated, whatever eval_body or EVAL_BODY_REQUESTED :param metadata: list of metadata to evaluate ('pre', 'post'...) :return: value of the evaluation or error """ - if concept.get_metadata().is_evaluated: - return concept + failed_to_evaluate_body = False + + if concept.get_hints().is_evaluated: + return self.apply_ret(concept, eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)) + + # if concept.get_hints().use_copy: + # raise Exception("Use copy") + # concept = concept.copy() + # concept.get_hints().use_copy = False # I cannot use cache because of concept like 'number'. # They don't have variables, but their values change every time they are instantiated @@ -526,9 +622,9 @@ class SheerkaEvaluateConcept(BaseService): # need_body = eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) # if need_body and len(concept.get_metadata().variables) == 0 and context.sheerka.has_id(concept.id): # from_cache = context.sheerka.get_by_id(concept.id) - # if from_cache.get_metadata().is_evaluated: + # if from_cache.get_hints().is_evaluated: # concept.set_value(ConceptParts.BODY, from_cache.body) - # concept.get_metadata().is_evaluated = True + # concept.get_hints().is_evaluated = True # return concept desc = f"Evaluating concept {concept}" @@ -538,6 +634,10 @@ class SheerkaEvaluateConcept(BaseService): # ask for body evaluation sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + if validation_only: + # Never eval the body + sub_context.protected_hints.add(BuiltinConcepts.VALIDATION_ONLY_REQUESTED) + # auto evaluate commands if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)): sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) @@ -563,10 +663,23 @@ class SheerkaEvaluateConcept(BaseService): if isinstance(prop_ast, list): # Do not send the current concept for the properties - resolved = self.resolve_list(sub_context, prop_ast, var_name, None, True, w_clause) + resolved = self.resolve_list( + sub_context, + prop_ast, + var_name, + None, + True, + not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED), + w_clause) else: # Do not send the current concept for the properties - resolved = self.resolve(sub_context, prop_ast, var_name, None, True, w_clause) + resolved = self.resolve(sub_context, + prop_ast, + var_name, + None, + True, + not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED), + w_clause) if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved): resolved.set_value("concept", concept) # since current concept was not sent @@ -592,11 +705,25 @@ class SheerkaEvaluateConcept(BaseService): force_concept_eval = False if part_key == ConceptParts.BODY else True # resolve - resolved = self.resolve(sub_context, metadata_ast, part_key, concept, force_concept_eval, None) + resolved = self.resolve(sub_context, + metadata_ast, + part_key, + concept, + force_concept_eval, + not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED), + None) # 'FATAL' error is detected, let's stop if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved): - return resolved + if not (part_key == ConceptParts.BODY and + self.sheerka.has_error(context, resolved, body=BuiltinConcepts.METHOD_ACCESS_ERROR) and + sub_context.in_context(BuiltinConcepts.VALIDATION_ONLY_REQUESTED)): + return resolved + else: + # BuiltinConcepts.METHOD_ACCESS_ERROR is returned only when the access to side effect + # method is forbidden (during validation or ast initialisation) + resolved = NotInit + failed_to_evaluate_body = True concept.set_value(part_key, self.get_infinite_recursion_resolution(resolved) or resolved) @@ -614,8 +741,8 @@ class SheerkaEvaluateConcept(BaseService): concept.init_key() # Necessary for old unit tests. To remove someday - if ConceptParts.BODY in all_metadata_to_eval: - concept.get_metadata().is_evaluated = True + if ConceptParts.BODY in all_metadata_to_eval and not failed_to_evaluate_body: + concept.get_hints().is_evaluated = True # # update the cache for concepts with no variables # Cannot use cache. See the comment at the beginning of this method @@ -626,10 +753,7 @@ class SheerkaEvaluateConcept(BaseService): self.sheerka.register_object(sub_context, concept.name, concept) # manage RET metadata - if sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED) and ConceptParts.RET in concept.values(): - return concept.get_value(ConceptParts.RET) - else: - return concept + return self.apply_ret(concept, sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED)) def call_concept(self, context, concept, *args, **kwargs): """ @@ -656,7 +780,7 @@ class SheerkaEvaluateConcept(BaseService): needed, variables, body = self.get_needed_metadata(concept, ConceptParts.PRE, True, True) to_eval.extend(needed) - if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.get_metadata().need_validation: + if context.in_context(BuiltinConcepts.EVAL_WHERE_REQUESTED) or concept.get_hints().need_validation: # What are the cases where we do not need a validation ? # see test_sheerka_non_reg::test_i_can_evaluate_bnf_concept_with_where_clause() # res = sheerka.evaluate_user_input("foobar") diff --git a/src/core/sheerka/services/SheerkaEvaluateRules.py b/src/core/sheerka/services/SheerkaEvaluateRules.py index 53ecb00..607e4fa 100644 --- a/src/core/sheerka/services/SheerkaEvaluateRules.py +++ b/src/core/sheerka/services/SheerkaEvaluateRules.py @@ -111,13 +111,23 @@ class SheerkaEvaluateRules(BaseService): return expect_one(context, results) - def evaluate_conditions(self, context, conditions, bag): + def evaluate_conditions(self, context, conditions, bag, missing_vars=None): + """ + Evaluate the conditions + :param context: + :param conditions: + :param bag: variables that are supposed to be in short term memory + :param missing_vars: if initialized to a set, keeps tracks of the missing variables + :return: + """ bag_variables = set(bag.keys()) results = [] for compiled_condition in conditions: if compiled_condition.variables.intersection(bag_variables) != compiled_condition.variables: + if isinstance(missing_vars, set): + missing_vars.update(compiled_condition.variables - bag_variables) continue if compiled_condition.not_variables.intersection(bag_variables): @@ -131,11 +141,12 @@ class SheerkaEvaluateRules(BaseService): # do not forget to reset the 'is_evaluated' in the case of a concept for concept in compiled_condition.concepts_to_reset: - concept.get_metadata().is_evaluated = False + concept.get_hints().is_evaluated = False evaluator = self.evaluators_by_name[compiled_condition.evaluator_type] res = evaluator.eval(context, compiled_condition.return_value) - if res.status and isinstance(res.body, bool) and res.body: + value = context.sheerka.objvalue(res.body) + if res.status and isinstance(value, bool) and value: # one successful value found. No need to look any further results = [res] # don't we care about the other failing results ? break diff --git a/src/core/sheerka/services/SheerkaExecute.py b/src/core/sheerka/services/SheerkaExecute.py index 28550e1..272185d 100644 --- a/src/core/sheerka/services/SheerkaExecute.py +++ b/src/core/sheerka/services/SheerkaExecute.py @@ -2,7 +2,7 @@ import core.utils from cache.FastCache import FastCache from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept from core.concept import ConceptParts -from core.global_symbols import NotFound, NO_MATCH +from core.global_symbols import NotFound, NO_MATCH, EVENT_CONCEPT_CREATED, EVENT_CONCEPT_MODIFIED, EVENT_CONCEPT_DELETED from core.sheerka.services.sheerka_service import BaseService from core.tokenizer import Tokenizer, TokenKind, Token, Keywords @@ -15,6 +15,8 @@ ALL_STEPS = PARSE_AND_EVAL_STEPS + [BuiltinConcepts.BEFORE_RENDERING, BuiltinConcepts.AFTER_RENDERING, BuiltinConcepts.BEFORE_RULES_EVALUATION, BuiltinConcepts.AFTER_RULES_EVALUATION] +STM_PARSER_NAME = "ShortTermMemory" +DEFAULT = "__default" class ParserInput: @@ -190,17 +192,20 @@ class SheerkaExecute(BaseService): # order must be after SheerkaEvaluateRules because of self.rules_evaluation_service # order must be after ConceptManager because it needs concept bnf definitions super().__init__(sheerka, order=15) - self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=20) + self.pi_cache = FastCache(default=lambda key: ParserInput(key), max_size=200) + self.parsers_cache = FastCache(max_size=2000) + self.instantiated_evaluators = None self.evaluators_by_name = None self.instantiated_parsers = None self.parsers_by_name = None self.preprocessed_items_old_values = [] + self.question_parsers = [] # parsers to use when BuiltinConcepts.EVAL_QUESTION_REQUESTED is set # cache for all preregistered evaluator combination - # the key is the concatenation of the step and the name of evaluators in the group - # ex : BEFORE_EVALUATION|Python|Sya|Bnf + # the key is a tuple with the name of the step the names of the evaluators in the group + # ex : (BEFORE_EVALUATION, "Python|Sya|Bnf") # The value is a tuple, # The first entry is the grouped evaluators # ex : {60 : [PythonEvaluator(), SyaEvaluator()], 50: [BnfEvaluator()]} @@ -209,7 +214,7 @@ class SheerkaExecute(BaseService): # cache for preregistered parsers # Same construction than the evaluators - # Except 1 : the key does not have a step component. It is simple the list of parsers' names + # Except 1 : the key is a tuple (concept_hint or DEFAULT, names of the parsers) # Except 2 : we store the type of the parser, not its instance self.grouped_parsers_cache = {} @@ -229,8 +234,13 @@ class SheerkaExecute(BaseService): from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules self.rules_eval_service = self.sheerka.services[SheerkaEvaluateRules.NAME] + self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.on_concepts_modified) + self.sheerka.subscribe(EVENT_CONCEPT_MODIFIED, self.on_concepts_modified) + self.sheerka.subscribe(EVENT_CONCEPT_DELETED, self.on_concepts_modified) + def reset_state(self): self.pi_cache.clear() + self.parsers_cache.clear() def reset_registered_evaluators(self): # instantiate evaluators, once for all, only keep when it's enabled @@ -240,7 +250,7 @@ class SheerkaExecute(BaseService): # get default evaluators by process step for process_step in ALL_STEPS: - self.grouped_evaluators_cache[f"{process_step}|__default"] = self.get_grouped( + self.grouped_evaluators_cache[(process_step, DEFAULT)] = self.get_grouped( [e for e in self.instantiated_evaluators if process_step in e.steps]) def reset_registered_parsers(self): @@ -249,10 +259,20 @@ class SheerkaExecute(BaseService): :return: """ self.instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()] - self.instantiated_parsers = [p for p in self.instantiated_parsers if p.enabled] + self.instantiated_parsers = [p for p in self.instantiated_parsers if p.enabled and p.name != STM_PARSER_NAME] self.parsers_by_name = {p.short_name: p for p in self.instantiated_parsers} - self.grouped_parsers_cache["__default"] = self.get_grouped(self.instantiated_parsers, use_classes=True) + default_parsers = [p for p in self.instantiated_parsers if p.hints is None] + self.grouped_parsers_cache[(DEFAULT, DEFAULT)] = self.get_grouped(default_parsers, use_classes=True) + + # By default, we use the same parsers when it's a question + question_parsers = [p for p in self.instantiated_parsers if + p.hints is not None and BuiltinConcepts.EVAL_QUESTION_REQUESTED in p.hints] + self.grouped_parsers_cache[(BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT)] = self.get_grouped( + # default_parsers, + question_parsers, + use_classes=True) + self.question_parsers = [p.name for p in question_parsers] @staticmethod def get_grouped(evaluators, use_classes=False): @@ -302,12 +322,12 @@ class SheerkaExecute(BaseService): """ # Normal case, the evaluators are the default one if not context.preprocess_evaluators and not context.preprocess: - return self.grouped_evaluators_cache[f"{process_step}|__default"] + return self.grouped_evaluators_cache[(process_step, DEFAULT)] # Other case, only use a subset of evaluators selected = context.preprocess_evaluators if selected and not context.preprocess: - key = str(process_step) + "|" + "|".join(selected) + key = (process_step, "|".join(selected)) try: return self.grouped_evaluators_cache[key] except KeyError: @@ -357,6 +377,15 @@ class SheerkaExecute(BaseService): groups, sorted_priorities = self.get_grouped(parsers, use_classes=True) return key, *get_instances((groups, sorted_priorities)) + def get_input_as_text(self, text): + if isinstance(text, str): + return text + + if isinstance(text, ParserInput): + return text.as_text() + + raise NotImplementedError() + def get_parser_input(self, text, tokens=None): """ Returns new or existing parser input @@ -386,17 +415,44 @@ class SheerkaExecute(BaseService): """ From the context.preprocess_parsers and context.preprocess, try to find a key to store the further results of the parsings + The key is a two values tuple + * The first part indicates the parsers to use (__default for the hardcoded default behaviour) + * The second part indicates some context hint, like for instance if it's question :param context: :return: """ - if not context.preprocess_parsers and not context.preprocess: - return "__default" + # as of now, EVAL_QUESTION_REQUESTED is the only hint that can alter the parsing + if context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED): + in_context = BuiltinConcepts.EVAL_QUESTION_REQUESTED + else: + in_context = DEFAULT - if context.preprocess_parsers and not context.preprocess: - return "|".join(context.preprocess_parsers) + if context.preprocess: + from parsers.BaseParser import BaseParser + preprocess = [p for p in context.preprocess if p.preprocess_name.startswith(BaseParser.PREFIX)] + else: + preprocess = None + + if not context.preprocess_parsers and not preprocess: + return in_context, DEFAULT + + if context.preprocess_parsers and not preprocess: + return in_context, "|".join(context.preprocess_parsers) return None + def add_to_parser_cache(self, parsers_key, text, return_value): + if parsers_key is None: + return + + key = (parsers_key, text) + if key in self.parsers_cache: + old = self.parsers_cache.get(key) + old.append((return_value.who, return_value.status, return_value.value)) + self.parsers_cache.put(key, old) + else: + self.parsers_cache.put(key, [(return_value.who, return_value.status, return_value.value)]) + def call_parsers(self, context, return_values): """ Call all the parsers, ordered by priority @@ -414,7 +470,7 @@ class SheerkaExecute(BaseService): if not isinstance(return_values, list): return_values = [return_values] - # first make the distinguish between what is for the parsers and what is not + # 1. Make the distinguish between what is for the parsers and what is not result = [] to_process = [] for r in return_values: @@ -431,58 +487,110 @@ class SheerkaExecute(BaseService): parsers_key, grouped_parsers, sorted_priorities = self.get_parsers(context) - stop_processing = False - for priority in sorted_priorities: - inputs_for_this_group = to_process[:] + # 2. Try the stm parser, as it depends on the context + from parsers.ShortTermMemoryParser import ShortTermMemoryParser + if parsers_key is None or parsers_key[1] == DEFAULT or ShortTermMemoryParser.NAME in parsers_key[1]: + try: + stm_parser = self.parsers_by_name[STM_PARSER_NAME] + if stm_parser.enabled: + processed = [] + for return_value in to_process: + to_parse = self.get_parser_input(return_value.body.body) \ + if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \ + else return_value.body - for parser in grouped_parsers[priority]: - - for return_value in inputs_for_this_group: - - to_parse = self.get_parser_input(return_value.body.body) \ - if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \ - else return_value.body - - # if self.sheerka.log.isEnabledFor(logging.DEBUG): - # debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \ - # else "'" + core.utils.get_text_from_tokens(to_parse) + "' as tokens" - # context.log(f"Parsing {debug_text}") - - with context.push(BuiltinConcepts.PARSING, - {"parser": parser.name}, - desc=f"Parsing using {parser.name}") as sub_context: - sub_context.add_inputs(to_parse=to_parse) - res = parser.parse(sub_context, to_parse) - if res is not None: - if hasattr(res, "__iter__"): - for r in res: - if r is None: - continue - r.parents = [return_value] - result.append(r) - if self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT): - # if a ParserResultConcept is returned, it will be used by the parsers - # of the following groups - to_process.append(r) - if r.status: - stop_processing = True - - else: + with context.push(BuiltinConcepts.PARSING, + {"parser": stm_parser.name}, + desc=f"Parsing using {stm_parser.name}") as sub_context: + sub_context.add_inputs(to_parse=to_parse) + res = stm_parser.parse(sub_context, to_parse) + if res.status: res.parents = [return_value] result.append(res) - if self.sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT): - # if a ParserResultConcept is returned, it will be used by the parsers - # of the following groups - to_process.append(res) - if res.status: - stop_processing = True - sub_context.add_values(return_values=res) + processed.append(return_value) + sub_context.add_values(return_values=res) - if stop_processing: - break # Do not try the other priorities if a match is found + to_process = core.utils.remove_list_from_list(to_process, processed) + + except KeyError: + # stm_parser may not exist in some unit tests + pass + + # 3. Try the cache + if to_process and parsers_key: + processed = [] + for return_value in to_process: + to_parse_as_str = self.get_input_as_text(return_value.body.body) \ + if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \ + else return_value.body.source + + key_to_use = (parsers_key, to_parse_as_str) + if key_to_use in self.parsers_cache: + for who, status, value in self.parsers_cache.get(key_to_use): + ret = self.sheerka.ret(who, status, value) + ret.parents = [return_value] + result.append(ret) + processed.append(return_value) + to_process = core.utils.remove_list_from_list(to_process, processed) + + # 4. Call the parsers + if to_process: + stop_processing = False + for priority in sorted_priorities: + inputs_for_this_group = to_process[:] + + for parser in grouped_parsers[priority]: + + for return_value in inputs_for_this_group: + + if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT): + to_parse_as_str = self.get_input_as_text(return_value.body.body) + to_parse = self.get_parser_input(return_value.body.body) + else: + to_parse = return_value.body + to_parse_as_str = return_value.body.source + + # if self.sheerka.log.isEnabledFor(logging.DEBUG): + # debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \ + # else "'" + core.utils.get_text_from_tokens(to_parse) + "' as tokens" + # context.log(f"Parsing {debug_text}") + + with context.push(BuiltinConcepts.PARSING, + {"parser": parser.name}, + desc=f"Parsing using {parser.name}") as sub_context: + sub_context.add_inputs(to_parse=to_parse) + res = parser.parse(sub_context, to_parse) + if res is not None: + if hasattr(res, "__iter__"): + for r in res: + if r is None: + continue + r.parents = [return_value] + result.append(r) + self.add_to_parser_cache(parsers_key, to_parse_as_str, r) + if self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT): + # if a ParserResultConcept is returned, it will be used by the parsers + # of the following groups + to_process.append(r) + if r.status: + stop_processing = True + + else: + res.parents = [return_value] + result.append(res) + self.add_to_parser_cache(parsers_key, to_parse_as_str, res) + if self.sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT): + # if a ParserResultConcept is returned, it will be used by the parsers + # of the following groups + to_process.append(res) + if res.status: + stop_processing = True + sub_context.add_values(return_values=res) + + if stop_processing: + break # Do not try the other priorities if a match is found result = core.utils.remove_list_from_list(result, user_inputs) - return result def call_evaluators(self, context, return_values, process_step): @@ -687,7 +795,12 @@ class SheerkaExecute(BaseService): else: return parser_or_evaluator_name == preprocessor_name - def parse_unrecognized(self, context, source, parsers, who=None, prop=None, filter_func=None): + def parse_unrecognized(self, context, source, parsers, + who=None, + prop=None, + filter_func=None, + is_question=False, + possible_variables=None): """ Try to recognize concepts or code from source using the given parsers :param context: @@ -696,6 +809,8 @@ class SheerkaExecute(BaseService): :param who: who is asking the parsing ? :param prop: Extra info, when parsing a property :param filter_func: Once the result are found, call this function to filter them + :param is_question: Force EVAL_QUESTION_REQUESTED + :param possible_variables: concepts that must be considered as variables :return: """ sheerka = context.sheerka @@ -708,14 +823,18 @@ class SheerkaExecute(BaseService): desc = f"Parsing '{source}'" with context.push(BuiltinConcepts.PARSING, action_context, who=who, desc=desc) as sub_context: + + if (prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN) or + is_question): + sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + # disable all parsers but the requested ones if parsers != "all": sub_context.preprocess_parsers = parsers else: sub_context.preprocess_parsers = None - if prop in (Keywords.WHERE, Keywords.PRE, ConceptParts.WHERE, ConceptParts.PRE, Keywords.WHEN): - sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + sub_context.possible_variables = possible_variables sub_context.add_inputs(source=source) to_parse = sheerka.ret(context.who, @@ -779,7 +898,7 @@ class SheerkaExecute(BaseService): python_parser = PythonParser() return python_parser.parse(sub_context, parser_input) - def parse_expression(self, context, source, desc=None): + def parse_expression(self, context, source, desc=None, auto_compile=False): """ Helper function to parser expressions with AND, OR and NOT """ @@ -787,5 +906,8 @@ class SheerkaExecute(BaseService): with context.push(BuiltinConcepts.PARSE_CODE, source, desc) as sub_context: parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(source) from parsers.ExpressionParser import ExpressionParser - expr_parser = ExpressionParser() + expr_parser = ExpressionParser(auto_compile=auto_compile) return expr_parser.parse(sub_context, parser_input) + + def on_concepts_modified(self, *args, **kwargs): + self.parsers_cache.clear() diff --git a/src/core/sheerka/services/SheerkaIsAManager.py b/src/core/sheerka/services/SheerkaIsAManager.py index 54350a6..c149d63 100644 --- a/src/core/sheerka/services/SheerkaIsAManager.py +++ b/src/core/sheerka/services/SheerkaIsAManager.py @@ -1,12 +1,12 @@ import core.builtin_helpers from cache.Cache import Cache from cache.SetCache import SetCache -from core.ast_helpers import UnreferencedVariablesVisitor from core.builtin_concepts import BuiltinConcepts -from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF +from core.concept import Concept, DEFINITION_TYPE_BNF from core.global_symbols import NotFound from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from core.sheerka.services.sheerka_service import BaseService +from core.tokenizer import Tokenizer, TokenKind from core.utils import merge_sets @@ -153,16 +153,7 @@ class SheerkaIsAManager(BaseService): # apply the where clause if any if sub_concept.get_metadata().where: - new_condition = self._validate_where_clause(context, sub_concept) - if not new_condition: - return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=sub_concept) - - # This methods sucks, but I don't have enough tools (like proper AST manipulation functions) - # to do it properly now. It will be enhanced later - globals_ = {"xx__concepts__xx": concepts, "sheerka": self.sheerka} - locals_ = {} - exec(new_condition, globals_, locals_) - concepts = locals_["result"] + concepts = self._filter_results(context, sub_concept, concepts) return concepts @@ -201,7 +192,7 @@ class SheerkaIsAManager(BaseService): return False for c in a.get_metadata().props[BuiltinConcepts.ISA]: - if c == b: + if c.id == b.id: return True if self.isa(self.sheerka.get_by_id(c.id), b): return True @@ -237,28 +228,25 @@ class SheerkaIsAManager(BaseService): return self.isaset(context, concept.body) - def _validate_where_clause(self, context, concept): - python_parser_result = [r for r in concept.get_compiled()[ConceptParts.WHERE] if r.who == "parsers.Python"] - if not python_parser_result or not python_parser_result[0].status: - return None + def _filter_results(self, context, concept, results): + """ + Filter the list of results, according to the specification of the concept + ex: def concept sub_concept as number where number < 4 + We want to return the numbers that are < 4 + :param context: + :param concept: + :param results: + :return: + """ + # first get the pivot variable + possibles_variables = [t.value for t in Tokenizer(concept.get_metadata().body, yield_eof=False) if + t.type in (TokenKind.IDENTIFIER, TokenKind.KEYWORD)] + if len(possibles_variables) != 1: + return self.sheerka.new(BuiltinConcepts.CONDITION_FAILED, body=concept) - ast_ = python_parser_result[0].body.body.ast_ - visitor = UnreferencedVariablesVisitor(context) - names = list(visitor.get_names(ast_)) - if len(names) != 1 or names[0] != concept.get_metadata().body: - return None - - condition = concept.get_metadata().where.replace(concept.get_metadata().body, "sheerka.objvalue(x)") - expression = f""" -result=[] -for x in xx__concepts__xx: - try: - if {condition}: - result.append(x) - except Exception: - pass -""" - return expression + predicate = concept.get_metadata().where.replace(possibles_variables[0], "sheerka.objvalue(self)") + res = self.sheerka.filter_objects(context, results, predicate) + return res def _get_concepts(self, context, ids, evaluate): """ diff --git a/src/core/sheerka/services/SheerkaMemory.py b/src/core/sheerka/services/SheerkaMemory.py index 261e299..b5f270a 100644 --- a/src/core/sheerka/services/SheerkaMemory.py +++ b/src/core/sheerka/services/SheerkaMemory.py @@ -222,12 +222,14 @@ class SheerkaMemory(BaseService): """ name_to_use = name.name if isinstance(name, Concept) else name self.unregister_object(context, name_to_use) + + # first try direct access obj = self.get_last_from_memory(context, name_to_use) if obj is not NotFound: return obj.obj - all_objects = self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY) - all_objects_copy = [] + all_objects = self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY) # not always a list of list + all_objects_copy = [] # to transform into list of list for obj in all_objects: if isinstance(obj, list): all_objects_copy.append(obj.copy()) @@ -242,15 +244,15 @@ class SheerkaMemory(BaseService): if len(obj) > 0: temp.append(obj) - all_objects_copy = temp + all_objects_copy = temp # list constructed with the last item of each item current_list = sorted(current_list, key=lambda o: o.timestamp, reverse=True) current_objects = [o.obj for o in current_list] - res = self.sheerka.filter_objects(context, current_objects, name) + res = self.sheerka.filter_objects(context, current_objects, name_to_use) if len(res) > 0: return res[0] # only the first, as it should have a better timestamp - return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name}) + return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"#name": name_to_use}) def mem(self): keys = sorted([k for k in self.sheerka.om.list(SheerkaMemory.OBJECTS_ENTRY)]) diff --git a/src/core/sheerka/services/SheerkaQueryManager.py b/src/core/sheerka/services/SheerkaQueryManager.py index 6b5205e..dadf891 100644 --- a/src/core/sheerka/services/SheerkaQueryManager.py +++ b/src/core/sheerka/services/SheerkaQueryManager.py @@ -19,7 +19,7 @@ class SheerkaQueryManager(BaseService): QUERY_PARAMETER_PREFIX = "__xxx__query_parameter__xx__" def __init__(self, sheerka): - super().__init__(sheerka) + super().__init__(sheerka, order=16) self.queries = FastCache() self.conditions = FastCache() self.lexer = Lexer() diff --git a/src/core/sheerka/services/SheerkaRuleManager.py b/src/core/sheerka/services/SheerkaRuleManager.py index f12e173..cd505d2 100644 --- a/src/core/sheerka/services/SheerkaRuleManager.py +++ b/src/core/sheerka/services/SheerkaRuleManager.py @@ -1,610 +1,38 @@ import operator -import re from dataclasses import dataclass from typing import Union, Set, List, Tuple from cache.Cache import Cache from cache.ListIfNeededCache import ListIfNeededCache +from core.ast_helpers import UnreferencedNamesVisitor from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept from core.builtin_helpers import ensure_evaluated, expect_one, evaluate_from_source, \ - get_possible_variables_from_concept, is_a_question + get_possible_variables_from_concept, is_a_question, only_successful, is_only_successful, evaluate_return_values from core.concept import Concept from core.global_symbols import EVENT_RULE_PRECEDENCE_MODIFIED, RULE_COMPARISON_CONTEXT, NotFound, ErrorObj, \ - EVENT_RULE_CREATED, EVENT_RULE_DELETED, NotInit + EVENT_RULE_CREATED, EVENT_RULE_DELETED, NotInit, INIT_AST_PARSERS from core.rule import Rule, ACTION_TYPE_PRINT from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID -from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError -from core.tokenizer import Keywords, TokenKind, Token, IterParser -from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets, get_safe_str_value +from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, UnknownVariableError +from core.tokenizer import Keywords, TokenKind, Token +from core.utils import merge_dictionaries, merge_sets, get_safe_str_value from evaluators.PythonEvaluator import PythonEvaluator from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode, \ ComparisonType, NotNode, NameExprNode from parsers.BaseNodeParser import ConceptNode +from parsers.FormatRuleActionParser import FormatRuleActionParser from parsers.LogicalOperatorParser import LogicalOperatorParser from sheerkapython.python_wrapper import Expando, sheerka_globals from sheerkarete.common import V from sheerkarete.conditions import AndConditions, Condition, NegatedCondition, NegatedConjunctiveConditions from sheerkarete.network import FACT_NAME, FACT_SELF -CONCEPTS_ONLY_PARSERS = ["ExactConcept", "Bnf", "Sya", "Sequence"] -CONDITIONS_VISITOR_EVALUATORS = ["Python", "Concept", "LexerNode"] - -identifier_regex = re.compile(r"[\w _.]+") - - -@dataclass -class FormatRuleError(ErrorObj): - pass - - -@dataclass -class BraceMismatch(FormatRuleError): - lbrace: Token - - -@dataclass -class UnexpectedEof(FormatRuleError): - message: str - token: Token = None - - def __eq__(self, other): - if id(self) == id(other): - return True - - if not isinstance(other, UnexpectedEof): - return False - - return self.message == other.message and (other.token is None or other.token == self.token) - - def __hash__(self): - return hash(self.message, self.token) - - -@dataclass -class FormatRuleSyntaxError(FormatRuleError): - message: str - token: Token - - -@dataclass -class FormatAstNode: - @staticmethod - def repr_value(items): - if items is None: - return "" - - return ", ".join(repr(item) for item in items) - - def clone(self, instance, props, **kwargs): - for prop_name in props: - setattr(instance, prop_name, getattr(self, prop_name)) - - for k, v in kwargs.items(): - setattr(instance, k, v) - - return instance - - -@dataclass -class FormatAstRawText(FormatAstNode): - text: str - - -@dataclass -class FormatAstVariable(FormatAstNode): - name: str - format: Union[str, None] = None - debug: bool = False - value: object = None - index: object = None - - def clone(self, **kwargs): - return super().clone(FormatAstVariable(self.name), - ("format", "debug", "value", "index"), - **kwargs) - - -@dataclass -class FormatAstVariableNotFound(FormatAstNode): - name: str - - -@dataclass -class FormatAstGrid(FormatAstNode): - pass - - -@dataclass -class FormatAstList(FormatAstNode): - variable: str - items_prop: str = None # where to search the list if variable does not resolve to an iterable - recurse_on: str = None - recursion_depth: int = 0 - debug: bool = False - prefix: str = None - suffix: str = None - show_index: bool = False - index: object = None - - items: object = None - - def clone(self, **kwargs): - return super().clone( - FormatAstList(self.variable), - ( - "items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "index", - "items"), - **kwargs) - - -@dataclass -class FormatAstDict(FormatAstNode): - variable: str - items_prop: str = None # where to search the dict if variable does not resolve to an iterable - debug: bool = False - prefix: str = None - suffix: str = None - - items: object = None - - def clone(self, **kwargs): - return super().clone( - FormatAstDict(self.variable), - ("items_prop", "debug", "prefix", "suffix", "items"), - **kwargs) - - -@dataclass -class FormatAstColor(FormatAstNode): - color: str - format_ast: FormatAstNode - - def __repr__(self): - return f"{self.color}({self.format_ast})" - - def clone(self, **kwargs): - return super().clone( - FormatAstColor(self.color, self.format_ast), - (), - **kwargs) - - -@dataclass -class FormatAstFunction(FormatAstNode): - name: str - args: list = None - kwargs: dict = None - - -@dataclass -class FormatAstSequence(FormatAstNode): - items: list - debug: bool = False - - def __repr__(self): - return "FormatAstSequence(" + self.repr_value(self.items) + ")" - - def clone(self, **kwargs): - return super().clone( - FormatAstSequence(self.items), - ("debug",), - **kwargs) - - -@dataclass -class FormatAstMulti(FormatAstNode): - """ - Used when there are multiple out to print, but they are not related - Just print them one by one - """ - variable: str - items: list = None - - def __repr__(self): - return f"FormatAstMulti({self.variable}, items={self.items})" - - def clone(self, **kwargs): - return super().clone( - FormatAstMulti(self.variable), - ("items",), - **kwargs) - - -class FormatRuleActionParser(IterParser): - - @staticmethod - def to_text(list_or_dict_of_tokens): - """ - Works on list of list of tokens - or dict of list of tokens - :param list_or_dict_of_tokens: - :return: - """ - get_text = get_text_from_tokens - if isinstance(list_or_dict_of_tokens, list): - return [get_text(i) for i in list_or_dict_of_tokens] - if isinstance(list_or_dict_of_tokens, dict): - return {k: get_text(v) for k, v in list_or_dict_of_tokens.items()} - raise NotImplementedError("") - - def to_value(self, tokens): - """ - Works on list of tokens - return string or numeric value of the tokens - :return: - """ - - value = get_text_from_tokens(tokens) - if value[0] in ("'", '"'): - return value[1:-1] - - if value in ("True", "False"): - return bool(value) - - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - self.error_sink = FormatRuleSyntaxError(f"'{value}' is not numeric", None) - - def parse(self): - """ - Parses the print part of the format rule - format ::= {variable'} | function(...) | rawtext - :return: - """ - - if self.source == "": - return FormatAstRawText("") - - buffer = [] - result = [] - res = None - escaped = False - - def _flush_buffer(): - if len(buffer) > 0: - result.append(FormatAstRawText(get_text_from_tokens(buffer))) - buffer.clear() - - while self.next_token(skip_whitespace=False): - if not escaped: - if self.token.type == TokenKind.IDENTIFIER and self.the_token_after().type == TokenKind.LPAR: - _flush_buffer() - res = self.parse_function(self.token) - elif self.token.type == TokenKind.LBRACE: - _flush_buffer() - res = self.parse_variable(self.token) - elif self.token.type == TokenKind.BACK_SLASH: - escaped = True - else: - buffer.append(self.token) - else: - escaped = False - buffer.append(self.token) - - if self.error_sink: - break - - if res: - result.append(res) - res = None - - _flush_buffer() - - return [] if len(result) == 0 else result[0] if len(result) == 1 else FormatAstSequence(result) - - def parse_function(self, func_name): - self.next_token() - self.next_token() - - if self.token.type == TokenKind.EOF: - self.error_sink = UnexpectedEof("while parsing function", func_name) - return None - - param_buffer = [] - args = [] - kwargs = {} - get_text = get_text_from_tokens - - def _process_parameters(): - if len(param_buffer) == 0: - self.error_sink = FormatRuleSyntaxError("no parameter found", self.token) - return None - if (index := index_tokens(param_buffer, "=")) > 0: - kwargs[get_text(param_buffer[:index])] = param_buffer[index + 1:] - else: - args.append(param_buffer.copy()) - param_buffer.clear() - - while True: - if self.token.type == TokenKind.RPAR: - if len(param_buffer) > 0: - _process_parameters() - break - - elif self.token.type == TokenKind.COMMA: - _process_parameters() - if self.error_sink: - break - - else: - param_buffer.append(self.token) - - if not self.next_token(): - break - - if self.error_sink: - return None - - if self.token.type != TokenKind.RPAR: - self.error_sink = UnexpectedEof("while parsing function", func_name) - return None - - if func_name.value in COLORS: - return self.return_color(func_name.value, args, kwargs) - elif func_name.value == "list": - return self.return_list(args, kwargs) - elif func_name.value == "dict": - return self.return_dict(args, kwargs) - elif func_name.value == "multi": - return self.return_multi(args, kwargs) - - return FormatAstFunction(func_name.value, self.to_text(args), self.to_text(kwargs)) - - def parse_variable(self, lbrace): - self.next_token() - - if self.token.type == TokenKind.EOF: - self.error_sink = UnexpectedEof("while parsing variable", lbrace) - return None - - buffer = [] - while True: - if self.token.type == TokenKind.RBRACE: - break - buffer.append(self.token) - - if not self.next_token(): - break - - # if self.error_sink: - # return None - - if self.token.type != TokenKind.RBRACE: - self.error_sink = UnexpectedEof("while parsing variable", lbrace) - return None - - if len(buffer) == 0: - self.error_sink = FormatRuleSyntaxError("variable name not found", None) - return None - - variable = get_text_from_tokens(buffer) - try: - index = variable.index(":") - return FormatAstVariable(variable[:index], variable[index + 1:]) - except ValueError: - return FormatAstVariable(variable) - - def return_color(self, color, args, kwargs): - if len(kwargs) > 0: - self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None) - return None - - if len(args) == 0: - return FormatAstColor(color, FormatAstRawText("")) - - if len(args) > 1: - self.error_sink = FormatRuleSyntaxError("only one parameter supported", args[1][0]) - return None - - source = get_text_from_tokens(args[0]) - if len(source) > 1 and source[0] in ("'", '"') and source[-1] in ("'", '"'): - source = source[1:-1] - parser = FormatRuleActionParser(source) - res = parser.parse() - self.error_sink = parser.error_sink - return FormatAstColor(color, res) - else: - try: - index = source.index(":") - variable, vformat = source[:index], source[index + 1:] - except ValueError: - variable, vformat = source, None - - if not identifier_regex.fullmatch(variable): - self.error_sink = FormatRuleSyntaxError("Invalid identifier", None) - return None - return FormatAstColor(color, FormatAstVariable(variable, vformat)) - - def return_list(self, args, kwargs): - """ - Looking for greeting_var, [recurse_on], [recursion_depth], [items_prop] - :param args: - :param kwargs: - :return: - """ - len_args = len(args) - if len_args < 1: - self.error_sink = FormatRuleSyntaxError("variable name not found", None) - return None - - if len_args > 4: - self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[4][0]) - return None - - variable_name = get_text_from_tokens(args[0]) - recurse_on, recursion_depth, items_prop = None, 0, None - - if len_args == 2: - recursion_depth = self.to_value(args[1]) - elif len_args == 3: - recursion_depth = self.to_value(args[1]) - recurse_on = self.to_value(args[2]) - elif len_args == 4: - recursion_depth = self.to_value(args[1]) - recurse_on = self.to_value(args[2]) - items_prop = self.to_value(args[3]) - - if "recurse_on" in kwargs: - recurse_on = self.to_value(kwargs["recurse_on"]) - - if "recursion_depth" in kwargs: - recursion_depth = self.to_value(kwargs["recursion_depth"]) - - if "items_prop" in kwargs: - items_prop = self.to_value(kwargs["items_prop"]) - - if self.error_sink: - return None - - if not isinstance(recursion_depth, int): - self.error_sink = FormatRuleSyntaxError("'recursion_depth' must be an integer", None) - return None - - return FormatAstList(variable_name, items_prop, recurse_on, recursion_depth) - - def return_dict(self, args, kwargs): - len_args = len(args) - if len_args < 1: - self.error_sink = FormatRuleSyntaxError("variable name not found", None) - return None - - if len_args > 1: - self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0]) - return None - - variable_name = get_text_from_tokens(args[0]) - - kwargs_parameters = {} - for prop in ("items_prop", "prefix", "suffix", "debug"): - if prop in kwargs: - kwargs_parameters[prop] = self.to_value(kwargs[prop]) - - if "debug" in kwargs_parameters: - if "prefix" not in kwargs_parameters: - kwargs_parameters["prefix"] = "{" - if "suffix" not in kwargs_parameters: - kwargs_parameters["suffix"] = "}" - - if self.error_sink: - return None - - return FormatAstDict(variable_name, **kwargs_parameters) - - def return_multi(self, args, kwargs): - if len(kwargs) > 0: - self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None) - return None - - if len(args) > 1: - self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0]) - return None - - return FormatAstMulti(get_text_from_tokens(args[0])) - @dataclass class EmitPythonCodeException(Exception): error: object -class PythonCodeEmitter: - - def __init__(self, context, text=None): - self.context = context - self.text = text or "" - self.var_counter = 0 - self.variables = [] - - def add(self, text): - self.text += f" and {text}" if self.text else text - return self - - def recognize(self, obj, as_name, root=True): - if isinstance(obj, str): - return self.recognize_str(obj, as_name) - elif isinstance(obj, (int, float)): - return self.recognize_int(obj, as_name) - elif isinstance(obj, Concept): - return self.recognize_concept(obj, as_name, root) - elif isinstance(obj, Expando): - return self.recognize_expando(obj, as_name, root) - else: - raise NotImplementedError() - - def recognize_str(self, text, as_name): - if self.text: - self.text += " and " - - if "'" in text and '"' in text: - self.text += f"{as_name} == '{text}'" - elif "'" in text: - self.text += f'{as_name} == "{text}"' - else: - self.text += f"{as_name} == '{text}'" - return self - - def recognize_int(self, value, as_name): - if self.text: - self.text += " and " - - self.text += f"{as_name} == {value}" - return self - - def recognize_expando(self, value, as_name, root=True): - if self.text: - self.text += " and " - - if not root: - as_name = self.add_variable(as_name) - - self.text += f"isinstance({as_name}, Expando) and {as_name}.get_name() == '{value.get_name()}'" - return self - - def recognize_concept(self, concept, as_name, root=True): - if self.text: - self.text += " and " - - if not root: - as_name = self.add_variable(as_name) - - if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME: - self.text += f"isinstance({as_name}, Concept) and {as_name}.name == '{concept.name}'" - elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID: - self.text += f"isinstance({as_name}, Concept) and {as_name}.id == '{concept.id}'" - else: - self.text += f"isinstance({as_name}, Concept) and {as_name}.key == '{concept.key}'" - if len(concept.get_metadata().variables) > 0: - # add variables constraints - evaluated = ensure_evaluated(self.context, concept, eval_body=False, metadata=["variables"]) - - if not self.context.sheerka.is_success(evaluated) and evaluated.key != concept.key: - raise EmitPythonCodeException(evaluated) - - for k, v in concept.variables().items(): - self.recognize(v, f"{as_name}.get_value('{k}')", root=False) - - return self - - def add_variable(self, target): - var_name = f"__x_{self.var_counter:02}__" - self.var_counter += 1 - self.variables.append((var_name, target)) - return var_name - - def get_text(self): - if self.variables: - variables_as_str = '\n'.join([f"{k} = {v}" for k, v in self.variables]) - return variables_as_str + "\n" + self.text - - return self.text - - class NoConditionFound(ErrorObj): def __eq__(self, other): return isinstance(other, NoConditionFound) @@ -1149,7 +577,6 @@ class GetConditionExprVisitor(ExpressionVisitor): def evaluate_from_source(self, source, is_question=False, return_body=False): res = evaluate_from_source(self.context, source, - evaluators=CONDITIONS_VISITOR_EVALUATORS, desc=None, eval_body=not is_question, eval_where=False, @@ -1160,6 +587,9 @@ class GetConditionExprVisitor(ExpressionVisitor): if return_body: if not res.status: + python_eval_error = self.context.sheerka.get_errors(self.context, res, __type="PythonEvalError") + if python_eval_error and isinstance(python_eval_error[0].error, NameError): + raise UnknownVariableError(python_eval_error[0].source) raise FailedToCompileError(res.body) return res.body @@ -1268,7 +698,11 @@ class ReteConditionExprVisitor(GetConditionExprVisitor): def visit_ComparisonNode(self, expr_node: ComparisonNode): if isinstance(expr_node.left, VariableNode): conditions = [] - value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True) + try: + value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True) + except UnknownVariableError: + value = expr_node.right.unpack() + self.add_to_condition(expr_node.left.unpack(), value, conditions) return conditions else: @@ -1334,7 +768,6 @@ class ReteConditionExprVisitor(GetConditionExprVisitor): def visit_NameExprNode(self, expr_node: NameExprNode): res = evaluate_from_source(self.context, expr_node.get_source(), - evaluators=CONDITIONS_VISITOR_EVALUATORS, desc=None, eval_body=True, eval_where=False, @@ -1367,9 +800,9 @@ class ReteConditionExprVisitor(GetConditionExprVisitor): variable = self.init_or_get_variable_from_attr(variable_path, conditions) conditions.append(Condition(variable, "__is_concept__", True)) - if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME: + if concept.get_hints().recognized_by == RECOGNIZED_BY_NAME: conditions.append(Condition(variable, "name", concept.name)) - elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID: + elif concept.get_hints().recognized_by == RECOGNIZED_BY_ID: conditions.append(Condition(variable, "id", concept.id)) else: conditions.append(Condition(variable, "key", concept.key)) @@ -1392,6 +825,9 @@ class ReteConditionExprVisitor(GetConditionExprVisitor): elif isinstance(value, Concept): res = self.recognize_concept(var_path, value, {}) conditions.extend(res) + elif isinstance(value, list): + var_root, var_attr = self.init_or_get_variable_from_name(value, conditions) + conditions.append(Condition(left, attr, var_root)) else: conditions.append(Condition(left, attr, value)) @@ -1483,12 +919,25 @@ class PythonConditionExprVisitorObj: node.variables, node.not_variables) + @staticmethod + def create_condition(text, op, left, right): + def get_source(a, b): + if op == ComparisonType.EQUALS and b == "sheerka": + return f"is_sheerka({a})" + else: + return ComparisonNode.rebuild_source(a, op, b) + + return PythonConditionExprVisitorObj(text, + get_source(left.source, right.source), + merge_dictionaries(left.objects, right.objects), + merge_sets(left.variables, right.variables), + merge_sets(left.not_variables, right.not_variables)) + class PythonConditionExprVisitor(GetConditionExprVisitor): def __init__(self, context): super().__init__(context) - self.know_object_variables = {} self.check_variable_existence_only = True self.concepts_to_reset = set() @@ -1537,7 +986,8 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): return var_name def unpack_variable(self, variable_path: List[str], obj_variables): - obj_variables.add(variable_path[0]) + if self.is_a_possible_variable(variable_path[0]): + obj_variables.add(variable_path[0]) return self.inner_unpack_variable(variable_path) @staticmethod @@ -1551,49 +1001,49 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): # try to recognize a concept res = self.evaluate_from_source(expr_node.name, is_question=True) if res.status and isinstance(res.value, Concept): + if self.context.possible_variables and res.value.name in self.context.possible_variables: + variable_name = expr_node.get_source() + return PythonConditionExprVisitorObj(variable_name, variable_name, {}, {variable_name}, set()) + + # else / otherwise self.check_variable_existence_only = False - if is_a_question(self.context, res.value): - return self.evaluate_concept_as_question(expr_node.name, res.value) - else: - return self.evaluate_concept(expr_node.name, res.value) + return self.manage_concept(expr_node.get_source(), res.value) else: if self.context.sheerka.has_error(self.context, res, __type=BuiltinConcepts.TOO_MANY_SUCCESS): raise FailedToCompileError([res]) variable_name = expr_node.get_source() - variables = {variable_name} if not res.status else set() + if res.status: + variables = set() + self.check_variable_existence_only = False + else: + variables = {variable_name} return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables, set()) - variable_name = expr_node.get_source() - variables_detected = {variable_name} if self.is_a_possible_variable(variable_name) else set() - return PythonConditionExprVisitorObj(variable_name, variable_name, {}, variables_detected, set()) + if self.check_variable_existence_only: + # special case where we want to check the existence of the whole string + variable_name = expr_node.get_source() + possible_variables = {variable_name} if self.is_a_possible_variable(variable_name) else set() + else: + # try to detect the variable + possible_variables = set() + var_root, var_attr = self.unpack_variable(expr_node.unpack(), possible_variables) + variable_name = self.construct_variable(var_root, var_attr) + return PythonConditionExprVisitorObj(variable_name, variable_name, {}, possible_variables, set()) def visit_ComparisonNode(self, expr_node: ComparisonNode): self.check_variable_existence_only = False - - if not isinstance(expr_node.left, VariableNode): - # KSI 2021-04-22. Not quite sure of the reason why I have this piece of code - left = self.visit(expr_node.left) - source = expr_node.get_source() - return PythonConditionExprVisitorObj(source, source, {}, left.variables, left.not_variables) - + left = self.visit(expr_node.left) right = self.visit(expr_node.right) - if right.source in right.objects: - return self.create_comparison_condition(expr_node.left.unpack(), - expr_node.comp, - right.source, - right.objects[right.source], - right.objects, - expr_node.get_source()) + + # special case when we call recognize concept when an equality with a concept is found + right_value = right.objects[right.source] if right.source in right.objects else None + if expr_node.comp == ComparisonType.EQUALS and isinstance(right_value, Concept): + res = self.recognize_concept(expr_node.left.unpack(), right_value, {}, expr_node.get_source()) else: - value = self.evaluate_from_source(expr_node.right.get_source(), return_body=True) - object_name, objects = self.get_object_name(value) - return self.create_comparison_condition(expr_node.left.unpack(), - expr_node.comp, - object_name, - value, - objects, - expr_node.get_source()) + res = PythonConditionExprVisitorObj.create_condition(expr_node.get_source(), expr_node.comp, left, right) + + return res def visit_AndNode(self, expr_node: AndNode): current_visitor_obj = self.visit(expr_node.parts[0]) @@ -1636,13 +1086,43 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): def visit_NameExprNode(self, expr_node: NameExprNode): self.check_variable_existence_only = False source = expr_node.get_source() - res = self.evaluate_from_source(source, is_question=False) - if res.status: - obj_name, objects = self.get_object_name(res.value) - return PythonConditionExprVisitorObj(source, obj_name, objects, set(), set()) - else: + res = self.context.sheerka.parse_unrecognized(self.context, + source, + INIT_AST_PARSERS, + filter_func=only_successful, + is_question=True) + if not res.status: raise FailedToCompileError([expr_node]) + return_values = res.body.body if is_only_successful(self.context.sheerka, res) else [res] + # if the result is a concept, create an object for it, otherwise just leave the source as it is + res = evaluate_return_values(self.context, source, return_values, is_question=True) + if res.status: + if isinstance(res.value, Concept): + return self.manage_concept(expr_node.get_source(), res.value) + else: + obj_name, objects = self.get_object_name(res.value) + return PythonConditionExprVisitorObj(source, obj_name, objects, set(), set()) + + else: + if len(return_values) == 1: + body = return_values[0].body.body + if hasattr(body, "get_python_node"): + python_node = return_values[0].body.body.get_python_node() + unreferenced_names_visitor = UnreferencedNamesVisitor(self.context) + variables = unreferenced_names_visitor.get_names(python_node.ast_) + variables = variables - set(python_node.objects.keys()) + return PythonConditionExprVisitorObj(python_node.original_source, + python_node.source, + python_node.objects, + variables, + set()) + elif isinstance(body, Concept): + return self.manage_concept(expr_node.get_source(), body) + + else: + raise FailedToCompileError([expr_node]) + def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict, original_source=None): if not isinstance(concept_to_recognize, Concept): concept_as_str = concept_to_recognize.get_source() @@ -1659,9 +1139,9 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): source = f"isinstance({var_name}, Concept)" - if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME: + if concept.get_hints().recognized_by == RECOGNIZED_BY_NAME: source += f" and {var_name}.name == '{concept.name}'" - elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID: + elif concept.get_hints().recognized_by == RECOGNIZED_BY_ID: source += f" and {var_name}.id == '{concept.id}'" else: source += f" and {var_name}.key == '{concept.key}'" @@ -1676,17 +1156,24 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): ComparisonType.EQUALS, obj_name, var_value, - objects) + objects, + set()) source += " and " + variable_condition.source text += " and " + variable_condition.text return PythonConditionExprVisitorObj(original_source or text, source, objects, obj_variables, set()) + def manage_concept(self, source, concept): + if is_a_question(self.context, concept): + return self.evaluate_concept_as_question(source, concept) + else: + return self.evaluate_concept(source, concept) + def evaluate_concept_as_question(self, original_text, concept): concept_var_name, objects = self.get_object_name(concept) source = f"evaluate_question({concept_var_name})" variables = get_possible_variables_from_concept(self.context, concept) - self.concepts_to_reset.add(concept) + self.concepts_to_reset.update(self.get_concepts_to_reset(concept)) return PythonConditionExprVisitorObj(original_text, source, objects, variables, set()) def evaluate_concept(self, original_text, concept): @@ -1694,13 +1181,62 @@ class PythonConditionExprVisitor(GetConditionExprVisitor): source, objects = self.get_object_name(concept) return PythonConditionExprVisitorObj(original_text, source, objects, set(), set()) - def create_comparison_condition(self, left_path, op, right_name, right_value, objects, original_source=None): - possible_variables = set() + def get_concepts_to_reset(self, concept): + """ + Returns all the concept that might be reset before a second evaluation + The algo is empirical, there must be a theory to make sure that no concept is missed + :param concept: + :return: + """ + + res = set() + + def _inner_get_concept_to_reset(_concept): + if _concept in res: # prevent circular references + return + + res.add(_concept) + assert not _concept.get_hints().use_copy + for part_name, asts in _concept.get_compiled().items(): + if isinstance(asts, Concept): + if is_a_question(self.context, asts): + res.update(self.get_concepts_to_reset(asts)) + else: + for ret_val in asts: + body_as_list = ret_val.body.body # go through the ParserResult + if not isinstance(body_as_list, list): + body_as_list = [body_as_list] + + for body in body_as_list: + + if hasattr(body, "get_concept"): # to manage Concept and ConceptNode + c = body.get_concept() + if is_a_question(self.context, c): + res.update(self.get_concepts_to_reset(c)) + + elif hasattr(body, "get_python_node"): # to manage PythonNode and SourceCodeNode like + python_node = body.get_python_node() + for obj in python_node.objects.values(): + if isinstance(obj, Concept) and is_a_question(self.context, obj): + res.update(self.get_concepts_to_reset(obj)) + + _inner_get_concept_to_reset(concept) + return res + + def create_comparison_condition(self, + left_path, + op, + right_name, + right_value, + objects, + variables, + original_source=None): + possible_variables = variables.copy() var_root, var_attr = self.unpack_variable(left_path, possible_variables) left = self.construct_variable(var_root, var_attr) if original_source is None: - right = get_safe_str_value(right_value) + right = get_safe_str_value(right_value or right_name) original_source = ComparisonNode.rebuild_source(left, op, right) if op == ComparisonType.EQUALS: diff --git a/src/core/sheerka/services/sheerka_service.py b/src/core/sheerka/services/sheerka_service.py index c620792..e9b6a28 100644 --- a/src/core/sheerka/services/sheerka_service.py +++ b/src/core/sheerka/services/sheerka_service.py @@ -1,5 +1,6 @@ from dataclasses import dataclass +from core.concept import Concept from core.global_symbols import NotFound, ErrorObj from core.utils import sheerka_deepcopy @@ -52,3 +53,13 @@ class BaseService: @dataclass() class FailedToCompileError(Exception, ErrorObj): cause: list + + +@dataclass() +class UnknownVariableError(Exception, ErrorObj): + variable: str + + +@dataclass +class ChickenAndEggException(Exception, ErrorObj): + error: Concept diff --git a/src/evaluators/ConceptEvaluator.py b/src/evaluators/ConceptEvaluator.py index 95ec990..89e4693 100644 --- a/src/evaluators/ConceptEvaluator.py +++ b/src/evaluators/ConceptEvaluator.py @@ -51,6 +51,10 @@ class ConceptEvaluator(OneReturnValueEvaluator): return sheerka.ret(self.name, True, value, parents=[return_value]) + if concept.get_hints().use_copy: + concept = concept.copy() + concept.get_hints().use_copy = False + evaluated = sheerka.evaluate_concept(context, concept) if not sheerka.is_success(evaluated) and evaluated.key != concept.key: @@ -67,4 +71,3 @@ class ConceptEvaluator(OneReturnValueEvaluator): return sheerka.ret(self.name, True, evaluated.body, parents=[return_value]) else: return sheerka.ret(self.name, True, evaluated, parents=[return_value]) - diff --git a/src/evaluators/DefConceptEvaluator.py b/src/evaluators/DefConceptEvaluator.py index 8a016a5..b6b221b 100644 --- a/src/evaluators/DefConceptEvaluator.py +++ b/src/evaluators/DefConceptEvaluator.py @@ -8,6 +8,7 @@ from core.global_symbols import NotInit from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind, Tokenizer from evaluators.BaseEvaluator import OneReturnValueEvaluator +from parsers.BaseExpressionParser import ExprNode from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor from parsers.DefConceptParser import DefConceptNode, NameNode from parsers.PythonParser import get_python_node @@ -255,6 +256,15 @@ class DefConceptEvaluator(OneReturnValueEvaluator): debugger.debug_var("names", visitor.variables, hint="from BNF") return visitor.variables + # + # case of ExprNode + # + if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, ExprNode): + variables = set() + for compiled in ret_value.value.value.compiled: + variables.update(compiled.variables) + return [PossibleVariable(v) for v in variables] + # # Case of python code # diff --git a/src/evaluators/ExpressionEvaluator.py b/src/evaluators/ExpressionEvaluator.py new file mode 100644 index 0000000..0708dfd --- /dev/null +++ b/src/evaluators/ExpressionEvaluator.py @@ -0,0 +1,56 @@ +from core.builtin_concepts import ParserResultConcept +from core.builtin_concepts_ids import BuiltinConcepts +from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules +from evaluators.BaseEvaluator import OneReturnValueEvaluator +from parsers.BaseExpressionParser import ExprNode +from sheerkapython.python_wrapper import create_namespace + + +class ExpressionEvaluator(OneReturnValueEvaluator): + NAME = "Expression" + + def __init__(self): + super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50) + + def matches(self, context, return_value): + if not return_value.status: + return False + + if not isinstance(return_value.value, ParserResultConcept): + return False + + return isinstance(return_value.value.body, ExprNode) + + def eval(self, context, return_value): + sheerka = context.sheerka + conditions = return_value.value.value.compiled + rule_evaluator = sheerka.services[SheerkaEvaluateRules.NAME] + + errors = [] + success = False + + with context.push(BuiltinConcepts.EXEC_CODE, return_value.value.value.source) as sub_context: + # sub condition is created only to add a namespace + + requested_vars = set() + for c in conditions: + requested_vars.update(c.variables) + namespace = create_namespace(context, self.NAME, requested_vars, set(), {}, True, False) + # TODO: ADD NAMESPACE TO STM + missing_vars = set() + results = rule_evaluator.evaluate_conditions(sub_context, conditions, namespace, missing_vars) + + if not results and missing_vars: + errors = [NameError(v) for v in missing_vars] + + for res in results: + if not res.status: + errors.append(res.body) + value = context.sheerka.objvalue(res.body) + if isinstance(value, bool) and value: + success = True + + if errors: + return sheerka.ret(self.name, False, sheerka.err(errors)) + + return sheerka.ret(self.name, True, success) diff --git a/src/evaluators/MultipleErrorsEvaluator.py b/src/evaluators/MultipleErrorsEvaluator.py index 9c2a566..42e3abe 100644 --- a/src/evaluators/MultipleErrorsEvaluator.py +++ b/src/evaluators/MultipleErrorsEvaluator.py @@ -52,4 +52,4 @@ class MultipleErrorsEvaluator(AllReturnValuesEvaluator): self.name, False, sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=self.return_values_in_error.copy()), - parents=self.eaten) + parents=self.eaten.copy()) diff --git a/src/evaluators/MultipleOutEvaluator.py b/src/evaluators/MultipleOutEvaluator.py index 195ed09..cccf02c 100644 --- a/src/evaluators/MultipleOutEvaluator.py +++ b/src/evaluators/MultipleOutEvaluator.py @@ -41,4 +41,4 @@ class MultipleOutEvaluator(AllReturnValuesEvaluator): def eval(self, context, return_values): to_multi = context.sheerka.new(BuiltinConcepts.TO_MULTI, body=[r.body for r in self.success]) - return context.sheerka.ret(self.name, True, to_multi, parents=self.eaten) + return context.sheerka.ret(self.name, True, to_multi, parents=self.eaten.copy()) diff --git a/src/evaluators/MultipleSuccessEvaluator.py b/src/evaluators/MultipleSuccessEvaluator.py index b3d5837..5a34029 100644 --- a/src/evaluators/MultipleSuccessEvaluator.py +++ b/src/evaluators/MultipleSuccessEvaluator.py @@ -81,7 +81,7 @@ class MultipleSuccessEvaluator(AllReturnValuesEvaluator): self.name, True, sheerka.new(BuiltinConcepts.MULTIPLE_SUCCESS, body=self.successful_return_values.copy()), - parents=self.eaten) + parents=self.eaten.copy()) def already_seen(self, context, ret_val): for successful in self.successful_return_values: diff --git a/src/evaluators/MutipleSameSuccessEvaluator.py b/src/evaluators/MutipleSameSuccessEvaluator.py index 8cec1ff..2ee9fd9 100644 --- a/src/evaluators/MutipleSameSuccessEvaluator.py +++ b/src/evaluators/MutipleSameSuccessEvaluator.py @@ -70,18 +70,18 @@ class MultipleSameSuccessEvaluator(AllReturnValuesEvaluator): # give the priority to the ConceptEvaluator for s in self.success: if isinstance(s.value, Concept) and s.who == ConceptEvaluator().name: - return sheerka.ret(self.name, True, s.value, parents=self.eaten) + return sheerka.ret(self.name, True, s.value, parents=self.eaten.copy()) # Then the PythonEvaluator for s in self.success: if isinstance(s.value, Concept) and s.who == PythonEvaluator().name: - return sheerka.ret(self.name, True, s.value, parents=self.eaten) + return sheerka.ret(self.name, True, s.value, parents=self.eaten.copy()) # Then the first concept. # It's not predictable, so I guess that it's not a good implementation choice for s in self.success: if isinstance(s.value, Concept): - return sheerka.ret(self.name, True, s.value, parents=self.eaten) + return sheerka.ret(self.name, True, s.value, parents=self.eaten.copy()) - return sheerka.ret(self.name, True, self.success[0].value, parents=self.eaten) + return sheerka.ret(self.name, True, self.success[0].value, parents=self.eaten.copy()) diff --git a/src/evaluators/OneErrorEvaluator.py b/src/evaluators/OneErrorEvaluator.py index 089c2e8..0a2cf66 100644 --- a/src/evaluators/OneErrorEvaluator.py +++ b/src/evaluators/OneErrorEvaluator.py @@ -47,4 +47,4 @@ class OneErrorEvaluator(AllReturnValuesEvaluator): context.log(f"{self.return_value_in_error}", who=self) sheerka = context.sheerka - return sheerka.ret(self.name, False, self.return_value_in_error.value, parents=self.eaten) + return sheerka.ret(self.name, False, self.return_value_in_error.value, parents=self.eaten.copy()) diff --git a/src/evaluators/OneSuccessEvaluator.py b/src/evaluators/OneSuccessEvaluator.py index 68ecc12..d96e0a7 100644 --- a/src/evaluators/OneSuccessEvaluator.py +++ b/src/evaluators/OneSuccessEvaluator.py @@ -48,4 +48,4 @@ class OneSuccessEvaluator(AllReturnValuesEvaluator): context.log(f"{self.successful_return_value}", who=self) sheerka = context.sheerka - return sheerka.ret(self.name, True, self.value_to_return, parents=self.eaten) + return sheerka.ret(self.name, True, self.value_to_return, parents=self.eaten.copy()) diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index 240025c..7fde950 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -79,7 +79,8 @@ class PythonEvaluator(OneReturnValueEvaluator): # or if EVAL_QUESTION_REQUESTED is explicit # We need to disable the functions that may alter the state # It's a poor way to have source code security check - expression_only = context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + expression_only = context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED) or \ + context.in_context(BuiltinConcepts.VALIDATION_ONLY_REQUESTED) if not expression_only: attr_under_eval = context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_ATTRIBUTE) @@ -92,6 +93,11 @@ class PythonEvaluator(OneReturnValueEvaluator): my_globals = self.get_globals(context, node, expression_only) debugger.debug_var("globals", my_globals) except MethodAccessError as ex: + # Quick and dirty, + # When VALIDATION_ONLY_REQUESTED is enabled, it's normal to have some NameError exceptions + if context.in_context(BuiltinConcepts.VALIDATION_ONLY_REQUESTED): + return sheerka.ret(self.name, False, BuiltinConcepts.METHOD_ACCESS_ERROR, parents=[return_value]) + eval_error = PythonEvalError(ex, node.source, traceback.format_exc() if get_trace_back else None, diff --git a/src/evaluators/RetEvaluator.py b/src/evaluators/RetEvaluator.py index 7984282..0203a31 100644 --- a/src/evaluators/RetEvaluator.py +++ b/src/evaluators/RetEvaluator.py @@ -25,7 +25,7 @@ # concept = return_value.value # context.log(f"Processing ret value for concept {concept}.", self.name) # -# if not concept.get_metadata().is_evaluated: +# if not concept.get_hints().is_evaluated: # evaluated = ensure_evaluated(context, concept) # if evaluated.key != concept.key: # context.log(f"Failed to evaluate concept '{concept}'") diff --git a/src/evaluators/TooManySuccessEvaluator.py b/src/evaluators/TooManySuccessEvaluator.py index ea20228..da52338 100644 --- a/src/evaluators/TooManySuccessEvaluator.py +++ b/src/evaluators/TooManySuccessEvaluator.py @@ -55,7 +55,7 @@ class TooManySuccessEvaluator(AllReturnValuesEvaluator): if not same_success: context.log(f"Values are different. Raising {BuiltinConcepts.TOO_MANY_SUCCESS}.", self.name) too_many_success = sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS, body=self.success.copy()) - return sheerka.ret(self.name, False, too_many_success, parents=self.eaten) + return sheerka.ret(self.name, False, too_many_success, parents=self.eaten.copy()) context.log(f"Values are the same. Nothing to do.", self.name) return None diff --git a/src/evaluators/ValidateConceptEvaluator.py b/src/evaluators/ValidateConceptEvaluator.py new file mode 100644 index 0000000..4334f14 --- /dev/null +++ b/src/evaluators/ValidateConceptEvaluator.py @@ -0,0 +1,109 @@ +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, ConceptParts +from evaluators.BaseEvaluator import OneReturnValueEvaluator +from parsers.BaseNodeParser import ConceptNode +from parsers.BaseParser import BaseParser + + +class ValidateConceptEvaluator(OneReturnValueEvaluator): + """ + To recognize when the user input is a question + """ + + NAME = "ValidateConcept" + + def __init__(self, **kwargs): + super().__init__(self.NAME, [BuiltinConcepts.AFTER_PARSING], 90) + + def matches(self, context, return_value): + if not return_value.status or \ + not context.sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT) or \ + not return_value.who.startswith(BaseParser.PREFIX): + return False + + value = return_value.body.body + if isinstance(return_value.body.body, Concept): # case of ExactConceptParser + return value.get_hints().need_validation and ( + value.get_metadata().pre is not None and value.get_metadata().pre != "" or + value.get_metadata().where is not None and value.get_metadata().where != "" + ) + + # simple case first, cases with a longer list may be managed later if needed + if isinstance(value, list) and len(value) == 1 and isinstance(value[0], ConceptNode): + concept = value[0].concept + return concept.get_hints().need_validation and ( + concept.get_metadata().pre is not None and concept.get_metadata().pre != "" or + concept.get_metadata().where is not None and concept.get_metadata().where != "" + ) + + return False + + def eval(self, context, return_value): + """ + This evaluator returns None is the concept validates its PRE and POST constraint or if the constraint cannot + be validated + If the constraint can be validated, but fails, an error is returned + :param context: + :param return_value: + :return: + """ + sheerka = context.sheerka + + if isinstance(return_value.body.body, list): + concept_node = return_value.body.body[0] + concept = concept_node.concept + else: + concept_node = None + concept = return_value.body.body + + if context.get_parents(lambda ec: ec.action == BuiltinConcepts.EVALUATING_CONCEPT and + ec.action_context.id == concept.id): + # we are in an infinite loop trying to validate ourself + return None + + if concept.get_hints().use_copy: + use_copy = True + concept = concept.copy() + concept.get_hints().use_copy = False + else: + use_copy = False + + res = sheerka.evaluate_concept(context, + concept, + eval_body=False, + validation_only=True, + metadata=["variables", ConceptParts.PRE, ConceptParts.WHERE]) + + # either the 'pre' or the 'where' condition is not fulfilled + if sheerka.isinstance(res, BuiltinConcepts.CONDITION_FAILED): + filtered = sheerka.new(BuiltinConcepts.FILTERED, filtered=return_value.body.body, reason=res) + failed = sheerka.ret(self.name, False, filtered) + return failed + + # CAUTION + # Make sure that the return value and the parser result are not modified + # As they might be cached + if concept_node: + node = concept_node.clone() + node.concept = concept + return self.new_ret_val(context, return_value, [node]) + + if use_copy: + return self.new_ret_val(context, return_value, concept) + + else: + return None # tells the Execute() engine to use the old return value + + def new_ret_val(self, context, old_ret_value, new_value): + parser_result_copy = context.sheerka.new( + BuiltinConcepts.PARSER_RESULT, + parser=old_ret_value.body.parser, + source=old_ret_value.body.source, + body=new_value, + try_parsed=new_value) + + return context.sheerka.ret( + self.name, + True, + parser_result_copy, + parents=[old_ret_value]) diff --git a/src/out/AsStrVisitor.py b/src/out/AsStrVisitor.py index 251d997..6008be3 100644 --- a/src/out/AsStrVisitor.py +++ b/src/out/AsStrVisitor.py @@ -1,6 +1,6 @@ import re -from core.sheerka.services.SheerkaRuleManager import FormatAstNode +from parsers.FormatRuleActionParser import FormatAstNode from core.utils import CONSOLE_COLORS_MAP as CCM, no_color_str, CONSOLE_COLUMNS from out.OutVisitor import OutVisitor diff --git a/src/out/DeveloperVisitor.py b/src/out/DeveloperVisitor.py index e15ccf4..48f5087 100644 --- a/src/out/DeveloperVisitor.py +++ b/src/out/DeveloperVisitor.py @@ -1,5 +1,5 @@ from core.builtin_helpers import evaluate_expression -from core.sheerka.services.SheerkaRuleManager import FormatAstVariable, FormatAstVariableNotFound, FormatAstColor, \ +from parsers.FormatRuleActionParser import FormatAstVariable, FormatAstVariableNotFound, FormatAstColor, \ FormatAstList, FormatAstRawText, FormatAstDict from core.utils import as_bag diff --git a/src/parsers/BaseExpressionParser.py b/src/parsers/BaseExpressionParser.py index 44373d2..c66537f 100644 --- a/src/parsers/BaseExpressionParser.py +++ b/src/parsers/BaseExpressionParser.py @@ -44,6 +44,7 @@ class ExprNode(Node): self.end = end self.tokens = tokens self.source = None + self.compiled = None def eval(self, obj): return True @@ -426,6 +427,8 @@ class BaseExpressionParser(BaseParser): False, context.sheerka.new(BuiltinConcepts.ERROR, body=error_sink.sink)) + # Do not compile the node here, as it merely be useless + # The node is compiled in ExpressionParser.parse() or FunctionParser.parse(), depending of the requirement node = self.parse_input(context, parser_input, error_sink) token = parser_input.token @@ -582,7 +585,9 @@ class TrueifyVisitor(ExpressionVisitor): is_question_tokens = list(Tokenizer("is_question()")) -eval_question_requested_in_context = list(Tokenizer("context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)")) +eval_question_requested_in_context_tokens = list( + Tokenizer("context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)")) +in_context_tokens = list(Tokenizer("in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)")) class IsAQuestionVisitor(ExpressionVisitor): @@ -593,13 +598,13 @@ class IsAQuestionVisitor(ExpressionVisitor): def visit_NameExprNode(self, expr_node): if tokens_are_matching(expr_node.tokens, is_question_tokens) or \ - tokens_are_matching(expr_node.tokens, eval_question_requested_in_context): + tokens_are_matching(expr_node.tokens, eval_question_requested_in_context_tokens): return True return None def visit_FunctionNode(self, expr_node: FunctionNode): if tokens_are_matching(expr_node.tokens, is_question_tokens) or \ - tokens_are_matching(expr_node.tokens, eval_question_requested_in_context): + tokens_are_matching(expr_node.tokens, in_context_tokens): return True return None diff --git a/src/parsers/BaseNodeParser.py b/src/parsers/BaseNodeParser.py index 2902dd4..adb00ec 100644 --- a/src/parsers/BaseNodeParser.py +++ b/src/parsers/BaseNodeParser.py @@ -223,6 +223,8 @@ class ConceptNode(LexerNode): def to_short_str(self): return f'CN({self.concept})' + def get_concept(self): + return self.concept class SourceCodeNode(LexerNode): """ @@ -388,7 +390,8 @@ class SourceCodeWithConceptNode(LexerNode): return self._all_nodes def clone(self): - clone = SourceCodeWithConceptNode(self.first, self.last, self.nodes.copy(), self.has_unrecognized) + nodes = [n.clone() for n in self.nodes] + clone = SourceCodeWithConceptNode(self.first.clone(), self.last.clone(), nodes, self.has_unrecognized) clone.python_node = self.python_node clone.return_value = self.return_value return clone diff --git a/src/parsers/BaseParser.py b/src/parsers/BaseParser.py index b045a3e..a6dcbe9 100644 --- a/src/parsers/BaseParser.py +++ b/src/parsers/BaseParser.py @@ -71,7 +71,15 @@ class UnexpectedEofParsingError(ParsingError): class BaseParser: PREFIX = "parsers." - def __init__(self, name, priority: int, enabled=True, yield_eof=False): + def __init__(self, name, priority: int, enabled=True, yield_eof=False, hints=None): + """ + + :param name: + :param priority: + :param enabled: + :param yield_eof: + :param hints: Dictionary context_hint: priority. When not null, priority is taken from it + """ # self.log = get_logger("parsers." + self.__class__.__name__) # self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__) # self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__) @@ -81,6 +89,7 @@ class BaseParser: self.priority = priority self.enabled = enabled self.yield_eof = yield_eof + self.hints = hints def __eq__(self, other): if not isinstance(other, self.__class__): diff --git a/src/parsers/BnfNodeParser.py b/src/parsers/BnfNodeParser.py index 942a429..9658581 100644 --- a/src/parsers/BnfNodeParser.py +++ b/src/parsers/BnfNodeParser.py @@ -1227,6 +1227,18 @@ class BnfNodeConceptExpressionVisitor(ParsingExpressionVisitor): self.references.append(pe.concept) +class HasAChoiceExpressionVisitor(ParsingExpressionVisitor): + def __init__(self): + super().__init__() + self.result = False + + def visit_OrderedChoice(self, parsing_expression): + self.result = True + + def visit_UnOrderedChoice(self, parsing_expression): + self.result = True + + class BnfConceptParserHelper: def __init__(self, parser, debugger): self.parser = parser @@ -1480,6 +1492,7 @@ class BnfConceptParserHelper: key = (template.key, template.id) if template.id else template.key concept = sheerka.new(key) concept = self.finalize_concept(sheerka, concept, underlying) + concept.get_hints().use_copy = True concept_node = ConceptNode(concept, underlying.start, underlying.end, @@ -1555,7 +1568,7 @@ class BnfConceptParserHelper: _underlying.parsing_expression.rule_name not in _concept.get_compiled()): var_value = _get_underlying_value(_underlying) _add_compiled(_concept, _underlying.parsing_expression.rule_name, var_value) - _concept.get_metadata().need_validation = True + _concept.get_hints().need_validation = True elif isinstance(_underlying, NonTerminalNode): for child in _underlying.children: @@ -1567,7 +1580,7 @@ class BnfConceptParserHelper: concept.get_compiled()[ConceptParts.BODY] = value if underlying.parsing_expression.rule_name: _add_compiled(concept, underlying.parsing_expression.rule_name, value) - # KSI : Why don't we set concept.get_metadata().need_validation to True ? + # KSI : Why don't we set concept.get_hints().need_validation to True ? # then recursively browse children to update concept variables if isinstance(underlying, NonTerminalNode) and not isinstance(underlying.parsing_expression, ConceptExpression): diff --git a/src/parsers/DefConceptParser.py b/src/parsers/DefConceptParser.py index aa2c35b..ddd063a 100644 --- a/src/parsers/DefConceptParser.py +++ b/src/parsers/DefConceptParser.py @@ -3,9 +3,11 @@ from dataclasses import dataclass import core.builtin_helpers import core.utils from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept -from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF +from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, ConceptParts from core.global_symbols import NotInit +from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept, ChickenAndEggException from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute +from core.sheerka.services.sheerka_service import FailedToCompileError from core.tokenizer import TokenKind, Keywords from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, SyntaxErrorNode, NameNode, CustomGrammarParserNode from parsers.BaseParser import ParsingError, UnexpectedTokenParsingError @@ -54,7 +56,7 @@ class DefConceptParser(BaseCustomGrammarParser): """ Parse sheerka specific grammar (like def concept) """ - + NAME = "DefConcept" KEYWORDS = [Keywords.CONCEPT, Keywords.FROM, Keywords.AS, @@ -67,7 +69,7 @@ class DefConceptParser(BaseCustomGrammarParser): KEYWORDS_VALUES = [k.value for k in KEYWORDS] def __init__(self, **kwargs): - BaseCustomGrammarParser.__init__(self, "DefConcept", 60) + BaseCustomGrammarParser.__init__(self, DefConceptParser.NAME, 60) def parse(self, context, parser_input: ParserInput): # default parser can only manage string text @@ -140,11 +142,11 @@ class DefConceptParser(BaseCustomGrammarParser): node.definition_type, node.definition = self.get_concept_definition(node, parts) # get the bodies - node.body = self.get_ast(Keywords.AS, parts) - node.where = self.get_ast(Keywords.WHERE, parts) - node.pre = self.get_ast(Keywords.PRE, parts) - node.post = self.get_ast(Keywords.POST, parts) - node.ret = self.get_ast(Keywords.RET, parts) + node.body = self.get_ast(Keywords.AS, ConceptParts.BODY, parts) + node.where = self.get_ast(Keywords.WHERE, ConceptParts.WHERE, parts) + node.pre = self.get_ast(Keywords.PRE, ConceptParts.PRE, parts) + node.post = self.get_ast(Keywords.POST, ConceptParts.POST, parts) + node.ret = self.get_ast(Keywords.RET, ConceptParts.RET, parts) # other information node.auto_eval = self.get_concept_auto_eval(parts) @@ -263,7 +265,7 @@ class DefConceptParser(BaseCustomGrammarParser): return DEFINITION_TYPE_DEF, NameNode(tokens) - def get_ast(self, keyword, parts): + def get_ast(self, keyword, concept_part, parts): if keyword not in parts: return NotInit @@ -273,12 +275,26 @@ class DefConceptParser(BaseCustomGrammarParser): return None source = self.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens[1:]) - parsed = self.sheerka.parse_unrecognized(self.context, - source, - parsers="all", - who=self.name, - prop=keyword, - filter_func=core.builtin_helpers.expect_one) + + try: + parsed = self.sheerka.services[SheerkaEvaluateConcept.NAME].get_asts(self.context, + self.NAME, + source, + None, + concept_part, + None) + + parsed = core.builtin_helpers.expect_one(self.context, parsed) + if not parsed.status: + self.add_error(parsed.value) + return None + + except ChickenAndEggException as ex: + self.add_error(ex.error) + return None + except FailedToCompileError as ex: + self.add_error(ex.cause[0] if len(ex.cause) == 1 else ex.cause) + return None if not parsed.status: self.add_error(parsed.value) diff --git a/src/parsers/DefRuleParser.py b/src/parsers/DefRuleParser.py index 365547b..74c3aac 100644 --- a/src/parsers/DefRuleParser.py +++ b/src/parsers/DefRuleParser.py @@ -6,11 +6,12 @@ from core.builtin_concepts import ReturnValueConcept from core.builtin_concepts_ids import BuiltinConcepts from core.global_symbols import NotInit from core.sheerka.services.SheerkaExecute import ParserInput -from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatAstNode, CompiledCondition +from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition from core.sheerka.services.sheerka_service import FailedToCompileError from core.tokenizer import Keywords, TokenKind from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, NameNode, KeywordNotFound, SyntaxErrorNode from parsers.BaseParser import Node, UnexpectedEofParsingError +from parsers.FormatRuleActionParser import FormatAstNode from sheerkarete.conditions import AndConditions diff --git a/src/parsers/ExactConceptParser.py b/src/parsers/ExactConceptParser.py index 9b4aad3..2c78f62 100644 --- a/src/parsers/ExactConceptParser.py +++ b/src/parsers/ExactConceptParser.py @@ -4,18 +4,18 @@ from core.concept import VARIABLE_PREFIX from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind from core.utils import str_concept -from parsers.BaseParser import BaseParser, BaseParserInputParser +from parsers.BaseParser import BaseParserInputParser class ExactConceptParser(BaseParserInputParser): """ Tries to recognize a single concept """ - + NAME = "ExactConcept" MAX_WORDS_SIZE = 6 def __init__(self, max_word_size=None, **kwargs): - BaseParserInputParser.__init__(self, "ExactConcept", 80) + BaseParserInputParser.__init__(self, ExactConceptParser.NAME, 80) self.max_word_size = max_word_size def parse(self, context, parser_input: ParserInput): @@ -47,7 +47,7 @@ class ExactConceptParser(BaseParserInputParser): for combination in self.combinations(words): concept_key = " ".join(combination) - result = sheerka.get_by_key(concept_key) # use new(), not get() because we need a new instance + result = sheerka.get_by_key(concept_key) if sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT): continue @@ -74,13 +74,15 @@ class ExactConceptParser(BaseParserInputParser): index = int(token[len(VARIABLE_PREFIX):]) value = words[i] concept.def_var_by_index(index, str_concept(value) if isinstance(value, tuple) else value) - concept.get_metadata().need_validation = True + concept.get_hints().need_validation = True already_recognized.append(concept) by_name = sheerka.fast_resolve(parser_input.as_text()) core.builtin_helpers.set_is_evaluated(by_name) recognized = self.merge_concepts(already_recognized, by_name) + for c in recognized: + c.get_hints().use_copy = True if len(recognized) == 0: ret = sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, diff --git a/src/parsers/ExpressionParser.py b/src/parsers/ExpressionParser.py index 1e36927..2114eb4 100644 --- a/src/parsers/ExpressionParser.py +++ b/src/parsers/ExpressionParser.py @@ -1,3 +1,7 @@ +from core.builtin_concepts_ids import BuiltinConcepts +from core.sheerka.services.SheerkaExecute import ParserInput +from core.sheerka.services.SheerkaRuleManager import PythonConditionExprVisitor +from core.sheerka.services.sheerka_service import FailedToCompileError from parsers.BaseExpressionParser import BaseExpressionParser from parsers.FunctionParser import FunctionParser from parsers.LogicalOperatorParser import LogicalOperatorParser @@ -12,15 +16,45 @@ class ExpressionParser(BaseExpressionParser): NAME = "Expression" - def __init__(self, **kwargs): - super().__init__(ExpressionParser.NAME, 60, False, yield_eof=False) + def __init__(self, auto_compile=True, **kwargs): + super().__init__(ExpressionParser.NAME, + 0, + True, + yield_eof=False, + hints={BuiltinConcepts.EVAL_QUESTION_REQUESTED: 60}) self.variable_parser = VariableOrNamesParser() self.function_parser = FunctionParser(expr_parser=self, tokens_parser=self.variable_parser) self.relational_parser = RelationalOperatorParser(expr_parser=self.function_parser) self.logical_parser = LogicalOperatorParser(expr_parser=self.relational_parser) + self.auto_compile = auto_compile def parse_input(self, context, parser_input, error_sink): return self.logical_parser.parse_input(context, parser_input, error_sink) def parse_tokens_stop_condition(self, token, parser_input): pass + + def parse(self, context, parser_input: ParserInput): + ret = super().parse(context, parser_input) + + if not self.auto_compile: + return ret + + if ret is None: + return None + + if not ret.status: + return ret + + # The parsing is successful + # let's validate it + try: + python_visitor = PythonConditionExprVisitor(context) + python_conditions = python_visitor.get_conditions(ret.body.body) + ret.body.body.compiled = python_conditions + return ret + except FailedToCompileError as ex: + return context.sheerka.ret( + self.name, + False, + context.sheerka.err(ex.cause)) diff --git a/src/parsers/FormatRuleActionParser.py b/src/parsers/FormatRuleActionParser.py new file mode 100644 index 0000000..e4990b9 --- /dev/null +++ b/src/parsers/FormatRuleActionParser.py @@ -0,0 +1,485 @@ +import re +from dataclasses import dataclass +from typing import Union + +from core.global_symbols import ErrorObj +from core.tokenizer import IterParser, TokenKind, Token +from core.utils import get_text_from_tokens, COLORS, index_tokens + +identifier_regex = re.compile(r"[\w _.]+") + + +@dataclass +class FormatRuleError(ErrorObj): + pass + + +@dataclass +class BraceMismatch(FormatRuleError): + lbrace: Token + + +@dataclass +class UnexpectedEof(FormatRuleError): + message: str + token: Token = None + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, UnexpectedEof): + return False + + return self.message == other.message and (other.token is None or other.token == self.token) + + def __hash__(self): + return hash(self.message, self.token) + + +@dataclass +class FormatRuleSyntaxError(FormatRuleError): + message: str + token: Token + + +@dataclass +class FormatAstNode: + @staticmethod + def repr_value(items): + if items is None: + return "" + + return ", ".join(repr(item) for item in items) + + def clone(self, instance, props, **kwargs): + for prop_name in props: + setattr(instance, prop_name, getattr(self, prop_name)) + + for k, v in kwargs.items(): + setattr(instance, k, v) + + return instance + + +@dataclass +class FormatAstRawText(FormatAstNode): + text: str + + +@dataclass +class FormatAstVariable(FormatAstNode): + name: str + format: Union[str, None] = None + debug: bool = False + value: object = None + index: object = None + + def clone(self, **kwargs): + return super().clone(FormatAstVariable(self.name), + ("format", "debug", "value", "index"), + **kwargs) + + +@dataclass +class FormatAstVariableNotFound(FormatAstNode): + name: str + + +@dataclass +class FormatAstGrid(FormatAstNode): + pass + + +@dataclass +class FormatAstList(FormatAstNode): + variable: str + items_prop: str = None # where to search the list if variable does not resolve to an iterable + recurse_on: str = None + recursion_depth: int = 0 + debug: bool = False + prefix: str = None + suffix: str = None + show_index: bool = False + index: object = None + + items: object = None + + def clone(self, **kwargs): + return super().clone( + FormatAstList(self.variable), + ( + "items_prop", "recurse_on", "recursion_depth", "debug", "prefix", "suffix", "show_index", "index", + "items"), + **kwargs) + + +@dataclass +class FormatAstDict(FormatAstNode): + variable: str + items_prop: str = None # where to search the dict if variable does not resolve to an iterable + debug: bool = False + prefix: str = None + suffix: str = None + + items: object = None + + def clone(self, **kwargs): + return super().clone( + FormatAstDict(self.variable), + ("items_prop", "debug", "prefix", "suffix", "items"), + **kwargs) + + +@dataclass +class FormatAstColor(FormatAstNode): + color: str + format_ast: FormatAstNode + + def __repr__(self): + return f"{self.color}({self.format_ast})" + + def clone(self, **kwargs): + return super().clone( + FormatAstColor(self.color, self.format_ast), + (), + **kwargs) + + +@dataclass +class FormatAstFunction(FormatAstNode): + name: str + args: list = None + kwargs: dict = None + + +@dataclass +class FormatAstSequence(FormatAstNode): + items: list + debug: bool = False + + def __repr__(self): + return "FormatAstSequence(" + self.repr_value(self.items) + ")" + + def clone(self, **kwargs): + return super().clone( + FormatAstSequence(self.items), + ("debug",), + **kwargs) + + +@dataclass +class FormatAstMulti(FormatAstNode): + """ + Used when there are multiple out to print, but they are not related + Just print them one by one + """ + variable: str + items: list = None + + def __repr__(self): + return f"FormatAstMulti({self.variable}, items={self.items})" + + def clone(self, **kwargs): + return super().clone( + FormatAstMulti(self.variable), + ("items",), + **kwargs) + + +class FormatRuleActionParser(IterParser): + + @staticmethod + def to_text(list_or_dict_of_tokens): + """ + Works on list of list of tokens + or dict of list of tokens + :param list_or_dict_of_tokens: + :return: + """ + get_text = get_text_from_tokens + if isinstance(list_or_dict_of_tokens, list): + return [get_text(i) for i in list_or_dict_of_tokens] + if isinstance(list_or_dict_of_tokens, dict): + return {k: get_text(v) for k, v in list_or_dict_of_tokens.items()} + raise NotImplementedError("") + + def to_value(self, tokens): + """ + Works on list of tokens + return string or numeric value of the tokens + :return: + """ + + value = get_text_from_tokens(tokens) + if value[0] in ("'", '"'): + return value[1:-1] + + if value in ("True", "False"): + return bool(value) + + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + self.error_sink = FormatRuleSyntaxError(f"'{value}' is not numeric", None) + + def parse(self): + """ + Parses the print part of the format rule + format ::= {variable'} | function(...) | rawtext + :return: + """ + + if self.source == "": + return FormatAstRawText("") + + buffer = [] + result = [] + res = None + escaped = False + + def _flush_buffer(): + if len(buffer) > 0: + result.append(FormatAstRawText(get_text_from_tokens(buffer))) + buffer.clear() + + while self.next_token(skip_whitespace=False): + if not escaped: + if self.token.type == TokenKind.IDENTIFIER and self.the_token_after().type == TokenKind.LPAR: + _flush_buffer() + res = self.parse_function(self.token) + elif self.token.type == TokenKind.LBRACE: + _flush_buffer() + res = self.parse_variable(self.token) + elif self.token.type == TokenKind.BACK_SLASH: + escaped = True + else: + buffer.append(self.token) + else: + escaped = False + buffer.append(self.token) + + if self.error_sink: + break + + if res: + result.append(res) + res = None + + _flush_buffer() + + return [] if len(result) == 0 else result[0] if len(result) == 1 else FormatAstSequence(result) + + def parse_function(self, func_name): + self.next_token() + self.next_token() + + if self.token.type == TokenKind.EOF: + self.error_sink = UnexpectedEof("while parsing function", func_name) + return None + + param_buffer = [] + args = [] + kwargs = {} + get_text = get_text_from_tokens + + def _process_parameters(): + if len(param_buffer) == 0: + self.error_sink = FormatRuleSyntaxError("no parameter found", self.token) + return None + if (index := index_tokens(param_buffer, "=")) > 0: + kwargs[get_text(param_buffer[:index])] = param_buffer[index + 1:] + else: + args.append(param_buffer.copy()) + param_buffer.clear() + + while True: + if self.token.type == TokenKind.RPAR: + if len(param_buffer) > 0: + _process_parameters() + break + + elif self.token.type == TokenKind.COMMA: + _process_parameters() + if self.error_sink: + break + + else: + param_buffer.append(self.token) + + if not self.next_token(): + break + + if self.error_sink: + return None + + if self.token.type != TokenKind.RPAR: + self.error_sink = UnexpectedEof("while parsing function", func_name) + return None + + if func_name.value in COLORS: + return self.return_color(func_name.value, args, kwargs) + elif func_name.value == "list": + return self.return_list(args, kwargs) + elif func_name.value == "dict": + return self.return_dict(args, kwargs) + elif func_name.value == "multi": + return self.return_multi(args, kwargs) + + return FormatAstFunction(func_name.value, self.to_text(args), self.to_text(kwargs)) + + def parse_variable(self, lbrace): + self.next_token() + + if self.token.type == TokenKind.EOF: + self.error_sink = UnexpectedEof("while parsing variable", lbrace) + return None + + buffer = [] + while True: + if self.token.type == TokenKind.RBRACE: + break + buffer.append(self.token) + + if not self.next_token(): + break + + # if self.error_sink: + # return None + + if self.token.type != TokenKind.RBRACE: + self.error_sink = UnexpectedEof("while parsing variable", lbrace) + return None + + if len(buffer) == 0: + self.error_sink = FormatRuleSyntaxError("variable name not found", None) + return None + + variable = get_text_from_tokens(buffer) + try: + index = variable.index(":") + return FormatAstVariable(variable[:index], variable[index + 1:]) + except ValueError: + return FormatAstVariable(variable) + + def return_color(self, color, args, kwargs): + if len(kwargs) > 0: + self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None) + return None + + if len(args) == 0: + return FormatAstColor(color, FormatAstRawText("")) + + if len(args) > 1: + self.error_sink = FormatRuleSyntaxError("only one parameter supported", args[1][0]) + return None + + source = get_text_from_tokens(args[0]) + if len(source) > 1 and source[0] in ("'", '"') and source[-1] in ("'", '"'): + source = source[1:-1] + parser = FormatRuleActionParser(source) + res = parser.parse() + self.error_sink = parser.error_sink + return FormatAstColor(color, res) + else: + try: + index = source.index(":") + variable, vformat = source[:index], source[index + 1:] + except ValueError: + variable, vformat = source, None + + if not identifier_regex.fullmatch(variable): + self.error_sink = FormatRuleSyntaxError("Invalid identifier", None) + return None + return FormatAstColor(color, FormatAstVariable(variable, vformat)) + + def return_list(self, args, kwargs): + """ + Looking for greeting_var, [recurse_on], [recursion_depth], [items_prop] + :param args: + :param kwargs: + :return: + """ + len_args = len(args) + if len_args < 1: + self.error_sink = FormatRuleSyntaxError("variable name not found", None) + return None + + if len_args > 4: + self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[4][0]) + return None + + variable_name = get_text_from_tokens(args[0]) + recurse_on, recursion_depth, items_prop = None, 0, None + + if len_args == 2: + recursion_depth = self.to_value(args[1]) + elif len_args == 3: + recursion_depth = self.to_value(args[1]) + recurse_on = self.to_value(args[2]) + elif len_args == 4: + recursion_depth = self.to_value(args[1]) + recurse_on = self.to_value(args[2]) + items_prop = self.to_value(args[3]) + + if "recurse_on" in kwargs: + recurse_on = self.to_value(kwargs["recurse_on"]) + + if "recursion_depth" in kwargs: + recursion_depth = self.to_value(kwargs["recursion_depth"]) + + if "items_prop" in kwargs: + items_prop = self.to_value(kwargs["items_prop"]) + + if self.error_sink: + return None + + if not isinstance(recursion_depth, int): + self.error_sink = FormatRuleSyntaxError("'recursion_depth' must be an integer", None) + return None + + return FormatAstList(variable_name, items_prop, recurse_on, recursion_depth) + + def return_dict(self, args, kwargs): + len_args = len(args) + if len_args < 1: + self.error_sink = FormatRuleSyntaxError("variable name not found", None) + return None + + if len_args > 1: + self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0]) + return None + + variable_name = get_text_from_tokens(args[0]) + + kwargs_parameters = {} + for prop in ("items_prop", "prefix", "suffix", "debug"): + if prop in kwargs: + kwargs_parameters[prop] = self.to_value(kwargs[prop]) + + if "debug" in kwargs_parameters: + if "prefix" not in kwargs_parameters: + kwargs_parameters["prefix"] = "{" + if "suffix" not in kwargs_parameters: + kwargs_parameters["suffix"] = "}" + + if self.error_sink: + return None + + return FormatAstDict(variable_name, **kwargs_parameters) + + def return_multi(self, args, kwargs): + if len(kwargs) > 0: + self.error_sink = FormatRuleSyntaxError("keyword arguments are not supported", None) + return None + + if len(args) > 1: + self.error_sink = FormatRuleSyntaxError("too many positional arguments", args[1][0]) + return None + + return FormatAstMulti(get_text_from_tokens(args[0])) diff --git a/src/parsers/FunctionParser.py b/src/parsers/FunctionParser.py index 5d66019..cd7e1cb 100644 --- a/src/parsers/FunctionParser.py +++ b/src/parsers/FunctionParser.py @@ -62,6 +62,10 @@ class FunctionParser(BaseExpressionParser): if not ret.status: return ret + # FunctionParser returns LexerNodes, rather than an ExprNode + # I know that is is not very logical, but at the beginning, the FunctionParser was + # uses exclusively by the SyaNodeParser. + # It has been refactored to fit in ExpressionParser. So it has two main usages node = ret.body.body source_code_nodes = self.to_source_code_node(context, node) res = [] diff --git a/src/parsers/LogicalOperatorParser.py b/src/parsers/LogicalOperatorParser.py index 2374db4..c6dd823 100644 --- a/src/parsers/LogicalOperatorParser.py +++ b/src/parsers/LogicalOperatorParser.py @@ -1,15 +1,10 @@ -from itertools import product - -from core.builtin_helpers import only_successful, get_inner_body, get_lexer_nodes_using_positions from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.sheerka_service import FailedToCompileError -from core.tokenizer import TokenKind, Tokenizer, Keywords +from core.tokenizer import TokenKind, Tokenizer from core.utils import get_text_from_tokens -from parsers.BaseExpressionParser import ParenthesisNode, OrNode, AndNode, NotNode, ExprNode, VariableNode, \ +from parsers.BaseExpressionParser import ParenthesisNode, OrNode, AndNode, NotNode, VariableNode, \ ComparisonNode, BaseExpressionParser -from parsers.BaseNodeParser import UnrecognizedTokensNode from parsers.BaseParser import UnexpectedEofParsingError, ErrorSink -from parsers.PythonWithConceptsParser import PythonWithConceptsParser from sheerkarete.common import V from sheerkarete.conditions import Condition, AndConditions @@ -154,96 +149,96 @@ class LogicalOperatorParser(BaseExpressionParser): return token.type == TokenKind.IDENTIFIER and token.value in ("and", "or") or \ token.value == "not" and parser_input.the_token_after(True).value != "in" - def compile_conjunctions(self, context, conjunctions, who): - """ - Transform a list of conjunctions (AND and OR) into one or multiple CompiledExpr - :param context: - :param conjunctions: list of ExprNode - :param who: service that calls the method - :returns: List Of CompiledExpr - May throw FailedToRecognized if a conjunction cannot be parsed - """ - recognized = [] - for conjunction in conjunctions: - # try to recognize conjunction, one by one - # negative conjunction can be a concept starting with 'not' - parsed_ret = context.sheerka.parse_unrecognized( - context, - conjunction.get_value(), # we remove the 'NOT' part when needed to ease the recognition - parsers="all", - who=who, - prop=Keywords.WHEN, - filter_func=only_successful) - - if parsed_ret.status: - recognized.append(get_inner_body(context, parsed_ret.body)) - else: - raise FailedToCompileError(parsed_ret.body) - - # for each conjunction, we have a list of recognized concepts (or python node) - # we need a cartesian product of the results - # Explanation for later - # conjunction[0] : 'x is a y' that can be resolved with two concepts c:|1001: and c:|1002: - # conjunction[1] : 'y is an z' that can also be resolved with two concepts (c:|1003: and c:|1004) - # so to understand the full question 'x is a y and y is an z' - # we can have c:|1001: then c:|1003: - # or c:|1001: then c:|1004: - # or c:|1002: then c:|1003: - # or c:|1002: then c:|1004: - # if one of this combination works, it means that the question 'x is a y and y is an z' was matched - # hence the cartesian product - product_of_recognized = list(product(*recognized)) - - return_values = [] - for recognized_conjunctions in product_of_recognized: - if len(recognized_conjunctions) == 1 and not isinstance(conjunctions[0], NotNode): - return_values.append(recognized_conjunctions[0]) - elif len(recognized_conjunctions) == 1 and recognized_conjunctions[0].who == "parsers.Python": - # it is a negated python Node. Need to parse again - ret = context.sheerka.parse_python(context, source=str(conjunctions[0])) - if ret.status: - return_values.append(ret) - else: - # find a way to track the failure - pass - else: - # complex result. Use PythonWithNode - lexer_nodes = get_lexer_nodes_using_positions(recognized_conjunctions, - self._get_positions(conjunctions)) - - # put back the 'and' / 'not' node - for i in range(len(lexer_nodes) - 1, 0, -1): - end = lexer_nodes[i].start - 1 - start = lexer_nodes[i - 1].end + 1 - if isinstance(conjunctions[i], NotNode): - lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_not_tokens)) - else: - lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_tokens)) - - # add the starting 'not' if needed - # and reindex the following positions - if isinstance(conjunctions[0], NotNode): - lexer_nodes[0].start = 2 - lexer_nodes.insert(0, UnrecognizedTokensNode(0, 1, self.not_tokens)) - - python_with_concept_node_ret = PythonWithConceptsParser().parse_nodes(context, lexer_nodes) - if not python_with_concept_node_ret.status: - # find a way to track the failure - pass - return_values.append(python_with_concept_node_ret) - - rete_cond_emitter = ReteConditionsEmitter(context) - rete_disjunctions = rete_cond_emitter.get_conditions(conjunctions) - - return return_values, rete_disjunctions - - @staticmethod - def _get_positions(expr_nodes): - """ - simply manage NotNodes to address the fact that the 'not' part in removed - """ - for expr in expr_nodes: - if isinstance(expr, NotNode): - yield ExprNode(expr.start + 2, expr.end, expr.tokens[2:]) - else: - yield expr + # def compile_conjunctions(self, context, conjunctions, who): + # """ + # Transform a list of conjunctions (AND and OR) into one or multiple CompiledExpr + # :param context: + # :param conjunctions: list of ExprNode + # :param who: service that calls the method + # :returns: List Of CompiledExpr + # May throw FailedToRecognized if a conjunction cannot be parsed + # """ + # recognized = [] + # for conjunction in conjunctions: + # # try to recognize conjunction, one by one + # # negative conjunction can be a concept starting with 'not' + # parsed_ret = context.sheerka.parse_unrecognized( + # context, + # conjunction.get_value(), # we remove the 'NOT' part when needed to ease the recognition + # parsers="all", + # who=who, + # prop=Keywords.WHEN, + # filter_func=only_successful) + # + # if parsed_ret.status: + # recognized.append(get_inner_body(context, parsed_ret.body)) + # else: + # raise FailedToCompileError(parsed_ret.body) + # + # # for each conjunction, we have a list of recognized concepts (or python node) + # # we need a cartesian product of the results + # # Explanation for later + # # conjunction[0] : 'x is a y' that can be resolved with two concepts c:|1001: and c:|1002: + # # conjunction[1] : 'y is an z' that can also be resolved with two concepts (c:|1003: and c:|1004) + # # so to understand the full question 'x is a y and y is an z' + # # we can have c:|1001: then c:|1003: + # # or c:|1001: then c:|1004: + # # or c:|1002: then c:|1003: + # # or c:|1002: then c:|1004: + # # if one of this combination works, it means that the question 'x is a y and y is an z' was matched + # # hence the cartesian product + # product_of_recognized = list(product(*recognized)) + # + # return_values = [] + # for recognized_conjunctions in product_of_recognized: + # if len(recognized_conjunctions) == 1 and not isinstance(conjunctions[0], NotNode): + # return_values.append(recognized_conjunctions[0]) + # elif len(recognized_conjunctions) == 1 and recognized_conjunctions[0].who == "parsers.Python": + # # it is a negated python Node. Need to parse again + # ret = context.sheerka.parse_python(context, source=str(conjunctions[0])) + # if ret.status: + # return_values.append(ret) + # else: + # # find a way to track the failure + # pass + # else: + # # complex result. Use PythonWithNode + # lexer_nodes = get_lexer_nodes_using_positions(recognized_conjunctions, + # self._get_positions(conjunctions)) + # + # # put back the 'and' / 'not' node + # for i in range(len(lexer_nodes) - 1, 0, -1): + # end = lexer_nodes[i].start - 1 + # start = lexer_nodes[i - 1].end + 1 + # if isinstance(conjunctions[i], NotNode): + # lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_not_tokens)) + # else: + # lexer_nodes.insert(i, UnrecognizedTokensNode(start, end, self.and_tokens)) + # + # # add the starting 'not' if needed + # # and reindex the following positions + # if isinstance(conjunctions[0], NotNode): + # lexer_nodes[0].start = 2 + # lexer_nodes.insert(0, UnrecognizedTokensNode(0, 1, self.not_tokens)) + # + # python_with_concept_node_ret = PythonWithConceptsParser().parse_nodes(context, lexer_nodes) + # if not python_with_concept_node_ret.status: + # # find a way to track the failure + # pass + # return_values.append(python_with_concept_node_ret) + # + # rete_cond_emitter = ReteConditionsEmitter(context) + # rete_disjunctions = rete_cond_emitter.get_conditions(conjunctions) + # + # return return_values, rete_disjunctions + # + # @staticmethod + # def _get_positions(expr_nodes): + # """ + # simply manage NotNodes to address the fact that the 'not' part in removed + # """ + # for expr in expr_nodes: + # if isinstance(expr, NotNode): + # yield ExprNode(expr.start + 2, expr.end, expr.tokens[2:]) + # else: + # yield expr diff --git a/src/parsers/PythonParser.py b/src/parsers/PythonParser.py index 96cd40f..0b33f38 100644 --- a/src/parsers/PythonParser.py +++ b/src/parsers/PythonParser.py @@ -134,7 +134,7 @@ class PythonParser(BaseParserInputParser): if self.reset_parser(context, parser_input): source_code = parser_input.as_text(python_switcher, tracker) - source_code = source_code.strip() + source_code = source_code.lstrip() # right side spaces must be kept # first, try to parse an expression res, tree, error = self.try_parse_expression(source_code) diff --git a/src/parsers/PythonWithConceptsParser.py b/src/parsers/PythonWithConceptsParser.py index 724b92d..1208c75 100644 --- a/src/parsers/PythonWithConceptsParser.py +++ b/src/parsers/PythonWithConceptsParser.py @@ -2,7 +2,7 @@ from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import CreateObjectIdentifiers from parsers.BaseNodeParser import ConceptNode, RuleNode, VariableNode from parsers.BaseNodeParser import SourceCodeWithConceptNode -from parsers.BaseParser import BaseParser, BaseParserInputParser +from parsers.BaseParser import BaseParserInputParser from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser unrecognized_nodes_parser = UnrecognizedNodeParser() @@ -76,7 +76,6 @@ class PythonWithConceptsParser(BaseParserInputParser): python_ids_mappings[python_id] = var_ref last_token_index = node.end - else: source += node.source to_parse += node.get_source_to_parse() diff --git a/src/parsers/SequenceNodeParser.py b/src/parsers/SequenceNodeParser.py index c6563d1..49248fc 100644 --- a/src/parsers/SequenceNodeParser.py +++ b/src/parsers/SequenceNodeParser.py @@ -380,9 +380,11 @@ class SequenceNodeParser(BaseNodeParser): for node in parser_helper.sequence: # if isinstance(node, ConceptNode): # if len(node.concept.get_metadata().variables) > 0: - # node.concept.get_metadata().is_evaluated = True # Do not try to evaluate those concepts + # node.concept.get_hints().is_evaluated = True # Do not try to evaluate those concepts node.tokens = self.parser_input.tokens[node.start:node.end + 1] node.fix_source() + if isinstance(node, ConceptNode): + node.concept.get_hints().use_copy = True parser_helper_hash_code = compute_hash_code(parser_helper) if parser_helper_hash_code in already_seen: @@ -426,7 +428,7 @@ class SequenceNodeParser(BaseNodeParser): self.sheerka.new( BuiltinConcepts.PARSER_RESULT, parser=self, - source=parser_input, + source=parser_input.as_text(), body=parser_helper.sequence, try_parsed=parser_helper.sequence))) diff --git a/src/parsers/ShortTermMemoryParser.py b/src/parsers/ShortTermMemoryParser.py index cd5615d..021b1bb 100644 --- a/src/parsers/ShortTermMemoryParser.py +++ b/src/parsers/ShortTermMemoryParser.py @@ -8,9 +8,10 @@ class ShortTermMemoryParser(BaseParser): """ This parser is used to recognize concept that are already instantiated """ + NAME = "ShortTermMemory" def __init__(self, **kwargs): - super().__init__("ShortTermMemory", 85) + super().__init__(ShortTermMemoryParser.NAME, 85) # priority is irrelevant for ShortTermMemory parser def parse(self, context, parser_input): """ diff --git a/src/parsers/SyaNodeParser.py b/src/parsers/SyaNodeParser.py index 08f5489..75322d0 100644 --- a/src/parsers/SyaNodeParser.py +++ b/src/parsers/SyaNodeParser.py @@ -5,6 +5,7 @@ from typing import List from core import builtin_helpers from core.builtin_concepts import BuiltinConcepts +from core.builtin_helpers import update_compiled from core.concept import Concept, DEFINITION_TYPE_BNF from core.global_symbols import CONCEPT_COMPARISON_CONTEXT, SyaAssociativity from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager @@ -1270,8 +1271,10 @@ class SyaNodeParser(BaseNodeParser): while len(item.nodes) > 0: res = self.postfix_to_item(sheerka, item.nodes) if isinstance(res, PostFixToItem): - items.append( - ConceptNode(res.concept, res.start, res.end, self.parser_input.tokens[res.start: res.end + 1])) + items.append(ConceptNode(res.concept, + res.start, + res.end, + self.parser_input.tokens[res.start: res.end + 1])) else: items.append(res) item.has_unrecognized |= hasattr(res, "has_unrecognized") and res.has_unrecognized or \ @@ -1314,6 +1317,8 @@ class SyaNodeParser(BaseNodeParser): assert meta_orig[0] == meta_new[0] # ---- Sanity check. To remove at some point concept.get_metadata().variables = concept_metadata + concept.get_hints().use_copy = True + concept.get_hints().need_validation = True source = get_text_from_tokens(self.parser_input.tokens[start:end + 1]) return PostFixToItem(concept, start, end, has_unrecognized, source) @@ -1354,6 +1359,7 @@ class SyaNodeParser(BaseNodeParser): for infix_to_postfix in valid_infix_to_postfixs: sequence = [] has_unrecognized = False + errors = [] while len(infix_to_postfix.out) > 0: item = self.postfix_to_item(context.sheerka, infix_to_postfix.out) has_unrecognized |= hasattr(item, "has_unrecognized") and item.has_unrecognized or \ @@ -1363,10 +1369,30 @@ class SyaNodeParser(BaseNodeParser): item.start, item.end, self.parser_input.tokens[item.start: item.end + 1]) + + # validate the concept + update_compiled(context, item.concept, errors) + if errors: + break else: to_insert = item sequence.insert(0, to_insert) + if errors: + if len(errors) == 1: + ret.append( + self.sheerka.ret( + self.name, + False, + errors[0])) + else: + ret.append( + self.sheerka.ret( + self.name, + False, + self.sheerka.err([e.body for e in errors]))) + continue + if has_unrecognized: # Manage some sick cases where missing parenthesis mess the order or the sequence # example "foo bar(one plus two" @@ -1380,7 +1406,7 @@ class SyaNodeParser(BaseNodeParser): self.sheerka.new( BuiltinConcepts.PARSER_RESULT, parser=self, - source=parser_input, + source=parser_input.as_text(), body=sequence, try_parsed=sequence))) diff --git a/src/parsers/UnrecognizedNodeParser.py b/src/parsers/UnrecognizedNodeParser.py index 718cd66..321d8b9 100644 --- a/src/parsers/UnrecognizedNodeParser.py +++ b/src/parsers/UnrecognizedNodeParser.py @@ -6,6 +6,7 @@ from core.builtin_helpers import only_successful, get_lexer_nodes, update_compil from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode, SourceCodeWithConceptNode from parsers.BaseParser import BaseParser, ParsingError, BaseParserInputParser from parsers.BnfNodeParser import BnfNodeParser +from parsers.PythonParser import PythonParser from parsers.SequenceNodeParser import SequenceNodeParser from parsers.SyaNodeParser import SyaNodeParser @@ -14,7 +15,7 @@ PARSERS = ["EmptyString", SequenceNodeParser.NAME, BnfNodeParser.NAME, SyaNodeParser.NAME, - "Python"] + PythonParser.NAME] @dataclass() diff --git a/src/sheerkaql/symbols.py b/src/sheerkaql/symbols.py index dff2644..f9ca0d2 100644 --- a/src/sheerkaql/symbols.py +++ b/src/sheerkaql/symbols.py @@ -258,6 +258,7 @@ def comparison_value(value1, op, value2): return op(value1(namespace), value2(namespace)) object.__setattr__(where, '__objquery__', True) + object.__setattr__(where, '__objcond__', True) return where @@ -297,6 +298,7 @@ def set_expression_value(val, op, s): return op(val(namespace), s(namespace)) object.__setattr__(where, '__objquery__', True) + object.__setattr__(where, '__objcond__', True) return where @@ -309,6 +311,7 @@ def boolean_expression_value(value1, op, value2): return op(value1, value2, namespace) object.__setattr__(where, '__objquery__', True) + object.__setattr__(where, '__objcond__', True) return where @@ -321,6 +324,7 @@ def unary_expression_value(op, val): return op(val(namespace)) object.__setattr__(where, '__objquery__', True) + object.__setattr__(where, '__objcond__', True) return where @@ -345,6 +349,8 @@ def where_value(val): return val(namespace) object.__setattr__(where, '__objquery__', True) + if hasattr(val, "__objcond__"): + object.__setattr__(where, '__objcond__', True) return where @@ -414,12 +420,11 @@ def query_value(q): index = object.__getattribute__(current_namespace, '_objquery__i') attr_name, where = attrs_path[index] - # if sheerka_hasattr(current_namespace, attr_name): # the current object has the attr attr_value = sheerka_getattr(current_namespace, attr_name) # it is iterable if not isinstance(attr_value, str) and hasattr(attr_value, '__iter__'): # # try to use where clause as an indexer - if where is not None: + if where is not None and not hasattr(where, "__objcond__"): try: namespace_copy = dict(namespace) res = where(namespace_copy) diff --git a/src/sheerkarete/network.py b/src/sheerkarete/network.py index cea853c..c917afb 100644 --- a/src/sheerkarete/network.py +++ b/src/sheerkarete/network.py @@ -262,7 +262,7 @@ class ReteNetwork: # Manage list of requested attributes when bounding a new variable if (cond.identifier in vars_ids_mappings and - isinstance(cond.attribute, str) and + (isinstance(cond.attribute, str) and cond.attribute != FACT_SELF) and isinstance(cond.value, V)): vars_ids_mappings[cond.value] = f"{vars_ids_mappings[cond.identifier]}.{cond.attribute}" @@ -272,6 +272,11 @@ class ReteNetwork: if identifier: attr = "*" if isinstance(cond.attribute, V) else cond.attribute self.attributes_by_id.setdefault(identifier, []).append(attr) + + # to manage conditions like V(x) == V(y) + if cond.attribute == FACT_SELF and cond.value in vars_ids_mappings: + self.attributes_by_id.setdefault(vars_ids_mappings[cond.value], []).append(FACT_SELF) + elif not isinstance(cond.attribute, V): self.default_attributes.add(cond.attribute) diff --git a/tests/BaseTest.py b/tests/BaseTest.py index bd0aab6..a4ae650 100644 --- a/tests/BaseTest.py +++ b/tests/BaseTest.py @@ -45,7 +45,7 @@ class InitTestHelper: c.get_metadata().definition_type = DEFINITION_TYPE_BNF else: raise Exception(f"Error in bnf definition '{c.get_metadata().definition}'", - self.sheerka.get_errors(res)) + self.sheerka.get_errors(self.context, res)) if create_new: self.sheerka.create_new_concept(self.context, c) @@ -264,3 +264,7 @@ class BaseTest: assert res[0].status, f"Error while executing '{expression}'" return sheerka + + @staticmethod + def successful_return_values(return_values): + return [ret_val for ret_val in return_values if ret_val.status] diff --git a/tests/core/test_ExecutionContext.py b/tests/core/test_ExecutionContext.py index 6185524..5033177 100644 --- a/tests/core/test_ExecutionContext.py +++ b/tests/core/test_ExecutionContext.py @@ -47,6 +47,7 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka): a.preprocess_evaluators = ["list of evaluators"] a.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) a.global_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) + a.possible_variables = {"a", "b", "c"} b = a.push(BuiltinConcepts.EVALUATION, "sub action context", desc="sub description") @@ -65,8 +66,9 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka): assert b.preprocess_evaluators == a.preprocess_evaluators assert b.protected_hints == {BuiltinConcepts.EVAL_BODY_REQUESTED} assert b.global_hints == a.global_hints + assert b.possible_variables == a.possible_variables - def test_children_i_created_when_i_push(self): + def test_children_are_created_when_i_push(self): sheerka = self.get_sheerka() e = ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None) @@ -75,19 +77,9 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka): e.push(BuiltinConcepts.NOP, None, who="c", desc="I do something else") assert len(e._children) == 3 - assert e._children[0].who, e._children[0].who == ("a", "I do something") - assert e._children[1].who, e._children[1].who == ("b", "oups! I did a again") - assert e._children[2].who, e._children[2].who == ("c", "I do something else") - - # def test_i_can_add_variable_when_i_push(self): - # sheerka = self.get_sheerka() - # - # e = ExecutionContext("who_", Event("event"), sheerka, BuiltinConcepts.NOP, None) - # sub_e = e.push(BuiltinConcepts.NOP, None, who="a", my_new_var="new var value") - # - # assert sub_e.my_new_var == "new var value" - # with pytest.raises(AttributeError): - # assert e.my_new_var == "" # my_new_var does not exist in parent + assert (e._children[0].who, e._children[0].desc) == ("a", "I do something") + assert (e._children[1].who, e._children[1].desc) == ("b", "oups! I did a again") + assert (e._children[2].who, e._children[2].desc) == ("c", "I do something else") def test_local_hints_are_local_and_global_hints_are_global(self): sheerka = self.get_sheerka() diff --git a/tests/core/test_SheerkaConceptManager.py b/tests/core/test_SheerkaConceptManager.py index fcf00bf..1fb54c6 100644 --- a/tests/core/test_SheerkaConceptManager.py +++ b/tests/core/test_SheerkaConceptManager.py @@ -5,7 +5,7 @@ from core.builtin_concepts import BuiltinConcepts from core.builtin_helpers import ensure_bnf from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs, \ DEFINITION_TYPE_BNF -from core.global_symbols import NotInit, NotFound, SyaAssociativity +from core.global_symbols import NotInit, NotFound, SyaAssociativity, CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \ UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced, NoFirstTokenError from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore, \ @@ -293,8 +293,6 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): "definition", "definition_type", "desc", - "is_evaluated", - "need_validation", "full_serialization", ]) def test_i_can_modify_a_metadata_attribute(self, attr): @@ -1422,6 +1420,33 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka): res = sheerka.smart_get_attr(table_instance, size) assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND) + def test_i_can_set_concept_precedence(self): + sheerka, context, one, two, three = self.init_concepts("one", "two", "three") + + res = sheerka.set_precedence(context, one, two, three) + assert sheerka.isinstance(res, BuiltinConcepts.SUCCESS) + + weights = sheerka.get_weights(BuiltinConcepts.PRECEDENCE, comparison_context=CONCEPT_COMPARISON_CONTEXT) + assert weights == {'c:one|1001:': 3, 'c:two|1002:': 2, 'c:three|1003:': 1} + + def test_i_cannot_set_precedence_when_too_few_argument(self): + sheerka, context, one = self.init_concepts("one") + + res = sheerka.set_precedence(context) + assert res == sheerka.err("Not enough elements") + + res = sheerka.set_precedence(context, one) + assert res == sheerka.err("Not enough elements") + + def test_i_cannot_set_precedence_when_error(self): + sheerka, context, one, two = self.init_concepts("one", "two") + + ret = sheerka.set_precedence(context, one, two, one) + assert not ret.status + + weights = sheerka.get_weights(BuiltinConcepts.PRECEDENCE, comparison_context=CONCEPT_COMPARISON_CONTEXT) + assert weights == {'c:one|1001:': 2, 'c:two|1002:': 1} + class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): def test_i_can_add_several_concepts(self): diff --git a/tests/core/test_SheerkaDebugManager.py b/tests/core/test_SheerkaDebugManager.py index 78ed925..d52276a 100644 --- a/tests/core/test_SheerkaDebugManager.py +++ b/tests/core/test_SheerkaDebugManager.py @@ -672,16 +672,15 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka): res = sheerka.inspect(context, return_values[0], values=True) concept_debug_obj = ConceptDebugObj(return_values[0].body) - assert res.body == { - 'body': concept_debug_obj, - '#type#': 'ReturnValueConcept', - 'id': f'{self.return_value_id}', - 'key': '__RETURN_VALUE', - 'name': '__RETURN_VALUE', - 'parents': [concept_debug_obj], - 'status': True, - 'value': concept_debug_obj, - 'who': 'evaluators.OneSuccess'} + assert res.body['body'] == concept_debug_obj + assert res.body['#type#'] == 'ReturnValueConcept' + assert res.body['id'] == f'{self.return_value_id}' + assert res.body['key'] == '__RETURN_VALUE' + assert res.body['name'] == '__RETURN_VALUE' + assert isinstance(res.body['parents'], list) + assert res.body['status'] == True + assert res.body['value'] == concept_debug_obj + assert res.body['who'] == 'evaluators.OneSuccess' # I also can print it using bag res = sheerka.inspect(context, return_values[0], '#type#', "who", "status", "value", values=True, as_bag=True) diff --git a/tests/core/test_SheerkaEvaluateConcept.py b/tests/core/test_SheerkaEvaluateConcept.py index cbb7138..5256568 100644 --- a/tests/core/test_SheerkaEvaluateConcept.py +++ b/tests/core/test_SheerkaEvaluateConcept.py @@ -5,9 +5,13 @@ from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionR DEFINITION_TYPE_DEF from core.global_symbols import NotInit, NotFound from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept +from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.SheerkaMemory import SheerkaMemory from parsers.BaseParser import BaseParser +from parsers.ExactConceptParser import ExactConceptParser +from parsers.ExpressionParser import ExpressionParser from parsers.PythonParser import PythonNode, PythonParser +from parsers.SyaNodeParser import SyaNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, python_ret_val from tests.parsers.parsers_utils import CB, compare_with_test_object @@ -37,7 +41,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.get_metadata().post is None assert evaluated.get_metadata().where is None assert evaluated.variables() == {} - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated assert len(evaluated.values()) == 0 if body is None else 1 assert "foo" in sheerka.services[SheerkaMemory.NAME].registration @@ -70,7 +74,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.get_metadata().where is None assert evaluated.get_value(ConceptParts.POST) == expected assert evaluated.variables() == {} - assert not evaluated.get_metadata().is_evaluated + assert not evaluated.get_hints().is_evaluated assert len(evaluated.values) == 0 if expr is None else 1 @pytest.mark.parametrize("expr, expected", [ @@ -94,7 +98,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.get_metadata().post is None assert evaluated.get_metadata().where is None assert evaluated.variables() == {"a": expected} - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated def test_i_can_evaluate_when_the_body_is_the_name_of_the_concept(self): # to prove that I can distinguish from a string @@ -114,7 +118,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.body == "do not resolve" - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated def test_i_can_evaluate_variable_using_do_not_resolve(self): sheerka, context, concept = self.init_concepts(Concept("foo").def_var("a"), eval_body=True) @@ -123,7 +127,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.get_value("a") == "do not resolve" - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated def test_original_value_is_overridden_when_using_do_no_resolve(self): concept = Concept("foo", body="original value").def_var("a", "original value") @@ -135,7 +139,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.body == "do not resolve" assert evaluated.get_value("a") == "do not resolve" - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated def test_variables_are_evaluated_before_body(self): sheerka, context, concept = self.init_concepts(Concept("foo", body="a+1").def_var("a", "10"), eval_body=True) @@ -153,8 +157,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept) compare_with_test_object(evaluated, CB("foo", CB("a", NotInit))) - assert evaluated.get_metadata().is_evaluated - assert evaluated.body.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated + assert evaluated.body.get_hints().is_evaluated def test_i_can_evaluate_when_the_referenced_concept_has_a_body(self): sheerka, context, concept_a, concept = self.init_concepts( @@ -166,8 +170,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == concept.key compare_with_test_object(evaluated.body, CB("a", 1)) - assert not concept_a.get_metadata().is_evaluated - assert evaluated.get_metadata().is_evaluated + assert not concept_a.get_hints().is_evaluated + assert evaluated.get_hints().is_evaluated def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(self): sheerka, context, concept_a, concept_b, concept_c, concept_d = self.init_concepts( @@ -183,7 +187,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): expected = CB("c", CB("b", CB("a", "a"))) compare_with_test_object(evaluated.body, expected) assert sheerka.objvalue(evaluated) == 'a' - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(self): sheerka, context, concept_a, concept_b, concept_c, concept_d = self.init_concepts( @@ -199,7 +203,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): expected = CB("c", CB("b", CB("a", NotInit))) compare_with_test_object(evaluated.body, expected) compare_with_test_object(sheerka.objvalue(evaluated), CB("a", NotInit)) - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated def test_i_can_evaluate_concept_when_variables_reference_others_concepts_1(self): """ @@ -349,7 +353,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, add_instance) assert evaluated.key == add_instance.key - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated assert sheerka.objvalue(evaluated) == 3 def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_and_different_names(self): @@ -363,7 +367,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, add_instance) assert evaluated.key == add_instance.key - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated assert sheerka.objvalue(evaluated) == 3 def test_i_can_evaluate_when_body_is_a_concept_with_its_own_variables_multiple_levels(self): @@ -378,7 +382,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, inc_instance) assert evaluated.key == inc_instance.key - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated assert sheerka.objvalue(evaluated) == 2 def test_i_can_evaluate_a_concept_that_references_another_concept_twice(self): @@ -556,7 +560,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): sheerka, context, one_1, one_str, is_an_int, plus = self.init_concepts( Concept("one", body="1"), Concept("one", body="'one'"), - Concept("x is an int", body="isinstance(x, int)").def_var("x"), + Concept("x is an int", body="isinstance(x, int)", pre="is_question()").def_var("x"), Concept("a plus b", body="a + b", where="a is an int").def_var("a", "one").def_var("b", "2"), eval_body=True, eval_where=True, @@ -570,8 +574,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): sheerka, context, one_1, one_str, is_an_int, is_an_integer, plus = self.init_concepts( Concept("one", body="1"), Concept("one", body="'one'"), - Concept("x is an int", body="isinstance(x, int)").def_var("x"), - Concept("y is an integer", body="y is an int").def_var("y"), + Concept("x is an int", body="isinstance(x, int)", pre="is_question()").def_var("x"), + Concept("y is an integer", body="y is an int", pre="is_question()").def_var("y"), Concept("a plus b", body="a + b", where="a is an integer").def_var("a", "one").def_var("b", "2"), eval_body=True, eval_where=True, @@ -600,14 +604,15 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == plus.key assert evaluated.body == 3 - @pytest.mark.skip("Not ready for that") def test_i_can_apply_intermediate_where_condition_when_multiple_variables(self): + # The test does not work because the and condition is not correctly supported + # We need the ExpressionParser sheerka, context, one_1, one_str, two_2, two_str, is_an_int, plus = self.init_concepts( Concept("one", body="1"), Concept("one", body="'one'"), Concept("two", body="2"), Concept("two", body="'two'"), - Concept("x is an int", body="isinstance(x, int)").def_var("x"), + Concept("x is an int", body="isinstance(x, int)", pre="is_question()").def_var("x"), Concept("a plus b", body="a + b", where="a is an int and isinstance(b, int)").def_var("a", "one").def_var("b", "two"), @@ -622,7 +627,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): def test_i_cannot_evaluate_when_intermediate_where_condition_fails(self): sheerka, context, one_1, is_an_int, plus = self.init_concepts( Concept("one", body="'one'"), - Concept("x is an int", body="isinstance(x, int)").def_var("x"), + Concept("x is an int", body="isinstance(x, int)", pre="is_question()").def_var("x"), Concept("a plus b", body="a + b", where="a is an int").def_var("a", "one").def_var("b", "2"), eval_body=True, eval_where=True, @@ -685,10 +690,10 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): eval_body=True ) - for concept in (c1, c2, c3, c4): + for concept, expected in ((c1, 3), (c2, 1), (c3, 2), (c4, 3)): evaluated = sheerka.evaluate_concept(context, concept) assert evaluated.key == concept.key - assert evaluated.body == InfiniteRecursionResolved(3) + assert evaluated.body == InfiniteRecursionResolved(expected) def test_i_can_detect_infinite_recursion_when_no_constant(self): sheerka, context, foo, bar, baz, qux = self.init_concepts( @@ -877,26 +882,31 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): evaluated = sheerka.evaluate_concept(context, concept, eval_body=True) assert evaluated.key == concept.key - assert concept.get_metadata().is_evaluated == expected + assert concept.get_hints().is_evaluated == expected def test_i_only_compute_the_requested_metadata(self): sheerka, context, concept = self.init_concepts( - Concept("foo", pre="'pre'", post="'post'", ret="'ret'", where="'where'", body="'body'").def_var("a", "'a'") + Concept("foo", pre="True", post="'post'", ret="'ret'", where="True", body="'body'").def_var("a", "'a'") ) context.protected_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED) # to prove that we do not care context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED) # to prove that we do not care evaluated = sheerka.evaluate_concept(context, concept, metadata=[ConceptParts.PRE]) - assert evaluated.values() == {"a": NotInit, ConceptParts.PRE: 'pre'} + assert evaluated.values() == {"a": NotInit, ConceptParts.PRE: True} def test_i_can_manage_ret(self): sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="foo")) res = sheerka.evaluate_concept(context, bar) - assert res.id == bar.id + assert sheerka.isinstance(res, "bar") res = sheerka.evaluate_concept(context, bar, eval_body=True) - assert res.id == foo.id + assert sheerka.isinstance(res, "foo") + + # And the result is still the same after a second call + assert bar.get_hints().is_evaluated + res = sheerka.evaluate_concept(context, bar, eval_body=True) + assert sheerka.isinstance(res, "foo") def test_ret_is_evaluated_only_is_body_is_requested(self): sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="__NOT_FOUND")) @@ -930,7 +940,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): # 'def concept foo as foo' return_values = [pr_ret_val(foo, parser="ExactConcept"), python_ret_val("foo")] - res = evaluator.get_recursive_definitions(foo, return_values) + res = evaluator.get_recursive_definitions(context, foo, return_values) assert list(res) == [BaseParser.get_name("ExactConcept")] @@ -941,7 +951,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): # 'def concept foo as bar' return_values = [pr_ret_val(bar, parser="ExactConcept"), python_ret_val("foo")] - res = evaluator.get_recursive_definitions(foo, return_values) + res = evaluator.get_recursive_definitions(context, foo, return_values) assert list(res) == [] @@ -953,18 +963,120 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): # i dunno how to construct the return value return_values = [pr_ret_val(q, parser="ExactConcept")] - res = evaluator.get_recursive_definitions(q, return_values) + res = evaluator.get_recursive_definitions(context, q, return_values) assert list(res) == [] - # I cannot implement value cache for now - # def test_values_when_no_variables_are_computed_only_once(self): - # sheerka, context, foo = self.init_concepts(Concept("foo", body="10")) - # - # evaluated = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True) - # assert evaluated.body == 10 - # assert len(evaluated.get_compiled()) > 0 - # - # evaluated_2 = sheerka.evaluate_concept(context, sheerka.new("foo"), eval_body=True) - # assert evaluated_2.body == 10 - # assert len(evaluated_2.get_compiled()) == 0 + def test_i_do_not_mess_up_use_copy_when_exact_concept(self): + sheerka, context, one, number, isa = self.init_concepts( + "one", + "number", + Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y")) + + evaluator = SheerkaEvaluateConcept(sheerka) + + parsed_return_value = ExactConceptParser().parse(context, ParserInput("one is a number")) + concept = parsed_return_value[0].body.body + + # just get the compiled + context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + evaluated = evaluator.evaluate_concept(context, concept) + assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy + assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy + + # get the body + evaluated = evaluator.evaluate_concept(context, concept, eval_body=True) + assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy + assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy + assert not evaluated.get_value("x").get_hints().use_copy + assert not evaluated.get_value("y").get_hints().use_copy + + def test_i_do_not_mess_up_use_copy_when_expression_parser(self): + sheerka, context, one, number, isa = self.init_concepts( + "one", + "number", + Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y")) + + evaluator = SheerkaEvaluateConcept(sheerka) + + parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number")) + concept = next(iter(parsed_return_value.body.body.compiled[0].return_value.body.body.objects.values())) + assert concept.get_compiled()["x"][0].body.body.get_hints().use_copy + assert concept.get_compiled()["y"][0].body.body.get_hints().use_copy + + # get the body + context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + evaluated = evaluator.evaluate_concept(context, concept, eval_body=True) + assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy + assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy + assert not evaluated.get_value("x").get_hints().use_copy + assert not evaluated.get_value("y").get_hints().use_copy + + def test_i_do_not_evaluate_the_body_when_validation_only_is_set(self): + sheerka, context, red, shirt, a_x, red_x = self.init_concepts( + "red", + Concept("shirt", body="set_attr(self, 'body_shirt_is_evaluated', True)"), + Concept("a x", body="set_attr(x, 'body_ax_is_evaluated', True)", ret="x").def_var("x"), + Concept("red x", body="set_attr(x, 'color', 'red')", ret="x").def_var("x"), + create_new=True) + + parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt")) + + # Sanity check for normal behaviour + to_evaluate1 = parsed_ret_val.body.body[0].concept.copy() + evaluated1 = sheerka.evaluate_concept(context, to_evaluate1, eval_body=True, validation_only=False) + + assert sheerka.isinstance(evaluated1, shirt) + assert evaluated1.get_value("body_ax_is_evaluated") == True + assert evaluated1.get_value("body_shirt_is_evaluated") == True + assert evaluated1.get_value("color") == "red" + assert evaluated1.body == sheerka.new(BuiltinConcepts.SUCCESS) + + # check validation_only behaviour + to_evaluate2 = parsed_ret_val.body.body[0].concept.copy() + evaluated2 = sheerka.evaluate_concept(context, to_evaluate2, eval_body=True, validation_only=True) + + assert sheerka.isinstance(evaluated2, shirt) + assert evaluated2.get_value("body_ax_is_evaluated") == NotInit + assert evaluated2.get_value("body_shirt_is_evaluated") == NotInit + assert evaluated2.get_value("color") == NotInit + assert evaluated2.body == NotInit + + def test_methods_with_side_effect_are_not_called_when_eval_body_is_false(self): + sheerka, context, red, shirt, a_x, red_x = self.init_concepts( + "red", + Concept("shirt", body="set_attr(self, 'body_shirt_is_evaluated', True)"), + Concept("a x", body="set_attr(x, 'body_ax_is_evaluated', True)", ret="x").def_var("x"), + Concept("red x", body="set_attr(x, 'color', 'red')", ret="x").def_var("x"), + create_new=True, + ) + + parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt")) + to_evaluate = parsed_ret_val.body.body[0].concept + + evaluated = sheerka.evaluate_concept(context, to_evaluate, eval_body=False) + + assert sheerka.isinstance(evaluated, a_x) + assert "x" in evaluated.get_compiled() + assert ConceptParts.BODY in evaluated.get_compiled() + assert ConceptParts.RET in evaluated.get_compiled() + assert sheerka.isinstance(evaluated.get_compiled()["x"], red_x) + assert evaluated.get_compiled()["x"].get_compiled()["x"] == shirt # so, it's not evaluated + + # sanity check + parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt")) + to_evaluate = parsed_ret_val.body.body[0].concept + evaluated = sheerka.evaluate_concept(context, to_evaluate, eval_body=True) + + assert sheerka.isinstance(evaluated, shirt) + assert evaluated.get_value("body_ax_is_evaluated") == True + assert evaluated.get_value("body_shirt_is_evaluated") == True + assert evaluated.get_value("color") == "red" + + def test_concept_is_not_evaluated_when_method_access_error(self): + sheerka, context, foo = self.init_concepts(Concept("foo", body="set_attr(self, 'prop_name', 'prop_value')")) + + evaluated = sheerka.evaluate_concept(context, foo, eval_body=True, validation_only=True) + + assert sheerka.isinstance(evaluated, foo) + assert not foo.get_hints().is_evaluated diff --git a/tests/core/test_SheerkaEvaluateRules.py b/tests/core/test_SheerkaEvaluateRules.py index ab5db60..acfb64e 100644 --- a/tests/core/test_SheerkaEvaluateRules.py +++ b/tests/core/test_SheerkaEvaluateRules.py @@ -2,14 +2,14 @@ import operator import pytest -from core.builtin_concepts_ids import BuiltinConcepts from core.concept import Concept, DEFINITION_TYPE_DEF from core.rule import Rule, ACTION_TYPE_EXEC from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules, LOW_PRIORITY_RULES, DISABLED_RULES from core.sheerka.services.SheerkaExecute import ParserInput -from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition +from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition, PythonConditionExprVisitor from evaluators.PythonEvaluator import PythonEvaluator +from parsers.ExpressionParser import ExpressionParser from parsers.PythonParser import PythonParser from sheerkapython.python_wrapper import Expando from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -40,7 +40,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): DISABLED_RULES: [r7] } - @pytest.mark.skip("Not ready for that") + @pytest.mark.skip("Rete is not ready for that") def test_i_can_evaluate_question_concept_rules(self): sheerka, context, concept, r1, r2, r3, r4, r5, r6, r7, r8, r9 = self.init_test().with_concepts( Concept("x equals y", body="x == y", pre="is_question()").def_var("x").def_var("y"), @@ -152,7 +152,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): there_instance = sheerka.new_from_template(there, there.key) if recognized_by: - there_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, recognized_by) + there_instance.get_hints().recognized_by = recognized_by ret = sheerka.ret("evaluator", True, sheerka.new(greetings, a=there_instance)) res = service.evaluate_rules(context, [rule], {"__ret": ret}, set()) assert res == {True: [rule]} @@ -175,7 +175,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): my_friend_instance = sheerka.new_from_template(my_friend, my_friend.key) if recognized_by: - my_friend_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, recognized_by) + my_friend_instance.get_hints().recognized_by = recognized_by ret = sheerka.ret("evaluator", True, sheerka.new(greetings, a=my_friend_instance)) res = service.evaluate_rules(context, [rule], {"__ret": ret}, set()) assert res == {True: [rule]} @@ -211,7 +211,8 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): def test_i_can_evaluate_concept_rule_with_the_same_name_when_the_second_concept_is_declared_after(self): sheerka, context, g1, rule, g2 = self.init_test().with_concepts( Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), - create_new=True).with_format_rules(Rule(predicate="recognize(__ret.body, greetings)", action="")).with_concepts( + create_new=True).with_format_rules( + Rule(predicate="recognize(__ret.body, greetings)", action="")).with_concepts( Concept("greetings", definition="hi a", definition_type=DEFINITION_TYPE_DEF).def_var("a")).unpack() service = sheerka.services[SheerkaEvaluateRules.NAME] @@ -249,3 +250,22 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka): assert rule in evaluate_rule_service.network.rules assert rule.rete_net == evaluate_rule_service.network + + def test_i_can_get_missing_variables_when_evaluate_conditions(self): + sheerka, context = self.init_test().unpack() + + expression = "isinstance(a, int)" + parser = ExpressionParser() + ret_val = parser.parse(context, ParserInput(expression)) + parsed = ret_val.body.body + + visitor = PythonConditionExprVisitor(context) + conditions = visitor.get_conditions(parsed) + + missing_vars = set() + service = sheerka.services[SheerkaEvaluateRules.NAME] + res = service.evaluate_conditions(context, conditions, {}, missing_vars) + + assert res == [] + assert missing_vars == {"a"} + diff --git a/tests/core/test_SheerkaIsAManager.py b/tests/core/test_SheerkaIsAManager.py index 0b4da52..cd80863 100644 --- a/tests/core/test_SheerkaIsAManager.py +++ b/tests/core/test_SheerkaIsAManager.py @@ -84,7 +84,19 @@ class TestSheerkaIsAManager(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(error, BuiltinConcepts.NOT_A_SET) assert error.body == one - def test_isa_and_isa_group(self): + def test_isa(self): + sheerka, context, blue, color = self.init_concepts(Concept("blue"), Concept("color")) + + assert not sheerka.isa(blue, color) + + sheerka.set_isa(context, blue, color) + assert sheerka.isa(blue, color) + + # isa tests the id of a concept, not it's content + another_color_instance_but_with_a_body = sheerka.new(color, body="a body") + assert sheerka.isa(blue, another_color_instance_but_with_a_body) + + def test_isaset(self): sheerka, context, group, foo = self.init_concepts(Concept("group"), Concept("foo")) assert not sheerka.isaset(context, group) diff --git a/tests/core/test_SheerkaRuleManager.py b/tests/core/test_SheerkaRuleManager.py index b1662a8..af7d4b3 100644 --- a/tests/core/test_SheerkaRuleManager.py +++ b/tests/core/test_SheerkaRuleManager.py @@ -4,28 +4,18 @@ from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept from core.concept import Concept, DEFINITION_TYPE_DEF from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound, EVENT_RULE_DELETED from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC -from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME from core.sheerka.services.SheerkaExecute import ParserInput -from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, FormatRuleActionParser, \ - FormatAstRawText, FormatAstVariable, FormatAstSequence, FormatAstFunction, \ - FormatRuleSyntaxError, FormatAstList, UnexpectedEof, FormatAstColor, FormatAstDict, \ - FormatAstMulti, \ - PythonCodeEmitter, FormatAstNode, ReteConditionExprVisitor +from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, ReteConditionExprVisitor from core.tokenizer import Token, TokenKind from parsers.BaseParser import ErrorSink from parsers.ExpressionParser import ExpressionParser +from parsers.FormatRuleActionParser import FormatAstNode from sheerkarete.conditions import FilterCondition from sheerkarete.network import ReteNetwork from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND, NCCOND -seq = FormatAstSequence -raw = FormatAstRawText -var = FormatAstVariable -func = FormatAstFunction -lst = FormatAstList - PYTHON_EVALUATOR_NAME = "Python" CONCEPT_EVALUATOR_NAME = "Concept" @@ -175,7 +165,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): sheerka, context = self.init_test(cache_only=False).unpack() service = sheerka.services[SheerkaRuleManager.NAME] - rule = Rule(ACTION_TYPE_EXEC, "name", "cannot build = False", "'Hello back at you !'") + rule = Rule(action_type, "name", "cannot build = False", action) rule.metadata.is_enabled = True # it should be disabled rule = service.init_rule(context, rule) @@ -209,71 +199,6 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): assert not rule.metadata.is_enabled assert rule.compiled_action is None - @pytest.mark.parametrize("text, expected", [ - ("", FormatAstRawText("")), - (" ", FormatAstRawText(" ")), - (" raw text ", FormatAstRawText(" raw text ")), - ("{variable}", FormatAstVariable("variable")), - ("{ variable }", FormatAstVariable("variable")), - (" xy {v} z", seq([raw(" xy "), var("v"), raw(" z")])), - (r"\{variable}", FormatAstRawText("{variable}")), - (r"\\{variable}", seq([raw("\\"), var("variable")])), - (r"\\\{variable}", FormatAstRawText(r"\{variable}")), - (r"{var1}{var2}", seq([var("var1"), var("var2")])), - ("func()", FormatAstFunction("func", [], {})), - ("func(a, 'string value', c)", FormatAstFunction("func", ["a", "'string value'", "c"], {})), - ("func(a=10, b='string value')", FormatAstFunction("func", [], {"a": "10", "b": "'string value'"})), - ("func('string value'='another string value')", func("func", [], {"'string value'": "'another string value'"})), - ("red(' xy {v}')", FormatAstColor("red", seq([raw(" xy "), var("v")]))), - ('blue(" xy {v}")', FormatAstColor("blue", seq([raw(" xy "), var("v")]))), - ('green( xy )', FormatAstColor("green", var("xy"))), - ('green()', FormatAstColor("green", raw(""))), - ('green("")', FormatAstColor("green", raw(""))), - ("list(var_name, 2, 'children')", FormatAstList("var_name", recurse_on="children", recursion_depth=2)), - ("list(var_name, recursion_depth=2, recurse_on='children')", FormatAstList("var_name", - recurse_on="children", - recursion_depth=2)), - ("list(var_name, recursion_depth=2, 'children')", FormatAstList("var_name", recursion_depth=2)), - ("list(var_name, 'children', recursion_depth=2)", FormatAstList("var_name", recursion_depth=2)), - ("list(var_name)", FormatAstList("var_name")), - ("{obj.prop1.prop2[0].prop3['value']}", FormatAstVariable("obj.prop1.prop2[0].prop3['value']")), - ("[{id}]", seq([raw("["), var("id"), raw("]")])), - ("{variable:format}", FormatAstVariable("variable", "format")), - ("{variable:3}", FormatAstVariable("variable", "3")), - (r"\not_a_function(a={var})", seq([raw("not_a_function(a="), var("var"), raw(")")])), - ("dict(var_name)", FormatAstDict("var_name")), - ("dict(var_name, items_prop='props')", FormatAstDict("var_name", items_prop='props')), - ("dict(var_name, debug=True)", FormatAstDict("var_name", debug=True, prefix="{", suffix="}")), - ("multi(var_name)", FormatAstMulti("var_name")), - ]) - def test_i_can_parse_format_rule(self, text, expected): - assert FormatRuleActionParser(text).parse() == expected - - @pytest.mark.parametrize("text, expected_error", [ - ("{", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))), - ("{var_name", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))), - ("{}", FormatRuleSyntaxError("variable name not found", None)), - ("func(", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))), - ("func(a,b,c", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))), - ("func(a,,c", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))), - ("func(a,,c)", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))), - ("red(a,b)", FormatRuleSyntaxError("only one parameter supported", Token(TokenKind.IDENTIFIER, "b", 6, 1, 7))), - ("red(a=b)", FormatRuleSyntaxError("keyword arguments are not supported", None)), - ("red(xy {v})", FormatRuleSyntaxError("Invalid identifier", None)), - ("list()", FormatRuleSyntaxError("variable name not found", None)), - ("list(recursion_depth=2)", FormatRuleSyntaxError("variable name not found", None)), - ("list(a,b,c,d,e)", FormatRuleSyntaxError("too many positional arguments", - Token(TokenKind.IDENTIFIER, "e", 13, 1, 14))), - ("list(a, recursion_depth=hello)", FormatRuleSyntaxError("'hello' is not numeric", None)), - ("list(a, recursion_depth='hello')", FormatRuleSyntaxError("'recursion_depth' must be an integer", None)), - ("dict()", FormatRuleSyntaxError("variable name not found", None)), - ]) - def test_i_cannot_parse_invalid_format(self, text, expected_error): - parser = FormatRuleActionParser(text) - parser.parse() - - assert parser.error_sink == expected_error - def test_i_can_get_rule_priorities(self): sheerka, context, rule_true, rule_false = self.init_test().with_format_rules(("True", "True"), ("False", "False")).unpack() @@ -337,106 +262,6 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka): unresolved.metadata.id_is_unresolved = True assert sheerka.resolve_rule(context, unresolved) == rule - @pytest.mark.parametrize("obj, expected", [ - ("text value", "var == 'text value'"), - ("text 'value'", '''var == "text 'value'"'''), - ('text "value"', """var == 'text "value"'"""), - (10, "var == 10"), - (10.01, "var == 10.01"), - ]) - def test_i_can_test_python_code_emitter_for_basic_types(self, obj, expected): - sheerka, context = self.init_test().unpack() - - assert PythonCodeEmitter(context).recognize(obj, "var").get_text() == expected - assert PythonCodeEmitter(context, "status").recognize(obj, "var").get_text() == "status and " + expected - - @pytest.mark.parametrize("recognized_by, expected", [ - (RECOGNIZED_BY_ID, "isinstance(var, Concept) and var.id == '1001'"), - (RECOGNIZED_BY_NAME, "isinstance(var, Concept) and var.name == 'greetings'"), - (None, "isinstance(var, Concept) and var.key == 'hello'"), - ]) - def test_i_can_test_python_code_emitter_for_concepts(self, recognized_by, expected): - sheerka, context, foo = self.init_concepts( - Concept("greetings", definition="hello", definition_type=DEFINITION_TYPE_DEF)) - - instance = sheerka.new_from_template(foo, foo.key) - if recognized_by: - instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, recognized_by) - - assert PythonCodeEmitter(context).recognize(instance, "var").get_text() == expected - assert PythonCodeEmitter(context, "status").recognize(instance, "var").get_text() == "status and " + expected - - def test_i_can_test_python_code_emitter_for_concepts_with_variable(self): - sheerka, context, greetings, little, foo, bar, and_concept = self.init_concepts( - Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), - Concept("little x").def_var("x"), - "foo", - "bar", - Concept("a and b").def_var("a").def_var("b") - ) - - # variable is a string - greetings_instance = sheerka.new_from_template(greetings, greetings.key, a='sheerka') - expected = "isinstance(var, Concept) and var.key == 'hello __var__0' and var.get_value('a') == 'sheerka'" - text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() - assert text == expected - - # variable is a concept recognized by id - foo_instance = sheerka.new_from_template(foo, foo.key) - foo_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_ID) - greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance) - expected = """__x_00__ = var.get_value('a') -isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.id == '1003'""" - text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() - assert text == expected - - # variable is a concept recognized by name - foo_instance = sheerka.new_from_template(foo, foo.key) - foo_instance.set_hint(BuiltinConcepts.RECOGNIZED_BY, RECOGNIZED_BY_NAME) - greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance) - expected = """__x_00__ = var.get_value('a') -isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.name == 'foo'""" - text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() - assert text == expected - - # variable is a concept recognized by value - foo_instance = sheerka.new_from_template(foo, foo.key) - greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance) - expected = """__x_00__ = var.get_value('a') -isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.key == 'foo'""" - text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() - assert text == expected - - # variable is a concept witch has itself some variable - foo_instance = sheerka.new_from_template(foo, foo.key) - little_instance = sheerka.new_from_template(little, little.key, x=foo_instance) - greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=little_instance) - expected = """__x_00__ = var.get_value('a') -__x_01__ = __x_00__.get_value('x') -isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ - " and isinstance(__x_00__, Concept) and __x_00__.key == 'little __var__0'" + \ - " and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'""" - text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() - assert text == expected - - # concept with multiple variables (which are themselves concepts) - foo_instance = sheerka.new_from_template(foo, foo.key) - bar_instance = sheerka.new_from_template(bar, bar.key) - little_instance = sheerka.new_from_template(little, little.key, x=foo_instance) - and_instance = sheerka.new_from_template(and_concept, and_concept.key, a=bar_instance, b=little_instance) - greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=and_instance) - expected = """__x_00__ = var.get_value('a') -__x_01__ = __x_00__.get_value('a') -__x_02__ = __x_00__.get_value('b') -__x_03__ = __x_02__.get_value('x') -isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ - " and isinstance(__x_00__, Concept) and __x_00__.key == '__var__0 and __var__1'" + \ - " and isinstance(__x_01__, Concept) and __x_01__.key == 'bar'" + \ - " and isinstance(__x_02__, Concept) and __x_02__.key == 'little __var__0'" + \ - " and isinstance(__x_03__, Concept) and __x_03__.key == 'foo'" - text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() - assert text == expected - def test_i_can_get_format_rules(self): sheerka, context = self.init_test().unpack() service = sheerka.services[SheerkaRuleManager.NAME] @@ -674,3 +499,103 @@ class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka): assert rule.compiled_conditions == expected.compiled_conditions assert rule.priority is not None assert rule.priority == expected.priority + +# @pytest.mark.parametrize("obj, expected", [ +# ("text value", "var == 'text value'"), +# ("text 'value'", '''var == "text 'value'"'''), +# ('text "value"', """var == 'text "value"'"""), +# (10, "var == 10"), +# (10.01, "var == 10.01"), +# ]) +# def test_i_can_test_python_code_emitter_for_basic_types(self, obj, expected): +# sheerka, context = self.init_test().unpack() +# +# assert PythonCodeEmitter(context).recognize(obj, "var").get_text() == expected +# assert PythonCodeEmitter(context, "status").recognize(obj, "var").get_text() == "status and " + expected +# +# @pytest.mark.parametrize("recognized_by, expected", [ +# (RECOGNIZED_BY_ID, "isinstance(var, Concept) and var.id == '1001'"), +# (RECOGNIZED_BY_NAME, "isinstance(var, Concept) and var.name == 'greetings'"), +# (None, "isinstance(var, Concept) and var.key == 'hello'"), +# ]) +# def test_i_can_test_python_code_emitter_for_concepts(self, recognized_by, expected): +# sheerka, context, foo = self.init_concepts( +# Concept("greetings", definition="hello", definition_type=DEFINITION_TYPE_DEF)) +# +# instance = sheerka.new_from_template(foo, foo.key) +# if recognized_by: +# instance.get_hints().recognized_by = recognized_by +# +# assert PythonCodeEmitter(context).recognize(instance, "var").get_text() == expected +# assert PythonCodeEmitter(context, "status").recognize(instance, "var").get_text() == "status and " + expected +# +# def test_i_can_test_python_code_emitter_for_concepts_with_variable(self): +# sheerka, context, greetings, little, foo, bar, and_concept = self.init_concepts( +# Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), +# Concept("little x").def_var("x"), +# "foo", +# "bar", +# Concept("a and b").def_var("a").def_var("b") +# ) +# +# # variable is a string +# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a='sheerka') +# expected = "isinstance(var, Concept) and var.key == 'hello __var__0' and var.get_value('a') == 'sheerka'" +# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() +# assert text == expected +# +# # variable is a concept recognized by id +# foo_instance = sheerka.new_from_template(foo, foo.key) +# foo_instance.get_hints().recognized_by = RECOGNIZED_BY_ID +# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance) +# expected = """__x_00__ = var.get_value('a') +# isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.id == '1003'""" +# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() +# assert text == expected +# +# # variable is a concept recognized by name +# foo_instance = sheerka.new_from_template(foo, foo.key) +# foo_instance.get_hints().recognized_by = RECOGNIZED_BY_NAME +# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance) +# expected = """__x_00__ = var.get_value('a') +# isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.name == 'foo'""" +# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() +# assert text == expected +# +# # variable is a concept recognized by value +# foo_instance = sheerka.new_from_template(foo, foo.key) +# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=foo_instance) +# expected = """__x_00__ = var.get_value('a') +# isinstance(var, Concept) and var.key == 'hello __var__0' and isinstance(__x_00__, Concept) and __x_00__.key == 'foo'""" +# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() +# assert text == expected +# +# # variable is a concept witch has itself some variable +# foo_instance = sheerka.new_from_template(foo, foo.key) +# little_instance = sheerka.new_from_template(little, little.key, x=foo_instance) +# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=little_instance) +# expected = """__x_00__ = var.get_value('a') +# __x_01__ = __x_00__.get_value('x') +# isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ +# " and isinstance(__x_00__, Concept) and __x_00__.key == 'little __var__0'" + \ +# " and isinstance(__x_01__, Concept) and __x_01__.key == 'foo'""" +# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() +# assert text == expected +# +# # concept with multiple variables (which are themselves concepts) +# foo_instance = sheerka.new_from_template(foo, foo.key) +# bar_instance = sheerka.new_from_template(bar, bar.key) +# little_instance = sheerka.new_from_template(little, little.key, x=foo_instance) +# and_instance = sheerka.new_from_template(and_concept, and_concept.key, a=bar_instance, b=little_instance) +# greetings_instance = sheerka.new_from_template(greetings, greetings.key, a=and_instance) +# expected = """__x_00__ = var.get_value('a') +# __x_01__ = __x_00__.get_value('a') +# __x_02__ = __x_00__.get_value('b') +# __x_03__ = __x_02__.get_value('x') +# isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ +# " and isinstance(__x_00__, Concept) and __x_00__.key == '__var__0 and __var__1'" + \ +# " and isinstance(__x_01__, Concept) and __x_01__.key == 'bar'" + \ +# " and isinstance(__x_02__, Concept) and __x_02__.key == 'little __var__0'" + \ +# " and isinstance(__x_03__, Concept) and __x_03__.key == 'foo'" +# text = PythonCodeEmitter(context).recognize(greetings_instance, "var").get_text() +# assert text == expected diff --git a/tests/core/test_SheerkaRuleManagerRulesCompilation.py b/tests/core/test_SheerkaRuleManagerRulesCompilation.py index 6e3dd97..0bbe39d 100644 --- a/tests/core/test_SheerkaRuleManagerRulesCompilation.py +++ b/tests/core/test_SheerkaRuleManagerRulesCompilation.py @@ -4,6 +4,7 @@ import pytest from core.builtin_concepts import ReturnValueConcept from core.builtin_concepts_ids import BuiltinConcepts +from core.builtin_helpers import ensure_evaluated from core.concept import Concept, DEFINITION_TYPE_DEF from core.rule import Rule from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules @@ -23,6 +24,19 @@ from tests.parsers.parsers_utils import get_rete_conditions, NEGCOND class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka): + + @staticmethod + def get_conditions(context, expression): + parser = ExpressionParser() + error_sink = ErrorSink() + parser_input = ParserInput(expression) + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = PythonConditionExprVisitor(context) + conditions = visitor.get_conditions(parsed) + return conditions + @staticmethod def check_against_rete(rule_expression, rule_conditions, objects): """ @@ -72,6 +86,9 @@ class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka): sub_context.sheerka.add_many_to_short_term_memory(sub_context, objects) evaluator = PythonEvaluator() + for c in condition.concepts_to_reset: + c.get_hints().is_evaluated = False + return evaluator.eval(sub_context, condition.return_value) @classmethod @@ -130,15 +147,7 @@ class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka): expected_not_variables, expected_objects): sheerka = context.sheerka - - parser = ExpressionParser() - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = PythonConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) + conditions = BaseTestSheerkaRuleManagerRulesCompilation.get_conditions(context, expression) assert len(conditions) == 1 assert isinstance(conditions[0], CompiledCondition) @@ -312,6 +321,132 @@ class TestSheerkaRuleManagerRulesCompilationNotExists(BaseTestSheerkaRuleManager self.check_against_python(context, expression, conditions, objects) +class TestSheerkaRuleManagerRulesCompilationSimplePython(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing Python + True + False + 10 + 5 + 'hello world' + a + self + a + 10 + a + " world !" + a + foo + a + twenty one + a + my friend + """ + + @pytest.mark.parametrize("expression, e_compiled, e_text, e_variables, e_objects, e_result", [ + ( + "True", + "True", + "True", + set(), + set(), + True + ), + ( + "False", + "False", + "False", + set(), + set(), + False + ), + ( + "10 + 5", + "__o_00__", + "10 + 5", + set(), + {("__o_00__", 15)}, + 15 + ), + ( + "'hello world'", + "__o_00__", + "'hello world'", + set(), + {("__o_00__", 'hello world')}, + 'hello world' + ), + ( + "a + self", + "a + self", + "a + self", + {("a", 10), ("self", "foo")}, + set(), + 15 + ), + ( + "a + 10", + "a + 10", + "a + 10", + {("a", 10)}, + set(), + 20 + ), + ( + "a + 'world !'", + "a + 'world !'", + "a + 'world !'", + {("a", "hello ")}, + set(), + "hello world !" + ), + ( + "a + foo", + "a + foo", + "a + foo", + {("a", 10), ("foo", "foo")}, + set(), + 15 + ), + ( + "a + twenty one", + "a + __C__twenties__1004__C__", + "a + twenty one", + {("a", 10)}, + {"__C__twenties__1004__C__"}, + 31 + ), + ( + "a + my friend", + "a + __C__my0friend__1005__C__", + "a + my friend", + {("a", "hello ")}, + {'__C__my0friend__1005__C__'}, + "hello my friend" + ), + ]) + def test_python(self, expression, e_compiled, e_text, e_variables, e_objects, e_result): + sheerka, context, foo, one, two, twenties, my_friend = self.init_concepts( + Concept("foo", body="5"), + Concept("one", body="1"), + Concept("two", body="2"), + Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"), + Concept("my friend", body="'my friend'"), + create_new=True + ) + ensure_evaluated(context, foo, eval_body=True) + ensure_evaluated(context, my_friend, eval_body=True) + conditions = self.validate_python_test(context, + expression, + e_compiled, + e_text, + e_variables, + set(), + e_objects) + + # check against SheerkaEvaluateRules + variables_mapping = { + "foo": foo, + } + namespace = self.get_testing_objects(context, e_variables, variables_mapping) + res = self.evaluate_condition(context, expression, conditions[0], namespace) + assert res.status + assert sheerka.objvalue(res) == e_result + + class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerRulesCompilation): """ Testing simple equality: @@ -321,6 +456,7 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR self == sheerka self == BuiltinConcepts.TO_DICT self == hello 'my friend' + a == b """ @pytest.mark.parametrize("expression, expected_as_list_of_str, expected_variables", [ @@ -339,7 +475,12 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR "#__x_00__|key|'hello __var__0'", "#__x_00__|a|'my friend'"], {("self", "hello_my_friend")} - ) + ), + # ("a == b", + # ["#__x_00__|__name__|'a'", + # "#__x_01__|__name__|'b'", + # "#__x_00__|__self__|#__x_01__"], + # {("a", 10), ("b", 10)}), ]) def test_rete(self, expression, expected_as_list_of_str, expected_variables): sheerka, context, greetings = self.init_test().with_concepts( @@ -365,23 +506,27 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR objects = self.get_testing_objects(context, expected_variables, objects_mappings) self.check_against_rete(expression, conditions, objects) + # KSI: 2021-05-06 The last test done not produce any match because the WME (b, __self__, 10) + # is not added to memory. + @pytest.mark.parametrize("expression, expected_compiled, expected_variables, expected_objects", [ ("a == 10", "a == __o_00__", {("a", 10)}, {("__o_00__", 10)}), - ("__ret.status == True", "__ret.status == __o_00__", {"__ret"}, {("__o_00__", True)}), + ("__ret.status == True", "__ret.status == True", {"__ret"}, set()), ("self == 'a'", "self == __o_00__", {("self", 'a')}, {("__o_00__", 'a')}), ("self == sheerka", "is_sheerka(self)", {("self", "sheerka")}, {}), ( "self == BuiltinConcepts.TO_DICT", - "self == __o_00__", + "self == BuiltinConcepts.TO_DICT", {("self", BuiltinConcepts.TO_DICT)}, - {("__o_00__", BuiltinConcepts.TO_DICT)} + set() ), ( "self == hello 'my friend'", """isinstance(self, Concept) and self.key == 'hello __var__0' and self.a == __o_01__""", {("self", "hello_my_friend")}, {("__o_01__", "my friend")} - ) + ), + ("a == b", "a == b", {("a", 10), ("b", 10)}, {}), ]) def test_python(self, expression, expected_compiled, expected_variables, expected_objects): sheerka, context, greetings = self.init_test().with_concepts( @@ -403,6 +548,176 @@ class TestSheerkaRuleManagerRulesCompilationEquality(BaseTestSheerkaRuleManagerR self.check_against_python(context, expression, conditions, objects) +class TestSheerkaRuleManagerRulesCompilationOtherConditions(BaseTestSheerkaRuleManagerRulesCompilation): + """ + Testing other conditions than equality + a > 10 + a >= 10 + a < 10 + a <= 10 + a != 10 + a > 10 and b <= 5 + + __ret.value > 10 + 10 > __ret.value + + a + self > 10 + a + 10 > 10 + a + " world !" == "hello world !" + a + foo > 10 + a + twenty one > 21 + a + my friend == 'hello my friend' + + 10 < a + self + 10 < a + 10 + 'hello world !' == a + ' world !' + 10 < a + foo + 10 > a + twenty one + 'hello my friend' == a + my friend + """ + + @pytest.mark.parametrize("expression, e_compiled, e_variables, e_objects, e_result", [ + ("a > 10", "a > __o_00__", {("a", 10)}, {("__o_00__", 10)}, False), + ("a >= 10", "a >= __o_00__", {("a", 10)}, {("__o_00__", 10)}, True), + ("a < 10", "a < __o_00__", {("a", 10)}, {("__o_00__", 10)}, False), + ("a <= 10", "a <= __o_00__", {("a", 10)}, {("__o_00__", 10)}, True), + ("a != 10", "a != __o_00__", {("a", 10)}, {("__o_00__", 10)}, False), + ( + "a > 10 and b <= 5", + "a > __o_00__ and b <= __o_01__", + {("a", 11), ("b", 4)}, + {("__o_00__", 10), ("__o_01__", 5)}, + True + ), + ( + "__ret.value > 10", + "__ret.value > __o_00__", + {("__ret", 15)}, + {("__o_00__", 10)}, + True + ), + ( + "10 > __ret.value", + "__o_00__ > __ret.value", + {("__ret", 15)}, + {("__o_00__", 10)}, + False + ), + ( + "a + self > 10", + "a + self > __o_00__", + {("a", 6), ("self", "foo")}, + {("__o_00__", 10)}, + True + ), + ( + "a + 10 > 10", + "a + 10 > __o_00__", + {("a", 5)}, + {("__o_00__", 10)}, + True + ), + ( + "a + 'world !' == 'hello world !'", + "a + 'world !' == __o_00__", + {("a", "hello ")}, + {("__o_00__", 'hello world !')}, + True + ), + ( + "a + foo > 10", + "a + foo > __o_00__", + {("a", 6), ("foo", "foo")}, + {("__o_00__", 10)}, + True + ), + ( + "a + twenty one > 21", + "a + __C__twenties__1004__C__ > __o_00__", + {("a", 5)}, + {"__C__twenties__1004__C__", ("__o_00__", 21)}, + True + ), + ( + "a + my friend == 'hello my friend'", + "a + __C__my0friend__1005__C__ == __o_00__", + {("a", "hello ")}, + {"__C__my0friend__1005__C__", ("__o_00__", 'hello my friend')}, + True + ), + + ( + "10 < a + self", + "__o_00__ < a + self", + {("a", 6), ("self", "foo")}, + {("__o_00__", 10)}, + True + ), + ( + "10 > a + 10", + "__o_00__ > a + 10", + {("a", 5)}, + {("__o_00__", 10)}, + False + ), + ( + "'hello world !' != a + 'world !'", + "__o_00__ != a + 'world !'", + {("a", "hello ")}, + {("__o_00__", 'hello world !')}, + False + ), + ( + "10 < a + foo", + "__o_00__ < a + foo", + {("a", 6), ("foo", "foo")}, + {("__o_00__", 10)}, + True + ), + ( + "21 > a + twenty one", + "__o_00__ > a + __C__twenties__1004__C__", + {("a", 5)}, + {"__C__twenties__1004__C__", ("__o_00__", 21)}, + False + ), + ( + "'hello my friend' == a + my friend", + "__o_00__ == a + __C__my0friend__1005__C__", + {("a", "hello ")}, + {"__C__my0friend__1005__C__", ("__o_00__", 'hello my friend')}, + True + ), + + ]) + def test_python(self, expression, e_compiled, e_variables, e_objects, e_result): + sheerka, context, foo, one, two, twenties, my_friend = self.init_concepts( + Concept("foo", body="5"), + Concept("one", body="1"), + Concept("two", body="2"), + Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"), + Concept("my friend", body="'my friend'"), + create_new=True + ) + ensure_evaluated(context, foo, eval_body=True) + ensure_evaluated(context, my_friend, eval_body=True) + + conditions = self.validate_python_test(context, + expression, + e_compiled, + expression, + e_variables, + set(), + e_objects) + + # check against SheerkaEvaluateRules + variables_mapping = { + "foo": foo, + } + objects = self.get_testing_objects(context, e_variables, variables_mapping) + self.check_against_python(context, expression, conditions, objects, expected_result=e_result) + + class TestSheerkaRuleManagerRulesCompilationFunctionsCall(BaseTestSheerkaRuleManagerRulesCompilation): """ Testing functions @@ -795,6 +1110,8 @@ class TestSheerkaRuleManagerRulesCompilationEvalQuestionConcept(BaseTestSheerkaR with long concept : the little boy is a human being with long concept + variable : the little boy is a self with long concept + variable : self is a human being + with a special symbol : self is a 'human' + with a special symbol : the little boy is a 'human' """ def test_rete(self): @@ -826,6 +1143,16 @@ class TestSheerkaRuleManagerRulesCompilationEvalQuestionConcept(BaseTestSheerkaR "evaluate_question(__o_00__)", {"self"}, ), + ( + "self is a 'human'", + "evaluate_question(__o_00__)", + {"self"}, + ), + ( + "the little boy is a 'human'", + "evaluate_question(__o_00__)", + set(), + ), ]) def test_python(self, expression, expected_compiled, expected_variables): sheerka, context, girl, human, little_boy, human_being, isa = self.init_test().with_concepts( @@ -980,11 +1307,11 @@ class TestSheerkaRuleManagerRulesCompilationEvalConceptMixedWithOther(BaseTestSh expected_objects) # check against SheerkaEvaluateRules - variable_mapping = { + variables_mapping = { "girl": girl, "human being": human_being } - testing_objects = self.get_testing_objects(context, expected_variables, variable_mapping) + testing_objects = self.get_testing_objects(context, expected_variables, variables_mapping) self.check_against_python(context, expression, conditions, @@ -1005,6 +1332,8 @@ class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheer with function: func_identity(twenty two + twenty one) with function: func_identity(twenty two plus one) with function: func_identity(twenty two plus twenty one) + with special char : 'one' plus 'two' + with special char : twenty two plus 2 """ def test_rete(self): @@ -1081,6 +1410,20 @@ class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheer {"__o_00__"}, 43 ), + ( + "'one' plus 'two'", + "__o_00__", + "'one' plus 'two'", + {"__o_00__"}, + 'onetwo' + ), + ( + "twenty two plus 2", + "__o_00__", + "twenty two plus 2", + {"__o_00__"}, + 24 + ), ]) def test_python(self, expression, e_compiled, e_text, e_objects, e_result): sheerka, context, one, two, twenties, plus = self.init_test().with_concepts( @@ -1109,7 +1452,7 @@ class TestSheerkaRuleManagerRulesCompilationEvalNonQuestionConcept(BaseTestSheer class TestSheerkaRuleManagerRulesCompilationMultipleSameConcept(BaseTestSheerkaRuleManagerRulesCompilation): """ Test when a concept returns multiple results - The compilation should fail : No need to execute a condition if we are not sure of the meaning ? + The compilation should fail : No need to execute a condition if we are not sure of the meaning self is a bar """ @@ -1156,6 +1499,99 @@ class TestSheerkaRuleManagerRulesCompilationNot(BaseTestSheerkaRuleManagerRulesC Testing not not __ret.status == True not recognize(__ret.body, hello sheerka) + not a cat is a pet and not bird is an animal # where x is a y is a concept + not a cat is a pet and not x > 5 # concept mixed with python """ pass + + +class TestCompiledCondition(BaseTestSheerkaRuleManagerRulesCompilation): + @pytest.mark.parametrize("expression, expected", [ + ("self is a 'foo'", {"x is a y"}), + ("set self is a 'foo'", set()), + ]) + def test_i_can_get_concept_to_reset(self, expression, expected): + """ + When compiled conditions, sometimes there are concepts to reset between two usages + :param expression: + :param expected: + :return: + """ + sheerka, context, question, not_a_question = self.init_concepts( + Concept("x is a y", pre="is_question()").def_var("x").def_var("y"), + Concept("set x is a y").def_var("x").def_var("y"), + ) + + conditions = self.get_conditions(context, expression) + + assert len(conditions) == 1 + assert set(c.name for c in conditions[0].concepts_to_reset) == expected + + def test_i_can_reset_concepts_when_multiple_levels(self): + """ + When compiled conditions, sometimes there are concepts to reset between two usages + :return: + """ + sheerka, context, is_instance, is_int, is_integer = self.init_concepts( + Concept("x is an instance of y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + Concept("x is a int", pre="is_question()", body="x is an instance of int").def_var("x"), + Concept("x is an integer", pre="is_question()", body="x is a int").def_var("x"), + ) + + expression = "self is an integer" + conditions = conditions = self.get_conditions(context, expression) + + assert len(conditions) == 1 + assert set(c.name for c in conditions[0].concepts_to_reset) == {"x is an instance of y", + "x is a int", + "x is an integer"} + + # So I can evaluate multiple times + res = self.evaluate_condition(context, expression, conditions[0], {'self': 10}) + assert res.status + assert sheerka.objvalue(res.body) + + res = self.evaluate_condition(context, expression, conditions[0], {'self': "string"}) + assert res.status + assert not sheerka.objvalue(res.body) + + def test_i_can_reset_concepts_when_multiple_levels_and_concept_node(self): + """ + When compiled conditions, sometimes there are concepts to reset between two usages + :return: + """ + # in this example, x + 2 is an int won't be parsed as an ExactNodeConcept, but as a ConceptNode + sheerka, context, is_int, is_integer = self.init_concepts( + Concept("x is a int", pre="is_question()", body="isinstance(x, int)").def_var("x"), + Concept("x is an integer", pre="is_question()", body="x + 2 is a int").def_var("x"), + create_new=True + ) + + expression = "self is an integer" + conditions = self.get_conditions(context, expression) + + assert len(conditions) == 1 + assert set(c.name for c in conditions[0].concepts_to_reset) == {"x is a int", + "x is an integer"} + + # So I can evaluate multiple times + res = self.evaluate_condition(context, expression, conditions[0], {'self': 10}) + assert res.status + assert sheerka.objvalue(res.body) + + res = self.evaluate_condition(context, expression, conditions[0], {'self': "string"}) + assert not res.status + + def test_long_name_concept_set_are_not_considered_as_variables(self): + sheerka, context, one, number = self.init_concepts( + "one", + "all numbers", + ) + sheerka.set_isa(context, one, number) + + expression = "all numbers < 5" + conditions = self.get_conditions(context, expression) + + assert len(conditions) == 1 + assert conditions[0].return_value.body.body.source == '__o_00__ < __o_01__' diff --git a/tests/core/test_builtin_helpers.py b/tests/core/test_builtin_helpers.py index dd81b9e..adc677f 100644 --- a/tests/core/test_builtin_helpers.py +++ b/tests/core/test_builtin_helpers.py @@ -4,6 +4,13 @@ import core.builtin_helpers from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts from core.concept import Concept from core.global_symbols import NotInit +from core.sheerka.services.SheerkaExecute import ParserInput +from core.tokenizer import Tokenizer +from evaluators.BaseEvaluator import BaseEvaluator +from evaluators.ValidateConceptEvaluator import ValidateConceptEvaluator +from parsers.BaseNodeParser import ConceptNode +from parsers.BaseParser import BaseParser +from parsers.SyaNodeParser import SyaNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -162,6 +169,7 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka): (" is_question ( ) ", True), ("context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", True), (" context . in_context ( BuiltinConcepts . EVAL_QUESTION_REQUESTED ) ", True), + ("in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", True), (None, False), ("", False), (NotInit, False), @@ -193,6 +201,58 @@ class TestBuiltinHelpers(TestUsingMemoryBasedSheerka): evaluated = [r for r in res if r.status][0].body assert evaluated.body is NotInit + def test_i_can_evaluate_from_source_with_specific_evaluators(self): + sheerka, context, one = self.init_concepts(Concept("foo", body="'hello world'")) + + res = core.builtin_helpers.evaluate_from_source(context, "foo", eval_body=True, evaluators=["Python"]) + res = self.successful_return_values(res) + assert len(res) == 1 + assert res[0].who.startswith(BaseParser.PREFIX) # Cannot evaluate concept with PythonEvaluator + + res = core.builtin_helpers.evaluate_from_source(context, "foo", eval_body=True, evaluators=["Concept"]) + res = self.successful_return_values(res) + assert len(res) == 1 + assert res[0].who == BaseEvaluator.PREFIX + "Concept" + assert sheerka.isinstance(res[0].body.body, + BuiltinConcepts.PARSER_RESULT) # cannot eval 'hello world' without PythonEvaluator + + res = core.builtin_helpers.evaluate_from_source(context, "foo", eval_body=True, + evaluators=["Concept", "Python"]) + res = self.successful_return_values(res) + assert len(res) == 1 + assert res[0].who == BaseEvaluator.PREFIX + "Concept" + assert res[0].body.body == "hello world" + + def test_i_can_get_lexer_nodes_after_parsing_validation(self): + sheerka, context, the, foo = self.init_concepts( + Concept("the x", ret="x", where="isinstance(x, Concept)").def_var("x"), + "foo", + create_new=True) + + parsed_ret_val = SyaNodeParser().parse(context, ParserInput("the foo")) + validated_ret_val = ValidateConceptEvaluator().eval(context, parsed_ret_val) + + res = core.builtin_helpers.get_lexer_nodes([validated_ret_val], 0, list(Tokenizer("the foo", yield_eof=False))) + + assert isinstance(res, list) + assert isinstance(res[0][0], ConceptNode) + + def test_ensure_evaluated_returns_the_ret_value(self): + """ + When a concept has a RET defined, make sure to return it + :return: + """ + + sheerka, context, foo, bar = self.init_concepts( + "foo", + Concept("bar", ret="foo") + ) + + assert core.builtin_helpers.ensure_evaluated(context, bar) == foo + + # a second time, now that bar is already evaluated + assert core.builtin_helpers.ensure_evaluated(context, bar) == foo + # @pytest.mark.parametrize("return_values", [ # None, # [] diff --git a/tests/core/test_sheerka.py b/tests/core/test_sheerka.py index b86bdb0..eb1e9a0 100644 --- a/tests/core/test_sheerka.py +++ b/tests/core/test_sheerka.py @@ -75,7 +75,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): assert loaded is not None assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT) assert loaded.body == {"key": "key_that_does_not_exist"} - assert loaded.get_metadata().is_evaluated + assert loaded.get_hints().is_evaluated def test_i_cannot_get_when_id_is_not_found(self): sheerka = self.get_sheerka() @@ -85,7 +85,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): assert loaded is not None assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT) assert loaded.body == {"id": "id_that_does_not_exist"} - assert loaded.get_metadata().is_evaluated + assert loaded.get_hints().is_evaluated def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self): sheerka = self.get_sheerka() @@ -192,15 +192,15 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka): create_new=True).unpack() sheerka.evaluate_concept(context, sheerka.get_by_id(template.id)) - assert template.get_metadata().is_evaluated + assert template.get_hints().is_evaluated assert template.body == "foo body" new = sheerka.new(template.key) - assert not new.get_metadata().is_evaluated + assert not new.get_hints().is_evaluated assert new.body == NotInit new = sheerka.new((None, template.id)) - assert not new.get_metadata().is_evaluated + assert not new.get_hints().is_evaluated assert new.body == NotInit def test_i_cannot_instantiate_an_unknown_concept(self): diff --git a/tests/core/test_sheerka_call_evaluators.py b/tests/core/test_sheerka_call_evaluators.py index 2f4dab3..437859d 100644 --- a/tests/core/test_sheerka_call_evaluators.py +++ b/tests/core/test_sheerka_call_evaluators.py @@ -287,7 +287,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]} assert sorted_priorities == [20, 15] - key = BuiltinConcepts.EVALUATION + "|" + "|".join(evaluators_names) + key = (BuiltinConcepts.EVALUATION, "|".join(evaluators_names)) assert key in service.grouped_evaluators_cache groups, sorted_priorities = service.grouped_evaluators_cache[key] assert groups == {20: [EvaluatorOneWithPriority20()], 15: [EvaluatorAllWithPriority15()]} @@ -343,7 +343,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): assert groups == {99: [EvaluatorAllWithPriority15(99)]} assert sorted_priorities == [99] - key = BuiltinConcepts.EVALUATION + "|" + "|".join(evaluators_names) + key = (BuiltinConcepts.EVALUATION, "|".join(evaluators_names)) assert key not in service.grouped_evaluators_cache def test_i_can_revert_back_evaluators_alterations(self): @@ -570,7 +570,7 @@ class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka): # grouped evaluator is in cache service = sheerka.services[SheerkaExecute.NAME] - assert "__EVALUATION|all_priority10" in service.grouped_evaluators_cache + assert ("__EVALUATION", "all_priority10") in service.grouped_evaluators_cache def test_evaluators_priorities_can_be_tweaked_by_the_context(self): sheerka, context = self.init_concepts() diff --git a/tests/core/test_sheerka_call_parsers.py b/tests/core/test_sheerka_call_parsers.py index 4a157f0..c218f52 100644 --- a/tests/core/test_sheerka_call_parsers.py +++ b/tests/core/test_sheerka_call_parsers.py @@ -1,14 +1,38 @@ -from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept -from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute -from parsers.BaseParser import BaseParser +import pytest +from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept +from core.concept import Concept +from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute, DEFAULT, PARSE_STEPS +from parsers.BaseExpressionParser import ExprNode +from parsers.BaseParser import BaseParser +from parsers.BnfNodeParser import BnfNodeParser +from parsers.SequenceNodeParser import SequenceNodeParser +from parsers.SyaNodeParser import SyaNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -def get_ret_val(text, who="who"): +def get_user_input(text, who="who"): return ReturnValueConcept(who, True, UserInputConcept(text, "user_name")) +def check_same_results(result1, result2, user_input): + assert len(result1) == len(result2) + for previous, current in zip(result1, result2): + assert current.parents == user_input + assert current.who == previous.who + assert current.status == previous.status + assert id(current.body) == id(previous.body) + + +def check_same_values(sheerka, context, ret_val1, ret_val2): + previous = sheerka.execute(context, ret_val1, [BuiltinConcepts.EVALUATION]) + current = sheerka.execute(context, ret_val2, [BuiltinConcepts.EVALUATION]) + + assert len(previous) == len(current) + for p, c in zip(previous, current): + assert p == c + + class BaseTestParser(BaseParser): debug_out = [] @@ -136,12 +160,23 @@ class ListOfNoneParser(BaseTestParser): class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): + default_parsers = None + + @classmethod + def setup_class(cls): + cls.default_parsers = cls().get_sheerka().parsers + @classmethod def teardown_class(cls): # At the end of the tests, sheerka singleton instance will be corrupted # Ask for a new one TestUsingMemoryBasedSheerka.sheerka = None + def reset_parsers(self, sheerka): + sheerka.parsers = self.default_parsers + service = sheerka.services[SheerkaExecute.NAME] + service.reset_registered_parsers() + def test_i_can_get_parser_when_context_is_not_altered(self): sheerka, context = self.init_concepts() sheerka.parsers = { @@ -152,7 +187,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service.reset_registered_parsers() parsers_key, groups, sorted_priorities = service.get_parsers(context) - assert parsers_key == "__default" + assert parsers_key == ("__default", "__default") assert groups == {80: [Enabled80FalseParser()], 90: [Enabled90FalseParser()]} assert sorted_priorities == [90, 80] @@ -171,13 +206,13 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): context.preprocess_parsers = parsers_names parsers_key, groups, sorted_priorities = service.get_parsers(context) - assert parsers_key == "Enabled50True|Enabled70False|Disabled" + assert parsers_key == (DEFAULT, "Enabled50True|Enabled70False|Disabled") assert groups == {50: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]} assert sorted_priorities == [70, 50] # Disabled parser does not appear key = "|".join(parsers_names) - assert key in service.grouped_parsers_cache - groups, sorted_priorities = service.grouped_parsers_cache[key] + assert (DEFAULT, key) in service.grouped_parsers_cache + groups, sorted_priorities = service.grouped_parsers_cache[(DEFAULT, key)] assert groups == {50: [Enabled50TrueParser], 70: [Enabled70FalseParser]} assert sorted_priorities == [70, 50] @@ -202,8 +237,8 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): assert groups == {80: [Enabled50TrueParser()], 70: [Enabled70FalseParser()]} assert sorted_priorities == [80, 70] # Disabled parsers does not appear - key = "|".join(parsers_names) - assert key not in service.grouped_parsers_cache # not saved in cache + key = (DEFAULT, "|".join(parsers_names)) + assert key not in service.grouped_parsers_cache # not saved in cache def test_disabled_parsers_are_not_executed(self): sheerka = self.get_sheerka() @@ -214,7 +249,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) @@ -230,7 +265,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) @@ -254,7 +289,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) @@ -279,7 +314,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) @@ -310,7 +345,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) @@ -342,7 +377,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res] @@ -365,7 +400,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) @@ -391,7 +426,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) @@ -412,7 +447,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) @@ -432,7 +467,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'), ] - def test_i_can_manage_parser_with_multiple_results_and_a_sucess(self): + def test_i_can_manage_parser_with_multiple_results_and_a_success(self): sheerka = self.get_sheerka() sheerka.parsers = { "Enabled80MultipleTrue": Enabled80MultipleTrueParser, @@ -441,7 +476,7 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): service = sheerka.services[SheerkaExecute.NAME] service.reset_registered_parsers() - user_input = [get_ret_val("hello world")] + user_input = [get_user_input("hello world")] BaseTestParser.debug_out = [] res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING]) @@ -454,3 +489,390 @@ class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka): ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'), ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'), ] + + def test_parser_calls_are_put_in_cache(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaExecute.NAME] + self.reset_parsers(sheerka) + + user_input = [get_user_input("1")] + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + assert len(res) > 0 + assert len(service.parsers_cache.cache) == 1 + assert (('__default', '__default'), '1') in service.parsers_cache + + def test_parser_calls_are_put_in_cache_when_question(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaExecute.NAME] + self.reset_parsers(sheerka) + context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + + user_input = [get_user_input("1")] + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + assert len(res) > 0 + assert len(service.parsers_cache.cache) == 2 # one for EVAL_QUESTION, one to compile the Python + assert ((BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT), '1') in service.parsers_cache + + def test_parser_calls_are_not_put_in_cache_when_preprocess_parsers(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaExecute.NAME] + self.reset_parsers(sheerka) + context.preprocess_parsers = [SequenceNodeParser.NAME, BnfNodeParser.NAME, SyaNodeParser.NAME] + + user_input = [get_user_input("1")] + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + expected_key = (DEFAULT, "|".join([SequenceNodeParser.NAME, BnfNodeParser.NAME, SyaNodeParser.NAME])) + + assert len(res) == 3 + assert len(service.parsers_cache.cache) == 1 + assert (expected_key, '1') in service.parsers_cache + + def test_parser_calls_are_not_put_in_cache_when_preprocess(self): + sheerka, context = self.init_test().unpack() + service = sheerka.services[SheerkaExecute.NAME] + self.reset_parsers(sheerka) + context.add_preprocess(BaseParser.PREFIX + "parser_name", some_attribute='some_value') + + user_input = [get_user_input("1")] + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + assert len(res) > 0 + assert len(service.parsers_cache.cache) == 0 + + def test_i_can_use_parser_memoization_on_python(self): + sheerka, context = self.init_test().unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("1 + 1")] + first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + user_input_2 = [get_user_input("1 + 1")] + second_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + check_same_results(first_request, second_request, user_input_2) + check_same_values(sheerka, context, first_request, second_request) + + @pytest.mark.parametrize("concept", { + Concept("foo"), + Concept("foo", body="1"), + }) + def test_i_can_use_parser_memoization_when_exact_concept(self, concept): + sheerka, context, foo = self.init_test(eval_body=True, eval_where=True).with_concepts(concept).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("foo")] + first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + successful = [r.who for r in first_request if r.status] + assert successful == ['parsers.ExactConcept'] + + user_input_2 = [get_user_input("foo")] + second_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + check_same_results(first_request, second_request, user_input_2) + check_same_values(sheerka, context, first_request, second_request) + + def test_i_can_use_parser_memoization_when_multiple_results(self): + sheerka, context, *foo = self.init_test(eval_body=True, eval_where=True).with_concepts( + Concept("foo", body="1"), + Concept("foo", body="2")).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("foo")] + first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + successful = [r.who for r in first_request if r.status] + assert successful == ['parsers.ExactConcept', 'parsers.ExactConcept'] + + user_input_2 = [get_user_input("foo")] + second_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + check_same_results(first_request, second_request, user_input_2) + check_same_values(sheerka, context, first_request, second_request) + + def test_i_can_use_parser_memoization_when_bnf_node(self): + sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( + Concept("one", body="1"), + Concept("two", body="2"), + Concept("twenties", definition="'twenty' (one|two)=unit", body="20 + unit").def_var("unit"), + create_new=True).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("twenty one")] + first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + successful = [r.who for r in first_request if r.status] + assert successful == ['parsers.Bnf'] + + user_input_2 = [get_user_input("twenty one")] + second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) + + check_same_results(first_request, second_request, user_input_2) + check_same_values(sheerka, context, first_request, second_request) + + def test_i_can_use_parser_memoization_when_sequence_node(self): + sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( + Concept("one", body="1"), + Concept("two", body="2"), + create_new=True).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("one two")] + first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + successful = [r.who for r in first_request if r.status] + assert successful == ['parsers.Sequence'] + + user_input_2 = [get_user_input("one two")] + second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) + + check_same_results(first_request, second_request, user_input_2) + check_same_values(sheerka, context, first_request, second_request) + + def test_i_can_use_parser_memoization_when_sya_node(self): + sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( + Concept("one", body="1"), + Concept("twenty two", body="22"), + Concept("a plus b", body="a + b").def_var("a").def_var("b"), + create_new=True).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("one plus twenty two")] + first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + successful = [r.who for r in first_request if r.status] + assert successful == ['parsers.Sya'] + + user_input_2 = [get_user_input("one plus twenty two")] + second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) + + check_same_results(first_request, second_request, user_input_2) + check_same_values(sheerka, context, first_request, second_request) + + def test_i_can_use_parser_memoization_when_not_parser_input_from_sequence_node(self): + # In the test, 'one two' will be partially parsed by the SequenceNodeParser + # We test that the results of the UnrecognizedNodeParser and the PythonWithConceptParser + # which both do not used ParsingInput as an input (but a ParserResult) + # are added to the list of results + + sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( + Concept("one", body="1")).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("one two")] + first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + successful = [r.who for r in first_request if r.status] + assert successful == [] + + user_input_2 = [get_user_input("one two")] + second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) + + check_same_results(first_request, second_request, user_input_2) + check_same_values(sheerka, context, first_request, second_request) + + def test_i_can_use_parser_memoization_when_not_parser_input_from_bnf_node(self): + # check the comment from test_i_can_use_parser_memoization_when_not_parser_input_from_sequence_node + # for more information on this test + + sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( + Concept("one", body="1"), + Concept("two", body="2"), + Concept("twenties", definition="'twenty' (one|two)=unit", body="20 + unit"), + create_new=True).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("twenty two foo")] + first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + successful = [r.who for r in first_request if r.status] + assert successful == [] + + user_input_2 = [get_user_input("twenty two foo")] + second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) + + check_same_results(first_request, second_request, user_input_2) + check_same_values(sheerka, context, first_request, second_request) + + def test_i_can_use_parser_memoization_when_not_parser_input_from_sy_node(self): + # check the comment from test_i_can_use_parser_memoization_when_not_parser_input_from_sequence_node + # for more information on this test + + sheerka, context, *concepts = self.init_test(eval_body=True, eval_where=True).with_concepts( + Concept("one", body="1"), + Concept("twenty two", body="22"), + Concept("a plus b", body="a + b").def_var("a").def_var("b"), + create_new=True).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("twenty two plus one foo")] + first_request = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + + successful = [r.who for r in first_request if r.status] + assert successful == [] + + user_input_2 = [get_user_input("twenty two plus one foo")] + second_request = sheerka.execute(context, user_input_2, [BuiltinConcepts.PARSING]) + + check_same_results(first_request, second_request, user_input_2) + check_same_values(sheerka, context, first_request, second_request) + + def test_cache_is_reset_on_concept_creation(self): + sheerka, context, *concepts = self.init_test().with_concepts( + Concept("foo", body="1"), create_new=True).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("foo")] + + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + successful = [r for r in res if r.status] + assert len(successful) == 1 + + sheerka.create_new_concept(context, Concept("foo", body="2")) + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + successful = [r for r in res if r.status] + assert len(successful) == 2 + + def test_cache_is_reset_on_concept_deletion(self): + sheerka, context, foo1, foo2 = self.init_test().with_concepts( + Concept("foo", body="1"), + Concept("foo", body="2"), + create_new=True).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("foo")] + + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + successful = [r for r in res if r.status] + assert len(successful) == 2 + + sheerka.remove_concept(context, foo2) + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + successful = [r for r in res if r.status] + assert len(successful) == 1 + + def test_cache_is_reset_on_concept_modification(self): + sheerka, context, foo1, foo2 = self.init_test().with_concepts( + Concept("foo", body="1"), + Concept("foo", body="2"), + create_new=True).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("foo")] + + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + successful = [r for r in res if r.status] + assert len(successful) == 2 + + sheerka.modify_concept(context, foo2, to_add={'meta': {"name": "bar"}}) + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + successful = [r for r in res if r.status] + assert len(successful) == 1 + + def test_i_do_not_use_short_term_memory_when_not_requested(self): + sheerka, context, foo = self.init_test().with_concepts( + Concept("foo"), + create_new=True).unpack() + self.reset_parsers(sheerka) + + user_input = [get_user_input("foo")] + + # put something is STM + context.add_to_short_term_memory("foo", "value") + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + assert len(res) == 1 + assert res[0].body == "value" + + # specify the parsers to use + context.preprocess_parsers = ["ExactConcept"] + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + assert len(res) == 1 + assert res[0].body.body == foo + + # I can force STM + context.preprocess_parsers = ["ExactConcept", "ShortTermMemory"] # order in not relevant for STM parser + res = sheerka.execute(context, user_input, [BuiltinConcepts.PARSING]) + assert len(res) == 1 + assert res[0].body == "value" + + def test_i_can_compute_parsers_key(self): + sheerka = self.get_sheerka() + + context = self.get_context(sheerka) + key = SheerkaExecute.get_parsers_key(context) + assert key == (DEFAULT, DEFAULT) + + context = self.get_context(sheerka) + context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + key = SheerkaExecute.get_parsers_key(context) + assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT) + + context = self.get_context(sheerka) + context.preprocess_parsers = ["foo", "bar", "baz"] + key = SheerkaExecute.get_parsers_key(context) + assert key == (DEFAULT, 'foo|bar|baz') + + context = self.get_context(sheerka) + context.preprocess_parsers = ["foo", "bar", "baz"] + context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + key = SheerkaExecute.get_parsers_key(context) + assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, 'foo|bar|baz') + + context = self.get_context(sheerka) + context.add_preprocess(BaseParser.PREFIX + "foo", key="some_value") + key = SheerkaExecute.get_parsers_key(context) + assert key is None + + context = self.get_context(sheerka) + context.add_preprocess(BaseParser.PREFIX + "foo", key="some_value") + context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + key = SheerkaExecute.get_parsers_key(context) + assert key is None + + context = self.get_context(sheerka) + context.add_preprocess("foo", key="some_value") + key = SheerkaExecute.get_parsers_key(context) + assert key == (DEFAULT, DEFAULT) + + context = self.get_context(sheerka) + context.add_preprocess("foo", key="some_value") + context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + key = SheerkaExecute.get_parsers_key(context) + assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, DEFAULT) + + context = self.get_context(sheerka) + context.preprocess_parsers = ["foo", "bar", "baz"] + context.add_preprocess("foo", key="some_value") + key = SheerkaExecute.get_parsers_key(context) + assert key == (DEFAULT, 'foo|bar|baz') + + context = self.get_context(sheerka) + context.preprocess_parsers = ["foo", "bar", "baz"] + context.add_preprocess("foo", key="some_value") + context.add_to_private_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + key = SheerkaExecute.get_parsers_key(context) + assert key == (BuiltinConcepts.EVAL_QUESTION_REQUESTED, 'foo|bar|baz') + + def test_i_use_expression_parser_when_needed(self): + sheerka, context, one, number, isa_q, isa = self.init_concepts( + "one", + "number", + Concept("x is a y ", pre="is_question()").def_var("x").def_var("y"), + Concept("x is a y ").def_var("x").def_var("y"), + ) + user_input = get_user_input("one is a number") + + res = sheerka.execute(context, [user_input], PARSE_STEPS) + res = self.successful_return_values(res) + assert len(res) == 1 + assert res[0].status + assert isinstance(res[0].body.body, Concept) + + context = self.get_context(sheerka) + context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + res = sheerka.execute(context, [user_input], PARSE_STEPS) + assert len(res) == 1 + assert res[0].status + assert isinstance(res[0].body.body, ExprNode) diff --git a/tests/evaluators/EvaluatorTestsUtils.py b/tests/evaluators/EvaluatorTestsUtils.py index 2230b34..9aa3a94 100644 --- a/tests/evaluators/EvaluatorTestsUtils.py +++ b/tests/evaluators/EvaluatorTestsUtils.py @@ -11,53 +11,112 @@ reduced_requested = ReturnValueConcept("Sheerka", True, Concept(name=BuiltinConc def ret_val(value="value", who="who", status=True): + """ + ReturnValueConcept + :param value: + :param who: + :param status: + :return: + """ return ReturnValueConcept(who, status, value) def p_ret_val(value="value", parser="parser", status=True): + """ + ReturnValueConcept from parser + :param value: + :param parser: + :param status: + :return: + """ return ReturnValueConcept(BaseParser.get_name(parser), status, value) def e_ret_val(value="value", evaluator="evaluator", status=True): + """ + ReturnValueConcept from evaluator + :param value: + :param evaluator: + :param status: + :return: + """ return ReturnValueConcept(BaseEvaluator.PREFIX + evaluator, status, value) def p_ret_val_false(value="value", parser="parser"): + """ + Failed ReturnValueConcept from parser + :param value: + :param parser: + :return: + """ return p_ret_val(value, parser, status=False) def p_ret_val_true(value="value", parser="parser"): + """ + Successful ReturnValueConcept from parser + :param value: + :param parser: + :return: + """ return p_ret_val(value, parser, status=True) def e_ret_val_false(value="value", parser="parser"): + """ + Failed ReturnValueConcept from evaluator + :param value: + :param parser: + :return: + """ return e_ret_val(value, parser, status=False) def e_ret_val_true(value="value", parser="parser"): + """ + Successful ReturnValueConcept from evaluator + :param value: + :param parser: + :return: + """ return e_ret_val(value, parser, status=True) def e_ret_val_new(key, evaluator="evaluator", status=True, **kwargs): + """ + Successful ReturnValueConcept from evaluator that returns a concept + :param key: + :param evaluator: + :param status: + :param kwargs: + :return: + """ body = new_concept(key, **kwargs) return e_ret_val(body, evaluator, status) -def pr_ret_val(value, parser="parser", source=None): +def pr_ret_val(value, parser="parser", source=None, status=True): """ ParserResult ReturnValue eg: ReturnValue with a ParserResult :param value: :param parser: :param source: + :param status: :return: """ source = source or (value.name if isinstance(value, Concept) else "source") parser_result = ParserResultConcept(BaseParser.get_name(parser), source=source, value=value) - return p_ret_val(parser_result, parser) + return p_ret_val(value=parser_result, parser=parser, status=status) def python_ret_val(source): + """ + ReturnValueConcept with a PythonNode + :param source: + :return: + """ python_node = PythonNode(source.strip(), ast.parse(source.strip(), f"", 'eval')) return pr_ret_val(python_node, parser="Python", source=source) @@ -68,5 +127,5 @@ def new_concept(key, **kwargs): to_use = "#" + k + "#" if k in ("body", "pre", "post", "ret") else k res.set_value(to_use, v) - res.get_metadata().is_evaluated = True + res.get_hints().is_evaluated = True return res diff --git a/tests/evaluators/test_ConceptEvaluator.py b/tests/evaluators/test_ConceptEvaluator.py index d5281e2..2cccce6 100644 --- a/tests/evaluators/test_ConceptEvaluator.py +++ b/tests/evaluators/test_ConceptEvaluator.py @@ -47,7 +47,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): concept = Concept(name="foo", body="'I have a value'", where="True", - pre="2", + pre="True", post="3").set_value("a", "4").set_value("b", "5") evaluator = ConceptEvaluator(return_body=True) @@ -64,7 +64,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): concept = Concept(name="foo", body="'I have a value'", where="True", - pre="2", + pre="True", post="3").set_value("a", "4").set_value("b", "5") evaluator = ConceptEvaluator(return_body=False) # which is the default behaviour diff --git a/tests/evaluators/test_DefConceptEvaluator.py b/tests/evaluators/test_DefConceptEvaluator.py index b879bd2..166388c 100644 --- a/tests/evaluators/test_DefConceptEvaluator.py +++ b/tests/evaluators/test_DefConceptEvaluator.py @@ -204,7 +204,9 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka): assert DefConceptEvaluator.get_variables(context, concept_definition, []) == expected def test_i_can_recognize_variables_when_referencing_other_concepts(self): - sheerka, context, isa_concept = self.init_concepts(Concept("x is an y").def_var("x").def_var("y")) + sheerka, context, isa_concept = self.init_concepts( + Concept("x is an y", pre="is_question()").def_var("x").def_var("y") + ) text = "def concept what x is y pre is_question() where x is an adjective as get_attr(x, y)" def_ret_val = DefConceptParser().parse(context, ParserInput(text)) diff --git a/tests/evaluators/test_ExpressionEvaluator.py b/tests/evaluators/test_ExpressionEvaluator.py new file mode 100644 index 0000000..5f93c01 --- /dev/null +++ b/tests/evaluators/test_ExpressionEvaluator.py @@ -0,0 +1,83 @@ +import pytest + +from core.builtin_concepts_ids import BuiltinConcepts +from core.concept import Concept +from core.sheerka.services.SheerkaExecute import ParserInput +from evaluators.BaseEvaluator import BaseEvaluator +from evaluators.ExpressionEvaluator import ExpressionEvaluator +from parsers.BaseExpressionParser import AndNode +from parsers.ExpressionParser import ExpressionParser +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, e_ret_val_true + + +class TestExpressionEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("return_value, expected", [ + (pr_ret_val(AndNode(0, 0, [])), True), + (pr_ret_val(AndNode(0, 0, []), status=False), False), + (pr_ret_val("not a ExprNode", status=True), False), + (e_ret_val_true("no parser result"), False), + ]) + def test_i_can_match(self, return_value, expected): + sheerka, context = self.init_concepts() + evaluator = ExpressionEvaluator() + + assert evaluator.matches(context, return_value) == expected + + def test_i_can_eval(self): + sheerka, context, one, number, isa = self.init_concepts( + "one", + "number", + Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y")) + evaluator = ExpressionEvaluator() + + context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + + parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number")) + + # first time, it returns False + res = evaluator.eval(context, parsed_return_value) + assert res.status + assert res.who == BaseEvaluator.PREFIX + ExpressionEvaluator.NAME + assert not res.body + + # second time + sheerka.set_isa(context, one, number) + parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number")) + res = evaluator.eval(context, parsed_return_value) + assert res.status + assert res.body + + def test_i_do_not_mess_up_use_copy(self): + sheerka, context, one, number, isa = self.init_concepts( + "one", + "number", + Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y")) + evaluator = ExpressionEvaluator() + + parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number")) + concept = next(iter(parsed_return_value.body.body.compiled[0].return_value.body.body.objects.values())) + assert concept.get_compiled()["x"][0].body.body.get_hints().use_copy + assert concept.get_compiled()["y"][0].body.body.get_hints().use_copy + + evaluator.eval(context, parsed_return_value) + concept = next(iter(parsed_return_value.body.body.compiled[0].return_value.body.body.objects.values())) + assert concept.get_compiled()["x"][0].body.body.get_hints().use_copy + assert concept.get_compiled()["y"][0].body.body.get_hints().use_copy + + def test_i_cannot_eval_when_variables_are_missing(self): + sheerka, context, one, number, isa = self.init_concepts( + "one", + "number", + Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y")) + evaluator = ExpressionEvaluator() + + parsed_return_value = ExpressionParser().parse(context, ParserInput("self is a number")) + res = evaluator.eval(context, parsed_return_value) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR) + assert len(res.body.body) == 1 + assert isinstance(res.body.body[0], NameError) + assert res.body.body[0].args == ("self",) diff --git a/tests/evaluators/test_ValidateConceptEvaluator.py b/tests/evaluators/test_ValidateConceptEvaluator.py new file mode 100644 index 0000000..87f06e9 --- /dev/null +++ b/tests/evaluators/test_ValidateConceptEvaluator.py @@ -0,0 +1,198 @@ +import pytest + +from core.builtin_concepts import ParserResultConcept +from core.builtin_concepts_ids import BuiltinConcepts +from core.concept import Concept +from core.sheerka.services.SheerkaExecute import ParserInput +from evaluators.BaseEvaluator import BaseEvaluator +from evaluators.ValidateConceptEvaluator import ValidateConceptEvaluator +from parsers.BaseNodeParser import ConceptNode +from parsers.BaseParser import BaseParser +from parsers.BnfNodeParser import BnfNodeParser +from parsers.SyaNodeParser import SyaNodeParser +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka +from tests.evaluators.EvaluatorTestsUtils import p_ret_val, pr_ret_val, ret_val + + +class TestValidateConceptEvaluator(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("return_value, need_validation, expected", [ + (pr_ret_val(Concept("foo", where="something"), status=True), True, True), + (pr_ret_val(Concept("foo", pre="something"), status=True), True, True), + (pr_ret_val(Concept("foo", where="something"), status=True), False, False), + (pr_ret_val(Concept("foo", pre="something"), status=True), False, False), + (pr_ret_val(Concept("foo"), status=True), True, False), + (pr_ret_val(Concept("foo"), status=True), True, False), + (pr_ret_val(Concept("foo", where=None), status=True), True, False), + (pr_ret_val(Concept("foo", pre=None), status=True), True, False), + (pr_ret_val(Concept("foo", where=""), status=True), True, False), + (pr_ret_val(Concept("foo", pre=""), status=True), True, False), + (pr_ret_val(Concept("foo", where="something"), status=False), True, False), + (pr_ret_val(Concept("foo", pre="something"), status=False), True, False), + (pr_ret_val("Not a concept", status=True), True, False), + (pr_ret_val("Not a concept", status=False), True, False), + (p_ret_val("Not a parser result", status=True), True, False), + (p_ret_val("Not a parser result", status=False), True, False), + ]) + def test_i_can_match(self, return_value, need_validation, expected): + sheerka, context = self.init_concepts() + evaluator = ValidateConceptEvaluator() + if (sheerka.isinstance(return_value.body, BuiltinConcepts.PARSER_RESULT) and + isinstance(return_value.body.body, Concept)): + return_value.body.body.get_hints().need_validation = need_validation + + assert evaluator.matches(context, return_value) == expected + + def test_i_cannot_match_when_the_return_value_is_not_a_direct_parsing_result(self): + # I match only if the return_value comes from a parser (not from an after_parsing evaluator) + sheerka, context = self.init_concepts() + + concept = Concept("foo", pre="something") + parser_result = ParserResultConcept(BaseParser.get_name("parser"), source=concept.name, value=concept) + return_value = ret_val(value=parser_result, who="evaluators.something", status=True) + return_value.body.body.get_hints().need_validation = True + + evaluator = ValidateConceptEvaluator() + assert not evaluator.matches(context, return_value) + + @pytest.mark.parametrize("concept", [ + Concept("foo", pre="False"), + Concept("foo", where="False"), + ]) + def test_i_can_eval_when_constraint_is_false(self, concept): + sheerka, context, foo = self.init_concepts(concept) + + ret_val = pr_ret_val(foo, status=True) + res = ValidateConceptEvaluator().eval(context, ret_val) + + assert not res.status + assert res.who == BaseEvaluator.PREFIX + ValidateConceptEvaluator.NAME + assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED) + assert res.body.body == foo + + @pytest.mark.parametrize("concept", [ + Concept("foo"), + Concept("foo", pre="True"), + Concept("foo", where="True"), + ]) + def test_i_can_eval_when_constraint_is_true_or_when_no_constrain(self, concept): + sheerka, context, foo = self.init_concepts(concept) + + ret_val = pr_ret_val(foo, status=True) + res = ValidateConceptEvaluator().eval(context, ret_val) + + assert res is None + + @pytest.mark.parametrize("concept", [ + Concept("foo"), + Concept("foo", pre="True"), + Concept("foo", where="True"), + ]) + def test_i_can_eval_when_constraint_is_true_or_when_no_constrain_use_copy(self, concept): + sheerka, context, foo = self.init_concepts(concept) + + foo.get_hints().use_copy = True + ret_val = pr_ret_val(foo, status=True) + res = ValidateConceptEvaluator().eval(context, ret_val) + + assert res.status + assert res.who == BaseEvaluator.PREFIX + ValidateConceptEvaluator.NAME + assert res.body.body.id == foo.id + assert not res.body.body.get_hints().use_copy + + def test_i_can_eval_when_is_question(self): + sheerka, context, foo = self.init_concepts(Concept("foo", pre="is_question()")) + + ret_val = pr_ret_val(foo, status=True) + + res = ValidateConceptEvaluator().eval(context, ret_val) + assert not res.status + + context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED) + res = ValidateConceptEvaluator().eval(context, ret_val) + assert res is None + + def test_i_can_eval_when_unknown_variables_but_no_constraint(self): + sheerka, context, foo = self.init_concepts(Concept("a plus b").def_var("a").def_var("b")) + + ret_val = pr_ret_val(foo, status=True) + + res = ValidateConceptEvaluator().eval(context, ret_val) + assert res is None + + def test_i_can_eval_when_unknown_variables(self): + # ValidateConceptEvaluator must filter only if all the variables are known + # In this example, 'b' is not set, so the return_value must not be filtered + sheerka, context, foo = self.init_concepts( + Concept("a plus b", where="isinstance(b, int)").def_var("a", "10").def_var("b")) + + ret_val = pr_ret_val(foo, status=True) + + res = ValidateConceptEvaluator().eval(context, ret_val) + assert res is None + + def test_i_can_eval_when_constraint_on_variable_fails(self): + sheerka, context, foo = self.init_concepts( + Concept("a plus b", where="isinstance(b, int)").def_var("a").def_var("b", "'a string'")) + + ret_val = pr_ret_val(foo, status=True) + + res = ValidateConceptEvaluator().eval(context, ret_val) + assert not res.status + + def test_i_can_eval_bnf_concepts(self): + sheerka, context, quantify_x = self.init_concepts( + Concept("quantify x", definition="('one'|'two') x", where="x != 'one'").def_var("x"), + create_new=True) + evaluator = ValidateConceptEvaluator() + + # success + ret_val = BnfNodeParser().parse(context, ParserInput("one 'two'")) + assert evaluator.matches(context, ret_val) + res = evaluator.eval(context, ret_val) + assert isinstance(res.body.body, list) + assert len(res.body.body) == 1 + assert isinstance(res.body.body[0], ConceptNode) + assert res.body.body[0].concept.id == ret_val.body.body[0].concept.id + + # failure + ret_val = BnfNodeParser().parse(context, ParserInput("one 'one'")) + assert evaluator.matches(context, ret_val) + res = evaluator.eval(context, ret_val) + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED) + assert res.body.body == ret_val.body.body + + def test_i_can_eval_sya_concepts(self): + sheerka, context, quantify_x = self.init_concepts( + Concept("a plus b", where="a < 10").def_var("a").def_var("b"), + create_new=True) + evaluator = ValidateConceptEvaluator() + + # success + ret_val = SyaNodeParser().parse(context, ParserInput("5 plus 3")) + assert evaluator.matches(context, ret_val) + res = evaluator.eval(context, ret_val) + assert isinstance(res.body.body, list) + assert len(res.body.body) == 1 + assert isinstance(res.body.body[0], ConceptNode) + assert res.body.body[0].concept.id == ret_val.body.body[0].concept.id + + # failure + ret_val = SyaNodeParser().parse(context, ParserInput("15 plus 3")) + assert evaluator.matches(context, ret_val) + res = evaluator.eval(context, ret_val) + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED) + assert res.body.body == ret_val.body.body + + def test_i_can_manage_infinite_recursion(self): + sheerka, context, a_and_b = self.init_concepts( + Concept("a and b", where="is_question()", body="a and b").def_var("a").def_var("b"), + create_new=True) + evaluator = ValidateConceptEvaluator() + + ret_val = pr_ret_val(a_and_b) + res = evaluator.eval(context, ret_val) + + assert res is None # infinite recursion detected, res is None to drop the validator diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index 249e28c..3634f69 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -7,7 +7,6 @@ from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator from evaluators.OneSuccessEvaluator import OneSuccessEvaluator from evaluators.PythonEvaluator import PythonEvalError -from sheerkapython.python_wrapper import MethodAccessError from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, CB @@ -211,9 +210,9 @@ as: # sanity check evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value) assert evaluated.body == "hello foo" - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated compare_with_test_object(evaluated.get_value("a"), CB("foo", "foo")) - assert evaluated.get_value("a").get_metadata().is_evaluated + assert evaluated.get_value("a").get_hints().is_evaluated def test_i_can_recognize_duplicate_concepts_with_same_value(self): # when multiple result, choose the one that is the more specific (that has the less variables) @@ -296,9 +295,9 @@ as: # sanity check evaluated = sheerka.evaluate_concept(self.get_context(sheerka=sheerka, eval_body=True), return_value) assert evaluated.body == "one three" - assert evaluated.get_metadata().is_evaluated + assert evaluated.get_hints().is_evaluated assert evaluated.get_value("a") == sheerka.new(concept_a.key, body="one").init_key() - assert evaluated.get_value("a").get_metadata().is_evaluated + assert evaluated.get_value("a").get_hints().is_evaluated @pytest.mark.parametrize("user_input", [ "def concept greetings from def hello a where a", @@ -314,7 +313,7 @@ as: concept_found = res[0].value assert sheerka.isinstance(concept_found, greetings) assert concept_found.get_value("a") == "foo" - assert concept_found.get_metadata().need_validation + assert concept_found.get_hints().need_validation res = sheerka.evaluate_user_input("greetings") assert len(res) == 1 @@ -322,7 +321,7 @@ as: concept_found = res[0].value assert sheerka.isinstance(concept_found, greetings) assert concept_found.get_value("a") == NotInit - assert not concept_found.get_metadata().need_validation + assert not concept_found.get_hints().need_validation @pytest.mark.parametrize("desc, definitions", [ ("Simple form", [ @@ -681,14 +680,9 @@ as: assert res[0].body == 22 res = sheerka.evaluate_user_input("twenty three") - assert len(res) == 1 - assert not res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.MULTIPLE_ERRORS) assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED) res = sheerka.evaluate_user_input("eval twenty three") - assert len(res) == 1 - assert not res[0].status assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED) def test_i_can_manage_some_type_of_infinite_recursion(self): @@ -725,20 +719,20 @@ as: assert res[0].body == "hello world" res = sheerka.evaluate_user_input("foo baz") - assert len(res) == 1 - assert not res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED) + assert sheerka.has_error(self.get_context(sheerka), res, __type=BuiltinConcepts.CONDITION_FAILED) res = sheerka.evaluate_user_input("eval foo baz") - assert len(res) == 1 - assert not res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED) + assert sheerka.has_error(self.get_context(sheerka), res, __type=BuiltinConcepts.CONDITION_FAILED) - res = sheerka.evaluate_user_input("foobar") + res = sheerka.evaluate_user_input("c:foobar:") # where clause must not be evaluated assert len(res) == 1 assert res[0].status - res = sheerka.evaluate_user_input("eval foobar") + res = sheerka.evaluate_user_input("foobar") # where clause must not be evaluated + assert len(res) == 1 + assert res[0].status + + res = sheerka.evaluate_user_input("eval foobar") # where clause is forced assert len(res) == 1 # error assert not res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED) @@ -835,7 +829,6 @@ as: def test_concepts_parsed_by_atom_parser_must_not_be_evaluated(self): definitions = [ "def concept mult from a mult b as a * b", - "def concept a mult b as a * b", ] sheerka = self.init_scenario(definitions) @@ -844,9 +837,17 @@ as: assert res[0].status assert isinstance(res[0].body, Concept) - # res = sheerka.evaluate_user_input("eval a mult b") - # assert res[0].status - # assert isinstance(res[0].body, Concept) + @pytest.mark.skip("Need to be fixed") + def test_concepts_parsed_by_atom_parser_must_not_be_evaluated_2(self): + definitions = [ + "def concept a mult b as a * b", + ] + + sheerka = self.init_scenario(definitions) + + res = sheerka.evaluate_user_input("eval a mult b") + assert res[0].status + assert isinstance(res[0].body, Concept) def test_i_can_express_comparison(self): definitions = [ @@ -1059,8 +1060,7 @@ as: "def concept foo", "def concept number", "set_isa(one, number)", - "def concept q from q ? as question(q)", - "set_auto_eval(q)", + "def concept q from q ? as question(q) auto_eval True", "def concept is_a from x is a y as isa(x,y) pre in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", "set_is_greater_than(BuiltinConcepts.PRECEDENCE, c:is_a:, c:q:, 'Sya')", ] @@ -1069,18 +1069,16 @@ as: res = sheerka.evaluate_user_input("one is a number ?") # automatically evaluated assert len(res) == 1 assert res[0].status - assert res[0].body == True # the body MUST be a boolean + assert isinstance(res[0].body, bool) and res[0].body # the body MUST be a boolean res = sheerka.evaluate_user_input("foo is a number ?") # automatically evaluated assert len(res) == 1 assert res[0].status - assert res[0].body == False # the body MUST be a boolean + assert isinstance(res[0].body, bool) and not res[0].body # the body MUST be a boolean # x is a y is supposed to be a question. It cannot be used if not in a context of a question res = sheerka.evaluate_user_input("one is a number") - assert len(res) == 1 - assert not res[0].status - assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED) + assert sheerka.has_error(self.get_context(sheerka), res, __type=BuiltinConcepts.CONDITION_FAILED) def test_i_can_evaluate_source_code_with_concept(self): init = [ @@ -1299,7 +1297,7 @@ as: assert sheerka.objvalue(res[0].body.get_value("qty")) == 2 def test_i_can_implement_the_concept_and(self): - # Normally, redefining and leads to a circular ref between the concept and the python + # Normally, redefining 'and' leads to a circular ref between the concept and the python init = [ "def concept x and y as x and y", "set_is_lesser(__PRECEDENCE, c:x and y:, 'Sya')", @@ -1362,3 +1360,47 @@ as: res = sheerka.evaluate_user_input("eval foo") assert sheerka.has_error(context, res, __type="MethodAccessError") + + def test_i_can_use_function_parser_and_complex_concept(self): + init = [ + "def concept the x ret x", + "def concept foo", + "def concept bar", + ] + sheerka = self.init_scenario(init) + + res = sheerka.evaluate_user_input("smart_get_attr(the foo, bar)") + + assert len(res) == 1 + assert res[0].status + + def test_i_can_get_smart_get_attr_for_complex_concepts(self): + init = [ + "def concept q from q ? as question(q) auto_eval True", + "set_is_lesser(__PRECEDENCE, q, 'Sya')", + "def concept a x ret x where isinstance(x, Concept)", + "def concept the x ret memory(x)", + "def concept short", + "def concept color", + "def concept red", + "def concept blue", + "def concept adjective", + "set_isa(red, color)", + "set_isa(blue, color)", + "set_isa(color, adjective)", + "def concept what is the x of y pre is_question() as smart_get_attr(y, x)", + "def concept qualify x from bnf adjective x as set_attr(x, c:adjective:, adjective) ret x", + "eval a red short", + ] + sheerka = self.init_scenario(init) + + res = sheerka.evaluate_user_input("what is the color of the short ?") + assert len(res) == 1 + assert res[0].status + assert res[0].value == "red" + + res = sheerka.evaluate_user_input("eval a blue short") + res = sheerka.evaluate_user_input("what is the color of the short ?") + assert len(res) == 1 + assert res[0].status + assert res[0].value == "blue" diff --git a/tests/non_reg/test_sheerka_non_reg2.py b/tests/non_reg/test_sheerka_non_reg2.py new file mode 100644 index 0000000..fce91bf --- /dev/null +++ b/tests/non_reg/test_sheerka_non_reg2.py @@ -0,0 +1,18 @@ +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaNonRegMemory2(TestUsingMemoryBasedSheerka): + + def test_i_can_select_the_correct_concept_when_ambiguity(self): + init = [ + "def concept foo", + "def concept x is a y pre is_question() as isa(x,y)", + "def concept x is a foo pre is_question() as isinstance(x, foo)", + ] + sheerka = self.init_scenario(init) + + res = sheerka.evaluate_user_input("question(foo is a foo)") + + # assert len(res) == 1 + # assert res[0].status + # assert res[0].value diff --git a/tests/out/test_AsStrVisitor.py b/tests/out/test_AsStrVisitor.py index 41150d8..d46a8aa 100644 --- a/tests/out/test_AsStrVisitor.py +++ b/tests/out/test_AsStrVisitor.py @@ -1,5 +1,5 @@ import pytest -from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, FormatAstVariable, FormatAstVariableNotFound, \ +from parsers.FormatRuleActionParser import FormatAstRawText, FormatAstVariable, FormatAstVariableNotFound, \ FormatAstSequence, FormatAstList, FormatAstDict from out.AsStrVisitor import AsStrVisitor diff --git a/tests/out/test_ConsoleVisitor.py b/tests/out/test_ConsoleVisitor.py index 2da5cbb..28b4074 100644 --- a/tests/out/test_ConsoleVisitor.py +++ b/tests/out/test_ConsoleVisitor.py @@ -1,5 +1,5 @@ import pytest -from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, FormatAstVariable, FormatAstVariableNotFound, \ +from parsers.FormatRuleActionParser import FormatAstRawText, FormatAstVariable, FormatAstVariableNotFound, \ FormatAstSequence, FormatAstList, FormatAstDict from out.ConsoleVisistor import ConsoleVisitor diff --git a/tests/out/test_DeveloperVisitor.py b/tests/out/test_DeveloperVisitor.py index 853f0d7..0d1a16d 100644 --- a/tests/out/test_DeveloperVisitor.py +++ b/tests/out/test_DeveloperVisitor.py @@ -1,7 +1,7 @@ from core.builtin_concepts import BuiltinConcepts from core.sheerka.services.SheerkaDebugManager import NullDebugLogger from core.sheerka.services.SheerkaOut import SheerkaOut -from core.sheerka.services.SheerkaRuleManager import FormatAstList, FormatAstVariable, FormatAstDict, FormatAstMulti +from parsers.FormatRuleActionParser import FormatAstList, FormatAstVariable, FormatAstDict, FormatAstMulti from out.DeveloperVisitor import DeveloperVisitor from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka diff --git a/tests/out/test_SheerkaOut.py b/tests/out/test_SheerkaOut.py index 969a58c..fbaead6 100644 --- a/tests/out/test_SheerkaOut.py +++ b/tests/out/test_SheerkaOut.py @@ -8,8 +8,9 @@ from core.sheerka.Sheerka import Sheerka from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager from core.sheerka.services.SheerkaComparisonManager import SheerkaComparisonManager from core.sheerka.services.SheerkaOut import SheerkaOut -from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, FormatAstVariable, FormatAstSequence, \ - FormatAstColor, FormatAstVariableNotFound, FormatAstList, FormatAstDict, SheerkaRuleManager +from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager +from parsers.FormatRuleActionParser import FormatAstRawText, FormatAstVariable, FormatAstSequence, \ + FormatAstColor, FormatAstVariableNotFound, FormatAstList, FormatAstDict from core.utils import flatten_all_children from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka diff --git a/tests/parsers/parsers_utils.py b/tests/parsers/parsers_utils.py index 36d9330..a9f7cdb 100644 --- a/tests/parsers/parsers_utils.py +++ b/tests/parsers/parsers_utils.py @@ -230,7 +230,7 @@ class CC: self.exclude_body, **compiled) - raise NotImplementedError(f"CC, {other=}") + raise Exception(f"Expecting Concept but received {other=}") class CB: @@ -269,7 +269,7 @@ class CB: body = other.body return CB(concept, body) - raise NotImplementedError(f"CB, {other=}") + raise Exception(f"Expecting Concept but received {other=}") class CV: @@ -310,7 +310,7 @@ class CV: values = get_test_obj_delegate(other.values(), self.values, get_test_obj_delegate) return CV(concept, **values) - raise NotImplementedError(f"CV, {other=}") + raise Exception(f"Expecting Concept but received {other=}") class CMV: @@ -361,7 +361,7 @@ class CMV: variables = {name: value for name, value in other.get_metadata().variables} return CMV(concept, **variables) - raise NotImplementedError(f"CMV, {other=}") + raise Exception(f"Expecting Concept but received {other=}") class CIO: @@ -408,7 +408,7 @@ class CIO: if isinstance(other, Concept): return CIO(other) - raise NotImplementedError(f"CIO, {other=}") + raise Exception(f"Expecting Concept but received {other=}") class HelperWithPos: @@ -496,7 +496,7 @@ class SCN(HelperWithPos): other.start if self.start is not None else None, other.end if self.end is not None else None) - raise NotImplementedError(f"SCN, {other=}") + raise Exception(f"Expecting SourceCodeNode but received {other=}") class SCWC(HelperWithPos): @@ -572,7 +572,7 @@ class SCWC(HelperWithPos): res.end = other.end return res - raise NotImplementedError(f"SCWC, {other=}") + raise Exception(f"Expecting SourceCodeWithConceptNode but received {other=}") @property def source(self): @@ -663,7 +663,7 @@ class CN(HelperWithPos): other.start if self.start is not None else None, other.end if self.end is not None else None) - raise NotImplementedError(f"CN, {other=}") + raise Exception(f"Expecting ConceptNode but received {other=}") class CNC(CN): @@ -737,8 +737,7 @@ class CNC(CN): other.end if self.end is not None else None, self.exclude_body, **compiled) - - raise NotImplementedError(f"CNC, {other=}") + raise Exception(f"Expecting ConceptNode but received {other=}") class UTN(HelperWithPos): @@ -799,7 +798,7 @@ class UTN(HelperWithPos): other.start, other.end) - raise NotImplementedError(f"UTN, {other=}") + raise Exception(f"Expecting UnrecognizedTokensNode but received {other=}") class RN(HelperWithPos): @@ -863,7 +862,7 @@ class RN(HelperWithPos): other.start if self.start is not None else None, other.end if self.end is not None else None) - raise NotImplementedError(f"RN, {other=}") + raise Exception(f"Expecting RuleNode but received {other=}") class FN: @@ -930,7 +929,7 @@ class FN: return FN(other.first.value, other.last.value, params) - raise NotImplementedError(f"FN, {other=}") + raise Exception(f"Expecting FunctionNode but received {other=}") @dataclass() diff --git a/tests/parsers/test_BnfNodeParser.py b/tests/parsers/test_BnfNodeParser.py index 668e71b..b08893b 100644 --- a/tests/parsers/test_BnfNodeParser.py +++ b/tests/parsers/test_BnfNodeParser.py @@ -1927,6 +1927,27 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka): assert res.status compare_with_test_object(res.value.value, expected_array) + def test_i_cannot_parse_regex_concept_mixed_with_unrecognized_sya(self): + my_map = { + "hex": self.bnf_concept("hex", RegExMatch("[a-f0-9]{8}")), + "isa": Concept("x is an y", body="isinstance(x, y)", pre="is_question()").def_var("x").def_var("y"), + "isafoo": Concept("x is an foo", body="False", pre="is_question()").def_var("x"), + "q": Concept("q ?", body="question(a)").def_var("q") + } + + # I need the concept isafoo to fool SyaNodeParser when parsing the sub text 'is an hex ?'" + # The parser will try to recognize 'is an foo', will fail and will revert the result to UTN() + # It's this UTN that need to be properly handled + + sheerka, context, parser = self.init_parser(my_map, init_from_sheerka=True, create_new=True) + sheerka.set_precedence(context, my_map["isa"], my_map["q"]) + sheerka.set_precedence(context, my_map["isafoo"], my_map["q"]) + + text = "01234567 is an hexadecimal ?" + res = parser.parse(context, ParserInput(text)) + assert not res.status + + # @pytest.mark.parametrize("parser_input, expected", [ # ("one", [ # (True, [CNC("bnf_one", source="one", one="one", body="one")]), diff --git a/tests/parsers/test_DefConceptParser.py b/tests/parsers/test_DefConceptParser.py index 204ac1d..dfc04aa 100644 --- a/tests/parsers/test_DefConceptParser.py +++ b/tests/parsers/test_DefConceptParser.py @@ -8,16 +8,18 @@ from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept from core.global_symbols import NotInit from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Keywords, Tokenizer, LexerError +from parsers.BaseExpressionParser import VariableNode, ExprNode from parsers.BaseParser import UnexpectedEofParsingError from parsers.BnfDefinitionParser import BnfDefinitionParser from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence, RegExMatch, OneOrMore, \ VariableExpression from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode, CannotHandleParsingError from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode +from parsers.ExpressionParser import ExpressionParser from parsers.FunctionParser import FunctionParser from parsers.PythonParser import PythonParser, PythonNode from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import compute_expected_array, SCWC, CV, compare_with_test_object +from tests.parsers.parsers_utils import compute_expected_array, SCWC, compare_with_test_object, CIO def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None, ret=None): @@ -26,9 +28,9 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition if body: def_concept.body = get_concept_part(body) if where: - def_concept.where = get_concept_part(where) + def_concept.where = get_concept_part(where, use_expression=True) if pre: - def_concept.pre = get_concept_part(pre) + def_concept.pre = get_concept_part(pre, use_expression=True) if post: def_concept.post = get_concept_part(post) if ret: @@ -46,11 +48,21 @@ def get_def_concept(name, where=None, pre=None, post=None, body=None, definition return def_concept -def get_concept_part(part): - if isinstance(part, str): - node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval")) +def get_concept_part(part, use_expression=False): + if use_expression: + node = VariableNode(0, 0, [], part) return ReturnValueConcept( - who="parsers.DefConcept", + who="parsers.Expression", + status=True, + value=ParserResultConcept( + source=part, + parser=ExpressionParser(), + value=node)) + + if isinstance(part, str): + node = PythonNode(part.lstrip(), ast.parse(part.lstrip(), mode="eval")) + return ReturnValueConcept( + who="parsers.Python", status=True, value=ParserResultConcept( source=part, @@ -61,7 +73,7 @@ def get_concept_part(part): # node = PythonNode(part.strip(), ast.parse(part.strip(), mode="eval")) nodes = compute_expected_array({}, part.source, [SCWC(part.first, part.last, *part.content)]) return ReturnValueConcept( - who="parsers.DefConcept", + who="parsers.Python", status=True, value=ParserResultConcept( source=part.source, @@ -70,9 +82,9 @@ def get_concept_part(part): try_parsed=nodes[0])) if isinstance(part, PN): - node = PythonNode(part.source.strip(), ast.parse(part.source.strip(), mode=part.mode)) + node = PythonNode(part.source.lstrip(), ast.parse(part.source.lstrip(), mode=part.mode)) return ReturnValueConcept( - who="parsers.DefConcept", + who="parsers.Python", status=True, value=ParserResultConcept( source=part.source, @@ -81,7 +93,7 @@ def get_concept_part(part): if isinstance(part, PythonNode): return ReturnValueConcept( - who="parsers.DefConcept", + who="parsers.Python", status=True, value=ParserResultConcept( source=part.source, @@ -248,7 +260,7 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka): assert isinstance(res.value, ParserResultConcept) part_mapping = "body" if part == "as" else part - args = {part_mapping: get_concept_part("True")} + args = {part_mapping: "True"} expected = get_def_concept("foo", **args) assert node == expected @@ -260,28 +272,6 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS) - def test_i_can_parse_complex_def_concept_statement(self): - text = """def concept a mult b -where a,b -pre isinstance(a, int) and isinstance(b, int) -as res = a * b -ret a if isinstance(a, Concept) else self -""" - sheerka, context, parser, *concepts = self.init_parser() - res = parser.parse(context, ParserInput(text)) - return_value = res.value - expected_concept = get_def_concept( - name="a mult b", - where="a,b\n", - pre="isinstance(a, int) and isinstance(b, int)\n", - body=PN("res = a * b\n", "exec"), - ret="a if isinstance(a, Concept) else self\n" - ) - - assert res.status - assert isinstance(return_value, ParserResultConcept) - assert return_value.value == expected_concept - def test_i_can_parse_mutilines_declarations(self): text = """ def concept add one to a as @@ -547,7 +537,10 @@ from give me the date ! assert isinstance(res.value, ParserResultConcept) assert isinstance(node, DefConceptNode) assert sheerka.isinstance(node.where, BuiltinConcepts.RETURN_VALUE) - compare_with_test_object(node.where.body.body, CV(concepts[0], pre=True)) + where_condition = node.where.body.body + assert isinstance(where_condition, ExprNode) + concept_found = where_condition.compiled[0].objects["__o_00__"] + compare_with_test_object(concept_found, CIO(concepts[0])) text = "def concept foo x y pre x is a y" res = parser.parse(context, ParserInput(text)) @@ -559,7 +552,10 @@ from give me the date ! assert isinstance(res.value, ParserResultConcept) assert isinstance(node, DefConceptNode) assert sheerka.isinstance(node.pre, BuiltinConcepts.RETURN_VALUE) - compare_with_test_object(node.pre.body.body, CV(concepts[0], pre=True)) + pre_condition = node.pre.body.body + assert isinstance(pre_condition, ExprNode) + concept_found = pre_condition.compiled[0].objects["__o_00__"] + compare_with_test_object(concept_found, CIO(concepts[0])) def test_i_can_parse_bnf_concept_with_regex(self): sheerka, context, parser, number = self.init_parser("number") diff --git a/tests/parsers/test_DefRuleParser.py b/tests/parsers/test_DefRuleParser.py index 087e6f1..1b2e240 100644 --- a/tests/parsers/test_DefRuleParser.py +++ b/tests/parsers/test_DefRuleParser.py @@ -3,12 +3,13 @@ import pytest from core.builtin_concepts_ids import BuiltinConcepts from core.concept import Concept from core.sheerka.services.SheerkaExecute import ParserInput -from core.sheerka.services.SheerkaRuleManager import FormatAstNode, CompiledCondition +from core.sheerka.services.SheerkaRuleManager import CompiledCondition from core.tokenizer import Tokenizer, Keywords from core.utils import tokens_are_matching from parsers.BaseCustomGrammarParser import KeywordNotFound, NameNode, SyntaxErrorNode from parsers.BaseParser import UnexpectedEofParsingError from parsers.DefRuleParser import DefRuleParser, DefExecRuleNode, DefFormatRuleNode +from parsers.FormatRuleActionParser import FormatAstNode from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka cmap = { diff --git a/tests/parsers/test_ExactConceptParser.py b/tests/parsers/test_ExactConceptParser.py index 845ca43..7bdd8ff 100644 --- a/tests/parsers/test_ExactConceptParser.py +++ b/tests/parsers/test_ExactConceptParser.py @@ -62,8 +62,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status assert concept_found == concept - assert not concept_found.get_metadata().need_validation - assert not concept_found.get_metadata().is_evaluated + assert not concept_found.get_hints().need_validation + assert not concept_found.get_hints().is_evaluated def test_i_can_parse_concepts_defined_several_times(self): sheerka = self.get_sheerka(singleton=True) @@ -80,11 +80,11 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert results[0].status assert results[0].value.value.name == "hello a" assert variable_def(results[0].value.value, "a") == "world" - assert results[0].value.value.get_metadata().need_validation + assert results[0].value.value.get_hints().need_validation assert results[1].status assert results[1].value.value.name == "hello world" - assert not results[1].value.value.get_metadata().need_validation + assert not results[1].value.value.get_hints().need_validation def test_i_can_parse_a_concept_with_variables(self): sheerka = self.get_sheerka(singleton=True) @@ -99,8 +99,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): concept_found = results[0].value.value compare_with_test_object(concept_found, CMV(concept, a="10", b="5")) - assert concept_found.get_metadata().need_validation - assert not concept_found.get_metadata().is_evaluated + assert concept_found.get_hints().need_validation + assert not concept_found.get_hints().is_evaluated def test_i_can_parse_a_concept_with_duplicate_variables(self): sheerka = self.get_sheerka(singleton=True) @@ -115,7 +115,7 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): concept_found = results[0].value.value compare_with_test_object(concept_found, CMV(concept, a="10", b="5")) - assert concept_found.get_metadata().need_validation + assert concept_found.get_hints().need_validation def test_i_can_parse_concept_when_defined_using_from_def(self): sheerka, context, plus = self.init_concepts( @@ -129,8 +129,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status compare_with_test_object(concept_found, CMV(plus, a="10", b="5")) - assert concept_found.get_metadata().need_validation - assert not concept_found.get_metadata().is_evaluated + assert concept_found.get_hints().need_validation + assert not concept_found.get_hints().is_evaluated def test_i_can_parse_concept_token(self): sheerka, context, foo = self.init_concepts("foo") @@ -142,8 +142,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status assert concept_found == foo - assert not concept_found.get_metadata().need_validation - assert concept_found.get_metadata().is_evaluated + assert not concept_found.get_hints().need_validation + assert concept_found.get_hints().is_evaluated def test_i_can_parse_concept_with_concept_tokens(self): sheerka, context, one, two, plus = self.init_concepts( @@ -159,8 +159,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status compare_with_test_object(concept_found, CMV(plus, a="c:one:", b="c:two:")) - assert concept_found.get_metadata().need_validation - assert not concept_found.get_metadata().is_evaluated + assert concept_found.get_hints().need_validation + assert not concept_found.get_hints().is_evaluated def test_i_can_parse_when_expression_contains_keyword(self): sheerka, context, isa, def_concept = self.init_concepts( @@ -175,8 +175,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status compare_with_test_object(concept_found, CMV(isa, c="z")) - assert concept_found.get_metadata().need_validation - assert not concept_found.get_metadata().is_evaluated + assert concept_found.get_hints().need_validation + assert not concept_found.get_hints().is_evaluated source = "def concept z" results = ExactConceptParser().parse(context, ParserInput(source)) @@ -185,8 +185,8 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): assert len(results) == 1 assert results[0].status compare_with_test_object(concept_found, CMV(def_concept, a="z")) - assert concept_found.get_metadata().need_validation - assert not concept_found.get_metadata().is_evaluated + assert concept_found.get_hints().need_validation + assert not concept_found.get_hints().is_evaluated def test_i_can_manage_unknown_concept(self): context = self.get_context(self.get_sheerka(singleton=True)) @@ -219,4 +219,4 @@ class TestExactConceptParser(TestUsingMemoryBasedSheerka): # assert len(results) == 1 # assert results[0].status # assert results[0].value.value == concept - # assert not results[0].value.value.get_metadata().need_validation + # assert not results[0].value.value.get_hints().need_validation diff --git a/tests/parsers/test_ExpressionParser.py b/tests/parsers/test_ExpressionParser.py index 2dbc669..96cc9e0 100644 --- a/tests/parsers/test_ExpressionParser.py +++ b/tests/parsers/test_ExpressionParser.py @@ -3,8 +3,8 @@ import pytest from core.builtin_concepts_ids import BuiltinConcepts from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer -from parsers.BaseExpressionParser import VariableNode, ComparisonNode -from parsers.BaseParser import ErrorSink +from parsers.BaseExpressionParser import VariableNode, ComparisonNode, ExprNode +from parsers.BaseParser import ErrorSink, BaseParser from parsers.ExpressionParser import ExpressionParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, FN, AND, NOT, OR, GT, GTE, LT, LTE, EQ, \ @@ -71,7 +71,7 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka): "var.attr1.attr2", "var . attr1 . attr2", ]) - def test_i_can_parse_variable(self, expression): + def test_i_can_parse_input_variable(self, expression): sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression) parsed = parser.parse_input(context, parser_input, error_sink) @@ -80,7 +80,7 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka): assert parsed.name == "var" assert parsed.attributes == ["attr1", "attr2"] - def test_i_can_parse_sub_tokens(self): + def test_i_can_parse_input_sub_tokens(self): sheerka, context, parser = self.init_parser() expression = "do not care var1 + var2 do not care either" @@ -105,3 +105,23 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka): new_source = ComparisonNode.rebuild_source("new_var", parsed.comp, parsed.right.get_source()) assert new_source == expected + + def test_i_cannot_parse_empty_string(self): + sheerka, context, parser = self.init_parser() + + res = parser.parse(context, ParserInput("")) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY) + + def test_i_can_compile(self): + sheerka, context, parser = self.init_parser() + + text = ParserInput("a > b and c < d") + res = parser.parse(context, text) + + assert res.who == BaseParser.PREFIX + ExpressionParser.NAME + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT) + assert isinstance(res.body.body, ExprNode) + assert res.body.body.compiled is not None diff --git a/tests/parsers/test_FormatRuleActionParser.py b/tests/parsers/test_FormatRuleActionParser.py new file mode 100644 index 0000000..8c43b3b --- /dev/null +++ b/tests/parsers/test_FormatRuleActionParser.py @@ -0,0 +1,81 @@ +import pytest + +from core.tokenizer import Token, TokenKind +from parsers.FormatRuleActionParser import FormatAstSequence, FormatAstRawText, FormatAstVariable, FormatAstFunction, \ + FormatAstList, FormatAstColor, FormatAstDict, FormatAstMulti, FormatRuleActionParser, UnexpectedEof, \ + FormatRuleSyntaxError +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + +seq = FormatAstSequence +raw = FormatAstRawText +var = FormatAstVariable +func = FormatAstFunction +lst = FormatAstList + + +class TestFormatRuleActionParser(TestUsingMemoryBasedSheerka): + + @pytest.mark.parametrize("text, expected", [ + ("", FormatAstRawText("")), + (" ", FormatAstRawText(" ")), + (" raw text ", FormatAstRawText(" raw text ")), + ("{variable}", FormatAstVariable("variable")), + ("{ variable }", FormatAstVariable("variable")), + (" xy {v} z", seq([raw(" xy "), var("v"), raw(" z")])), + (r"\{variable}", FormatAstRawText("{variable}")), + (r"\\{variable}", seq([raw("\\"), var("variable")])), + (r"\\\{variable}", FormatAstRawText(r"\{variable}")), + (r"{var1}{var2}", seq([var("var1"), var("var2")])), + ("func()", FormatAstFunction("func", [], {})), + ("func(a, 'string value', c)", FormatAstFunction("func", ["a", "'string value'", "c"], {})), + ("func(a=10, b='string value')", FormatAstFunction("func", [], {"a": "10", "b": "'string value'"})), + ("func('string value'='another string value')", func("func", [], {"'string value'": "'another string value'"})), + ("red(' xy {v}')", FormatAstColor("red", seq([raw(" xy "), var("v")]))), + ('blue(" xy {v}")', FormatAstColor("blue", seq([raw(" xy "), var("v")]))), + ('green( xy )', FormatAstColor("green", var("xy"))), + ('green()', FormatAstColor("green", raw(""))), + ('green("")', FormatAstColor("green", raw(""))), + ("list(var_name, 2, 'children')", FormatAstList("var_name", recurse_on="children", recursion_depth=2)), + ("list(var_name, recursion_depth=2, recurse_on='children')", FormatAstList("var_name", + recurse_on="children", + recursion_depth=2)), + ("list(var_name, recursion_depth=2, 'children')", FormatAstList("var_name", recursion_depth=2)), + ("list(var_name, 'children', recursion_depth=2)", FormatAstList("var_name", recursion_depth=2)), + ("list(var_name)", FormatAstList("var_name")), + ("{obj.prop1.prop2[0].prop3['value']}", FormatAstVariable("obj.prop1.prop2[0].prop3['value']")), + ("[{id}]", seq([raw("["), var("id"), raw("]")])), + ("{variable:format}", FormatAstVariable("variable", "format")), + ("{variable:3}", FormatAstVariable("variable", "3")), + (r"\not_a_function(a={var})", seq([raw("not_a_function(a="), var("var"), raw(")")])), + ("dict(var_name)", FormatAstDict("var_name")), + ("dict(var_name, items_prop='props')", FormatAstDict("var_name", items_prop='props')), + ("dict(var_name, debug=True)", FormatAstDict("var_name", debug=True, prefix="{", suffix="}")), + ("multi(var_name)", FormatAstMulti("var_name")), + ]) + def test_i_can_parse_format_rule(self, text, expected): + assert FormatRuleActionParser(text).parse() == expected + + @pytest.mark.parametrize("text, expected_error", [ + ("{", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))), + ("{var_name", UnexpectedEof("while parsing variable", Token(TokenKind.LBRACE, "{", 0, 1, 1))), + ("{}", FormatRuleSyntaxError("variable name not found", None)), + ("func(", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))), + ("func(a,b,c", UnexpectedEof("while parsing function", Token(TokenKind.IDENTIFIER, "func", 0, 1, 1))), + ("func(a,,c", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))), + ("func(a,,c)", FormatRuleSyntaxError("no parameter found", Token(TokenKind.COMMA, ",", 7, 1, 8))), + ("red(a,b)", FormatRuleSyntaxError("only one parameter supported", Token(TokenKind.IDENTIFIER, "b", 6, 1, 7))), + ("red(a=b)", FormatRuleSyntaxError("keyword arguments are not supported", None)), + ("red(xy {v})", FormatRuleSyntaxError("Invalid identifier", None)), + ("list()", FormatRuleSyntaxError("variable name not found", None)), + ("list(recursion_depth=2)", FormatRuleSyntaxError("variable name not found", None)), + ("list(a,b,c,d,e)", FormatRuleSyntaxError("too many positional arguments", + Token(TokenKind.IDENTIFIER, "e", 13, 1, 14))), + ("list(a, recursion_depth=hello)", FormatRuleSyntaxError("'hello' is not numeric", None)), + ("list(a, recursion_depth='hello')", FormatRuleSyntaxError("'recursion_depth' must be an integer", None)), + ("dict()", FormatRuleSyntaxError("variable name not found", None)), + ]) + def test_i_cannot_parse_invalid_format(self, text, expected_error): + parser = FormatRuleActionParser(text) + parser.parse() + + assert parser.error_sink == expected_error diff --git a/tests/parsers/test_LogicalOperatorParser.py b/tests/parsers/test_LogicalOperatorParser.py index 5449235..99d57ec 100644 --- a/tests/parsers/test_LogicalOperatorParser.py +++ b/tests/parsers/test_LogicalOperatorParser.py @@ -1,21 +1,15 @@ -import ast - import pytest -from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept -from core.concept import Concept, DoNotResolve -from core.rule import Rule +from core.builtin_concepts import BuiltinConcepts from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind -from parsers.BaseExpressionParser import TrueifyVisitor, IsAQuestionVisitor, AndNode, LeftPartNotFoundError, \ +from parsers.BaseExpressionParser import TrueifyVisitor, IsAQuestionVisitor, LeftPartNotFoundError, \ ParenthesisMismatchError from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError from parsers.LogicalOperatorParser import LogicalOperatorParser -from parsers.PythonParser import PythonNode -from sheerkarete.network import ReteNetwork from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import compute_expected_array, resolve_test_concept, EXPR, OR, AND, NOT, \ - get_expr_node_from_test_node, get_rete_conditions, CMV, CNC, CC, compare_with_test_object +from tests.parsers.parsers_utils import EXPR, OR, AND, NOT, \ + get_expr_node_from_test_node class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka): @@ -181,281 +175,281 @@ class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka): assert IsAQuestionVisitor().visit(expr_node) == expected - @pytest.mark.parametrize("expression, expected", [ - ("foo", "foo"), - ("one two", "one two"), - ("foo is a bar", CMV("is a", x='foo', y='bar')), - ("one two is a bar", [CNC("is a", "one two is a bar", x="one two", y="bar")]), - ("foo is an foo bar", - [CNC("is an", "foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]), - ]) - def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected): - concepts_map = { - "foo": Concept("foo"), - "bar": Concept("bar"), - "one two": Concept("one two"), - "is a": Concept("x is a y").def_var("x").def_var("y"), - "is an": Concept("x is an y", definition="('foo'|'bar')=x 'is an' 'foo bar'").def_var("x"), - } - sheerka, context, *concepts = self.init_test().with_concepts(*concepts_map.values(), create_new=True).unpack() + # @pytest.mark.parametrize("expression, expected", [ + # ("foo", "foo"), + # ("one two", "one two"), + # ("foo is a bar", CMV("is a", x='foo', y='bar')), + # ("one two is a bar", [CNC("is a", "one two is a bar", x="one two", y="bar")]), + # ("foo is an foo bar", + # [CNC("is an", "foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]), + # ]) + # def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected): + # concepts_map = { + # "foo": Concept("foo"), + # "bar": Concept("bar"), + # "one two": Concept("one two"), + # "is a": Concept("x is a y").def_var("x").def_var("y"), + # "is an": Concept("x is an y", definition="('foo'|'bar')=x 'is an' 'foo bar'").def_var("x"), + # } + # sheerka, context, *concepts = self.init_test().with_concepts(*concepts_map.values(), create_new=True).unpack() + # + # parser = LogicalOperatorParser() + # expr_node = parser.parse(context, ParserInput(expression)).body.body + # return_values, _ = parser.compile_conjunctions(context, [expr_node], "test") + # + # assert len(return_values) == 1 + # ret = return_values[0] + # + # if isinstance(expected, list): + # expected_nodes = compute_expected_array(concepts_map, expression, expected) + # compare_with_test_object(ret.body.body, expected_nodes) + # else: + # expected_concept = resolve_test_concept(concepts_map, expected) + # compare_with_test_object(ret.body.body, expected_concept) - parser = LogicalOperatorParser() - expr_node = parser.parse(context, ParserInput(expression)).body.body - return_values, _ = parser.compile_conjunctions(context, [expr_node], "test") - - assert len(return_values) == 1 - ret = return_values[0] - - if isinstance(expected, list): - expected_nodes = compute_expected_array(concepts_map, expression, expected) - compare_with_test_object(ret.body.body, expected_nodes) - else: - expected_concept = resolve_test_concept(concepts_map, expected) - compare_with_test_object(ret.body.body, expected_concept) - - @pytest.mark.parametrize("expression", [ - "a == 5", - "foo > 5", - "func() == 5", - "not a == 5", - "not foo > 5", - "not func() == 5", - "isinstance(a, int)", - "func()", - "not isinstance(a, int)", - "not func()" - ]) - def test_i_can_get_compiled_expr_from_simple_python_expressions(self, expression): - sheerka, context, = self.init_test().unpack() - - parser = LogicalOperatorParser() - expr_node = parser.parse(context, ParserInput(expression)).body.body - return_values, _ = parser.compile_conjunctions(context, [expr_node], "test") - - assert len(return_values) == 1 - ret = return_values[0] - - assert ret.status - python_node = ret.body.body.get_python_node() - _ast = ast.parse(expression, mode="eval") - expected_python_node = PythonNode(expression, _ast) - assert python_node == expected_python_node - - @pytest.mark.parametrize("expression", [ - "a and not b", - "not b and a", - "__ret and not __ret.status", - ]) - def test_i_can_compile_negative_conjunctions_when_pure_python(self, expression): - sheerka, context, *concepts = self.init_concepts("foo") - - parser = LogicalOperatorParser() - expr_node = parser.parse(context, ParserInput(expression)).body.body - return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test") - - ast_ = ast.parse(expression, "", 'eval') - expected_python_node = PythonNode(expression, ast_) - - assert len(return_values) == 1 - ret = return_values[0] - - assert sheerka.objvalue(ret) == expected_python_node - - @pytest.mark.parametrize("expression, text_to_compile", [ - ("foo bar == 5", "__C__foo0bar__1001__C__ == 5"), - ("not foo bar == 5", "not __C__foo0bar__1001__C__ == 5"), - ]) - def test_i_can_get_compiled_expr_from_python_and_concept(self, expression, text_to_compile): - sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack() - - parser = LogicalOperatorParser() - expr_node = parser.parse(context, ParserInput(expression)).body.body - return_values, _ = parser.compile_conjunctions(context, [expr_node], "test") - - assert len(return_values) == 1 - ret = return_values[0] - - assert ret.status - python_node = ret.body.body.get_python_node() - _ast = ast.parse(text_to_compile, mode="eval") - expected_python_node = PythonNode(text_to_compile, _ast, expression) - assert python_node == expected_python_node - - def test_i_can_get_compiled_expr_from__mix_of_concepts_and_python(self): - sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts( - Concept("animal"), - Concept("a cat"), - Concept("dog"), - Concept("pet"), - Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), - Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"), - create_new=True - ).unpack() - - parser = LogicalOperatorParser() - expression = "not a cat is a pet and not bird is an animal and not x > 5 and not dog is a pet" - expr_node = parser.parse(context, ParserInput(expression)).body.body - return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test") - - to_compile = 'not __C__00var0000is0a000var001__1005__C__' - to_compile += ' and not __C__00var0000is0an0y__1006__C__' - to_compile += ' and not x > 5' - to_compile += ' and not __C__00var0000is0a000var001__1005_1__C__' - ast_ = ast.parse(to_compile, "", 'eval') - expected_python_node = PythonNode(to_compile, ast_, expression) - - assert len(return_values) == 1 - ret = return_values[0] - python_node = ret.body.body - assert python_node == expected_python_node - compare_with_test_object(python_node.objects, { - "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet), - "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal), - "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"), - }) - - def test_i_can_get_compiled_expr_from_mix(self): - sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts( - Concept("animal"), - Concept("a cat"), - Concept("dog"), - Concept("pet"), - Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), - Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"), - create_new=True - ).unpack() - - expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet" - parser = LogicalOperatorParser() - expr_node = parser.parse(context, ParserInput(expression)).body.body - return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test") - - assert len(return_values) == 1 - ret = return_values[0] - - to_compile = '__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1006__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__' - ast_ = ast.parse(to_compile, "", 'eval') - expected_python_node = PythonNode(to_compile, ast_, expression) - - python_node = ret.body.body - assert python_node == expected_python_node - compare_with_test_object(python_node.objects, { - "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet), - "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal), - "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"), - }) - - def test_i_can_get_compiled_expr_when_multiple_choices(self): - sheerka, context, *concepts = self.init_test().with_concepts( - Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), - Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"), - create_new=True - ).unpack() - - parser = LogicalOperatorParser() - expression = "a is a b" - expr_node = parser.parse(context, ParserInput(expression)).body.body - return_values, _ = parser.compile_conjunctions(context, [expr_node], "test") - - assert len(return_values) == 2 - - ret = return_values[0] - compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[0], x="a", y="b")) - - ret = return_values[1] - compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[1], x="a", y="b")) - - def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self): - sheerka, context, *concepts = self.init_test().with_concepts( - Concept("animal"), - Concept("a cat"), - Concept("dog"), - Concept("pet"), - Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), - Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"), - Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"), - create_new=True - ).unpack() - - expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet" - parser = LogicalOperatorParser() - expr_node = parser.parse(context, ParserInput(expression)).body.body - return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test") - - assert len(return_values) == 4 - trimmed_source = "a cat is a pet and bird is an animal and x > 5 and dog is a pet" - - current_ret = return_values[0] - python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__" - ast_ = ast.parse(python_source, "", 'eval') - resolved_expected = PythonNode(python_source, ast_, trimmed_source) - assert sheerka.objvalue(current_ret) == resolved_expected - - current_ret = return_values[1] - assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE) - python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006__C__" - ast_ = ast.parse(python_source, "", 'eval') - resolved_expected = PythonNode(python_source, ast_, trimmed_source) - assert sheerka.objvalue(current_ret) == resolved_expected - - current_ret = return_values[2] - assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE) - python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005__C__" - ast_ = ast.parse(python_source, "", 'eval') - resolved_expected = PythonNode(python_source, ast_, trimmed_source) - assert sheerka.objvalue(current_ret) == resolved_expected - - current_ret = return_values[3] - python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006_1__C__" - ast_ = ast.parse(python_source, "", 'eval') - resolved_expected = PythonNode(python_source, ast_, trimmed_source) - assert sheerka.objvalue(current_ret) == resolved_expected - - @pytest.mark.skip - @pytest.mark.parametrize("expression, expected_conditions, test_obj", [ - ( - "__ret", - ["#__x_00__|__name__|'__ret'"], - ReturnValueConcept("Test", True, None) - ), - ( - "__ret.status == True", - ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"], - ReturnValueConcept("Test", True, None) - ), - ( - "__ret.status", - ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"], - ReturnValueConcept("Test", True, None) - ), - ( - "__ret and __ret.status", - ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"], - ReturnValueConcept("Test", True, None) - ), - ]) - def test_i_can_get_rete_condition_from_python(self, expression, expected_conditions, test_obj): - sheerka, context, = self.init_test().unpack() - expected_full_condition = get_rete_conditions(*expected_conditions) - - parser = LogicalOperatorParser() - expr_node = parser.parse(context, ParserInput(expression)).body.body - - nodes = expr_node.parts if isinstance(expr_node, AndNode) else [expr_node] - _, rete_disjunctions = parser.compile_conjunctions(context, nodes, "test") - - assert len(rete_disjunctions) == 1 - assert rete_disjunctions == [expected_full_condition] - - # check against a Rete network - network = ReteNetwork() - rule = Rule("test", expression, None) - rule.metadata.id = 9999 - rule.metadata.is_compiled = True - rule.metadata.is_enabled = True - rule.rete_disjunctions = rete_disjunctions - network.add_rule(rule) - - network.add_obj("__ret", test_obj) - matches = list(network.matches) - assert len(matches) > 0 + # @pytest.mark.parametrize("expression", [ + # "a == 5", + # "foo > 5", + # "func() == 5", + # "not a == 5", + # "not foo > 5", + # "not func() == 5", + # "isinstance(a, int)", + # "func()", + # "not isinstance(a, int)", + # "not func()" + # ]) + # def test_i_can_get_compiled_expr_from_simple_python_expressions(self, expression): + # sheerka, context, = self.init_test().unpack() + # + # parser = LogicalOperatorParser() + # expr_node = parser.parse(context, ParserInput(expression)).body.body + # return_values, _ = parser.compile_conjunctions(context, [expr_node], "test") + # + # assert len(return_values) == 1 + # ret = return_values[0] + # + # assert ret.status + # python_node = ret.body.body.get_python_node() + # _ast = ast.parse(expression, mode="eval") + # expected_python_node = PythonNode(expression, _ast) + # assert python_node == expected_python_node + # + # @pytest.mark.parametrize("expression", [ + # "a and not b", + # "not b and a", + # "__ret and not __ret.status", + # ]) + # def test_i_can_compile_negative_conjunctions_when_pure_python(self, expression): + # sheerka, context, *concepts = self.init_concepts("foo") + # + # parser = LogicalOperatorParser() + # expr_node = parser.parse(context, ParserInput(expression)).body.body + # return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test") + # + # ast_ = ast.parse(expression, "", 'eval') + # expected_python_node = PythonNode(expression, ast_) + # + # assert len(return_values) == 1 + # ret = return_values[0] + # + # assert sheerka.objvalue(ret) == expected_python_node + # + # @pytest.mark.parametrize("expression, text_to_compile", [ + # ("foo bar == 5", "__C__foo0bar__1001__C__ == 5"), + # ("not foo bar == 5", "not __C__foo0bar__1001__C__ == 5"), + # ]) + # def test_i_can_get_compiled_expr_from_python_and_concept(self, expression, text_to_compile): + # sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack() + # + # parser = LogicalOperatorParser() + # expr_node = parser.parse(context, ParserInput(expression)).body.body + # return_values, _ = parser.compile_conjunctions(context, [expr_node], "test") + # + # assert len(return_values) == 1 + # ret = return_values[0] + # + # assert ret.status + # python_node = ret.body.body.get_python_node() + # _ast = ast.parse(text_to_compile, mode="eval") + # expected_python_node = PythonNode(text_to_compile, _ast, expression) + # assert python_node == expected_python_node + # + # def test_i_can_get_compiled_expr_from__mix_of_concepts_and_python(self): + # sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts( + # Concept("animal"), + # Concept("a cat"), + # Concept("dog"), + # Concept("pet"), + # Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + # Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"), + # create_new=True + # ).unpack() + # + # parser = LogicalOperatorParser() + # expression = "not a cat is a pet and not bird is an animal and not x > 5 and not dog is a pet" + # expr_node = parser.parse(context, ParserInput(expression)).body.body + # return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test") + # + # to_compile = 'not __C__00var0000is0a000var001__1005__C__' + # to_compile += ' and not __C__00var0000is0an0y__1006__C__' + # to_compile += ' and not x > 5' + # to_compile += ' and not __C__00var0000is0a000var001__1005_1__C__' + # ast_ = ast.parse(to_compile, "", 'eval') + # expected_python_node = PythonNode(to_compile, ast_, expression) + # + # assert len(return_values) == 1 + # ret = return_values[0] + # python_node = ret.body.body + # assert python_node == expected_python_node + # compare_with_test_object(python_node.objects, { + # "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet), + # "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal), + # "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"), + # }) + # + # def test_i_can_get_compiled_expr_from_mix(self): + # sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts( + # Concept("animal"), + # Concept("a cat"), + # Concept("dog"), + # Concept("pet"), + # Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + # Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"), + # create_new=True + # ).unpack() + # + # expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet" + # parser = LogicalOperatorParser() + # expr_node = parser.parse(context, ParserInput(expression)).body.body + # return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test") + # + # assert len(return_values) == 1 + # ret = return_values[0] + # + # to_compile = '__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1006__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__' + # ast_ = ast.parse(to_compile, "", 'eval') + # expected_python_node = PythonNode(to_compile, ast_, expression) + # + # python_node = ret.body.body + # assert python_node == expected_python_node + # compare_with_test_object(python_node.objects, { + # "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet), + # "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal), + # "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"), + # }) + # + # def test_i_can_get_compiled_expr_when_multiple_choices(self): + # sheerka, context, *concepts = self.init_test().with_concepts( + # Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + # Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"), + # create_new=True + # ).unpack() + # + # parser = LogicalOperatorParser() + # expression = "a is a b" + # expr_node = parser.parse(context, ParserInput(expression)).body.body + # return_values, _ = parser.compile_conjunctions(context, [expr_node], "test") + # + # assert len(return_values) == 2 + # + # ret = return_values[0] + # compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[0], x="a", y="b")) + # + # ret = return_values[1] + # compare_with_test_object(sheerka.objvalue(ret)[0].concept, CMV(concepts[1], x="a", y="b")) + # + # def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self): + # sheerka, context, *concepts = self.init_test().with_concepts( + # Concept("animal"), + # Concept("a cat"), + # Concept("dog"), + # Concept("pet"), + # Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"), + # Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"), + # Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"), + # create_new=True + # ).unpack() + # + # expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet" + # parser = LogicalOperatorParser() + # expr_node = parser.parse(context, ParserInput(expression)).body.body + # return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test") + # + # assert len(return_values) == 4 + # trimmed_source = "a cat is a pet and bird is an animal and x > 5 and dog is a pet" + # + # current_ret = return_values[0] + # python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__" + # ast_ = ast.parse(python_source, "", 'eval') + # resolved_expected = PythonNode(python_source, ast_, trimmed_source) + # assert sheerka.objvalue(current_ret) == resolved_expected + # + # current_ret = return_values[1] + # assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE) + # python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006__C__" + # ast_ = ast.parse(python_source, "", 'eval') + # resolved_expected = PythonNode(python_source, ast_, trimmed_source) + # assert sheerka.objvalue(current_ret) == resolved_expected + # + # current_ret = return_values[2] + # assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE) + # python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005__C__" + # ast_ = ast.parse(python_source, "", 'eval') + # resolved_expected = PythonNode(python_source, ast_, trimmed_source) + # assert sheerka.objvalue(current_ret) == resolved_expected + # + # current_ret = return_values[3] + # python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006_1__C__" + # ast_ = ast.parse(python_source, "", 'eval') + # resolved_expected = PythonNode(python_source, ast_, trimmed_source) + # assert sheerka.objvalue(current_ret) == resolved_expected + # + # @pytest.mark.skip + # @pytest.mark.parametrize("expression, expected_conditions, test_obj", [ + # ( + # "__ret", + # ["#__x_00__|__name__|'__ret'"], + # ReturnValueConcept("Test", True, None) + # ), + # ( + # "__ret.status == True", + # ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"], + # ReturnValueConcept("Test", True, None) + # ), + # ( + # "__ret.status", + # ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"], + # ReturnValueConcept("Test", True, None) + # ), + # ( + # "__ret and __ret.status", + # ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"], + # ReturnValueConcept("Test", True, None) + # ), + # ]) + # def test_i_can_get_rete_condition_from_python(self, expression, expected_conditions, test_obj): + # sheerka, context, = self.init_test().unpack() + # expected_full_condition = get_rete_conditions(*expected_conditions) + # + # parser = LogicalOperatorParser() + # expr_node = parser.parse(context, ParserInput(expression)).body.body + # + # nodes = expr_node.parts if isinstance(expr_node, AndNode) else [expr_node] + # _, rete_disjunctions = parser.compile_conjunctions(context, nodes, "test") + # + # assert len(rete_disjunctions) == 1 + # assert rete_disjunctions == [expected_full_condition] + # + # # check against a Rete network + # network = ReteNetwork() + # rule = Rule("test", expression, None) + # rule.metadata.id = 9999 + # rule.metadata.is_compiled = True + # rule.metadata.is_enabled = True + # rule.rete_disjunctions = rete_disjunctions + # network.add_rule(rule) + # + # network.add_obj("__ret", test_obj) + # matches = list(network.matches) + # assert len(matches) > 0 diff --git a/tests/parsers/test_PythonWithConceptsParser.py b/tests/parsers/test_PythonWithConceptsParser.py index ee7fa25..b7704a7 100644 --- a/tests/parsers/test_PythonWithConceptsParser.py +++ b/tests/parsers/test_PythonWithConceptsParser.py @@ -9,8 +9,10 @@ from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Token, TokenKind, Tokenizer from core.var_ref import VariableRef from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode, VariableNode +from parsers.BnfNodeParser import BnfNodeParser from parsers.PythonParser import PythonNode from parsers.PythonWithConceptsParser import PythonWithConceptsParser +from parsers.SequenceNodeParser import SequenceNodeParser from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.parsers.parsers_utils import get_source_code_node @@ -226,3 +228,39 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka): assert result_python_node.ast_str == PythonNode.get_dump(expected_ast) assert result_python_node.original_source == "not foo == 1 and bar < 1" assert result_python_node.objects == {"__C__foo__C__": foo, "__C__bar__C__": bar} + + def test_can_parse_after_unrecognized_bnf(self): + sheerka, context, one, two, twenties = self.init_concepts( + Concept("one", body="1"), + Concept("two", body="2"), + Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"), + create_new=True + ) + + bnf_parser_ret_val = BnfNodeParser().parse(context, ParserInput("a + twenty one")) + unrec_node_ret_val = UnrecognizedNodeParser().parse(context, bnf_parser_ret_val.body) + + parser = PythonWithConceptsParser() + result = parser.parse(context, unrec_node_ret_val.body) + + assert result.status + python_node = result.body.body + assert isinstance(python_node, PythonNode) + assert python_node.source == 'a + __C__twenties__1003__C__' + assert "__C__twenties__1003__C__" in python_node.objects + + def test_i_cannot_parse_unrecognized_sequence(self): + sheerka, context, one, two, twenties = self.init_concepts( + Concept("one", body="1"), + Concept("two", body="2"), + Concept("twenties", definition="'twenty' (one|two)=n", body='20 + n').def_var("n"), + create_new=True + ) + + sequence_parser_ret_val = SequenceNodeParser().parse(context, ParserInput("a + twenty one")) + unrec_node_ret_val = UnrecognizedNodeParser().parse(context, sequence_parser_ret_val.body) + + parser = PythonWithConceptsParser() + result = parser.parse(context, unrec_node_ret_val.body) + + assert not result.status diff --git a/tests/parsers/test_SequenceNodeParser.py b/tests/parsers/test_SequenceNodeParser.py index 74ad842..f657946 100644 --- a/tests/parsers/test_SequenceNodeParser.py +++ b/tests/parsers/test_SequenceNodeParser.py @@ -9,7 +9,7 @@ from tests.parsers.parsers_utils import compute_expected_array, CN, CNC, SCN, ge UTN -class TestAtomsParser(TestUsingMemoryBasedSheerka): +class TestSequenceNodeParser(TestUsingMemoryBasedSheerka): def init_parser(self, my_map, create_new=False, singleton=True, use_sheerka=False): sheerka, context, *updated_concepts = self.init_test().with_concepts( *my_map.values(), @@ -283,8 +283,8 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("text, expected", [ ("hello foo bar", [ - (True, [CNC("hello1", "hello foo ", a="foo "), "bar"]), - (True, [CNC("hello2", "hello foo ", b="foo "), "bar"]), + ("a", [CN("hello1", "hello foo "), "bar"]), + ("b", [CN("hello2", "hello foo "), "bar"]), ]), ]) def test_i_can_parse_when_unrecognized_yield_multiple_values(self, text, expected): @@ -303,11 +303,12 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): wrapper = res.body lexer_nodes = res.body.body - assert res.status == expected[0] + assert res.status assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) expected_array = compute_expected_array(concepts_map, text, expected[1]) transformed_nodes = get_test_obj(lexer_nodes, expected_array) assert transformed_nodes == expected_array + assert lexer_nodes[0].concept.get_metadata().variables == [(expected[0], "foo ")] @pytest.mark.parametrize("text, expected", [ ("1 + twenty one", [SCN("1 + twenty "), "one"]), @@ -352,7 +353,7 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): lexer_nodes = res.body.body assert res.status - assert lexer_nodes[0].concept.get_metadata().is_evaluated == expected_is_evaluated + assert lexer_nodes[0].concept.get_hints().is_evaluated == expected_is_evaluated def test_the_parser_always_return_a_new_instance_of_the_concept(self): concepts_map = { @@ -383,3 +384,24 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka): assert not res.status assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME) + + @pytest.mark.parametrize("text", [ + "foo", + "foo bar", + "long concept", + "unrecognized foo", + "bar unrecognized", + ]) + def test_i_correctly_set_up_use_copy(self, text): + concepts_map = { + "foo": Concept("foo"), + "bar": Concept("bar"), + "long concept": Concept("long concept"), + } + + sheerka, context, parser = self.init_parser(concepts_map) + res = parser.parse(context, ParserInput(text)) + + for node in res.body.body: + if hasattr(node, "concept"): + assert node.concept.get_hints().use_copy diff --git a/tests/parsers/test_TestShortTermMemoryParser.py b/tests/parsers/test_ShortTermMemoryParser.py similarity index 100% rename from tests/parsers/test_TestShortTermMemoryParser.py rename to tests/parsers/test_ShortTermMemoryParser.py diff --git a/tests/parsers/test_SyaNodeParser.py b/tests/parsers/test_SyaNodeParser.py index fc555c0..98f110c 100644 --- a/tests/parsers/test_SyaNodeParser.py +++ b/tests/parsers/test_SyaNodeParser.py @@ -7,7 +7,6 @@ from core.global_symbols import CONCEPT_COMPARISON_CONTEXT from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import Tokenizer from core.utils import NextIdManager -from parsers.BaseNodeParser import UnrecognizedTokensNode from parsers.PythonParser import PythonNode from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \ NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError @@ -1081,29 +1080,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert isinstance(concept_plus_b[0].body.body, PythonNode) assert concept_suffixed_a == cmap["two"] - @pytest.mark.parametrize("text, expected_status, expected_result", [ - ("f1(one prefixed) plus f2(suffixed two)", False, [ - CNC("plus", - a=SCWC("f1(", ")", CNC("prefixed", a="one")), - b=SCWC("f2(", (")", 1), CNC("suffixed", a="two"))) - ]), - ("one is a concept", True, [CNC("is a concept", c="one")]), - ("a is a concept", False, [CNC("is a concept", c=UTN("a"))]), - ]) - def test_i_can_parse_when_one_result(self, text, expected_status, expected_result): - sheerka, context, parser = self.init_parser() - - res = parser.parse(context, ParserInput(text)) - wrapper = res.body - lexer_nodes = res.body.body - - expected_array = compute_expected_array(cmap, text, expected_result) - assert res.status == expected_status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - - transformed_nodes = get_test_obj(lexer_nodes, expected_array) - assert transformed_nodes == expected_array - @pytest.mark.parametrize("text", [ "function(suffixed one)", "function(one plus two mult three)", @@ -1144,8 +1120,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): ("one plus two a long other b", [CNC("plus", a="one", b="two"), UTN(" a long other b")]), ("one plus two a long infixed", [CNC("plus", a="one", b="two"), UTN(" a long infixed")]), ("one plus two a long", [CNC("plus", a="one", b="two"), UTN(" a long")]), - ("one ? a long infixed : two", [CNC("?", a="one", b=UTN("a long infixed"), c="two")]), - ("one ? a long infix : two", [CNC("?", a="one", b=UTN("a long infix"), c="two")]), ]) def test_i_can_almost_parse_when_one_part_is_recognized_but_not_the_rest(self, text, expected_result): """ @@ -1195,31 +1169,25 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert transformed_nodes == expected_array # assert lexer_nodes == expected_array - @pytest.mark.parametrize("text, expected_concept, expected_unrecognized", [ - ("x$!# prefixed", "prefixed", ["a"]), - ("suffixed x$!#", "suffixed", ["a"]), - ("one infix x$!#", "infix", ["b"]), - ("x$!# infix one", "infix", ["a"]), - ("x$!# infix z$!#", "infix", ["a", "b"]), + @pytest.mark.parametrize("text, expected_error", [ + ("x$!# prefixed", "Cannot parse 'x$!#'"), + ("suffixed x$!#", "Cannot parse 'x$!#'"), + ("one infix x$!#", "Cannot parse 'x$!#'"), + ("x$!# infix one", "Cannot parse 'x$!#'"), + ("x$!# infix z$!#", ["Cannot parse 'z$!#'", "Cannot parse 'x$!#'"]), + ("suffixed alpha beta", "Cannot parse 'alpha beta'"), + ("alpha beta prefixed", "Cannot parse 'alpha beta'"), + ("one plus alpha beta", "Cannot parse 'alpha beta'"), ]) - def test_i_cannot_parse_when_unrecognized(self, text, expected_concept, expected_unrecognized): + def test_i_cannot_parse_when_unrecognized(self, text, expected_error): sheerka, context, parser = self.init_parser() res = parser.parse(context, ParserInput(text)) wrapper = res.body - lexer_nodes = res.body.body - expected_end = len(list(Tokenizer(text))) - 2 assert not res.status - assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) - expected_array = [CN(cmap[expected_concept], text, 0, expected_end)] - transformed_nodes = get_test_obj(lexer_nodes, expected_array) - assert transformed_nodes == expected_array - # assert lexer_nodes == [CN(cmap[expected_concept], text, 0, expected_end)] - - concept_found = lexer_nodes[0].concept - for unrecognized in expected_unrecognized: - assert isinstance(concept_found.get_compiled()[unrecognized], UnrecognizedTokensNode) + assert sheerka.isinstance(wrapper, BuiltinConcepts.ERROR) + assert wrapper.body == expected_error @pytest.mark.parametrize("text, expected", [ ("x$!# suffixed one", [UTN("x$!# ", 0, 4), CN("suffixed __var__0", "suffixed one", 5, 7)]), @@ -1364,6 +1332,26 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka): assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT) compare_with_test_object(lexer_nodes, [CN(cmap["suffixed"], text, 0, 6)]) + def test_i_correctly_set_up_use_copy(self): + my_map = { + "shirt": Concept("shirt"), + "a x": Concept("a x", ret="x").def_var("x"), + "red x": Concept("red x", ret="x").def_var("x"), + } + + sheerka, context, parser = self.init_parser(my_map) + + res = parser.parse(context, ParserInput("a red shirt")) + concept_found = res.body.body[0].concept + + assert concept_found.get_hints().use_copy + + concept_found_x = concept_found.get_compiled()["x"] + assert concept_found_x.get_hints().use_copy + + concept_found_x_x = concept_found_x.get_compiled()["x"] + assert concept_found_x_x.get_hints().use_copy + class TestFileBaseSyaNodeParser(TestUsingFileBasedSheerka): def test_i_can_parse_after_restart(self): diff --git a/tests/parsers/test_UnrecognizedNodeParser.py b/tests/parsers/test_UnrecognizedNodeParser.py index 6ba20a0..08ccfd3 100644 --- a/tests/parsers/test_UnrecognizedNodeParser.py +++ b/tests/parsers/test_UnrecognizedNodeParser.py @@ -363,7 +363,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): assert parser_result.source == expression assert len(actual_nodes) == 1 assert actual_nodes[0].nodes[ - 0].concept.get_metadata().is_evaluated # 'a plus b' is recognized as concept definition + 0].concept.get_hints().is_evaluated # 'a plus b' is recognized as concept definition def test_i_can_parse_unrecognized_source_code_with_concept_node_when_var_in_short_term_memory(self): sheerka, context, parser = self.init_parser() @@ -373,7 +373,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): nodes = get_input_nodes_from(concepts_map, expression, source_code_concepts) parser_input = ParserResultConcept("parsers.xxx", source=expression, value=nodes) - context.add_to_short_term_memory("a", 1) + context.add_to_short_term_memory("a", 1) # -> a plus b is now an instance of the concept res = parser.parse(context, parser_input) parser_result = res.body actual_nodes = res.body.body @@ -382,7 +382,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka): assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT) assert parser_result.source == expression assert len(actual_nodes) == 1 - assert not actual_nodes[0].nodes[0].concept.get_metadata().is_evaluated # 'a plus b' need to be evaluated + assert not actual_nodes[0].nodes[0].concept.get_hints().is_evaluated # 'a plus b' need to be evaluated def test_i_can_parse_unrecognized_sya_concept_that_references_source_code(self): sheerka, context, parser = self.init_parser() diff --git a/tests/sheerkarete/test_conditions.py b/tests/sheerkarete/test_conditions.py index 146405a..4cfee7f 100644 --- a/tests/sheerkarete/test_conditions.py +++ b/tests/sheerkarete/test_conditions.py @@ -1,3 +1,5 @@ +import pytest + from sheerkarete.common import WME, V from sheerkarete.conditions import NotEqualsCondition, AndConditions, Condition, NegatedCondition, \ NegatedConjunctiveConditions, FilterCondition, BindCondition @@ -27,6 +29,31 @@ class TestReteConditions(TestUsingMemoryBasedSheerka): network.remove_wme(wme) assert len(list(network.matches)) == 0 + @pytest.mark.skip("Comparison between variables need to be implemented.") + def test_i_can_test_condition_between_variables(self): + network = ReteNetwork() + conditions = [Condition(V("x"), "__name__", "x"), + Condition(V("y"), "__name__", "y"), + Condition(V("x"), "__self__", V("y"))], + rule = RuleForTestingRete(AndConditions(conditions)) + network.add_rule(rule) + + assert len(list(network.matches)) == 0 + + wme_x = WME("x", "__self__", "10") + wme_y = WME("y", "__self__", "10") + network.add_wme(wme_x) + network.add_wme(wme_y) + + matches = list(network.matches) + assert len(matches) == 1 + assert matches[0].pnode.rules == [rule] + assert matches[0].token.wmes == [wme_x, wme_y] + + # remove wme + network.remove_wme(wme_x) + assert len(list(network.matches)) == 0 + def test_i_can_manage_not_equals_condition(self): network = ReteNetwork() diff --git a/tests/sheerkarete/test_network.py b/tests/sheerkarete/test_network.py index 12b6f22..d4a8b9c 100644 --- a/tests/sheerkarete/test_network.py +++ b/tests/sheerkarete/test_network.py @@ -88,6 +88,20 @@ class TestReteNetwork(TestUsingMemoryBasedSheerka): "fact_name.value.body": ["sub_value"], } + def test_i_can_update_conditions_attributes_when_value_is_a_variable(self): + network = ReteNetwork() + conditions = [Condition(V("x"), "__name__", "x"), + Condition(V("y"), "__name__", "y"), + Condition(V("x"), "__self__", V("y"))] + + rule = RuleForTestingRete(AndConditions(conditions)) + network.add_rule(rule) + + assert network.attributes_by_id == { + "x": ["__name__", "__self__"], + "y": ["__name__", "__self__"], + } + def test_adding_obj_when_no_rule_has_no_effect(self): network = ReteNetwork() ret = ReturnValueConcept("test", True, "value")