diff --git a/src/core/sheerka/services/SheerkaEvaluateRules.py b/src/core/sheerka/services/SheerkaEvaluateRules.py index e9dfba3..465b72e 100644 --- a/src/core/sheerka/services/SheerkaEvaluateRules.py +++ b/src/core/sheerka/services/SheerkaEvaluateRules.py @@ -159,7 +159,7 @@ class SheerkaEvaluateRules(BaseService): if compiled_condition.evaluator_type == ConceptEvaluator.NAME: compiled_condition.concept.get_metadata().is_evaluated = False - evaluator = self.evaluators_by_name[compiled_condition.evaluator] + evaluator = self.evaluators_by_name[compiled_condition.evaluator_type] res = evaluator.eval(context, compiled_condition.return_value) if res.status and isinstance(res.body, bool) and res.body: # one successful value found. No need to look any further diff --git a/src/core/sheerka/services/SheerkaRuleManager.py b/src/core/sheerka/services/SheerkaRuleManager.py index ec2927e..0cb29c4 100644 --- a/src/core/sheerka/services/SheerkaRuleManager.py +++ b/src/core/sheerka/services/SheerkaRuleManager.py @@ -1,6 +1,7 @@ import operator import re from dataclasses import dataclass +from itertools import product from typing import Union, Set, List from cache.Cache import Cache @@ -14,7 +15,7 @@ from core.rule import Rule, ACTION_TYPE_PRINT from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError from core.tokenizer import Keywords, TokenKind, Token, IterParser -from core.utils import index_tokens, COLORS, get_text_from_tokens +from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets from evaluators.ConceptEvaluator import ConceptEvaluator from evaluators.PythonEvaluator import PythonEvaluator, Expando from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode @@ -1309,19 +1310,166 @@ class ReteConditionExprVisitor(ExpressionVisitor): conditions.append(Condition(left, attr, value)) +@dataclass() +class PythonConditionExprVisitorObj: + text: Union[str, None] # human readable + source: Union[str, None] # python expression to compile + objects: dict + variables: set + + @staticmethod + def combine_with_and(left, right): + left_as_list = left if isinstance(left, list) else [left] + right_as_list = right if isinstance(right, list) else [right] + + def create_and(a, b): + if a is None and b is None: + return None + + if a is None or a == "": + return b + if b is None or b == "": + return a + return a + " and " + b + + left_right_product = list(product(left_as_list, right_as_list)) + res = [] + for left_obj, right_obj in left_right_product: + res.append(PythonConditionExprVisitorObj(create_and(left_obj.text, right_obj.text), + create_and(left_obj.source, right_obj.source), + merge_dictionaries(left_obj.objects, right_obj.objects), + merge_sets(left_obj.variables, right_obj.variables))) + + return res[0] if len(res) == 1 else res + + class PythonConditionExprVisitor(ExpressionVisitor): def __init__(self, context): self.context = context self.var_counter = 0 - self.variables = set() + self.variables = {} def get_conditions(self, expr_node): self.var_counter = 0 self.variables.clear() - condition = self.visit(expr_node) - return [condition] + visitor_obj = self.visit(expr_node) + if visitor_obj.source: + if self.variables: + variables_definitions = "\n".join([f"{v} = {k}" for k, v in self.variables.items()]) + source = variables_definitions + "\n" + visitor_obj.source + text = variables_definitions + "\n" + visitor_obj.text + else: + source = visitor_obj.source + text = visitor_obj.text + + ret = self.context.sheerka.parse_python(self.context, source) + if ret.status: + ret.body.body.original_source = text + ret.body.body.objects = visitor_obj.objects + return [CompiledCondition(PythonEvaluator.NAME, ret, visitor_obj.variables)] + else: + return [CompiledCondition(None, None, visitor_obj.variables)] + + def add_variable(self, target): + var_name = f"__x_{self.var_counter:02}__" + self.var_counter += 1 + self.variables[target] = var_name + return var_name + + def init_or_get_variable_from_name(self, variable_path: List[str], obj_variables): + + if len(variable_path) > 1: + left = variable_path[:-1] + right = [variable_path[-1]] + + while left: + var_name = ".".join(left) + if var_name in self.variables: + return self.variables[var_name], ".".join(right) + + right.insert(0, left.pop()) + + if variable_path[0] not in self.variables: + self.add_variable(variable_path[0]) + obj_variables.add(variable_path[0]) + + return self.variables[variable_path[0]], ".".join(variable_path[1:]) + + def init_or_get_variable_from_path(self, variable_path: List[str], obj_variables): + path = ".".join(variable_path) + if path in self.variables: + return self.variables[path] + + obj_variables.add(variable_path[0]) + return self.add_variable(path) def visit_VariableNode(self, expr_node: VariableNode): # no evaluator to call, simply check that the variable is in the bag - return CompiledCondition(None, None, {expr_node.name}) + if not expr_node.attributes and expr_node.name.startswith("__"): + return PythonConditionExprVisitorObj(None, None, {}, {expr_node.name}) + + source = expr_node.get_source() + " == True" + return PythonConditionExprVisitorObj(source, source, {}, {expr_node.name}) + + def visit_ComparisonNode(self, expr_node: ComparisonNode): + if isinstance(expr_node.left, VariableNode): + source = expr_node.get_source() + return PythonConditionExprVisitorObj(source, source, {}, {expr_node.left.name}) + else: + raise FailedToCompileError([expr_node]) + + def visit_AndNode(self, expr_node: AndNode): + current_visitor_obj = self.visit(expr_node.parts[0]) + for node in expr_node.parts[1:]: + visitor_obj = self.visit(node) + current_visitor_obj = PythonConditionExprVisitorObj.combine_with_and(current_visitor_obj, visitor_obj) + + return current_visitor_obj + + def visit_FunctionNode(self, expr_node: FunctionNode): + if expr_node.first.value == "recognize(": + if not isinstance(expr_node.parameters[0].value, VariableNode): + return FailedToCompileError([f"Cannot recognize '{expr_node.parameters[0].value}'"]) + + return self.recognize_concept(expr_node.parameters[0].value.unpack(), + expr_node.parameters[1].value, + {}) + + def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict): + if not isinstance(concept_to_recognize, Concept): + concept_as_str = concept_to_recognize.get_source() + if not concept_as_str: + return FailedToCompileError([f"Missing concept in for {variable_path}"]) + + res = evaluate(self.context, + concept_as_str, + evaluators=CONDITIONS_VISITOR_EVALUATORS, + desc=None, + eval_body=True, + eval_where=False, + is_question=False, + expect_success=False, + stm=None) + res = expect_one(self.context, res) + + if not res.status: + return FailedToCompileError([f"Unknown concept {concept_as_str}"]) + concept = res.body + else: + concept = concept_to_recognize + + obj_variables = set() + variable = self.init_or_get_variable_from_path(variable_path, obj_variables) + + source = f"isinstance({variable}, Concept)" + + if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME: + source += f" and {variable}.name == '{concept.name}'" + elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID: + source += f" and {variable}.id == '{concept.id}'" + else: + source += f" and {variable}.key == '{concept.key}'" + concept_variables.update({k: v for k, v in concept.variables().items() if v is not NotInit}) + + return PythonConditionExprVisitorObj(source, source, {}, obj_variables) diff --git a/src/core/utils.py b/src/core/utils.py index f63d72d..d52b787 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -305,6 +305,40 @@ def dict_product(a, b): return res +def merge_dictionaries(a, b): + """ + Returns a new dictionary which is the merge + :param a: + :param b: + :return: + """ + if a is None and b is None: + return None + + res = {} + if a: + res.update(a) + + if b: + res.update(b) + + return res + + +def merge_sets(a, b): + if a is None and b is None: + return None + + res = set() + if a: + res.update(a) + + if b: + res.update(b) + + return res + + def get_n_clones(obj, n): objs = [obj] for i in range(n - 1): diff --git a/src/parsers/BaseExpressionParser.py b/src/parsers/BaseExpressionParser.py index 71e6881..0bb55f9 100644 --- a/src/parsers/BaseExpressionParser.py +++ b/src/parsers/BaseExpressionParser.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Tuple, Union +from typing import List, Union from core.builtin_concepts_ids import BuiltinConcepts from core.sheerka.services.SheerkaExecute import ParserInput @@ -33,15 +33,11 @@ class ParenthesisMismatchError(ParsingError): token: Token -@dataclass(init=False) class ExprNode(Node): """ Base ExprNode eval() must be overridden """ - start: int # index of the first token - end: int # index of the last token - tokens: List[Token] def __init__(self, start: int, end: int, tokens: List[Token]): self.start = start @@ -118,10 +114,7 @@ class NameExprNode(ExprNode): return UnrecognizedTokensNode(self.start, self.end, [token]).fix_source() -@dataclass(init=False) class AndNode(ExprNode): - parts: Tuple[ExprNode] - def __init__(self, start, end, tokens, *parts: ExprNode): super().__init__(start, end, tokens) self.parts = parts @@ -154,10 +147,7 @@ class AndNode(ExprNode): return hash((self.start, self.end, self.parts)) -@dataclass(init=False) class OrNode(ExprNode): - parts: Tuple[ExprNode] - def __init__(self, start, end, tokens, *parts: ExprNode): super().__init__(start, end, tokens) self.parts = parts @@ -190,9 +180,10 @@ class OrNode(ExprNode): return hash((self.start, self.end, self.parts)) -@dataclass() class NotNode(ExprNode): - node: ExprNode + def __init__(self, start, end, tokens, node: ExprNode): + super().__init__(start, end, tokens) + self.node = node def eval(self, obj): return not self.node.eval(obj) @@ -222,13 +213,15 @@ class NotNode(ExprNode): return hash((self.start, self.end, self.node)) -@dataclass() class ParenthesisNode(ExprNode): """ Contains the boundaries of an expression inside parenthesis Need it, just to keep track of the boundaries of the parenthesis """ - node: ExprNode + + def __init__(self, start, end, tokens, node: ExprNode): + super().__init__(start, end, tokens) + self.node = node def __eq__(self, other): if not isinstance(other, ParenthesisNode): @@ -291,11 +284,12 @@ class VariableNode(ExprNode): return [self.name] + self.attributes -@dataclass class ComparisonNode(ExprNode): - comp: str - left: ExprNode - right: ExprNode + def __init__(self, start, end, tokens, comp: str, left: ExprNode, right: ExprNode): + super().__init__(start, end, tokens) + self.comp = comp + self.left = left + self.right = right def __eq__(self, other): if id(self) == id(other): @@ -338,11 +332,34 @@ class FunctionParameter: return UnrecognizedTokensNode(self.separator.start, self.separator.end, self.separator.tokens).fix_source() -@dataclass class FunctionNode(ExprNode): - first: NameExprNode # beginning of the function (it should represent the name of the function) - last: NameExprNode # last part of the function (it should be the trailing parenthesis) - parameters: Union[None, List[FunctionParameter]] + + def __init__(self, start, end, tokens, + first: NameExprNode, last: NameExprNode, parameters: Union[None, List[FunctionParameter]]): + super().__init__(start, end, tokens) + self.first = first + self.last = last + self.parameters = parameters + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, FunctionNode): + return False + + return (self.first == other.first and + self.last == other.last and + self.parameters == other.parameters) + + def __hash__(self): + return hash((self.first, self.last, self.parameters)) + + def __repr__(self): + return f"FunctionNode(start={self.start}, end={self.end}, {self.first!r} {self.last} {self.parameters!r})" + + def __str__(self): + return f"{self.first} {self.parameters} {self.last}" class BaseExpressionParser(BaseParser): diff --git a/tests/core/test_SheerkaRuleManager.py b/tests/core/test_SheerkaRuleManager.py index e1d1b22..a357189 100644 --- a/tests/core/test_SheerkaRuleManager.py +++ b/tests/core/test_SheerkaRuleManager.py @@ -1154,7 +1154,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ ), ]) - def test_i_can_get_rete_using_recognized_function(self, test_name, expression, variable_name, expected_as_str): + def test_i_can_get_rete_using_recognize_function(self, test_name, expression, variable_name, expected_as_str): sheerka, context, greetings, foo = self.init_test().with_concepts( Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), Concept("foo"), @@ -1290,42 +1290,13 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ matches = list(network.matches) assert len(matches) == 1 - def test_i_can_get_compiled_conditions_when_testing_data_existence(self): - sheerka, context = self.init_test().unpack() - expression = "__ret" - - parser = ExpressionParser() - error_sink = ErrorSink() - parser_input = ParserInput(expression) - parser.reset_parser_input(parser_input, error_sink) - parsed = parser.parse_input(context, parser_input, error_sink) - - visitor = PythonConditionExprVisitor(context) - conditions = visitor.get_conditions(parsed) - - assert len(conditions) == 1 - assert isinstance(conditions[0], CompiledCondition) - assert conditions[0].evaluator_type is None - assert conditions[0].return_value is None - assert conditions[0].concept is None - assert conditions[0].variables == {"__ret"} - - # check against SheerkaEvaluateRules - evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME] - bag = {"__ret": ReturnValueConcept("Test", True, None)} - rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression) - rule.compiled_conditions = conditions - res = evaluate_rules_service.evaluate_rule(context, rule, bag) - assert res.status - assert self.sheerka.is_success(self.sheerka.objvalue(res)) - - @pytest.mark.parametrize("expression", [ - "__ret", - "__ret.status == True", - "__ret.status", - "__ret and __ret.status", + @pytest.mark.parametrize("expression, expected_compiled", [ + ("__ret", None), + ("__ret.status == True", "__ret.status == True"), + ("__ret.status", "__ret.status == True"), + ("__ret and __ret.status", "__ret.status == True") ]) - def test_i_can_get_compiled_conditions(self, expression): + def test_i_can_get_compiled_conditions(self, expression, expected_compiled): sheerka, context = self.init_test().unpack() parser = ExpressionParser() @@ -1337,25 +1308,155 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \ visitor = PythonConditionExprVisitor(context) conditions = visitor.get_conditions(parsed) - ast_ = ast.parse(expression, "", 'eval') - expected_python_node = PythonNode(expression, ast_) - assert len(conditions) == 1 assert isinstance(conditions[0], CompiledCondition) - assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME - assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) - assert sheerka.objvalue(conditions[0].return_value) == expected_python_node + if expected_compiled: + ast_ = ast.parse(expected_compiled, "", 'eval') + expected_python_node = PythonNode(expected_compiled, ast_) + assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME + assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(conditions[0].return_value) == expected_python_node + else: + assert conditions[0].evaluator_type is None + assert conditions[0].return_value is None assert conditions[0].concept is None assert conditions[0].variables == {"__ret"} # check against SheerkaEvaluateRules evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME] bag = {"__ret": ReturnValueConcept("Test", True, None)} - rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression) - rule.compiled_conditions = conditions - res = evaluate_rules_service.evaluate_rule(context, rule, bag) - assert res.status - assert self.sheerka.is_success(self.sheerka.objvalue(res)) + with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context: + sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag) + rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression) + rule.compiled_conditions = conditions + res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag) + assert res.status + assert self.sheerka.is_success(self.sheerka.objvalue(res)) + + @pytest.mark.parametrize("expression, variable_name, expected_compiled", [ + ( + "recognize(__ret.body, greetings)", + None, + "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'" + ), + # ( + # "recognize(__ret.body, c:|1001:)", + # None, + # ["#__x_00__|__name__|'__ret'", + # "#__x_00__|body|#__x_01__", + # "#__x_01__|__is_concept__|True", + # "#__x_01__|id|'1001'"] + # ), + # ( + # "recognize(__ret.body, c:greetings:)", + # None, + # ["#__x_00__|__name__|'__ret'", + # "#__x_00__|body|#__x_01__", + # "#__x_01__|__is_concept__|True", + # "#__x_01__|name|'greetings'"] + # ), + # ( + # "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'", + # "my friend", + # ["#__x_00__|__name__|'__ret'", + # "#__x_00__|body|#__x_01__", + # "#__x_01__|__is_concept__|True", + # "#__x_01__|name|'greetings'", + # "#__x_01__|a|'my friend'"] + # ), + # ( + # "recognize(__ret.body, greetings) and __ret.body.a == sheerka", + # "sheerka", + # ["#__x_00__|__name__|'__ret'", + # "#__x_00__|body|#__x_01__", + # "#__x_01__|__is_concept__|True", + # "#__x_01__|name|'greetings'", + # "#__x_01__|a|'__sheerka__'"] + # ), + # ( + # "recognize(__ret.body, greetings) and __ret.body.a == foo", + # "foo", + # ["#__x_00__|__name__|'__ret'", + # "#__x_00__|body|#__x_01__", + # "#__x_01__|__is_concept__|True", + # "#__x_01__|name|'greetings'", + # "#__x_01__|a|#__x_02__", + # "#__x_02__|__is_concept__|True", + # "#__x_02__|key|'foo'"] + # ), + # ( + # "recognize(__ret.body, hello sheerka)", + # "sheerka", + # ["#__x_00__|__name__|'__ret'", + # "#__x_00__|body|#__x_01__", + # "#__x_01__|__is_concept__|True", + # "#__x_01__|key|'hello __var__0'", + # "#__x_01__|a|'__sheerka__'"] + # ), + # ( + # "recognize(__ret.body, hello 'my friend')", + # "my friend", + # ["#__x_00__|__name__|'__ret'", + # "#__x_00__|body|#__x_01__", + # "#__x_01__|__is_concept__|True", + # "#__x_01__|key|'hello __var__0'", + # "#__x_01__|a|'my friend'"] + # ), + # ( + # "recognize(__ret.body, hello foo)", + # "foo", + # ["#__x_00__|__name__|'__ret'", + # "#__x_00__|body|#__x_01__", + # "#__x_01__|__is_concept__|True", + # "#__x_01__|key|'hello __var__0'", + # "#__x_01__|a|#__x_02__", + # "#__x_02__|__is_concept__|True", + # "#__x_02__|key|'foo'", + # ] + # ), + + ]) + def test_i_can_get_compiled_using_recognize_function(self, expression, variable_name, expected_compiled): + sheerka, context, greetings, foo = self.init_test().with_concepts( + Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"), + Concept("foo"), + ).unpack() + + parser = ExpressionParser() + error_sink = ErrorSink() + parser_input = ParserInput(expression) + parser.reset_parser_input(parser_input, error_sink) + parsed = parser.parse_input(context, parser_input, error_sink) + + visitor = PythonConditionExprVisitor(context) + conditions = visitor.get_conditions(parsed) + + assert len(conditions) == 1 + assert isinstance(conditions[0], CompiledCondition) + if expected_compiled: + ast_ = ast.parse(expected_compiled, "", 'exec') + expected_python_node = PythonNode(expected_compiled, ast_, expected_compiled) + assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME + assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE) + assert sheerka.objvalue(conditions[0].return_value) == expected_python_node + else: + assert conditions[0].evaluator_type is None + assert conditions[0].return_value is None + assert conditions[0].concept is None + assert conditions[0].variables == {"__ret"} + + # check against SheerkaEvaluateRules + evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME] + variable = foo if variable_name == "foo" else sheerka if variable_name == "sheerka" else variable_name + to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable) + bag = {"__ret": ReturnValueConcept("Test", True, to_recognize)} + with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context: + sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag) + rule = Rule(name="test_i_can_get_compiled_using_recognize_function", predicate=expression) + rule.compiled_conditions = conditions + res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag) + assert res.status + assert self.sheerka.is_success(self.sheerka.objvalue(res)) class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):