Fixed #51 : I can compile simple recognize

src/core/sheerka/services/SheerkaEvaluateRules.py

@@ -159,7 +159,7 @@ class SheerkaEvaluateRules(BaseService):
                 if compiled_condition.evaluator_type == ConceptEvaluator.NAME:
                     compiled_condition.concept.get_metadata().is_evaluated = False
 
-                evaluator = self.evaluators_by_name[compiled_condition.evaluator]
+                evaluator = self.evaluators_by_name[compiled_condition.evaluator_type]
                 res = evaluator.eval(context, compiled_condition.return_value)
                 if res.status and isinstance(res.body, bool) and res.body:
                     # one successful value found. No need to look any further

src/core/sheerka/services/SheerkaRuleManager.py

@@ -1,6 +1,7 @@
 import operator
 import re
 from dataclasses import dataclass
+from itertools import product
 from typing import Union, Set, List
 
 from cache.Cache import Cache
@@ -14,7 +15,7 @@ from core.rule import Rule, ACTION_TYPE_PRINT
 from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
 from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError
 from core.tokenizer import Keywords, TokenKind, Token, IterParser
-from core.utils import index_tokens, COLORS, get_text_from_tokens
+from core.utils import index_tokens, COLORS, get_text_from_tokens, merge_dictionaries, merge_sets
 from evaluators.ConceptEvaluator import ConceptEvaluator
 from evaluators.PythonEvaluator import PythonEvaluator, Expando
 from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode
@@ -1309,19 +1310,166 @@ class ReteConditionExprVisitor(ExpressionVisitor):
             conditions.append(Condition(left, attr, value))
 
 
+@dataclass()
+class PythonConditionExprVisitorObj:
+    text: Union[str, None]  # human readable
+    source: Union[str, None]  # python expression to compile
+    objects: dict
+    variables: set
+
+    @staticmethod
+    def combine_with_and(left, right):
+        left_as_list = left if isinstance(left, list) else [left]
+        right_as_list = right if isinstance(right, list) else [right]
+
+        def create_and(a, b):
+            if a is None and b is None:
+                return None
+
+            if a is None or a == "":
+                return b
+            if b is None or b == "":
+                return a
+            return a + " and " + b
+
+        left_right_product = list(product(left_as_list, right_as_list))
+        res = []
+        for left_obj, right_obj in left_right_product:
+            res.append(PythonConditionExprVisitorObj(create_and(left_obj.text, right_obj.text),
+                                                     create_and(left_obj.source, right_obj.source),
+                                                     merge_dictionaries(left_obj.objects, right_obj.objects),
+                                                     merge_sets(left_obj.variables, right_obj.variables)))
+
+        return res[0] if len(res) == 1 else res
+
+
 class PythonConditionExprVisitor(ExpressionVisitor):
     def __init__(self, context):
         self.context = context
         self.var_counter = 0
-        self.variables = set()
+        self.variables = {}
 
     def get_conditions(self, expr_node):
         self.var_counter = 0
         self.variables.clear()
 
-        condition = self.visit(expr_node)
+        visitor_obj = self.visit(expr_node)
-        return [condition]
+        if visitor_obj.source:
+            if self.variables:
+                variables_definitions = "\n".join([f"{v} = {k}" for k, v in self.variables.items()])
+                source = variables_definitions + "\n" + visitor_obj.source
+                text = variables_definitions + "\n" + visitor_obj.text
+            else:
+                source = visitor_obj.source
+                text = visitor_obj.text
+
+            ret = self.context.sheerka.parse_python(self.context, source)
+            if ret.status:
+                ret.body.body.original_source = text
+                ret.body.body.objects = visitor_obj.objects
+                return [CompiledCondition(PythonEvaluator.NAME, ret, visitor_obj.variables)]
+        else:
+            return [CompiledCondition(None, None, visitor_obj.variables)]
+
+    def add_variable(self, target):
+        var_name = f"__x_{self.var_counter:02}__"
+        self.var_counter += 1
+        self.variables[target] = var_name
+        return var_name
+
+    def init_or_get_variable_from_name(self, variable_path: List[str], obj_variables):
+
+        if len(variable_path) > 1:
+            left = variable_path[:-1]
+            right = [variable_path[-1]]
+
+            while left:
+                var_name = ".".join(left)
+                if var_name in self.variables:
+                    return self.variables[var_name], ".".join(right)
+
+                right.insert(0, left.pop())
+
+        if variable_path[0] not in self.variables:
+            self.add_variable(variable_path[0])
+            obj_variables.add(variable_path[0])
+
+        return self.variables[variable_path[0]], ".".join(variable_path[1:])
+
+    def init_or_get_variable_from_path(self, variable_path: List[str], obj_variables):
+        path = ".".join(variable_path)
+        if path in self.variables:
+            return self.variables[path]
+
+        obj_variables.add(variable_path[0])
+        return self.add_variable(path)
 
     def visit_VariableNode(self, expr_node: VariableNode):
         # no evaluator to call, simply check that the variable is in the bag
-        return CompiledCondition(None, None, {expr_node.name})
+        if not expr_node.attributes and expr_node.name.startswith("__"):
+            return PythonConditionExprVisitorObj(None, None, {}, {expr_node.name})
+
+        source = expr_node.get_source() + " == True"
+        return PythonConditionExprVisitorObj(source, source, {}, {expr_node.name})
+
+    def visit_ComparisonNode(self, expr_node: ComparisonNode):
+        if isinstance(expr_node.left, VariableNode):
+            source = expr_node.get_source()
+            return PythonConditionExprVisitorObj(source, source, {}, {expr_node.left.name})
+        else:
+            raise FailedToCompileError([expr_node])
+
+    def visit_AndNode(self, expr_node: AndNode):
+        current_visitor_obj = self.visit(expr_node.parts[0])
+        for node in expr_node.parts[1:]:
+            visitor_obj = self.visit(node)
+            current_visitor_obj = PythonConditionExprVisitorObj.combine_with_and(current_visitor_obj, visitor_obj)
+
+        return current_visitor_obj
+
+    def visit_FunctionNode(self, expr_node: FunctionNode):
+        if expr_node.first.value == "recognize(":
+            if not isinstance(expr_node.parameters[0].value, VariableNode):
+                return FailedToCompileError([f"Cannot recognize '{expr_node.parameters[0].value}'"])
+
+            return self.recognize_concept(expr_node.parameters[0].value.unpack(),
+                                          expr_node.parameters[1].value,
+                                          {})
+
+    def recognize_concept(self, variable_path, concept_to_recognize, concept_variables: dict):
+        if not isinstance(concept_to_recognize, Concept):
+            concept_as_str = concept_to_recognize.get_source()
+            if not concept_as_str:
+                return FailedToCompileError([f"Missing concept in for {variable_path}"])
+
+            res = evaluate(self.context,
+                           concept_as_str,
+                           evaluators=CONDITIONS_VISITOR_EVALUATORS,
+                           desc=None,
+                           eval_body=True,
+                           eval_where=False,
+                           is_question=False,
+                           expect_success=False,
+                           stm=None)
+            res = expect_one(self.context, res)
+
+            if not res.status:
+                return FailedToCompileError([f"Unknown concept {concept_as_str}"])
+            concept = res.body
+        else:
+            concept = concept_to_recognize
+
+        obj_variables = set()
+        variable = self.init_or_get_variable_from_path(variable_path, obj_variables)
+
+        source = f"isinstance({variable}, Concept)"
+
+        if concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_NAME:
+            source += f" and {variable}.name == '{concept.name}'"
+        elif concept.get_hint(BuiltinConcepts.RECOGNIZED_BY) == RECOGNIZED_BY_ID:
+            source += f" and {variable}.id == '{concept.id}'"
+        else:
+            source += f" and {variable}.key == '{concept.key}'"
+            concept_variables.update({k: v for k, v in concept.variables().items() if v is not NotInit})
+
+        return PythonConditionExprVisitorObj(source, source, {}, obj_variables)

src/core/utils.py

@@ -305,6 +305,40 @@ def dict_product(a, b):
     return res
 
 
+def merge_dictionaries(a, b):
+    """
+    Returns a new dictionary which is the merge
+    :param a: 
+    :param b: 
+    :return: 
+    """
+    if a is None and b is None:
+        return None
+
+    res = {}
+    if a:
+        res.update(a)
+
+    if b:
+        res.update(b)
+
+    return res
+
+
+def merge_sets(a, b):
+    if a is None and b is None:
+        return None
+
+    res = set()
+    if a:
+        res.update(a)
+
+    if b:
+        res.update(b)
+
+    return res
+
+
 def get_n_clones(obj, n):
     objs = [obj]
     for i in range(n - 1):

src/parsers/BaseExpressionParser.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import List, Tuple, Union
+from typing import List, Union
 
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.sheerka.services.SheerkaExecute import ParserInput
@@ -33,15 +33,11 @@ class ParenthesisMismatchError(ParsingError):
     token: Token
 
 
-@dataclass(init=False)
 class ExprNode(Node):
     """
     Base ExprNode
     eval() must be overridden
     """
-    start: int  # index of the first token
-    end: int  # index of the last token
-    tokens: List[Token]
 
     def __init__(self, start: int, end: int, tokens: List[Token]):
         self.start = start
@@ -118,10 +114,7 @@ class NameExprNode(ExprNode):
         return UnrecognizedTokensNode(self.start, self.end, [token]).fix_source()
 
 
-@dataclass(init=False)
 class AndNode(ExprNode):
-    parts: Tuple[ExprNode]
-
     def __init__(self, start, end, tokens, *parts: ExprNode):
         super().__init__(start, end, tokens)
         self.parts = parts
@@ -154,10 +147,7 @@ class AndNode(ExprNode):
         return hash((self.start, self.end, self.parts))
 
 
-@dataclass(init=False)
 class OrNode(ExprNode):
-    parts: Tuple[ExprNode]
-
     def __init__(self, start, end, tokens, *parts: ExprNode):
         super().__init__(start, end, tokens)
         self.parts = parts
@@ -190,9 +180,10 @@ class OrNode(ExprNode):
         return hash((self.start, self.end, self.parts))
 
 
-@dataclass()
 class NotNode(ExprNode):
-    node: ExprNode
+    def __init__(self, start, end, tokens, node: ExprNode):
+        super().__init__(start, end, tokens)
+        self.node = node
 
     def eval(self, obj):
         return not self.node.eval(obj)
@@ -222,13 +213,15 @@ class NotNode(ExprNode):
         return hash((self.start, self.end, self.node))
 
 
-@dataclass()
 class ParenthesisNode(ExprNode):
     """
     Contains the boundaries of an expression inside parenthesis
     Need it, just to keep track of the boundaries of the parenthesis
     """
-    node: ExprNode
+
+    def __init__(self, start, end, tokens, node: ExprNode):
+        super().__init__(start, end, tokens)
+        self.node = node
 
     def __eq__(self, other):
         if not isinstance(other, ParenthesisNode):
@@ -291,11 +284,12 @@ class VariableNode(ExprNode):
         return [self.name] + self.attributes
 
 
-@dataclass
 class ComparisonNode(ExprNode):
-    comp: str
+    def __init__(self, start, end, tokens, comp: str, left: ExprNode, right: ExprNode):
-    left: ExprNode
+        super().__init__(start, end, tokens)
-    right: ExprNode
+        self.comp = comp
+        self.left = left
+        self.right = right
 
     def __eq__(self, other):
         if id(self) == id(other):
@@ -338,11 +332,34 @@ class FunctionParameter:
         return UnrecognizedTokensNode(self.separator.start, self.separator.end, self.separator.tokens).fix_source()
 
 
-@dataclass
 class FunctionNode(ExprNode):
-    first: NameExprNode  # beginning of the function (it should represent the name of the function)
+
-    last: NameExprNode  # last part of the function (it should be the trailing parenthesis)
+    def __init__(self, start, end, tokens,
-    parameters: Union[None, List[FunctionParameter]]
+                 first: NameExprNode, last: NameExprNode, parameters: Union[None, List[FunctionParameter]]):
+        super().__init__(start, end, tokens)
+        self.first = first
+        self.last = last
+        self.parameters = parameters
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, FunctionNode):
+            return False
+
+        return (self.first == other.first and
+                self.last == other.last and
+                self.parameters == other.parameters)
+
+    def __hash__(self):
+        return hash((self.first, self.last, self.parameters))
+
+    def __repr__(self):
+        return f"FunctionNode(start={self.start}, end={self.end}, {self.first!r} {self.last} {self.parameters!r})"
+
+    def __str__(self):
+        return f"{self.first} {self.parameters} {self.last}"
 
 
 class BaseExpressionParser(BaseParser):

tests/core/test_SheerkaRuleManager.py

@@ -1154,7 +1154,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         ),
 
     ])
-    def test_i_can_get_rete_using_recognized_function(self, test_name, expression, variable_name, expected_as_str):
+    def test_i_can_get_rete_using_recognize_function(self, test_name, expression, variable_name, expected_as_str):
         sheerka, context, greetings, foo = self.init_test().with_concepts(
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
             Concept("foo"),
@@ -1290,42 +1290,13 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         matches = list(network.matches)
         assert len(matches) == 1
 
-    def test_i_can_get_compiled_conditions_when_testing_data_existence(self):
+    @pytest.mark.parametrize("expression, expected_compiled", [
-        sheerka, context = self.init_test().unpack()
+        ("__ret", None),
-        expression = "__ret"
+        ("__ret.status == True", "__ret.status == True"),
-
+        ("__ret.status", "__ret.status == True"),
-        parser = ExpressionParser()
+        ("__ret and __ret.status", "__ret.status == True")
-        error_sink = ErrorSink()
-        parser_input = ParserInput(expression)
-        parser.reset_parser_input(parser_input, error_sink)
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        visitor = PythonConditionExprVisitor(context)
-        conditions = visitor.get_conditions(parsed)
-
-        assert len(conditions) == 1
-        assert isinstance(conditions[0], CompiledCondition)
-        assert conditions[0].evaluator_type is None
-        assert conditions[0].return_value is None
-        assert conditions[0].concept is None
-        assert conditions[0].variables == {"__ret"}
-
-        # check against SheerkaEvaluateRules
-        evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
-        bag = {"__ret": ReturnValueConcept("Test", True, None)}
-        rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression)
-        rule.compiled_conditions = conditions
-        res = evaluate_rules_service.evaluate_rule(context, rule, bag)
-        assert res.status
-        assert self.sheerka.is_success(self.sheerka.objvalue(res))
-
-    @pytest.mark.parametrize("expression", [
-        "__ret",
-        "__ret.status == True",
-        "__ret.status",
-        "__ret and __ret.status",
     ])
-    def test_i_can_get_compiled_conditions(self, expression):
+    def test_i_can_get_compiled_conditions(self, expression, expected_compiled):
         sheerka, context = self.init_test().unpack()
 
         parser = ExpressionParser()
@@ -1337,25 +1308,155 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         visitor = PythonConditionExprVisitor(context)
         conditions = visitor.get_conditions(parsed)
 
-        ast_ = ast.parse(expression, "<source>", 'eval')
-        expected_python_node = PythonNode(expression, ast_)
-
         assert len(conditions) == 1
         assert isinstance(conditions[0], CompiledCondition)
-        assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
+        if expected_compiled:
-        assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
+            ast_ = ast.parse(expected_compiled, "<source>", 'eval')
-        assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
+            expected_python_node = PythonNode(expected_compiled, ast_)
+            assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
+            assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
+            assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
+        else:
+            assert conditions[0].evaluator_type is None
+            assert conditions[0].return_value is None
         assert conditions[0].concept is None
         assert conditions[0].variables == {"__ret"}
 
         # check against SheerkaEvaluateRules
         evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
         bag = {"__ret": ReturnValueConcept("Test", True, None)}
-        rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression)
+        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
-        rule.compiled_conditions = conditions
+            sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
-        res = evaluate_rules_service.evaluate_rule(context, rule, bag)
+            rule = Rule(name="test_i_can_get_compiled_conditions", predicate=expression)
-        assert res.status
+            rule.compiled_conditions = conditions
-        assert self.sheerka.is_success(self.sheerka.objvalue(res))
+            res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
+            assert res.status
+            assert self.sheerka.is_success(self.sheerka.objvalue(res))
+
+    @pytest.mark.parametrize("expression, variable_name, expected_compiled", [
+        (
+            "recognize(__ret.body, greetings)",
+            None,
+            "__x_00__ = __ret.body\nisinstance(__x_00__, Concept) and __x_00__.name == 'greetings'"
+        ),
+        # (
+        #     "recognize(__ret.body, c:|1001:)",
+        #     None,
+        #     ["#__x_00__|__name__|'__ret'",
+        #      "#__x_00__|body|#__x_01__",
+        #      "#__x_01__|__is_concept__|True",
+        #      "#__x_01__|id|'1001'"]
+        # ),
+        # (
+        #     "recognize(__ret.body, c:greetings:)",
+        #     None,
+        #     ["#__x_00__|__name__|'__ret'",
+        #      "#__x_00__|body|#__x_01__",
+        #      "#__x_01__|__is_concept__|True",
+        #      "#__x_01__|name|'greetings'"]
+        # ),
+        # (
+        #     "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'",
+        #     "my friend",
+        #     ["#__x_00__|__name__|'__ret'",
+        #      "#__x_00__|body|#__x_01__",
+        #      "#__x_01__|__is_concept__|True",
+        #      "#__x_01__|name|'greetings'",
+        #      "#__x_01__|a|'my friend'"]
+        # ),
+        # (
+        #     "recognize(__ret.body, greetings) and __ret.body.a == sheerka",
+        #     "sheerka",
+        #     ["#__x_00__|__name__|'__ret'",
+        #      "#__x_00__|body|#__x_01__",
+        #      "#__x_01__|__is_concept__|True",
+        #      "#__x_01__|name|'greetings'",
+        #      "#__x_01__|a|'__sheerka__'"]
+        # ),
+        # (
+        #     "recognize(__ret.body, greetings) and __ret.body.a == foo",
+        #     "foo",
+        #     ["#__x_00__|__name__|'__ret'",
+        #      "#__x_00__|body|#__x_01__",
+        #      "#__x_01__|__is_concept__|True",
+        #      "#__x_01__|name|'greetings'",
+        #      "#__x_01__|a|#__x_02__",
+        #      "#__x_02__|__is_concept__|True",
+        #      "#__x_02__|key|'foo'"]
+        # ),
+        # (
+        #     "recognize(__ret.body, hello sheerka)",
+        #     "sheerka",
+        #     ["#__x_00__|__name__|'__ret'",
+        #      "#__x_00__|body|#__x_01__",
+        #      "#__x_01__|__is_concept__|True",
+        #      "#__x_01__|key|'hello __var__0'",
+        #      "#__x_01__|a|'__sheerka__'"]
+        # ),
+        # (
+        #     "recognize(__ret.body, hello 'my friend')",
+        #     "my friend",
+        #     ["#__x_00__|__name__|'__ret'",
+        #      "#__x_00__|body|#__x_01__",
+        #      "#__x_01__|__is_concept__|True",
+        #      "#__x_01__|key|'hello __var__0'",
+        #      "#__x_01__|a|'my friend'"]
+        # ),
+        # (
+        #     "recognize(__ret.body, hello foo)",
+        #     "foo",
+        #     ["#__x_00__|__name__|'__ret'",
+        #      "#__x_00__|body|#__x_01__",
+        #      "#__x_01__|__is_concept__|True",
+        #      "#__x_01__|key|'hello __var__0'",
+        #      "#__x_01__|a|#__x_02__",
+        #      "#__x_02__|__is_concept__|True",
+        #      "#__x_02__|key|'foo'",
+        #      ]
+        # ),
+
+    ])
+    def test_i_can_get_compiled_using_recognize_function(self, expression, variable_name, expected_compiled):
+        sheerka, context, greetings, foo = self.init_test().with_concepts(
+            Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
+            Concept("foo"),
+        ).unpack()
+
+        parser = ExpressionParser()
+        error_sink = ErrorSink()
+        parser_input = ParserInput(expression)
+        parser.reset_parser_input(parser_input, error_sink)
+        parsed = parser.parse_input(context, parser_input, error_sink)
+
+        visitor = PythonConditionExprVisitor(context)
+        conditions = visitor.get_conditions(parsed)
+
+        assert len(conditions) == 1
+        assert isinstance(conditions[0], CompiledCondition)
+        if expected_compiled:
+            ast_ = ast.parse(expected_compiled, "<source>", 'exec')
+            expected_python_node = PythonNode(expected_compiled, ast_, expected_compiled)
+            assert conditions[0].evaluator_type == PYTHON_EVALUATOR_NAME
+            assert sheerka.isinstance(conditions[0].return_value, BuiltinConcepts.RETURN_VALUE)
+            assert sheerka.objvalue(conditions[0].return_value) == expected_python_node
+        else:
+            assert conditions[0].evaluator_type is None
+            assert conditions[0].return_value is None
+        assert conditions[0].concept is None
+        assert conditions[0].variables == {"__ret"}
+
+        # check against SheerkaEvaluateRules
+        evaluate_rules_service = sheerka.services[SheerkaEvaluateRules.NAME]
+        variable = foo if variable_name == "foo" else sheerka if variable_name == "sheerka" else variable_name
+        to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable)
+        bag = {"__ret": ReturnValueConcept("Test", True, to_recognize)}
+        with context.push(BuiltinConcepts.RULES_EVALUATION, bag, desc="Evaluating rules...") as sub_context:
+            sub_context.sheerka.add_many_to_short_term_memory(sub_context, bag)
+            rule = Rule(name="test_i_can_get_compiled_using_recognize_function", predicate=expression)
+            rule.compiled_conditions = conditions
+            res = evaluate_rules_service.evaluate_rule(sub_context, rule, bag)
+            assert res.status
+            assert self.sheerka.is_success(self.sheerka.objvalue(res))
 
 
 class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):