Fixed #49 : ExpressionParser: Implement ExpressionParser

src/core/sheerka/services/SheerkaEvaluateRules.py

@@ -29,8 +29,6 @@ class SheerkaEvaluateRules(BaseService):
         self.sheerka.register_debug_vars(self.NAME, "evaluate_rules", "results")
         self.sheerka.register_debug_rules(self.NAME, "evaluate_rule", "*")
 
-
-
     def reset_evaluators(self):
         # instantiate evaluators, once for all, only keep when it's enabled
         evaluators = [e_class() for e_class in self.sheerka.evaluators]

src/core/sheerka/services/SheerkaRuleManager.py

@@ -14,10 +14,10 @@ from core.rule import Rule, ACTION_TYPE_PRINT
 from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
 from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError
 from core.tokenizer import Keywords, TokenKind, Token, IterParser
-from core.utils import index_tokens, COLORS, get_text_from_tokens
+from core.utils import index_tokens, COLORS, get_text_from_tokens, unstr_concept
 from evaluators.ConceptEvaluator import ConceptEvaluator
 from evaluators.PythonEvaluator import PythonEvaluator, Expando
-from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode
+from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode
 from parsers.BaseNodeParser import SourceCodeWithConceptNode, ConceptNode, SourceCodeNode
 from parsers.LogicalOperatorParser import LogicalOperatorParser
 from parsers.PythonParser import PythonNode
@@ -1127,7 +1127,6 @@ class ReteConditionExprVisitor(ExpressionVisitor):
         self.context = context
         self.var_counter = 0
         self.variables = {}
-        self.res = []
 
     def add_variable(self, target):
         var_name = f"__x_{self.var_counter:02}__"
@@ -1135,35 +1134,71 @@ class ReteConditionExprVisitor(ExpressionVisitor):
         self.variables[target] = var_name
         return var_name
 
-    def init_variable_if_needed(self, node):
+    def init_or_get_variable_from_name(self, node, conditions):
         if node.name not in self.variables:
             var_name = self.add_variable(node.name)
-            self.res.append(Condition(V(var_name), "__name__", node.name))
+            conditions.append(Condition(V(var_name), "__name__", node.name))
 
         return V(self.variables[node.name])
 
+    def init_or_get_variable_from_attr(self, node, conditions):
+        path = f"{node.name}.{node.attributes_str}"
+        if path in self.variables:
+            return self.variables[path]
+
+        root = self.init_or_get_variable_from_name(node, conditions)
+        var_name = self.add_variable(path)
+        variable = V(var_name)
+        conditions.append(Condition(root, node.attributes_str, variable))
+        return variable
+
     def get_conditions(self, expr_node):
-        self.res.clear()
         self.var_counter = 0
         self.variables.clear()
 
-        self.visit(expr_node)
+        conditions = self.visit(expr_node)
-        return AndConditions(self.res)
+        return [AndConditions(conditions)]
 
     def visit_VariableNode(self, expr_node):
-        var_name = self.init_variable_if_needed(expr_node)
+        conditions = []
+        var_name = self.init_or_get_variable_from_name(expr_node, conditions)
         if expr_node.attributes_str is not None:
-            self.res.append(Condition(var_name, expr_node.attributes_str, True))
+            conditions.append(Condition(var_name, expr_node.attributes_str, True))
+        return conditions
 
     def visit_AndNode(self, expr_node: AndNode):
+        conditions = []
         for node in expr_node.parts:
-            self.visit(node)
+            conditions.extend(self.visit(node))
+
+        return conditions
 
     def visit_ComparisonNode(self, expr_node: ComparisonNode):
         if isinstance(expr_node.left, VariableNode):
-            left = self.init_variable_if_needed(expr_node.left)
+            conditions = []
+            left = self.init_or_get_variable_from_name(expr_node.left, conditions)
             attr = expr_node.left.attributes_str or "__self__"
             right = eval(get_text_from_tokens(expr_node.right.tokens))
-            self.res.append(Condition(left, attr, right))
+            conditions.append(Condition(left, attr, right))
+            return conditions
         else:
-            raise FailedToCompileError(expr_node)
+            raise FailedToCompileError([expr_node])
+
+    def visit_FunctionNode(self, expr_node: FunctionNode):
+        if expr_node.first.value == "recognize(":
+            return self.function_recognize(expr_node.parameters[0].value, expr_node.parameters[1].value)
+
+    def function_recognize(self, source, target):
+        conditions = []
+        body_var = self.init_or_get_variable_from_attr(source, conditions)
+        conditions.append(Condition(body_var, "__is_concept__", True))
+        if isinstance(target, VariableNode):
+            conditions.append(Condition(body_var, "name", target.name))
+        else:
+            concept_key, concept_id = unstr_concept(target.value)
+            if concept_id:
+                conditions.append(Condition(body_var, "id", concept_id))
+            elif concept_key:
+                conditions.append(Condition(body_var, "name", concept_key))
+
+        return conditions

src/sheerkarete/network.py

@@ -230,10 +230,17 @@ class ReteNetwork:
         for cond in conditions:
             # update requested attributes for a fact
             if isinstance(cond, Condition):
-                # Manage list of requested attributes
+
+                # Manage list of requested attributes when using __name__ indirection
                 if isinstance(cond.identifier, V) and cond.attribute == "__name__":
                     vars_ids_mappings[cond.identifier] = cond.value
 
+                # Manage list of requested attributes when bounding a new variable
+                if (cond.identifier in vars_ids_mappings and
+                    isinstance(cond.attribute, str) and
+                    isinstance(cond.value, V)):
+                    vars_ids_mappings[cond.value] = f"{vars_ids_mappings[cond.identifier]}.{cond.attribute}"
+
                 identifier = vars_ids_mappings[cond.identifier] if cond.identifier in vars_ids_mappings else \
                     cond.identifier if not isinstance(cond.identifier, V) else \
                         None
@@ -365,22 +372,23 @@ class ReteNetwork:
         for wme in to_remove:
             self.remove_wme(wme)
 
-    def add_obj(self, name, obj, use_bag=False, root=True):
+    def add_obj(self, name, obj, fact_id=None, use_bag=False):
         """
         Adds a new object to the working memory
         """
 
-        def inner_add_vme(ident, attr, val):
+        def inner_add_vme(name_, fact_id_, attr_, value_):
-            if val is NotInit:
+            if value_ is NotInit:
                 pass
-            elif attr != "self" and isinstance(val, Concept):
+            elif attr_ != "self" and isinstance(value_, Concept):
-                new_name = f"{ident}.{attr}"
+                new_name = f"{name_}.{attr_}"
-                self.add_wme(WME(ident, attr, new_name))
+                new_fact_id = f"{fact_id_}.{attr_}"
-                self.add_obj(new_name, val, use_bag=True, root=False)
+                self.add_wme(WME(fact_id_, attr_, new_fact_id))
+                self.add_obj(new_name, value_, new_fact_id)
             else:
-                self.add_wme(WME(ident, attr, val))
+                self.add_wme(WME(fact_id_, attr_, value_))
 
-        if root:
+        if fact_id is None:
             if hasattr(obj, FACT_ID):
                 raise ValueError("Object already has an id, cannot add")
 
@@ -388,8 +396,6 @@ class ReteNetwork:
             setattr(obj, FACT_ID, fact_id)
             self.facts[fact_id] = obj
             self.fact_counter += 1
-        else:
-            fact_id = name
 
         requested_attributes = "*" if use_bag else \
             self.attributes_by_id[name] if name in self.attributes_by_id else \
@@ -399,13 +405,15 @@ class ReteNetwork:
             if attribute == "*":
                 bag = as_bag(obj)
                 for k, v in bag.items():
-                    inner_add_vme(fact_id, k, v)
+                    inner_add_vme(name, fact_id, k, v)
             elif attribute == "__name__":
                 self.add_wme(WME(fact_id, "__name__", name))
+            elif attribute == "__is_concept__":
+                self.add_wme(WME(fact_id, "__is_concept__", isinstance(obj, Concept)))
             else:
                 try:
                     value = getattr(obj, attribute)
-                    inner_add_vme(fact_id, attribute, value)
+                    inner_add_vme(name, fact_id, attribute, value)
                 except AttributeError:
                     pass
 

tests/core/test_SheerkaRuleManager.py

@@ -2,7 +2,7 @@ import ast
 
 import pytest
 
-from core.builtin_concepts import BuiltinConcepts
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept, DEFINITION_TYPE_DEF, DoNotResolve
 from core.global_symbols import RULE_COMPARISON_CONTEXT, NotFound, EVENT_RULE_DELETED
 from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
@@ -21,6 +21,7 @@ from parsers.ExpressionParser import ExpressionParser
 from parsers.PythonParser import PythonNode
 from sheerkarete.common import V
 from sheerkarete.conditions import Condition, AndConditions
+from sheerkarete.network import ReteNetwork
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, get_test_obj, get_rete_conditions
@@ -1029,7 +1030,7 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         ),
     ])
     def test_i_can_get_rete_conditions(self, expression, expected_as_str):
-        sheerka, context, = self.init_test().unpack()
+        sheerka, context = self.init_test().unpack()
         parser = ExpressionParser()
         expected = get_rete_conditions(*expected_as_str)
 
@@ -1041,7 +1042,142 @@ isinstance(var, Concept) and var.key == 'hello __var__0'""" + \
         visitor = ReteConditionExprVisitor(context)
         conditions = visitor.get_conditions(parsed)
 
-        assert conditions == expected
+        assert conditions == [expected]
+
+        # check against a Rete network
+        network = ReteNetwork()
+        rule = Rule("test", expression, None)
+        rule.metadata.id = 9999
+        rule.metadata.is_compiled = True
+        rule.metadata.is_enabled = True
+        rule.rete_disjunctions = conditions
+        network.add_rule(rule)
+
+        network.add_obj("__ret", ReturnValueConcept("Test", True, None))
+        matches = list(network.matches)
+        assert len(matches) == 1
+
+    @pytest.mark.parametrize("test_name, expression, variable_name, expected_as_str", [
+        (
+            "recognize by name",
+            "recognize(__ret.body, greetings)",
+            None,
+            ["#__x_00__|__name__|'__ret'",
+             "#__x_00__|body|#__x_01__",
+             "#__x_01__|__is_concept__|True",
+             "#__x_01__|name|'greetings'"]
+        ),
+        (
+            "recognize by id",
+            "recognize(__ret.body, c:|1001:)",
+            None,
+            ["#__x_00__|__name__|'__ret'",
+             "#__x_00__|body|#__x_01__",
+             "#__x_01__|__is_concept__|True",
+             "#__x_01__|id|'1001'"]
+        ),
+        (
+            "recognize by name using c_str",
+            "recognize(__ret.body, c:greetings:)",
+            None,
+            ["#__x_00__|__name__|'__ret'",
+             "#__x_00__|body|#__x_01__",
+             "#__x_01__|__is_concept__|True",
+             "#__x_01__|name|'greetings'"]
+        ),
+        (
+            "recognize by name and variable sheerka",
+            "recognize(__ret.body, greetings, a=sheerka)",
+            "sheerka",
+            ["#__x_00__|__name__|'__ret'",
+             "#__x_00__|body|#__x_01__",
+             "#__x_01__|__is_concept__|True",
+             "#__x_01__|name|'greetings'",
+             "#__x_01__|a|'__sheerka__'"]
+        ),
+        (
+            "recognize by name and str variable",
+            "recognize(__ret.body, greetings, a='my friend')",
+            "'my friend'",
+            ["#__x_00__|__name__|'__ret'",
+             "#__x_00__|body|#__x_01__",
+             "#__x_01__|__is_concept__|True",
+             "#__x_01__|name|'greetings'",
+             "#__x_01__|a|'my friend'"]
+        ),
+        (
+            "recognize by name and concept variable",
+            "recognize(__ret.body, greetings, a=foo)",
+            "foo",
+            ["#__x_00__|__name__|'__ret'",
+             "#__x_00__|body|#__x_01__",
+             "#__x_01__|__is_concept__|True",
+             "#__x_01__|name|'greetings'",
+             "#__x_01__|a.name|'foo'"]
+        ),
+        (
+            "recognize by name and add other conditions (str)",
+            "recognize(__ret.body, greetings) and __ret.body.a == 'my friend'",
+            "foo",
+            ["#__x_00__|__name__|'__ret'",
+             "#__x_00__|body|#__x_01__",
+             "#__x_01__|__is_concept__|True",
+             "#__x_01__|name|'greetings'",
+             "#__x_01__|a|'my friend'"]
+        ),
+        (
+            "recognize by name and add other conditions (sheerka)",
+            "recognize(__ret.body, greetings) and __ret.body.a == sheerka",
+            "foo",
+            ["#__x_00__|__name__|'__ret'",
+             "#__x_00__|body|#__x_01__",
+             "#__x_01__|__is_concept__|True",
+             "#__x_01__|name|'greetings'",
+             "#__x_01__|a|'kodjo'"]
+        ),
+        (
+            "recognize by name and add other conditions (concept)",
+            "recognize(__ret.body, greetings) and __ret.body.a == foo",
+            "foo",
+            ["#__x_00__|__name__|'__ret'",
+             "#__x_00__|body|#__x_01__",
+             "#__x_01__|__is_concept__|True",
+             "#__x_01__|name|'greetings'",
+             "#__x_01__|a|'kodjo'"]
+        ),
+    ])
+    def test_i_can_get_rete_conditions_from_recognized(self, test_name, expression, variable_name, expected_as_str):
+        sheerka, context, greetings, foo = self.init_test().with_concepts(
+            Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
+            Concept("foo"),
+        ).unpack()
+        parser = ExpressionParser()
+        expected = get_rete_conditions(*expected_as_str)
+
+        error_sink = ErrorSink()
+        parser_input = ParserInput(expression)
+        parser.reset_parser_input(parser_input, error_sink)
+        parsed = parser.parse_input(context, parser_input, error_sink)
+
+        visitor = ReteConditionExprVisitor(context)
+        conditions = visitor.get_conditions(parsed)
+
+        assert conditions == [expected]
+
+        # check against a Rete network
+        network = ReteNetwork()
+        rule = Rule("test", expression, None)
+        rule.metadata.id = 9999
+        rule.metadata.is_compiled = True
+        rule.metadata.is_enabled = True
+        rule.rete_disjunctions = conditions
+        network.add_rule(rule)
+
+        variable = foo if variable_name == "foo" else variable_name
+        to_recognize = sheerka.new_from_template(greetings, greetings.key, a=variable)
+        network.add_obj("__ret", ReturnValueConcept("Test", True, to_recognize))
+        matches = list(network.matches)
+        assert len(matches) == 1
 
 
 class TestSheerkaRuleManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):

tests/parsers/parsers_utils.py

@@ -8,13 +8,14 @@ from core.concept import Concept, ConceptParts, DoNotResolve, AllConceptParts
 from core.rule import Rule
 from core.tokenizer import Tokenizer, TokenKind, Token
 from core.utils import get_text_from_tokens, tokens_index, str_concept
+from parsers.BaseExpressionParser import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, \
+    ComparisonType, \
+    FunctionParameter
 from parsers.BaseNodeParser import UnrecognizedTokensNode, SourceCodeNode, RuleNode, ConceptNode, \
     SourceCodeWithConceptNode
 from parsers.FunctionParser import FunctionNode
 from parsers.PythonParser import PythonNode
 from parsers.SyaNodeParser import SyaConceptParserHelper
-from parsers.BaseExpressionParser import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, ComparisonType, \
-    FunctionParameter
 from sheerkarete.common import V
 from sheerkarete.conditions import Condition, AndConditions
 
@@ -1304,17 +1305,22 @@ def get_rete_conditions(*conditions_as_string):
     "identifier|__name__|'True'" -> Condition(identifier, '__name__', 'True')  # the string 'True'
     "identifier|__name__|True" -> Condition(identifier, '__name__', True) # the bool True
     """
+
+    def get_value(obj):
+        if obj.startswith("#"):
+            return V(obj[1:])
+        if obj.startswith("'"):
+            return obj[1:-1]
+        if obj in ("True", "False"):
+            return obj == "True"
+        return int(obj)
+
     res = []
     for as_string in conditions_as_string:
-        identifier, attribute, value = as_string.split("|")
+        parts = as_string.split("|")
-        if identifier.startswith("#"):
+        identifier = get_value(parts[0])
-            identifier = V(identifier[1:])
+        attribute = parts[1]
-        if value.startswith("'"):
+        value = get_value(parts[2])
-            value = value[1:-1]
-        elif value in ("True", "False"):
-            value = (value == "True")
-        else:
-            value = int(value)
 
         res.append(Condition(identifier, attribute, value))
     return AndConditions(res)

tests/parsers/test_ExpressionParser.py

@@ -56,6 +56,7 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         ("func(var1.attr1 > var2.attr2)", FN("func(", ")", [GT(VAR("var1.attr1"), VAR("var2.attr2"))])),
         ("func1(var1) and func2(var2)", AND(FN("func1(", ")", [VAR("var1")]), FN("func2(", (")", 1), [VAR("var2")]))),
         ("__ret", VAR("__ret")),
+        #("func1().func2()", [])
     ])
     def test_i_can_parse_input(self, expression, expected):
         sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)

tests/sheerkarete/test_network.py

@@ -65,16 +65,28 @@ class TestReteNetwork(TestUsingMemoryBasedSheerka):
         assert len(network.pnodes) == 1
         assert network.pnodes[0].rules == [rule1, rule2]
 
-    def test_i_can_update_conditions_attributes_by_id_when_constraint_on__name__(self):
+    def test_i_can_update_conditions_attributes_by_id_when_constraints(self):
         network = ReteNetwork()
         conditions = [Condition(V("x"), "__name__", "fact_name"),
                       Condition(V("x"), "attr1", "value1"),
-                      Condition(V("x"), "attr2", "value1")]
+                      Condition(V("x"), "body", V("y")),
+                      Condition(V("y"), "__is_concept__", True),
+                      Condition(V("y"), "name", "SubConcept"),
+                      Condition(V("x"), "value", V("z")),
+                      Condition(V("z"), "status", False),
+                      Condition(V("z"), "body", V("zz")),
+                      Condition(V("zz"), "sub_value", "sub_value"),
+                      ]
 
         rule = RuleForTestingRete(AndConditions(conditions))
         network.add_rule(rule)
 
-        assert network.attributes_by_id == {"fact_name": ["__name__", "attr1", "attr2"]}
+        assert network.attributes_by_id == {
+            "fact_name": ["__name__", "attr1", "body", "value"],
+            "fact_name.body": ["__is_concept__", "name"],
+            "fact_name.value": ["status", "body"],
+            "fact_name.value.body": ["sub_value"],
+        }
 
     def test_adding_obj_when_no_rule_has_no_effect(self):
         network = ReteNetwork()
@@ -193,20 +205,11 @@ class TestReteNetwork(TestUsingMemoryBasedSheerka):
             WME("f-00000", "__name__", "__ret"),
             WME("f-00000", "status", True),
             WME("f-00000", "body", "f-00000.body"),
-            WME("f-00000.body", "id", "1003"),
             WME("f-00000.body", "name", "greetings"),
-            WME("f-00000.body", "key", "hello __var__0"),
             WME("f-00000.body", "a", "f-00000.body.a"),
-            WME("f-00000.body", "self", ret.body),
             WME("f-00000.body.a", "id", "1002"),
-            WME("f-00000.body.a", "name", "the x"),
-            WME("f-00000.body.a", "key", "the __var__0"),
             WME("f-00000.body.a", "x", "f-00000.body.a.x"),
-            WME("f-00000.body.a", "self", the_boy),
-            WME("f-00000.body.a.x", "id", "1001"),
             WME("f-00000.body.a.x", "name", "boy"),
-            WME("f-00000.body.a.x", "key", "boy"),
-            WME("f-00000.body.a.x", "self", boy),
         }
 
         # sanity check that the WME produced match the condition
@@ -649,7 +652,8 @@ class TestReteNetwork(TestUsingMemoryBasedSheerka):
         assert len(rule.rete_p_nodes) > 0
 
     def test_format_rule_is_not_added_to_rete_network_when_it_is_created(self):
-        sheerka, context, rule = self.init_test().with_format_rules(("rule_name", "id.attr == 'value'", 'True')).unpack()
+        sheerka, context, rule = self.init_test().with_format_rules(
+            ("rule_name", "id.attr == 'value'", 'True')).unpack()
         evaluation_service = sheerka.services[SheerkaEvaluateRules.NAME]
         rete_network = evaluation_service.network