Implemented a first and basic version of a Rete rule engine

tests/parsers/parsers_utils.py

@@ -1,8 +1,193 @@
-from core.concept import CC, Concept, ConceptParts, DoNotResolve, CIO
+import ast
+from dataclasses import dataclass
+
+from core.builtin_helpers import CreateObjectIdentifiers
+from core.concept import CC, Concept, ConceptParts, DoNotResolve, CIO, CMV
 from core.tokenizer import Tokenizer, TokenKind, Token
-from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, SourceCodeWithConceptNode, CN, UTN, \
-    SCN, RN
+from core.utils import get_text_from_tokens, tokens_index
+from parsers.BaseNodeParser import scnode, utnode, cnode, SCWC, CNC, short_cnode, CN, UTN, \
+    SCN, RN, UnrecognizedTokensNode, SourceCodeNode
+from parsers.PythonParser import PythonNode
 from parsers.SyaNodeParser import SyaConceptParserHelper
+from parsers.expressions import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, ComparisonType
+from sheerkarete.common import V
+from sheerkarete.conditions import Condition, AndConditions
+
+
+@dataclass
+class Obj:
+    prop_a: object
+    prop_b: object = None
+    prop_c: object = None
+    parent: object = None
+
+
+class AND:
+    """ Test class for AndNode"""
+
+    def __init__(self, *parts, source=None):
+        self.parts = parts
+        self.source = source
+
+
+class OR:
+    """ Test class for OrNode"""
+
+    def __init__(self, *parts, source=None):
+        self.parts = parts
+        self.source = source
+
+
+@dataclass
+class NOT:
+    """ Test class for NotNode"""
+    expr: object
+    source: str = None
+
+
+@dataclass
+class EXPR:
+    """Test class for NameNode. E stands for Expression"""
+    source: str
+
+
+@dataclass
+class VAR:
+    """Test class for VarNode"""
+    full_name: str
+    source: str = None
+
+
+@dataclass
+class EQ:
+    left: object
+    right: object
+    source = None
+
+
+@dataclass
+class NEQ:
+    left: object
+    right: object
+    source = None
+
+
+@dataclass
+class GT:
+    left: object
+    right: object
+    source = None
+
+
+@dataclass
+class GTE:
+    left: object
+    right: object
+    source = None
+
+
+@dataclass
+class LT:
+    left: object
+    right: object
+    source = None
+
+
+@dataclass
+class LTE:
+    left: object
+    right: object
+    source = None
+
+
+@dataclass
+class IN:
+    left: object
+    right: object
+    source = None
+
+
+@dataclass
+class NIN:  # for NOT INT
+    left: object
+    right: object
+    source = None
+
+
+comparison_type_mapping = {
+    "EQ": ComparisonType.EQUALS,
+    "NEQ": ComparisonType.NOT_EQUAlS,
+    "LT": ComparisonType.LESS_THAN,
+    "LTE": ComparisonType.LESS_THAN_OR_EQUALS,
+    "GT": ComparisonType.GREATER_THAN,
+    "GTE": ComparisonType.GREATER_THAN_OR_EQUALS,
+    "IN": ComparisonType.IN,
+    "NIN": ComparisonType.NOT_IN,
+}
+
+
+def get_expr_node_from_test_node(full_text, test_node):
+    """
+    Returns EXPR, OR, NOT, AND object to ease the comparison with the real ExprNode
+    """
+    full_text_as_tokens = list(Tokenizer(full_text, yield_eof=False))
+
+    def get_pos(nodes):
+        start, end = None, None
+        for n in nodes:
+            if start is None or start > n.start:
+                start = n.start
+            if end is None or end < n.end:
+                end = n.end
+        return start, end
+
+    def get_pos_from_source(source):
+        source_as_node = list(Tokenizer(source, yield_eof=False))
+        start = tokens_index(full_text_as_tokens, source_as_node)
+        end = start + len(source_as_node) - 1
+        return start, end
+
+    def get_expr_node(node):
+
+        if isinstance(node, EXPR):
+            value_as_tokens = list(Tokenizer(node.source, yield_eof=False))
+            start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
+            end = start + len(value_as_tokens) - 1
+            return NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+
+        if isinstance(node, AND):
+            parts = [get_expr_node(part) for part in node.parts]
+            start, end = get_pos_from_source(node.source) if node.source else get_pos(parts)
+            return AndNode(start, end, full_text_as_tokens[start: end + 1], *parts)
+
+        if isinstance(node, OR):
+            parts = [get_expr_node(part) for part in node.parts]
+            start, end = get_pos_from_source(node.source) if node.source else get_pos(parts)
+            return OrNode(start, end, full_text_as_tokens[start: end + 1], *parts)
+
+        if isinstance(node, NOT):
+            part = get_expr_node(node.expr)
+            start, end = get_pos_from_source(node.source) if node.source else (part.start - 2, part.end)
+            return NotNode(start, end, full_text_as_tokens[start: end + 1], part)
+
+        if isinstance(node, VAR):
+            value_as_tokens = list(Tokenizer(node.source or node.full_name, yield_eof=False))
+            start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
+            end = start + len(value_as_tokens) - 1
+            parts = node.full_name.split(".")
+            if len(parts) == 1:
+                return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0])
+            else:
+                return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0], *parts[1:])
+
+        if isinstance(node, (EQ, NEQ, GT, GTE, LT, LTE, IN, NIN)):
+            node_type = comparison_type_mapping[type(node).__name__]
+            left_node, right_node = get_expr_node(node.left), get_expr_node(node.right)
+            start, end = get_pos_from_source(node.source) if node.source else get_pos([left_node, right_node])
+            return ComparisonNode(start, end, full_text_as_tokens[start: end + 1],
+                                  node_type, left_node, right_node)
+
+    return get_expr_node(test_node)
 
 
 def _index(tokens, expr, index):
@@ -101,7 +286,7 @@ def get_node(
         sub_expr.fix_pos(sub_expr.first)
         sub_expr.fix_pos(sub_expr.last)
         return sub_expr
-        #return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
+        # return SourceCodeWithConceptNode(first, last, content).pseudo_fix_source()
 
     if isinstance(sub_expr, SCN):
         node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya)
@@ -128,7 +313,8 @@ def get_node(
         sub_expr.fix_pos((concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start))
         if hasattr(sub_expr, "compiled"):
             for k, v in sub_expr.compiled.items():
-                node = get_node(concepts_map, expression_as_tokens, v, sya=sya, exclude_body=exclude_body)  # need to get start and end positions
+                node = get_node(concepts_map, expression_as_tokens, v, sya=sya,
+                                exclude_body=exclude_body)  # need to get start and end positions
                 if isinstance(v, str) and v in concepts_map:
                     new_value_concept = concepts_map[v]
                     new_value = CC(Concept().update_from(new_value_concept), exclude_body=exclude_body)
@@ -214,3 +400,85 @@ def compute_expected_array(concepts_map, expression, expected, sya=False, init_e
         sya=sya,
         init_empty_body=init_empty_body,
         exclude_body=exclude_body) for sub_expr in expected]
+
+
+def get_unrecognized_node(start, text):
+    tokens = list(Tokenizer(text, yield_eof=False))
+    return UnrecognizedTokensNode(start, start + len(tokens) - 1, tokens)
+
+
+def get_source_code_node(start, text, concepts_map, id_manager=None):
+    id_manager = id_manager or CreateObjectIdentifiers()
+    id_mapping = {}
+    concept_mapping_by_id = {}
+
+    # get the concepts, mapped by their new id
+    for concept_name, concept in concepts_map.items():
+        concept_identifier = id_manager.get_identifier(concept, "__C__")
+        id_mapping[concept_name] = concept_identifier
+        concept_mapping_by_id[concept_identifier] = concept
+
+    # transform the source code to use the new id
+    tokens = list(Tokenizer(text, yield_eof=False))
+    text_to_compile_tokens = []
+    for t in tokens:
+        if t.type == TokenKind.IDENTIFIER and t.value in id_mapping:
+            text_to_compile_tokens.append(Token(TokenKind.IDENTIFIER, id_mapping[t.value], -1, -1, -1))
+        else:
+            text_to_compile_tokens.append(t)
+    text_to_compile = get_text_from_tokens(text_to_compile_tokens)
+
+    # create the python node
+    ast_ = ast.parse(text_to_compile, "<source>", 'eval')
+    python_node = PythonNode(text_to_compile, ast_, text)
+    python_node.objects = concept_mapping_by_id
+
+    return SourceCodeNode(start, start + len(tokens) - 1, tokens, text, python_node)
+
+
+def resolve_test_concept(concept_map, hint):
+    if isinstance(hint, str):
+        return concept_map[hint]
+
+    if isinstance(hint, CC):
+        concept = concept_map[hint.concept_key]
+        compiled = {k: resolve_test_concept(concept_map, v) for k, v in hint.compiled.items()}
+        return CC(concept, source=hint.source, exclude_body=hint.exclude_body, **compiled)
+
+    if isinstance(hint, CMV):
+        concept = concept_map[hint.concept_key]
+        return CMV(concept, **hint.variables)
+
+    # CV
+    #
+    # CMV
+    #
+    # CIO
+    raise NotImplementedError()
+
+
+def get_rete_conditions(*conditions_as_string):
+    """
+    Transform a list of string into a list of Condition (Rete conditions)
+    :param conditions_as_string: conditions in the form 'identifier|attribute|value'
+    when one argument starts with "#" it means that it's a variables
+    ex : "#__x_00__|__name__|'__ret'" -> Condition(V('#__x_00__'), '__name__', '__ret')
+
+    Caution, the value part is evaluated
+    "identifier|__name__|'True'" -> Condition(identifier, '__name__', 'True')  # the string 'True'
+    "identifier|__name__|True" -> Condition(identifier, '__name__', True) # the bool True
+    """
+    res = []
+    for as_string in conditions_as_string:
+        identifier, attribute, value = as_string.split("|")
+        if identifier.startswith("#"):
+            identifier = V(identifier[1:])
+        if value.startswith("'"):
+            value = value[1:-1]
+        elif value in ("True", "False"):
+            value = (value == "True")
+        else:
+            value = int(value)
+
+        res.append(Condition(identifier, attribute, value))
+    return AndConditions(res)

tests/parsers/test_BaseCustomGrammarParser.py

@@ -1,9 +1,9 @@
 import pytest
+
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Keywords, Tokenizer, TokenKind
 from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, SyntaxErrorNode, KeywordNotFound
 from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
-
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -43,6 +43,7 @@ class TestBaseCustomGrammarParser(TestUsingMemoryBasedSheerka):
         ("when uuu vvv print xxx yyy", False, {Keywords.WHEN: "when uuu vvv ", Keywords.PRINT: "print xxx yyy"}),
         ("print xxx yyy when uuu vvv", False, {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy "}),
         ("  when xxx", False, {Keywords.WHEN: "when xxx"}),
+
         ("when xxx yyy", True, {Keywords.WHEN: "when xxx yyy"}),
         ("when uuu vvv print xxx yyy", True, {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy"}),
         ("print xxx yyy when uuu vvv", True, {Keywords.WHEN: "when uuu vvv", Keywords.PRINT: "print xxx yyy"}),
@@ -84,7 +85,7 @@ func(a)
 
         assert parser.get_parts(["when", "print"], Keywords.PRINT) is None
         assert parser.error_sink == [UnexpectedTokenParsingError(f"'print' keyword not found.",
-                                                              "when",
+                                                                 "when",
                                                                  [Keywords.PRINT])]
 
     def test_i_can_detect_when_a_keyword_appears_several_times(self):
@@ -214,7 +215,8 @@ print xxx"""
 
         sheerka, context, parser = self.init_parser("")
         assert parser.get_body(list(Tokenizer(text, yield_eof=False))) is None
-        assert parser.error_sink == [UnexpectedTokenParsingError("Indentation not found.", "zzz", [TokenKind.WHITESPACE])]
+        assert parser.error_sink == [
+            UnexpectedTokenParsingError("Indentation not found.", "zzz", [TokenKind.WHITESPACE])]
 
     def test_i_can_detect_invalid_indentation_when_get_body(self):
         sheerka, context, parser = self.init_parser("")

tests/parsers/test_BnfNodeParser.py

@@ -136,7 +136,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         sheerka.set_isa(context, sheerka.new("thousands"), sheerka.new("number"))
 
         cls.shared_ontology = sheerka.get_ontology(context)
-        sheerka.pop_ontology()
+        sheerka.pop_ontology(context)
 
     @staticmethod
     def update_bnf(context, concept):

tests/parsers/test_ComparisonParser.py

@@ -0,0 +1,98 @@
+import pytest
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import TokenKind, Tokenizer
+from parsers.ComparisonParser import ComparisonParser
+from parsers.ExpressionParser import ParenthesisMismatchError
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, EQ, NEQ, GT, GTE, LT, LTE, IN, NIN
+
+
+class TestComparisonParser(TestUsingMemoryBasedSheerka):
+    def init_parser(self):
+        sheerka, context = self.init_concepts()
+        parser = ComparisonParser()
+        return sheerka, context, parser
+
+    def test_i_can_detect_empty_expression(self):
+        sheerka, context, parser = self.init_parser()
+        res = parser.parse(context, ParserInput(""))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
+
+    @pytest.mark.parametrize("expression, expected", [
+        ("var_name", VAR("var_name")),
+        ("var_name.attr", VAR("var_name.attr")),
+        ("var_name .attr", VAR("var_name.attr", source="var_name .attr")),
+        ("var_name. attr", VAR("var_name.attr", source="var_name. attr")),
+        ("var_name . attr", VAR("var_name.attr", source="var_name . attr")),
+        ("var_name.attr.get_value(x)", VAR("var_name.attr.get_value(x)")),
+        ("var_name.attr == 10", EQ(VAR("var_name.attr"), EXPR("10"))),
+        ("var_name.attr != 10", NEQ(VAR("var_name.attr"), EXPR("10"))),
+        ("var_name.attr > 10", GT(VAR("var_name.attr"), EXPR("10"))),
+        ("var_name.attr >= 10", GTE(VAR("var_name.attr"), EXPR("10"))),
+        ("var_name.attr < 10", LT(VAR("var_name.attr"), EXPR("10"))),
+        ("var_name.attr <= 10", LTE(VAR("var_name.attr"), EXPR("10"))),
+        ("var_name.attr in (a, b)", IN(VAR("var_name.attr"), EXPR("(a, b)"))),
+        ("var_name.attr not in (a, b)", NIN(VAR("var_name.attr"), EXPR("(a, b)"))),
+        ("var1.attr1 == var2.attr2", EQ(VAR("var1.attr1"), VAR("var2.attr2"))),
+
+        ("not a var identifier", EXPR("not a var identifier")),
+        ("func()", EXPR("func()")),
+        ("func(a, not an identifier, x >5)", EXPR("func(a, not an identifier, x >5)")),
+    ])
+    def test_i_can_parse_simple_expressions(self, expression, expected):
+        sheerka, context, parser = self.init_parser()
+        expected = get_expr_node_from_test_node(expression, expected)
+
+        res = parser.parse(context, ParserInput(expression))
+        parser_result = res.body
+        parsed_expr = parser_result.body
+
+        assert res.status
+        assert res.who == parser.name
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert parsed_expr == expected
+
+    @pytest.mark.parametrize("expression, expected_error, parenthesis_type, index", [
+        ("(", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 0),
+        (")", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 0),
+        ("something (", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10),
+        ("something )", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
+        ("something == (", BuiltinConcepts.ERROR, TokenKind.LPAR, 13),
+        ("something == )", BuiltinConcepts.ERROR, TokenKind.RPAR, 13),
+        ("something (==", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10),
+        ("something )==", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
+    ])
+    def test_i_can_detect_unbalanced_parenthesis(self, expression, expected_error, parenthesis_type, index):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(expression))
+        assert not res.status
+        if expected_error == BuiltinConcepts.NOT_FOR_ME:
+            assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+            assert isinstance(res.body.reason[0], ParenthesisMismatchError)
+            assert res.body.reason[0].token.type == parenthesis_type
+            assert res.body.reason[0].token.index == index
+        else:
+            assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+            assert isinstance(res.body.body[0], ParenthesisMismatchError)
+            assert res.body.body[0].token.type == parenthesis_type
+            assert res.body.body[0].token.index == index
+
+    def test_i_can_parse_tokens_rather_than_parser_input(self):
+        sheerka, context, parser = self.init_parser()
+        expression = "var1.attr1 == var2.attr2"
+        expected = EQ(VAR("var1.attr1"), VAR("var2.attr2"))
+        expected = get_expr_node_from_test_node(expression, expected)
+
+        res = parser.parse(context, list(Tokenizer(expression)))
+        parser_result = res.body
+        parsed_expr = parser_result.body
+
+        assert res.status
+        assert res.who == parser.name
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert parsed_expr == expected

tests/parsers/test_DefConceptParser.py

@@ -4,10 +4,11 @@ from dataclasses import dataclass
 import pytest
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV
+from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.BaseNodeParser import SCWC
-from parsers.BaseParser import NotInitializedNode, UnexpectedEofParsingError
+from parsers.BaseParser import UnexpectedEofParsingError
 from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence
 from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode
@@ -222,11 +223,11 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka):
         assert res.status
         assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert isinstance(node, DefConceptNode)
-        assert node.body == NotInitializedNode()
-        assert node.where == NotInitializedNode()
-        assert node.pre == NotInitializedNode()
-        assert node.post == NotInitializedNode()
-        assert node.ret == NotInitializedNode()
+        assert node.body is NotInit
+        assert node.where is NotInit
+        assert node.pre is NotInit
+        assert node.post is NotInit
+        assert node.ret is NotInit
 
     @pytest.mark.parametrize("part", [
         "as",

tests/parsers/test_DefFormatRuleParser.py

@@ -1,143 +0,0 @@
-import pytest
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
-from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, RulePredicate, FormatAstVariable
-from core.tokenizer import Keywords, Tokenizer
-from parsers.BaseCustomGrammarParser import KeywordNotFound
-from parsers.DefFormatRuleParser import DefFormatRuleParser, FormatRuleNode
-
-from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-
-cmap = {
-    "is a": Concept("x is a y").def_var("x").def_var("y"),
-    "is a question": Concept("x is a y", pre="is_question()").def_var("x").def_var("y"),
-    "a is good": Concept("a is good").def_var("a"),
-    "b is good": Concept("b is good").def_var("b"),
-}
-
-
-class TestDefFormatRuleParser(TestUsingMemoryBasedSheerka):
-    shared_ontology = None
-
-    @classmethod
-    def setup_class(cls):
-        init_test_helper = cls().init_test(cache_only=False, ontology="#TestDefFormatRuleParser#")
-        sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
-        for i, concept_name in enumerate(cmap):
-            cmap[concept_name] = updated[i]
-
-        cls.shared_ontology = sheerka.get_ontology(context)
-        sheerka.pop_ontology()
-
-    def init_parser(self, my_concepts_map=None, **kwargs):
-        if my_concepts_map is None:
-            sheerka, context = self.init_test().unpack()
-            sheerka.add_ontology(context, self.shared_ontology)
-        else:
-            sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
-            for i, pair in enumerate(my_concepts_map):
-                my_concepts_map[pair] = updated[i]
-
-        parser = DefFormatRuleParser()
-        return sheerka, context, parser
-
-    def test_i_can_detect_empty_expression(self):
-        sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, ParserInput(""))
-
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
-
-    def test_input_must_be_a_parser_input(self):
-        sheerka, context, parser = self.init_parser()
-        parser.parse(context, "not a parser input") is None
-
-    def test_i_can_parse_a_simple_rule(self):
-        sheerka, context, parser = self.init_parser()
-
-        text = "when isinstance(last_value(), Concept) print hello world!"
-        res = parser.parse(context, ParserInput(text))
-        parser_result = res.body
-        format_rule = res.body.body
-        rules = format_rule.rule
-        format_ast = format_rule.format_ast
-
-        assert res.status
-        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
-        assert isinstance(format_rule, FormatRuleNode)
-        assert self.dump_tokens(format_rule.tokens[Keywords.WHEN][1:]) == self.dump_tokens(
-            Tokenizer("isinstance(last_value(), Concept)", False))
-        assert self.dump_tokens(format_rule.tokens[Keywords.PRINT][1:]) == self.dump_tokens(
-            Tokenizer("hello world!", False))
-
-        assert len(rules) == 1
-        assert isinstance(rules[0], RulePredicate)
-        assert format_ast == FormatAstRawText("hello world!")
-
-    def test_when_is_parsed_in_the_context_of_a_question(self):
-        sheerka, context, parser = self.init_parser()
-
-        text = "when foo is a bar print hello world"
-        res = parser.parse(context, ParserInput(text))
-        format_rule = res.body.body
-        rules = format_rule.rule
-
-        assert res.status
-        assert len(rules) == 1
-        assert isinstance(rules[0], RulePredicate)
-        assert rules[0].predicate.body.body.get_metadata().pre == "is_question()"
-
-    def test_when_can_support_multiple_possibilities(self):
-        sheerka, context, parser = self.init_parser()
-
-        text = "when foo is good print hello world"
-        res = parser.parse(context, ParserInput(text))
-        format_rule = res.body.body
-        rules = format_rule.rule
-
-        assert res.status
-        assert len(rules) == 2
-        assert rules[0].predicate.body.body.get_metadata().name == "a is good"
-        assert rules[1].predicate.body.body.get_metadata().name == "b is good"
-
-    @pytest.mark.parametrize("text, error", [
-        ("hello world", [KeywordNotFound(None, keywords=['when', 'print'])]),
-        ("when True", [KeywordNotFound([], keywords=['print'])]),
-        ("print True", [KeywordNotFound([], keywords=['when'])]),
-    ])
-    def test_cannot_parse_when_not_for_me(self, text, error):
-        sheerka, context, parser = self.init_parser()
-
-        res = parser.parse(context, ParserInput(text))
-        not_for_me = res.body
-
-        assert not res.status
-        assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME)
-        assert not_for_me.reason == error
-
-    @pytest.mark.parametrize("text, expected_error", [
-        ("when a b print hello world!", BuiltinConcepts.TOO_MANY_ERRORS),
-    ])
-    def test_i_cannot_parse_invalid_predicates(self, text, expected_error):
-        sheerka, context, parser = self.init_parser()
-
-        res = parser.parse(context, ParserInput(text))
-
-        assert not res.status
-        assert sheerka.isinstance(res.body, expected_error)
-
-    @pytest.mark.parametrize("expr, expected", [
-        ("hello world", FormatAstRawText("hello world")),
-        ("{id}", FormatAstVariable("id")),
-    ])
-    def test_i_can_parse_valid_print_expression(self, expr, expected):
-        sheerka, context, parser = self.init_parser()
-
-        text = "when True print " + expr
-        res = parser.parse(context, ParserInput(text))
-        format_rule = res.body.body
-        format_ast = format_rule.format_ast
-
-        assert res.status
-        assert format_ast == expected

tests/parsers/test_DefRuleParser.py

@@ -0,0 +1,213 @@
+import pytest
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.sheerka.services.SheerkaRuleManager import RuleCompiledPredicate, FormatAstNode
+from core.tokenizer import Tokenizer, Keywords
+from core.utils import tokens_are_matching
+from parsers.BaseCustomGrammarParser import KeywordNotFound, NameNode, SyntaxErrorNode
+from parsers.BaseParser import UnexpectedEofParsingError
+from parsers.DefRuleParser import DefRuleParser, DefExecRuleNode, DefFormatRuleNode
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+cmap = {
+    "foo": Concept("foo"),
+    "bar": Concept("bar"),
+    "is a": Concept("x is a y").def_var("x").def_var("y"),
+    "is a question": Concept("x is a y", pre="is_question()").def_var("x").def_var("y"),
+    "a is good": Concept("a is good", pre="is_question()").def_var("a"),
+    "b is good": Concept("b is good", pre="is_question()").def_var("b"),
+    "greetings": Concept("hello a").def_var("a")
+}
+
+
+class TestDefRuleParser(TestUsingMemoryBasedSheerka):
+    shared_ontology = None
+
+    @classmethod
+    def setup_class(cls):
+        init_test_ref = cls().init_test(cache_only=False, ontology="#TestDefRuleParser#")
+        sheerka, context, *updated = init_test_ref.with_concepts(*cmap.values(), create_new=True).unpack()
+        for i, concept_name in enumerate(cmap):
+            cmap[concept_name] = updated[i]
+
+        cls.shared_ontology = sheerka.get_ontology(context)
+        sheerka.pop_ontology(context)
+
+    def init_parser(self, my_concepts_map=None, **kwargs):
+        if my_concepts_map is None:
+            sheerka, context = self.init_test().unpack()
+            sheerka.add_ontology(context, self.shared_ontology)
+        else:
+            sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
+            for i, pair in enumerate(my_concepts_map):
+                my_concepts_map[pair] = updated[i]
+
+        parser = DefRuleParser()
+        return sheerka, context, parser
+
+    def test_i_cannot_parse_when_parser_input_is_initialized_from_token(self):
+        sheerka, context, parser = self.init_parser()
+        res = parser.parse(context, ParserInput("", list(Tokenizer("init from tokens"))))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+
+    def test_i_can_detect_empty_expression(self):
+        sheerka, context, parser = self.init_parser()
+        res = parser.parse(context, ParserInput(""))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
+
+    def test_input_must_be_a_parser_input(self):
+        sheerka, context, parser = self.init_parser()
+        assert parser.parse(context, "not a parser input") is None
+
+    def test_i_can_parse_simple_exec_rule_definition(self):
+        sheerka, context, parser = self.init_parser()
+        text = "when True then answer('that is true')"
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.body
+        parsed = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(parsed, DefExecRuleNode)
+
+        assert len(parsed.tokens) == 2
+        assert tokens_are_matching(parsed.tokens[Keywords.WHEN], Tokenizer("when True"))
+        assert tokens_are_matching(parsed.tokens[Keywords.THEN], Tokenizer("then answer('that is true')"))
+        assert isinstance(parsed.when, list)
+        assert len(parsed.when) == 1
+        assert isinstance(parsed.when[0], RuleCompiledPredicate)
+        assert sheerka.isinstance(parsed.then, BuiltinConcepts.RETURN_VALUE)
+
+    def test_i_can_parse_simple_format_rule_definition(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "when True print hello world!"
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.body
+        parsed = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(parsed, DefFormatRuleNode)
+
+        assert len(parsed.tokens) == 2
+        assert tokens_are_matching(parsed.tokens[Keywords.WHEN], Tokenizer("when True"))
+        assert tokens_are_matching(parsed.tokens[Keywords.PRINT], Tokenizer("print hello world!"))
+        assert isinstance(parsed.when, list)
+        assert len(parsed.when) == 1
+        assert isinstance(parsed.when[0], RuleCompiledPredicate)
+        assert isinstance(parsed.print, FormatAstNode)
+
+    def test_i_can_parse_exec_rule_with_name(self):
+        sheerka, context, parser = self.init_parser()
+        text = "def rule my rule as when True then answer('that is true')"
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.body
+        parsed = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(parsed, DefExecRuleNode)
+
+        assert parsed.name == NameNode(list(Tokenizer("my rule")))
+        assert len(parsed.tokens) == 2
+        assert tokens_are_matching(parsed.tokens[Keywords.WHEN], Tokenizer("when True"))
+        assert tokens_are_matching(parsed.tokens[Keywords.THEN], Tokenizer("then answer('that is true')"))
+        assert isinstance(parsed.when, list)
+        assert len(parsed.when) == 1
+        assert isinstance(parsed.when[0], RuleCompiledPredicate)
+        assert sheerka.isinstance(parsed.then, BuiltinConcepts.RETURN_VALUE)
+
+    def test_when_is_parsed_in_the_context_of_a_question(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "when foo is a bar print hello world"
+        res = parser.parse(context, ParserInput(text))
+        format_rule = res.body.body
+        rules = format_rule.when
+
+        assert res.status
+        assert len(rules) == 1
+        assert isinstance(rules[0], RuleCompiledPredicate)
+        assert rules[0].predicate.body.body.get_metadata().pre == "is_question()"
+
+    def test_when_can_support_multiple_possibilities_when_question_only(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "when foo is good print hello world"
+        res = parser.parse(context, ParserInput(text))
+        format_rule = res.body.body
+        rules = format_rule.when
+
+        assert res.status
+        assert len(rules) == 2
+        assert rules[0].predicate.body.body.get_metadata().name == "a is good"
+        assert rules[1].predicate.body.body.get_metadata().name == "b is good"
+
+    @pytest.mark.parametrize("text, error", [
+        ("def", [KeywordNotFound(None, keywords=['rule'])]),
+        ("def concept", [KeywordNotFound(None, keywords=['rule'])]),
+        ("def other", [KeywordNotFound(None, keywords=['rule'])]),
+        ("def rule name", [KeywordNotFound(None, keywords=['as'])]),
+        ("def rule complicated long name", [KeywordNotFound(None, keywords=['as'])]),
+        ("hello world", [KeywordNotFound(None, keywords=['when'])]),
+        ("when True", [KeywordNotFound([], keywords=['then', 'print'])]),
+        ("print True", [KeywordNotFound([], keywords=['when'])]),
+        ("when True print 'hello world' then answer('yes')",
+         [SyntaxErrorNode([], message="Cannot have both 'print' and 'then' keywords")])
+    ])
+    def test_i_can_detect_not_for_me(self, text, error):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(text))
+        not_for_me = res.body
+
+        assert not res.status
+        assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME)
+        assert not_for_me.reason == error
+
+    @pytest.mark.parametrize("text, expected_error", [
+        ("when x x print 'hello world'", BuiltinConcepts.TOO_MANY_ERRORS),
+
+    ])
+    def test_i_can_detect_errors(self, text, expected_error):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(text))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, expected_error)
+
+    @pytest.mark.parametrize("text, error_message", [
+        ("def rule rule_name as", "While parsing 'when'."),
+        ("def rule rule_name as ", "While parsing 'when'."),
+    ])
+    def test_i_cannot_parse_when_unexpected_eof(self, text, error_message):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(text))
+        not_for_me = res.body
+
+        assert not res.status
+        assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME)
+        assert isinstance(not_for_me.reason[0], UnexpectedEofParsingError)
+        assert not_for_me.reason[0].message == error_message
+
+    def test_rule_name_is_mandatory_when_using_def_rule(self):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput("def rule as when True print 'true'"))
+        error = res.body
+
+        assert not res.status
+        assert sheerka.isinstance(error, BuiltinConcepts.ERROR)
+        assert isinstance(error.error[0], SyntaxErrorNode)
+        assert error.error[0].message == "Name is mandatory"

tests/parsers/test_ExpressionParser.py

@@ -1,27 +1,21 @@
-from dataclasses import dataclass
+import ast
 
 import pytest
+
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
-from core.concept import Concept
+from core.concept import Concept, CMV, DoNotResolve, CC
+from core.rule import Rule
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Tokenizer, TokenKind
+from core.tokenizer import TokenKind
+from parsers.BaseNodeParser import CNC
 from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
-from parsers.ExpressionParser import PropertyEqualsNode, PropertyEqualsSequenceNode, PropertyContainsNode, AndNode, \
-    OrNode, NotNode, LambdaNode, IsaNode, NameExprNode, ExpressionParser, LeftPartNotFoundError, TrueifyVisitor
-
+from parsers.ExpressionParser import ExpressionParser, LeftPartNotFoundError, ParenthesisMismatchError
+from parsers.PythonParser import PythonNode
+from parsers.expressions import TrueifyVisitor, IsAQuestionVisitor, AndNode
+from sheerkarete.network import ReteNetwork
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-
-
-@dataclass
-class Obj:
-    prop_a: object
-    prop_b: object = None
-    prop_c: object = None
-    parent: object = None
-
-
-def n(value):
-    return NameExprNode(Tokenizer(value, yield_eof=False))
+from tests.parsers.parsers_utils import compute_expected_array, resolve_test_concept, EXPR, OR, AND, NOT, \
+    get_expr_node_from_test_node, get_rete_conditions
 
 
 class TestExpressionParser(TestUsingMemoryBasedSheerka):
@@ -32,19 +26,34 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         return sheerka, context, parser
 
     @pytest.mark.parametrize("expression, expected", [
-        ("one complicated expression", n("one complicated expression")),
-        ("function_call(a,b,c)", n("function_call(a,b,c)")),
-        ("one expression or another expression", OrNode(n("one expression"), n("another expression"))),
-        ("one expression and another expression", AndNode(n("one expression"), n("another expression"))),
-        ("one or two or three", OrNode(n("one"), n("two"), n("three"))),
-        ("one and two and three", AndNode(n("one"), n("two"), n("three"))),
-        ("one or two and three", OrNode(n("one"), AndNode(n("two"), n("three")))),
-        ("one and two or three", OrNode(AndNode(n("one"), n("two")), n("three"))),
-        ("one and (two or three)", AndNode(n("one"), OrNode(n("two"), n("three")))),
+        ("one complicated expression", EXPR("one complicated expression")),
+        ("function_call(a,b,c)", EXPR("function_call(a,b,c)")),
+        ("one expression or another expression", OR(EXPR("one expression"), EXPR("another expression"))),
+        ("one expression and another expression", AND(EXPR("one expression"), EXPR("another expression"))),
+        ("not one", NOT(EXPR("one"))),
+        ("one and not two", AND(EXPR("one"), NOT(EXPR("two")))),
+        ("not one and two", AND(NOT(EXPR("one")), EXPR("two"))),
+        ("one or not two", OR(EXPR("one"), NOT(EXPR("two")))),
+        ("not one or two", OR(NOT(EXPR("one")), EXPR("two"))),
+        ("one or two or three", OR(EXPR("one"), EXPR("two"), EXPR("three"))),
+        ("one and two and three", AND(EXPR("one"), EXPR("two"), EXPR("three"))),
+        ("one or two and three", OR(EXPR("one"), AND(EXPR("two"), EXPR("three")))),
+        ("one and two or three", OR(AND(EXPR("one"), EXPR("two")), EXPR("three"))),
+        ("one and (two or three)", AND(EXPR("one"), OR(EXPR("two"), EXPR("three")), source="one and (two or three)")),
+        ("not not one", NOT(NOT(EXPR("one")))),
+        ("not (one and two)", NOT(AND(EXPR("one"), EXPR("two")), source="not (one and two)")),
+        ("one 'and' two or three", OR(EXPR("one 'and' two"), EXPR("three"))),
+        ("not ((a or b) and (c or d))", NOT(AND(OR(EXPR("a"), EXPR("b")), OR(EXPR("c"), EXPR("d")),
+                                                source="(a or b) and (c or d)"),
+                                            source="not ((a or b) and (c or d))")),
+        ("not ((a and b) or (c and d))", NOT(OR(AND(EXPR("a"), EXPR("b")), AND(EXPR("c"), EXPR("d")),
+                                                source="(a and b) or (c and d)"),
+                                             source="not ((a and b) or (c and d))")),
     ])
     def test_i_can_parse_expression(self, expression, expected):
         sheerka, context, parser = self.init_parser()
 
+        expected = get_expr_node_from_test_node(expression, expected)
         res = parser.parse(context, ParserInput(expression))
         wrapper = res.body
         expressions = res.body.body
@@ -69,6 +78,15 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == expected_errors
 
+    def test_i_can_detect_unexpected_not_error(self):
+        sheerka, context, parser = self.init_parser()
+        expression = "a cat is not a human"
+
+        res = parser.parse(context, ParserInput(expression))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
+
     def test_i_can_detect_unbalanced_parenthesis(self):
         sheerka, context, parser = self.init_parser()
 
@@ -86,6 +104,27 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         assert res.body.reason[0].token.type == TokenKind.RPAR
         assert res.body.reason[0].expected_tokens == []
 
+        res = parser.parse(context, ParserInput("one and two("))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert isinstance(res.body.body[0], ParenthesisMismatchError)
+        assert res.body.body[0].token.type == TokenKind.LPAR
+        assert res.body.body[0].token.index == 11
+
+        res = parser.parse(context, ParserInput("one ("))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+        assert isinstance(res.body.reason[0], ParenthesisMismatchError)
+        assert res.body.reason[0].token.type == TokenKind.LPAR
+        assert res.body.reason[0].token.index == 4
+
+        res = parser.parse(context, ParserInput("one (and"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert isinstance(res.body.body[0], ParenthesisMismatchError)
+        assert res.body.body[0].token.type == TokenKind.LPAR
+        assert res.body.body[0].token.index == 4
+
         res = parser.parse(context, ParserInput("one and two)"))
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
@@ -93,7 +132,14 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         assert res.body.body[0].token.type == TokenKind.RPAR
         assert res.body.body[0].expected_tokens == []
 
-        res = parser.parse(context, ParserInput("one and two)"))
+        res = parser.parse(context, ParserInput("one )"))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
+        assert res.body.body[0].token.type == TokenKind.RPAR
+        assert res.body.body[0].expected_tokens == []
+
+        res = parser.parse(context, ParserInput("one ) and"))
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
@@ -107,90 +153,6 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
 
-    def test_i_can_test_property_equals(self):
-        node = PropertyEqualsNode("prop_a", "good value")
-
-        assert node.eval(Obj(prop_a="good value"))
-        assert not node.eval(Obj(prop_a="other value"))
-
-    def test_i_can_test_property_equals_for_int(self):
-        node = PropertyEqualsNode("prop_a", "1")
-
-        assert node.eval(Obj(prop_a=1))
-        assert node.eval(Obj(prop_a="1"))
-
-    def test_i_can_test_property_equals_sequence(self):
-        node = PropertyEqualsSequenceNode(["prop_b", "prop_a"], ["good parent", "good child"])
-
-        assert node.eval(Obj(prop_a="good child", parent=Obj(prop_a="Don't care", prop_b="good parent")))
-        assert not node.eval(Obj(prop_a="good child", parent=Obj(prop_a="Don't care", prop_b="wrong parent")))
-        assert not node.eval(Obj(prop_a="good child"))
-        assert not node.eval(Obj(prop_a="wrong child", parent=Obj(prop_a="Don't care", prop_b="good parent")))
-
-    def test_i_can_test_property_contains(self):
-        node = PropertyContainsNode("prop_a", "substring")
-
-        assert node.eval(Obj(prop_a="it contains substring in it"))
-        assert not node.eval(Obj(prop_a="it does not"))
-
-    def test_i_can_test_property_contains_for_int(self):
-        node = PropertyContainsNode("prop_a", "44")
-
-        assert node.eval(Obj(prop_a=123445))
-        assert not node.eval(Obj(prop_a=12435))
-
-    def test_i_can_test_and(self):
-        left = PropertyEqualsNode("prop_a", "good a")
-        right = PropertyEqualsNode("prop_b", "good b")
-        other = PropertyEqualsNode("prop_c", "good c")
-        and_node = AndNode(left, right, other)
-
-        assert and_node.eval(Obj("good a", "good b", "good c"))
-        assert not and_node.eval(Obj("wrong a", "good b", "good c"))
-        assert not and_node.eval(Obj("good a", "wrong b", "good c"))
-        assert not and_node.eval(Obj("good a", "good b", "wrong c"))
-
-    def test_i_can_test_or(self):
-        left = PropertyEqualsNode("prop_a", "good a")
-        right = PropertyEqualsNode("prop_b", "good b")
-        other = PropertyEqualsNode("prop_c", "good c")
-        or_node = OrNode(left, right, other)
-
-        assert or_node.eval(Obj("wrong a", "good b", "good c"))
-        assert or_node.eval(Obj("good a", "wrong b", "good c"))
-        assert or_node.eval(Obj("good a", "good b", "wrong c"))
-        assert not or_node.eval(Obj("wrong a", "wrong b", "wrong c"))
-
-    def test_i_can_test_not(self):
-        node = PropertyEqualsNode("prop_a", "good value")
-        not_node = NotNode(node)
-
-        assert not not_node.eval(Obj(prop_a="good value"))
-        assert not_node.eval(Obj(prop_a="wrong value"))
-
-    def test_i_can_test_lambda_node(self):
-        node = LambdaNode(lambda o: o.prop_a + o.prop_b == "ab")
-
-        assert node.eval(Obj(prop_a="a", prop_b="b"))
-        assert not node.eval(Obj(prop_a="wrong value", prop_b="wrong value"))
-        assert not node.eval(Obj(prop_a="wrong value"))  # exception is caught
-
-    def test_i_can_test_isa_node(self):
-        class_node = IsaNode(Obj)
-        assert class_node.eval(Obj(prop_a="value"))
-        assert not class_node.eval(TestExpressionParser())
-
-        concept_node = IsaNode(BuiltinConcepts.RETURN_VALUE)
-        assert concept_node.eval(ReturnValueConcept())
-        assert concept_node.eval(Concept(name="foo", key=BuiltinConcepts.RETURN_VALUE))
-        assert not concept_node.eval(Obj)
-        assert not concept_node.eval(Concept())
-
-        concept_node2 = IsaNode("foo")
-        assert concept_node2.eval(Concept("foo").init_key())
-        assert not concept_node2.eval(Obj)
-        assert not concept_node2.eval(Concept())
-
     @pytest.mark.parametrize("expression, to_trueify, to_skip, expected", [
         ("a", ["b"], ["a"], "a"),
         ("b", ["b"], ["a"], "True"),
@@ -208,3 +170,321 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
         translated_node = TrueifyVisitor(to_trueify, to_skip).visit(expr_node)
 
         assert str(translated_node) == expected
+
+    @pytest.mark.parametrize("expression, expected", [
+        ("foo", None),
+        ("", None),
+        ("is_question()", True),
+        (" is_question() ", True),
+        ("is_question ( ) ", True),
+        ("is _question()", None),
+        ("is_ question()", None),
+        ("is _ question()", None),
+        ("context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", True),
+        ("not is_question()", False),
+        ("not context.in_context(BuiltinConcepts.EVAL_QUESTION_REQUESTED)", False),
+        ("not foo", None),
+
+        ("not is_question() and not is_question()", False),
+        ("not is_question() and is_question()", False),
+        ("not is_question() and foo", False),
+        ("is_question() and not is_question()", False),
+        ("is_question() and is_question()", True),
+        ("is_question() and foo", True),
+        ("foo and not is_question()", False),
+        ("foo and is_question()", True),
+        ("foo and bar", None),
+
+        ("not is_question() or not is_question()", False),
+        ("not is_question() or is_question()", True),
+        ("not is_question() or foo", False),
+        ("is_question() or not is_question()", True),
+        ("is_question() or is_question()", True),
+        ("is_question() or foo", True),
+        ("foo or not is_question()", False),
+        ("foo or is_question()", True),
+        ("foo or bar", None),
+    ])
+    def test_is_a_question(self, expression, expected):
+        sheerka, context, parser = self.init_parser()
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+
+        assert IsAQuestionVisitor().visit(expr_node) == expected
+
+    @pytest.mark.parametrize("expression, expected", [
+        ("foo", "foo"),
+        ("one two", "one two"),
+        ("foo is a bar", CMV("is a", x='foo', y='bar')),
+        ("one two is a bar", [CNC("is a", source="one two is a bar", x="one two", y="bar")]),
+        ("foo is an foo bar",
+         [CNC("is an", source="foo is an foo bar", x=DoNotResolve(value='foo'), exclude_body=True)]),
+    ])
+    def test_i_can_get_compiled_expr_from_simple_concepts_expressions(self, expression, expected):
+        concepts_map = {
+            "foo": Concept("foo"),
+            "bar": Concept("bar"),
+            "one two": Concept("one two"),
+            "is a": Concept("x is a y").def_var("x").def_var("y"),
+            "is an": Concept("x is an y", definition="('foo'|'bar')=x 'is an' 'foo bar'").def_var("x"),
+        }
+        sheerka, context, *concepts = self.init_test().with_concepts(*concepts_map.values(), create_new=True).unpack()
+
+        parser = ExpressionParser()
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+        return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
+
+        assert len(return_values) == 1
+        ret = return_values[0]
+
+        if isinstance(expected, list):
+            expected_nodes = compute_expected_array(concepts_map, expression, expected)
+            assert ret.body.body == expected_nodes
+        else:
+            expected_concept = resolve_test_concept(concepts_map, expected)
+            assert ret.body.body == expected_concept
+
+    @pytest.mark.parametrize("expression", [
+        "a == 5",
+        "foo > 5",
+        "func() == 5",
+        "not a == 5",
+        "not foo > 5",
+        "not func() == 5",
+        "isinstance(a, int)",
+        "func()",
+        "not isinstance(a, int)",
+        "not func()"
+    ])
+    def test_i_can_get_compiled_expr_from_simple_python_expressions(self, expression):
+        sheerka, context, = self.init_test().unpack()
+
+        parser = ExpressionParser()
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+        return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
+
+        assert len(return_values) == 1
+        ret = return_values[0]
+
+        assert ret.status
+        python_node = ret.body.body.get_python_node()
+        _ast = ast.parse(expression, mode="eval")
+        expected_python_node = PythonNode(expression, _ast)
+        assert python_node == expected_python_node
+
+    @pytest.mark.parametrize("expression", [
+        "a and not b",
+        "not b and a",
+        "__ret and not __ret.status",
+    ])
+    def test_i_can_compile_negative_conjunctions_when_pure_python(self, expression):
+        sheerka, context, *concepts = self.init_concepts("foo")
+
+        parser = ExpressionParser()
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+        return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
+
+        ast_ = ast.parse(expression, "<source>", 'eval')
+        expected_python_node = PythonNode(expression, ast_)
+
+        assert len(return_values) == 1
+        ret = return_values[0]
+
+        assert sheerka.objvalue(ret) == expected_python_node
+
+    @pytest.mark.parametrize("expression, text_to_compile", [
+        ("foo bar == 5", "__C__foo0bar__1001__C__ == 5"),
+        ("not foo bar == 5", "not __C__foo0bar__1001__C__ == 5"),
+    ])
+    def test_i_can_get_compiled_expr_from_python_and_concept(self, expression, text_to_compile):
+        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo bar"), create_new=True).unpack()
+
+        parser = ExpressionParser()
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+        return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
+
+        assert len(return_values) == 1
+        ret = return_values[0]
+
+        assert ret.status
+        python_node = ret.body.body.get_python_node()
+        _ast = ast.parse(text_to_compile, mode="eval")
+        expected_python_node = PythonNode(text_to_compile, _ast, expression)
+        assert python_node == expected_python_node
+
+    def test_i_can_get_compiled_expr_from__mix_of_concepts_and_python(self):
+        sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
+            Concept("animal"),
+            Concept("a cat"),
+            Concept("dog"),
+            Concept("pet"),
+            Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+            Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
+            create_new=True
+        ).unpack()
+
+        parser = ExpressionParser()
+        expression = "not a cat is a pet and not bird is an animal and not x > 5 and not dog is a pet"
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+        return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
+
+        to_compile = 'not __C__00var0000is0a000var001__1005__C__'
+        to_compile += ' and not __C__00var0000is0an0y__1006__C__'
+        to_compile += ' and not x > 5'
+        to_compile += ' and not __C__00var0000is0a000var001__1005_1__C__'
+        ast_ = ast.parse(to_compile, "<source>", 'eval')
+        expected_python_node = PythonNode(to_compile, ast_, expression)
+
+        assert len(return_values) == 1
+        ret = return_values[0]
+        python_node = ret.body.body
+        assert python_node == expected_python_node
+        assert python_node.objects == {
+            "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
+            "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
+            "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
+        }
+
+    def test_i_can_get_compiled_expr_from_mix(self):
+        sheerka, context, animal, cat, dog, pet, is_a, is_an = self.init_test().with_concepts(
+            Concept("animal"),
+            Concept("a cat"),
+            Concept("dog"),
+            Concept("pet"),
+            Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+            Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
+            create_new=True
+        ).unpack()
+
+        expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
+        parser = ExpressionParser()
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+        return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
+
+        assert len(return_values) == 1
+        ret = return_values[0]
+
+        to_compile = '__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1006__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__'
+        ast_ = ast.parse(to_compile, "<source>", 'eval')
+        expected_python_node = PythonNode(to_compile, ast_, expression)
+
+        python_node = ret.body.body
+        assert python_node == expected_python_node
+        assert python_node.objects == {
+            "__C__00var0000is0a000var001__1005__C__": CC(is_a, x=cat, y=pet),
+            "__C__00var0000is0an0y__1006__C__": CC(is_an, exclude_body=True, x=DoNotResolve("bird"), animal=animal),
+            "__C__00var0000is0a000var001__1005_1__C__": CMV(is_a, x="dog", y="pet"),
+        }
+
+    def test_i_can_get_compiled_expr_when_multiple_choices(self):
+        sheerka, context, *concepts = self.init_test().with_concepts(
+            Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+            Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
+            create_new=True
+        ).unpack()
+
+        parser = ExpressionParser()
+        expression = "a is a b"
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+        return_values, _ = parser.compile_conjunctions(context, [expr_node], "test")
+
+        assert len(return_values) == 2
+
+        ret = return_values[0]
+        assert sheerka.objvalue(ret)[0].concept == CMV(concepts[0], x="a", y="b")
+
+        ret = return_values[1]
+        assert sheerka.objvalue(ret)[0].concept == CMV(concepts[1], x="a", y="b")
+
+    def test_i_can_get_compiled_expr_from_mix_when_multiple_choices(self):
+        sheerka, context, *concepts = self.init_test().with_concepts(
+            Concept("animal"),
+            Concept("a cat"),
+            Concept("dog"),
+            Concept("pet"),
+            Concept("x is a y", pre="is_question()", body="isinstance(x, y)").def_var("x").def_var("y"),
+            Concept("x is a y", pre="is_question()", body="isa(x, y)").def_var("x").def_var("y"),
+            Concept("x is an y", pre="is_question()", definition="('cat'|'bird')=x 'is an' animal").def_var("x"),
+            create_new=True
+        ).unpack()
+
+        expression = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
+        parser = ExpressionParser()
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+        return_values, _ = parser.compile_conjunctions(context, expr_node.parts, "test")
+
+        assert len(return_values) == 4
+        trimmed_source = "a cat is a pet and bird is an animal and x > 5 and dog is a pet"
+
+        current_ret = return_values[0]
+        python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005_1__C__"
+        ast_ = ast.parse(python_source, "<source>", 'eval')
+        resolved_expected = PythonNode(python_source, ast_, trimmed_source)
+        assert sheerka.objvalue(current_ret) == resolved_expected
+
+        current_ret = return_values[1]
+        assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
+        python_source = "__C__00var0000is0a000var001__1005__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006__C__"
+        ast_ = ast.parse(python_source, "<source>", 'eval')
+        resolved_expected = PythonNode(python_source, ast_, trimmed_source)
+        assert sheerka.objvalue(current_ret) == resolved_expected
+
+        current_ret = return_values[2]
+        assert sheerka.isinstance(current_ret, BuiltinConcepts.RETURN_VALUE)
+        python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1005__C__"
+        ast_ = ast.parse(python_source, "<source>", 'eval')
+        resolved_expected = PythonNode(python_source, ast_, trimmed_source)
+        assert sheerka.objvalue(current_ret) == resolved_expected
+
+        current_ret = return_values[3]
+        python_source = "__C__00var0000is0a000var001__1006__C__ and __C__00var0000is0an0y__1007__C__ and x > 5 and __C__00var0000is0a000var001__1006_1__C__"
+        ast_ = ast.parse(python_source, "<source>", 'eval')
+        resolved_expected = PythonNode(python_source, ast_, trimmed_source)
+        assert sheerka.objvalue(current_ret) == resolved_expected
+
+    @pytest.mark.parametrize("expression, expected_conditions, test_obj", [
+        (
+            "__ret",
+            ["#__x_00__|__name__|'__ret'"],
+            ReturnValueConcept("Test", True, None)
+        ),
+        (
+            "__ret.status == True",
+            ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
+            ReturnValueConcept("Test", True, None)
+        ),
+        (
+            "__ret.status",
+            ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
+            ReturnValueConcept("Test", True, None)
+        ),
+        (
+            "__ret and __ret.status",
+            ["#__x_00__|__name__|'__ret'", "#__x_00__|status|True"],
+            ReturnValueConcept("Test", True, None)
+        ),
+    ])
+    def test_i_can_get_rete_condition_from_python(self, expression, expected_conditions, test_obj):
+        sheerka, context, = self.init_test().unpack()
+        expected_full_condition = get_rete_conditions(*expected_conditions)
+
+        parser = ExpressionParser()
+        expr_node = parser.parse(context, ParserInput(expression)).body.body
+
+        nodes = expr_node.parts if isinstance(expr_node, AndNode) else [expr_node]
+        _, rete_disjunctions = parser.compile_conjunctions(context, nodes, "test")
+
+        assert len(rete_disjunctions) == 1
+        assert rete_disjunctions == [expected_full_condition]
+
+        # check against a Rete network
+        network = ReteNetwork()
+        rule = Rule("test", expression, None)
+        rule.metadata.id = 9999
+        rule.metadata.is_compiled = True
+        rule.metadata.is_enabled = True
+        rule.rete_disjunctions = rete_disjunctions
+        network.add_rule(rule)
+
+        network.add_obj("__ret", test_obj)
+        matches = list(network.matches)
+        assert len(matches) > 0

tests/parsers/test_FunctionParser.py

@@ -28,7 +28,7 @@ class TestFunctionParser(TestUsingMemoryBasedSheerka):
             cmap[concept_name] = updated[i]
 
         cls.shared_ontology = sheerka.get_ontology(context)
-        sheerka.pop_ontology()
+        sheerka.pop_ontology(context)
 
     def init_parser(self, my_concepts_map=None, **kwargs):
         if my_concepts_map is None:

tests/parsers/test_PythonWithConceptsParser.py

@@ -1,18 +1,18 @@
 import ast
 
 import pytest
-import core.utils
+
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept
 from core.rule import Rule
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Token, TokenKind, Tokenizer
-from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode
+from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, RuleNode, SourceCodeNode
 from parsers.PythonParser import PythonNode
 from parsers.PythonWithConceptsParser import PythonWithConceptsParser
 from parsers.UnrecognizedNodeParser import UnrecognizedNodeParser
-
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.parsers.parsers_utils import get_source_code_node
 
 unrecognized_nodes_parser = UnrecognizedNodeParser()
 
@@ -73,7 +73,8 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
         assert wrapper.source == "foo + 1"
         assert isinstance(return_value, PythonNode)
-        assert return_value.source == "foo + 1"
+        assert return_value.source == "__C__foo__C__ + 1"
+        assert return_value.original_source == "foo + 1"
         assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1")
         assert return_value.objects["__C__foo__C__"] == foo
 
@@ -93,7 +94,8 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
         assert wrapper.source == "foo + 1"
         assert isinstance(return_value, PythonNode)
-        assert return_value.source == "foo + 1"
+        assert return_value.source == "__C__foo__1001__C__ + 1"
+        assert return_value.original_source == "foo + 1"
         assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__1001__C__ + 1")
         assert return_value.objects["__C__foo__1001__C__"] == foo
 
@@ -124,7 +126,8 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         assert context.sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert parser_result.source == "func(foo, bar)"
         assert isinstance(return_value, PythonNode)
-        assert return_value.source == "func(foo, bar)"
+        assert return_value.source == "func(__C__foo__1001__C__, __C__bar__1002__C__)"
+        assert return_value.original_source == "func(foo, bar)"
         assert return_value.get_dump(return_value.ast_) == to_str_ast("func(__C__foo__1001__C__, __C__bar__1002__C__)")
         assert return_value.objects["__C__foo__1001__C__"] == foo
         assert return_value.objects["__C__bar__1002__C__"] == bar
@@ -144,7 +147,8 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
         assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
         assert wrapper.source == "r:|rule_id: + 1"
         assert isinstance(return_value, PythonNode)
-        assert return_value.source == "r:|rule_id: + 1"
+        assert return_value.source == "__R____rule_id__R__ + 1"
+        assert return_value.original_source == "r:|rule_id: + 1"
         assert return_value.get_dump(return_value.ast_) == to_str_ast("__R____rule_id__R__ + 1")
         assert return_value.objects["__R____rule_id__R__"] == rule
 
@@ -193,3 +197,26 @@ class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
 
         assert not result.status
         assert context.sheerka.isinstance(result.value, BuiltinConcepts.NOT_FOR_ME)
+
+    def test_i_can_parse_nodes_when_source_code_node(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        bar = Concept("bar")
+
+        nodes = [
+            UnrecognizedTokensNode(0, 1, list(Tokenizer("not ", yield_eof=False))),
+            get_source_code_node(2, "foo == 1", {"foo": foo}),
+            UnrecognizedTokensNode(7, 9, list(Tokenizer(" and ", yield_eof=False))),
+            get_source_code_node(10, "bar < 1", {"bar": bar}),
+        ]
+        expected_ast = ast.parse('not __C__foo__C__ == 1 and __C__bar__C__ < 1', "<source>", 'eval')
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse_nodes(context, nodes)
+
+        result_python_node = result.value.value
+        assert isinstance(result_python_node, PythonNode)
+        assert result_python_node.source == 'not __C__foo__C__ == 1 and __C__bar__C__ < 1'
+        assert result_python_node.ast_str == PythonNode.get_dump(expected_ast)
+        assert result_python_node.original_source == "not foo == 1 and bar < 1"
+        assert result_python_node.objects == {"__C__foo__C__": foo, "__C__bar__C__": bar}

tests/parsers/test_RuleParser.py

@@ -14,17 +14,17 @@ class TestRuleParser(TestUsingMemoryBasedSheerka):
     @classmethod
     def setup_class(cls):
         init_test_helper = cls().init_test(cache_only=False, ontology="#TestRuleParser#")
-        sheerka, context, *updated = init_test_helper.with_rules(*my_rules).unpack()
+        sheerka, context, *updated = init_test_helper.with_format_rules(*my_rules).unpack()
 
         cls.shared_ontology = sheerka.get_ontology(context)
-        sheerka.pop_ontology()
+        sheerka.pop_ontology(context)
 
     def init_parser(self, rules=None, **kwargs):
         if rules is None:
             sheerka, context = self.init_test().unpack()
             sheerka.add_ontology(context, self.shared_ontology)
         else:
-            sheerka, context, *updated = self.init_test().with_rules(*rules, **kwargs).unpack()
+            sheerka, context, *updated = self.init_test().with_format_rules(*rules, **kwargs).unpack()
 
         parser = RuleParser()
         return sheerka, context, parser

tests/parsers/test_SyaNodeParser.py

@@ -66,7 +66,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
                                                       CONCEPT_COMPARISON_CONTEXT)
 
         cls.shared_ontology = sheerka.get_ontology(context)
-        sheerka.pop_ontology()
+        sheerka.pop_ontology(context)
 
     def init_parser(self,
                     my_concepts_map=None,

tests/parsers/test_UnrecognizedNodeParser.py

@@ -85,7 +85,7 @@ class TestUnrecognizedNodeParser(TestUsingMemoryBasedSheerka):
                                     concepts_map["plus"], 'Sya')
 
         cls.shared_ontology = sheerka.get_ontology(context)
-        sheerka.pop_ontology()
+        sheerka.pop_ontology(context)
 
     def init_parser(self, my_concepts_map=None, **kwargs):
         if my_concepts_map is None: