Fixed #131 : Implement ExprToConditions

Fixed #130 : ArithmeticOperatorParser
Fixed #129 : python_wrapper : create_namespace
Fixed #128 : ExpressionParser: Cannot parse func(x) infixed concept 'xxx'

tests/core/test_ParserInput.py

@@ -1,6 +1,7 @@
 import pytest
+
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Tokenizer, TokenKind
+from core.tokenizer import TokenKind, Tokenizer
 
 
 @pytest.mark.parametrize("text, start, end, expected", [
@@ -141,3 +142,37 @@ def test_i_can_get_the_token_after(text, skip_whitespace, expected):
     parser_input = ParserInput(text).reset()
     parser_input.next_token()
     assert parser_input.the_token_after(skip_whitespace).repr_value == expected
+
+
+def test_i_can_define_a_sub_part():
+    text = "Hello Koffi the great guy."
+    tokens = list(Tokenizer(text))
+
+    parser_input = ParserInput(None, tokens, 2, 6)
+    assert repr(parser_input) == "ParserInput(from_tokens'Koffi the great')"
+
+    parser_input.reset()
+    assert repr(parser_input) == "ParserInput(from_tokens'Koffi the great')"
+    res = []
+    while parser_input.next_token():
+        res.append(f"{parser_input.token.repr_value}")
+
+    assert res == ["Koffi", "the", "great"]
+    assert repr(parser_input) == "ParserInput(from_tokens'Koffi the great')"
+
+
+def test_i_can_define_parse_input_from_tokens():
+    text = "Hello Koffi the great guy."
+    tokens = list(Tokenizer(text))
+
+    parser_input = ParserInput(None, tokens)
+    assert repr(parser_input) == "ParserInput(from_tokens'Hello Koffi the great guy.')"
+
+    parser_input.reset()
+    assert repr(parser_input) == "ParserInput(from_tokens'Hello Koffi the great guy.')"
+    res = []
+    while parser_input.next_token():
+        res.append(f"{parser_input.token.repr_value}")
+
+    assert res == ["Hello", "Koffi", "the", "great", "guy", "."]
+    assert repr(parser_input) == "ParserInput(from_tokens'Hello Koffi the great guy.')"

tests/core/test_SheerkaEvaluateConcept.py

@@ -1017,7 +1017,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
             "one",
             "number",
             Concept("x is a y", body="isa(x,y)", pre="is_question()").def_var("x").def_var("y"))
-
+        context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
         evaluator = SheerkaEvaluateConcept(sheerka)
 
         parsed_return_value = ExpressionParser().parse(context, ParserInput("one is a number"))
@@ -1026,7 +1026,6 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert concept.get_compiled()["y"][0].body.body.get_hints().use_copy
 
         # get the body
-        context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
         evaluated = evaluator.evaluate_concept(context, concept, hints=EvaluationHints(eval_body=True))
         assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy
         assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy

tests/core/test_SheerkaEvaluateRules.py

@@ -7,10 +7,11 @@ from core.rule import Rule, ACTION_TYPE_EXEC
 from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME
 from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules, LOW_PRIORITY_RULES, DISABLED_RULES
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition, PythonConditionExprVisitor
+from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager, CompiledCondition
 from evaluators.PythonEvaluator import PythonEvaluator
 from parsers.ExpressionParser import ExpressionParser
 from parsers.PythonParser import PythonParser
+from sheerkapython.ExprToConditions import ExprToConditionsVisitor
 from sheerkapython.python_wrapper import Expando
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -114,6 +115,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         res = service.evaluate_rules(context, [r1, r2, r3], {}, set())
         assert res == {True: [r1, r3], False: [r2]}
 
+    @pytest.mark.skip("recognize is not supported yet")
     @pytest.mark.parametrize("predicate", [
         "recognize(__ret.body, greetings)",
         "recognize(__ret.body, c:|1001:)",
@@ -134,6 +136,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         res = service.evaluate_rules(context, [rule], {"__ret": ret}, set())
         assert res == {True: [rule]}
 
+    @pytest.mark.skip("recognize is not supported yet")
     @pytest.mark.parametrize("recognized_by", [
         RECOGNIZED_BY_ID,
         RECOGNIZED_BY_NAME,
@@ -157,6 +160,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         res = service.evaluate_rules(context, [rule], {"__ret": ret}, set())
         assert res == {True: [rule]}
 
+    @pytest.mark.skip("recognize is not supported yet")
     @pytest.mark.parametrize("recognized_by", [
         RECOGNIZED_BY_ID,
         RECOGNIZED_BY_NAME,
@@ -180,6 +184,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         res = service.evaluate_rules(context, [rule], {"__ret": ret}, set())
         assert res == {True: [rule]}
 
+    @pytest.mark.skip("recognize is not supported yet")
     def test_i_can_evaluate_concept_rules_when_variable_is_an_expando(self):
         sheerka, context, greetings, rule = self.init_test().with_concepts(
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
@@ -208,6 +213,7 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         res = service.evaluate_rules(context, [rule], {"__ret": ret2}, set())
         assert res == {True: [rule]}
 
+    @pytest.mark.skip("recognize is not supported yet")
     def test_i_can_evaluate_concept_rule_with_the_same_name_when_the_second_concept_is_declared_after(self):
         sheerka, context, g1, rule, g2 = self.init_test().with_concepts(
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
@@ -255,11 +261,11 @@ class TestSheerkaEvaluateRules(TestUsingMemoryBasedSheerka):
         sheerka, context = self.init_test().unpack()
 
         expression = "isinstance(a, int)"
-        parser = ExpressionParser()
+        parser = ExpressionParser(auto_compile=False)
         ret_val = parser.parse(context, ParserInput(expression))
         parsed = ret_val.body.body
 
-        visitor = PythonConditionExprVisitor(context)
+        visitor = ExprToConditionsVisitor(context)
         conditions = visitor.get_conditions(parsed)
 
         missing_vars = set()

tests/core/test_SheerkaQueryManager.py

@@ -96,17 +96,17 @@ class TestSheerkaQueryManager(TestUsingMemoryBasedSheerka):
                Concept("foo", body="b").auto_init(),
                B("a31", "a32")]
 
-        assert sheerka.filter_objects(context, lst, "self.prop1 == 'a21'") == [lst[1]]
-        assert sheerka.filter_objects(context, lst, "self.prop2 >= 1") == [lst[0], lst[1], lst[2]]
-        assert sheerka.filter_objects(context, lst, "get_type(self) == 'foo' ") == [lst[4], lst[6]]
-        assert sheerka.filter_objects(context, lst, "self.fake_prop1 == 'a21' ") == [lst[5]]
-        assert sheerka.filter_objects(context, lst, "hasattr(self, 'fake_prop1')") == [lst[5], lst[7]]
+        assert sheerka.filter_objects(context, lst, predicate="self.prop1 == 'a21'") == [lst[1]]
+        assert sheerka.filter_objects(context, lst, predicate="self.prop2 >= 1") == [lst[0], lst[1], lst[2]]
+        assert sheerka.filter_objects(context, lst, predicate="get_type(self) == 'foo' ") == [lst[4], lst[6]]
+        assert sheerka.filter_objects(context, lst, predicate="self.fake_prop1 == 'a21' ") == [lst[5]]
+        assert sheerka.filter_objects(context, lst, predicate="hasattr(self, 'fake_prop1')") == [lst[5], lst[7]]
 
     def test_i_can_filter_object_using_predicate_and_sheerka_objects(self):
         sheerka, context, foo, bar = self.init_concepts("foo", "bar")
         lst = [foo, bar, A("a21", 3.14)]
 
-        assert sheerka.filter_objects(context, lst, "self == bar") == [lst[1]]
+        assert sheerka.filter_objects(context, lst, predicate="self == bar") == [lst[1]]
 
     def test_i_can_filter_objects_using_concept(self):
         sheerka, context, foo, bar, isa = self.init_concepts(
@@ -116,7 +116,7 @@ class TestSheerkaQueryManager(TestUsingMemoryBasedSheerka):
             create_new=True)
 
         lst = [foo, A("a21", 3.14), bar, B("a21", 3.14)]
-        assert sheerka.filter_objects(context, lst, "self is a concept") == [foo, bar]
+        assert sheerka.filter_objects(context, lst, predicate="self is a concept") == [foo, bar]
 
     def test_i_can_filter_objects_when_no_kwargs_and_no_predicate(self):
         sheerka, context, foo, bar = self.init_concepts("foo", "bar")
@@ -124,6 +124,25 @@ class TestSheerkaQueryManager(TestUsingMemoryBasedSheerka):
 
         assert sheerka.filter_objects(context, lst) == lst
 
+    def test_i_can_filter_objects_after_applying_a_mapping(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", 10),
+               A("a21", 3.14)]
+
+        res = sheerka.filter_objects(context, lst, mapping=lambda o: o.prop1, predicate="self[1] == '1'")
+        assert res == [A("a11", 10)]
+
+    def test_i_can_filter_objects_using_concept_after_applying_a_mapping(self):
+        sheerka, context, foo, bar, isa = self.init_concepts(
+            "foo",
+            "bar",
+            Concept("x is a concept", body="isinstance(x, Concept)", pre="is_question()").def_var("x"),
+            create_new=True)
+
+        lst = [foo, A(foo, 3.14), bar, A(bar, 3.14)]
+        res = sheerka.filter_objects(context, lst, mapping=lambda o: o.prop1, predicate="self is a concept")
+        assert res == [lst[1], lst[3]]
+
     def test_i_must_select_object_property_using_string(self):
         sheerka, context = self.init_test().unpack()
 

tests/core/test_SheerkaRuleManager.py

@@ -170,7 +170,6 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         rule = service.init_rule(context, rule)
 
         assert len(rule.error_sink["when"]) > 0
-        assert sheerka.has_error(context, rule.error_sink["when"][0])
         assert "print" not in rule.error_sink
         assert "then" not in rule.error_sink
         assert rule.metadata.is_compiled
@@ -362,7 +361,7 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
             Concept("foo"),
         ).unpack()
-        parser = ExpressionParser()
+        parser = ExpressionParser(old_style=True)
         expected_conditions = get_rete_conditions(*expected)
 
         error_sink = ErrorSink()

tests/core/test_sheerka.py

@@ -3,11 +3,11 @@ import os
 import pytest
 
 from conftest import SHEERKA_TEST_FOLDER
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept, UnknownConcept
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UnknownConcept, UserInputConcept
 from core.builtin_concepts_ids import AllBuiltinConcepts
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, get_concept_attrs
+from core.concept import Concept, ConceptParts, PROPERTIES_TO_SERIALIZE, get_concept_attrs
 from core.global_symbols import NotInit
-from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS
+from core.sheerka.Sheerka import BASE_NODE_PARSER_CLASS, Sheerka
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, ValueNotFound
 from core.tokenizer import Token, TokenKind
 from parsers.PythonParser import PythonErrorNode
@@ -541,6 +541,18 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         errors_found = sheerka.get_errors(context, ret_val, source="error source")
         assert errors_found == [python_error]
 
+    def test_i_can_filter_error_by_exception_name(self):
+        sheerka, context = self.init_test().unpack()
+
+        from evaluators.PythonEvaluator import PythonEvalError
+        name_error = NameError("foo")
+        python_eval_error = PythonEvalError(name_error, "foo", None, None)
+        error = sheerka.err([python_eval_error])
+        ret_val = ReturnValueConcept("Test", False, error)
+
+        errors_found = sheerka.get_errors(context, ret_val, __type="NameError")
+        assert errors_found == [name_error]
+
     def test_i_can_filter_error_on_multiple_criteria(self):
         sheerka, context = self.init_test().unpack()
 

tests/core/test_tokenizer.py

@@ -5,7 +5,7 @@ from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
 
 def test_i_can_tokenize():
     source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&<>c:name:"
-    source += "$£€!_identifier°~_^\\`==#__var__10r/regex\nregex/r:xxx|1:"
+    source += "$£€!_identifier°~_^\\`==#__var__10r/regex\nregex/r:xxx|1:**//%"
     tokens = list(Tokenizer(source))
     assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
     assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
@@ -58,8 +58,11 @@ def test_i_can_tokenize():
     assert tokens[48] == Token(TokenKind.VAR_DEF, '__var__10', 112, 6, 55)
     assert tokens[49] == Token(TokenKind.REGEX, '/regex\nregex/', 121, 6, 64)
     assert tokens[50] == Token(TokenKind.RULE, ("xxx", "1"), 135, 7, 7)
+    assert tokens[51] == Token(TokenKind.STARSTAR, "**", 143, 7, 15)
+    assert tokens[52] == Token(TokenKind.SLASHSLASH, "//", 145, 7, 17)
+    assert tokens[53] == Token(TokenKind.PERCENT, "%", 147, 7, 19)
 
-    assert tokens[51] == Token(TokenKind.EOF, '', 143, 7, 15)
+    assert tokens[54] == Token(TokenKind.EOF, '', 148, 7, 20)
 
 
 @pytest.mark.parametrize("text, expected", [

tests/core/test_utils.py

@@ -7,7 +7,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import evaluate_expression
 from core.concept import Concept
 from core.global_symbols import NotFound, NotInit, Removed
-from core.tokenizer import Token, TokenKind, Tokenizer, Keywords
+from core.tokenizer import Keywords, Token, TokenKind, Tokenizer
 
 
 @dataclass
@@ -512,3 +512,24 @@ def test_i_can_replace_after():
     with pytest.raises(KeyError):
         my_list3 = ["a", "b", "c", "d"]
         core.utils.replace_after(my_list3, "x", my_new_items)
+
+
+def test_i_can_tokens_index():
+    tokens = list(Tokenizer("a b c d e"))
+
+    with pytest.raises(ValueError):
+        core.utils.tokens_index(tokens, None)
+
+    with pytest.raises(ValueError):
+        core.utils.tokens_index(tokens, [])
+
+    with pytest.raises(ValueError):
+        core.utils.tokens_index(tokens, list(Tokenizer("f", yield_eof=False)))
+
+    assert core.utils.tokens_index(tokens, list(Tokenizer("a", yield_eof=False))) == 0
+    assert core.utils.tokens_index(tokens, list(Tokenizer("b", yield_eof=False))) == 2
+    assert core.utils.tokens_index(tokens, list(Tokenizer("b c", yield_eof=False))) == 2
+    assert core.utils.tokens_index(tokens, list(Tokenizer(" ", yield_eof=False))) == 1
+    assert core.utils.tokens_index(tokens, list(Tokenizer(" ", yield_eof=False)), skip=2) == 5
+    assert core.utils.tokens_index(tokens, list(Tokenizer(" ", yield_eof=False)), start_from_end=True) == 7
+    assert core.utils.tokens_index(tokens, list(Tokenizer(" ", yield_eof=False)), skip=2, start_from_end=True) == 3