Fixed #18 : Parsing and evaluating Python

tests/base.py

@@ -8,6 +8,24 @@ from core.Sheerka import Sheerka
 from sdp.sheerkaDataProvider import SheerkaDataProvider
 
 
+class DummyObj:
+    def __init__(self, a: str = "hello", b: str = "world"):
+        self.a = a
+        self.b = b
+
+    def __eq__(self, other):
+        if not isinstance(other, DummyObj):
+            return False
+
+        return self.a == other.a and self.b == other.b
+
+    def __hash__(self):
+        return hash((self.a, self.b))
+
+    def __repr__(self):
+        return f"Dummy('{self.a}', '{self.b}')"
+
+
 class BaseTest:
     @pytest.fixture()
     def sdp(self) -> SheerkaDataProvider:

tests/caching/test_FastCache.py

@@ -46,7 +46,7 @@ def test_i_can_put_the_same_key_several_times():
     assert cache.lru == ["key2", "key1"]
 
 
-def test_none_is_returned_when_not_found():
+def test_not_found_is_returned_when_not_found():
     cache = FastCache()
     assert cache.get("foo") is NotFound
 

tests/common/test_ast_utils.py

@@ -0,0 +1,50 @@
+import ast
+
+import pytest
+
+from common.ast_utils import NamesWithAttributesVisitor, UnreferencedNamesVisitor, UnreferencedVariablesVisitor
+
+
+@pytest.mark.parametrize("source, expected", [
+    ("a,b", {"a", "b"}),
+    ("isinstance(a, int)", {"isinstance", "a", "int"}),
+    ("date.today()", {"date"}),
+    ("test()", {"test"}),
+    ("sheerka.test()", {"sheerka"}),
+    ("for i in range(10): pass", set()),
+    ("func(x=a, y=b)", {"func", "a", "b"}),
+
+])
+def test_i_can_get_unreferenced_names_from_simple_expressions(context, source, expected):
+    ast_ = ast.parse(source)
+    visitor = UnreferencedNamesVisitor(context)
+    visitor.visit(ast_)
+
+    assert visitor.names == expected
+
+
+def test_name_with_attribute():
+    # Looks for all attributes for a given name
+    ast_ = ast.parse("foo.bar.baz", "<src>", mode="exec")
+    assert NamesWithAttributesVisitor().get_sequences(ast_, "foo") == [["foo", "bar", "baz"]]
+
+    # It parses all expressions / statements
+    ast_ = ast.parse("foo.bar.baz; one.two.three; foo.bar", "<src>", mode="exec")
+    assert NamesWithAttributesVisitor().get_sequences(ast_, "foo") == [["foo", "bar", "baz"], ["foo", "bar"]]
+
+
+@pytest.mark.parametrize("source, expected", [
+    ("a,b", {"a", "b"}),
+    ("isinstance(a, int)", {"a", "int"}),
+    ("date.today()", set()),
+    ("test()", set()),
+    ("sheerka.test()", set()),
+    ("for i in range(10): pass", set()),
+    ("func(x=a, y=b)", {"a", "b", "x", "y"}),
+])
+def test_i_can_get_unreferenced_variables_from_simple_expressions(context, source, expected):
+    ast_ = ast.parse(source)
+    visitor = UnreferencedVariablesVisitor(context)
+    visitor.visit(ast_)
+
+    assert visitor.names == expected

tests/common/test_utils.py

@@ -2,9 +2,9 @@ from dataclasses import dataclass
 
 import pytest
 
-from common.utils import decode_enum, get_class, to_dict, str_concept, unstr_concept
+from common.utils import decode_enum, dict_product, get_class, get_text_from_tokens, str_concept, to_dict, unstr_concept
 from helpers import get_concept
-from parsers.tokenizer import Keywords, Token, TokenKind
+from parsers.tokenizer import Keywords, Token, TokenKind, Tokenizer
 
 
 @dataclass
@@ -120,3 +120,55 @@ def test_i_can_decode_enum(text, expected):
 ])
 def test_i_can_to_dict(items, expected):
     assert to_dict(items, lambda obj: obj.prop1) == expected
+
+
+@pytest.mark.parametrize("text, expected_text", [
+    ("hello world", "hello world"),
+    ("'hello' 'world'", "'hello' 'world'"),
+    ("def concept a from", "def concept a from"),
+    ("()[]{}1=1.5+-/*><&é", "()[]{}1=1.5+-/*><&é"),
+    ("execute(c:concept_name:)", "execute(c:concept_name:)")
+
+])
+def test_i_can_get_text_from_tokens(text, expected_text):
+    tokens = list(Tokenizer(text))
+    assert get_text_from_tokens(tokens) == expected_text
+
+
+@pytest.mark.parametrize("text, custom, expected_text", [
+    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)")
+])
+def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
+    tokens = list(Tokenizer(text))
+    assert get_text_from_tokens(tokens, custom) == expected_text
+
+
+def test_i_can_track_tokens():
+    text = "execute(c:name1: if r:#id: else c:name2:)"
+    switcher = {TokenKind.CONCEPT: lambda t: f"__CONCEPT__{t.value[0]}",
+                TokenKind.RULE: lambda t: f"__RULE__{t.value[1]}"}
+    tracker = {}
+    tokens = list(Tokenizer(text))
+    get_text_from_tokens(tokens, switcher, tracker)
+    assert len(tracker) == 3
+    assert tracker["__CONCEPT__name1"] == tokens[2]
+    assert tracker["__RULE__id"] == tokens[6]
+    assert tracker["__CONCEPT__name2"] == tokens[10]
+
+
+@pytest.mark.parametrize("a,b,expected", [
+    ([], [], []),
+    ([{"a": "a", "b": "b"}], [], [{"a": "a", "b": "b"}]),
+    ([], [{"a": "a", "b": "b"}], [{"a": "a", "b": "b"}]),
+    ([{"a": "a", "b": "b"}], [{"d": "d1"}, {"d": "d2"}], [{"a": "a", "b": "b", "d": "d1"},
+                                                          {"a": "a", "b": "b", "d": "d2"}]),
+    ([{"d": "d1"}, {"d": "d2"}], [{"a": "a", "b": "b"}], [{"a": "a", "b": "b", "d": "d1"},
+                                                          {"a": "a", "b": "b", "d": "d2"}]),
+    ([{"a": "a", "b": "b"}], [{"d": "d", "e": "e"}], [{"a": "a", "b": "b", "d": "d", "e": "e"}]),
+    ([{"a": "a"}, {"b": "b"}], [{"d": "d"}, {"e": "e"}], [{"a": "a", "d": "d"},
+                                                          {"a": "a", "e": "e"},
+                                                          {"b": "b", "d": "d"},
+                                                          {"b": "b", "e": "e"}])
+])
+def test_dict_product(a, b, expected):
+    assert dict_product(a, b) == expected

tests/conftest.py

@@ -1,6 +1,7 @@
 import pytest
 
 from helpers import GetNextId
+from server.authentication import User
 
 
 @pytest.fixture(scope="session")
@@ -25,12 +26,12 @@ def on_new_module(sheerka, request):
     :rtype:
     """
     from core.Event import Event
-    from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+    from core.ExecutionContext import ExecutionContext, ContextActions
     module_name = request.module.__name__.split(".")[-1]
     context = ExecutionContext("test",
                                Event(message=f"Executing module {module_name}"),
                                sheerka,
-                               ExecutionContextActions.TESTING,
+                               ContextActions.TESTING,
                                None)
 
     ontology = sheerka.om.push_ontology(module_name)
@@ -41,12 +42,12 @@ def on_new_module(sheerka, request):
 @pytest.fixture(scope="function")
 def context(sheerka):
     from core.Event import Event
-    from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+    from core.ExecutionContext import ExecutionContext, ContextActions
 
     return ExecutionContext("test",
                             Event(message=""),
                             sheerka,
-                            ExecutionContextActions.TESTING,
+                            ContextActions.TESTING,
                             None)
 
 
@@ -55,6 +56,11 @@ def next_id():
     return GetNextId()
 
 
+@pytest.fixture()
+def user():
+    return User(username="johan doe", email="johan.doe@sheerka.com", firstname="johan", lastname="doe")
+
+
 class TestUsingFileBasedSheerka:
     @pytest.fixture(scope="class")
     def sheerka(self):

tests/core/test_concept.py

@@ -10,7 +10,7 @@ def test_i_can_retrieve_concept_properties():
     assert foo.id == "1001"
     assert foo.str_id == "c:#1001:"
     assert foo.all_attrs() == ('#where#', '#pre#', '#post#', '#body#', '#ret#', 'a', 'b')
-    assert foo.get_definition_digest() == "3a2cfcda8ffd0d99a7f8c7d2f1ffc4a99fc96162f3be7b9875f30751d3691af6"
+    assert foo.get_definition_digest() == "13b61f45934a802b5486a1bdd60e404b32378a801408769cd584e3b3b7518cc2"
 
     # sanity check to make sure that 'get_concept' works as expected
     assert foo.get_metadata().variables == (("a", NotInit), ("b", NotInit))
@@ -20,6 +20,7 @@ def test_i_can_set_and_get_value():
     foo = get_concept("foo", variables=["a"])
     foo.set_value("a", "some value")
     assert foo.get_value("a") == "some value"
+    assert foo.a == "some value"
 
 
 def test_i_can_set_and_get_value_from_bound_attr():

tests/core/test_execution_context.py

@@ -1,15 +1,15 @@
 from core.Event import Event
-from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+from core.ExecutionContext import ExecutionContext, ContextActions
 
 
 def test_i_can_create_execution_context(sheerka):
     event = Event("myEvent", "fake_userid")
-    context1 = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value1", "my desc")
+    context1 = ExecutionContext("who", event, sheerka, ContextActions.TESTING, "value1", "my desc")
 
     assert context1.who == "who"
     assert context1.event == event
     assert context1.sheerka == sheerka
-    assert context1.action == ExecutionContextActions.TESTING
+    assert context1.action == ContextActions.TESTING
     assert context1.action_context == "value1"
     assert context1.desc == "my desc"
     assert context1.id == 0
@@ -18,12 +18,12 @@ def test_i_can_create_execution_context(sheerka):
 
 def test_i_can_push(sheerka):
     event = Event("test")
-    context = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value")
-    with context.push("pusher", ExecutionContextActions.PARSING, "action_context", "my desc") as sub_context:
+    context = ExecutionContext("who", event, sheerka, ContextActions.TESTING, "value")
+    with context.push("pusher", ContextActions.PARSING, "action_context", "my desc") as sub_context:
         assert sub_context.who == "pusher"
         assert sub_context.event == event
         assert sub_context.sheerka == sheerka
-        assert sub_context.action == ExecutionContextActions.PARSING
+        assert sub_context.action == ContextActions.PARSING
         assert sub_context.action_context == "action_context"
         assert sub_context.desc == "my desc"
         assert sub_context.id == context.id + 1
@@ -34,11 +34,11 @@ def test_i_can_increment_ids(sheerka):
     # If the event is the same, the id is incremented
 
     event = Event("TEST::myEvent", "fake_userid")
-    context1 = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value")
-    context2 = context1.push("who1", ExecutionContextActions.TESTING, "value1")
-    context3 = context2.push("who2", ExecutionContextActions.TESTING, "value2")
-    context4 = context1.push("who1", ExecutionContextActions.TESTING, "value3")
-    context5 = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value4")
+    context1 = ExecutionContext("who", event, sheerka, ContextActions.TESTING, "value")
+    context2 = context1.push("who1", ContextActions.TESTING, "value1")
+    context3 = context2.push("who2", ContextActions.TESTING, "value2")
+    context4 = context1.push("who1", ContextActions.TESTING, "value3")
+    context5 = ExecutionContext("who", event, sheerka, ContextActions.TESTING, "value4")
 
     assert context1.id == 0
     assert context2.id == 1
@@ -47,15 +47,15 @@ def test_i_can_increment_ids(sheerka):
     assert context5.id == 4
 
     event2 = Event("TEST::myEvent2", "fake_userid")
-    context6 = ExecutionContext("who", event2, sheerka, ExecutionContextActions.TESTING, "value")
+    context6 = ExecutionContext("who", event2, sheerka, ContextActions.TESTING, "value")
     assert context6.id == 0
 
 
 def test_i_can_manage_global_hints(context):
-    context2 = context.push("pusher", ExecutionContextActions.TESTING, None)
-    context3 = context2.push("pusher", ExecutionContextActions.TESTING, None)
-    context4 = context3.push("pusher", ExecutionContextActions.TESTING, None)
-    context5 = context.push("pusher", ExecutionContextActions.TESTING, None)
+    context2 = context.push("pusher", ContextActions.TESTING, None)
+    context3 = context2.push("pusher", ContextActions.TESTING, None)
+    context4 = context3.push("pusher", ContextActions.TESTING, None)
+    context5 = context.push("pusher", ContextActions.TESTING, None)
 
     context.global_hints.add("new_hint")
     assert context.global_hints == {"new_hint"}
@@ -75,11 +75,11 @@ def test_i_can_manage_global_hints(context):
 def test_i_can_manage_protected_hint(context):
     # Note that protected hint only works if the hint is added BEFORE the creation of the child
     context.protected_hints.add("new_hint")
-    context2 = context.push("pusher", ExecutionContextActions.TESTING, None)
-    context3 = context2.push("pusher", ExecutionContextActions.TESTING, None)
+    context2 = context.push("pusher", ContextActions.TESTING, None)
+    context3 = context2.push("pusher", ContextActions.TESTING, None)
     context3.protected_hints.add("another_hint")
-    context4 = context3.push("pusher", ExecutionContextActions.TESTING, None)
-    context5 = context.push("pusher", ExecutionContextActions.TESTING, None)
+    context4 = context3.push("pusher", ContextActions.TESTING, None)
+    context5 = context.push("pusher", ContextActions.TESTING, None)
 
     assert context.protected_hints == {"new_hint"}
     assert context2.protected_hints == {"new_hint"}
@@ -90,11 +90,11 @@ def test_i_can_manage_protected_hint(context):
 
 def test_i_can_manage_private_hints(context):
     context.private_hints.add("new_hint")
-    context2 = context.push("pusher", ExecutionContextActions.TESTING, None)
-    context3 = context2.push("pusher", ExecutionContextActions.TESTING, None)
+    context2 = context.push("pusher", ContextActions.TESTING, None)
+    context3 = context2.push("pusher", ContextActions.TESTING, None)
     context3.private_hints.add("another_hint")
-    context4 = context3.push("pusher", ExecutionContextActions.TESTING, None)
-    context5 = context.push("pusher", ExecutionContextActions.TESTING, None)
+    context4 = context3.push("pusher", ContextActions.TESTING, None)
+    context5 = context.push("pusher", ContextActions.TESTING, None)
 
     assert context.private_hints == {"new_hint"}
     assert context2.private_hints == set()
@@ -103,10 +103,24 @@ def test_i_can_manage_private_hints(context):
     assert context5.private_hints == set()
 
 
+def test_i_can_check_if_hints_are_in_context(context):
+    context.private_hints.add("private_hint")
+    context.protected_hints.add("protected_hint")
+    context.global_hints.add("global_hint")
+    assert context.in_context("private_hint")
+    assert context.in_context("protected_hint")
+    assert context.in_context("global_hint")
+
+    context2 = context.push("pusher", ContextActions.TESTING, None)
+    assert not context2.in_context("private_hint")
+    assert context2.in_context("protected_hint")
+    assert context2.in_context("global_hint")
+
+
 def test_i_can_keep_track_of_children(context):
-    context2 = context.push("pusher", ExecutionContextActions.TESTING, None)
-    context3 = context.push("pusher", ExecutionContextActions.TESTING, None)
-    context4 = context2.push("pusher2", ExecutionContextActions.TESTING, None)
+    context2 = context.push("pusher", ContextActions.TESTING, None)
+    context3 = context.push("pusher", ContextActions.TESTING, None)
+    context4 = context2.push("pusher2", ContextActions.TESTING, None)
 
     assert len(context._children) == 2
     assert len(context2._children) == 1
@@ -115,13 +129,13 @@ def test_i_can_keep_track_of_children(context):
 
 
 def test_i_can_get_children(context):
-    context1 = context.push("child 1", ExecutionContextActions.TESTING, None)
-    context2 = context.push("child 2", ExecutionContextActions.TESTING, None)
-    context3 = context.push("child 3", ExecutionContextActions.TESTING, None)
-    context21 = context2.push("child 21", ExecutionContextActions.TESTING, None)
-    context22 = context2.push("child 22", ExecutionContextActions.TESTING, None)
-    context211 = context21.push("child 211", ExecutionContextActions.TESTING, None)
-    context31 = context3.push("child 31", ExecutionContextActions.TESTING, None)
+    context1 = context.push("child 1", ContextActions.TESTING, None)
+    context2 = context.push("child 2", ContextActions.TESTING, None)
+    context3 = context.push("child 3", ContextActions.TESTING, None)
+    context21 = context2.push("child 21", ContextActions.TESTING, None)
+    context22 = context2.push("child 22", ContextActions.TESTING, None)
+    context211 = context21.push("child 211", ContextActions.TESTING, None)
+    context31 = context3.push("child 31", ContextActions.TESTING, None)
 
     assert list(context1.get_children()) == []
 
@@ -149,3 +163,15 @@ def test_i_can_get_children(context):
         context3,
         context31,
     ]
+
+
+def test_i_can_get_parents(context):
+    context1 = context.push("child 1", ContextActions.TESTING, None)
+    context2 = context1.push("child 2", ContextActions.TESTING, None)
+    context3 = context2.push("child 3", ContextActions.TESTING, None)
+
+    assert list(context3.get_parents()) == [context2, context1, context]
+    assert list(context3.get_parents(level=1)) == [context2]
+    assert list(context3.get_parents(level=2)) == [context2, context1]
+    assert list(context3.get_parents(level=3)) == [context2, context1, context]
+    assert list(context3.get_parents(level=4)) == [context2, context1, context]

tests/evaluators/test_PythonEvaluator.py

@@ -0,0 +1,58 @@
+import pytest
+
+from base import BaseTest
+from conftest import NewOntology
+from core.BuiltinConcepts import BuiltinConcepts
+from core.error import ErrorContext
+from evaluators.PythonEvaluator import PythonEvaluator
+from evaluators.PythonParser import PythonParser
+from helpers import _rv, _rvf, define_new_concept, get_concepts, get_metadata
+from parsers.ParserInput import ParserInput
+
+
+def get_parser_input_from(sheerka, context, command):
+    pi = ParserInput(command)
+    pi.init()
+    parser_start = _rv(sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=pi))
+    ret = PythonParser().eval(context, None, parser_start)
+    return ret.new[0]
+
+
+class TestPythonEvaluator(BaseTest):
+    @pytest.fixture()
+    def evaluator(self, sheerka):
+        return sheerka.evaluators[PythonEvaluator.NAME]
+
+    def test_i_can_match(self, sheerka, context, evaluator):
+        ret_val = _rv(sheerka.newn(BuiltinConcepts.PYTHON_CODE))
+        assert evaluator.matches(context, ret_val).status is True
+
+        ret_val = _rv(sheerka.newn(BuiltinConcepts.UNKNOWN_CONCEPT))  # it responds to USER_INPUT only
+        assert evaluator.matches(context, ret_val).status is False
+
+        ret_val = _rvf(sheerka.newn(BuiltinConcepts.PYTHON_CODE))  # status should be true
+        assert evaluator.matches(context, ret_val).status is False
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1 + 1", 2),
+        ("echo('I have access to Sheerka !')", "I have access to Sheerka !"),
+        ("sheerka.echo('I have access to Sheerka !')", "I have access to Sheerka !"),
+        ("a=10\na", 10),
+    ])
+    def test_i_can_evaluate_simple_expression(self, sheerka, context, evaluator, text, expected):
+        start = get_parser_input_from(sheerka, context, text)
+        ret = evaluator.eval(context, None, start)
+        assert ret.eaten == [start]
+        assert len(ret.new) == 1
+        assert ret.new[0].status is True
+        assert ret.new[0].value == expected
+        assert ret.new[0].parents == [start]
+
+    def test_i_can_detect_evaluation_error(self, sheerka, context, evaluator):
+        start = get_parser_input_from(sheerka, context, "a")
+        ret = evaluator.eval(context, None, start)
+        assert ret.eaten == []
+        assert len(ret.new) == 1
+        assert ret.new[0].status is False
+        assert isinstance(ret.new[0].value, ErrorContext)
+        assert ret.new[0].parents == [start]

tests/evaluators/test_PythonParser.py

@@ -0,0 +1,75 @@
+import pytest
+
+from base import BaseTest
+from core.BuiltinConcepts import BuiltinConcepts
+from core.error import ErrorContext
+from evaluators.PythonParser import PythonParser
+from helpers import _rv, _rvf
+from parsers.ParserInput import ParserInput
+
+
+class TestPythonParser(BaseTest):
+    @pytest.fixture()
+    def evaluator(self, sheerka):
+        return sheerka.evaluators[PythonParser.NAME]
+
+    def test_i_can_match(self, sheerka, context, evaluator):
+        ret_val = _rv(sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=ParserInput("a command")))
+        assert evaluator.matches(context, ret_val).status is True
+
+        ret_val = _rv(sheerka.newn(BuiltinConcepts.UNKNOWN_CONCEPT))  # it responds to USER_INPUT only
+        assert evaluator.matches(context, ret_val).status is False
+
+        ret_val = _rvf(sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=ParserInput("a command")))  # status should be true
+        assert evaluator.matches(context, ret_val).status is False
+
+    @pytest.mark.parametrize("text", [
+        "1 + 1",
+        "a = 20"
+    ])
+    def test_i_can_parse_python(self, sheerka, context, evaluator, text):
+        pi = ParserInput(text)
+        pi.init()
+        start = _rv(sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=pi))
+
+        res = evaluator.eval(context, None, start)
+
+        assert res.eaten == [start]
+        assert len(res.new) == 1
+        ret_val = res.new[0]
+        assert ret_val.status is True
+        assert sheerka.isinstance(ret_val.value, BuiltinConcepts.PYTHON_CODE)
+        assert ret_val.parents == [start]
+
+    def test_invalid_python_are_rejected(self, sheerka, context, evaluator):
+        text = "1 + "
+        pi = ParserInput(text)
+        pi.init()
+        start = _rv(sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=pi))
+
+        res = evaluator.eval(context, None, start)
+
+        assert res.eaten == []
+        assert len(res.new) == 1
+        ret_val = res.new[0]
+        assert ret_val.status is False
+        assert isinstance(ret_val.value, ErrorContext)
+        assert ret_val.parents == [start]
+
+    def test_i_can_detect_concepts(self, sheerka, context, evaluator):
+        pi = ParserInput("c:one: + c:two:")
+        pi.init()
+        start = _rv(sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=pi))
+
+        res = evaluator.eval(context, None, start)
+
+        assert res.eaten == [start]
+        assert len(res.new) == 1
+        ret_val = res.new[0]
+        assert ret_val.status is True
+        assert sheerka.isinstance(ret_val.value, BuiltinConcepts.PYTHON_CODE)
+        assert ret_val.parents == [start]
+        assert len(ret_val.value.pf.namespace) == 2
+        assert ret_val.value.pf.namespace["__C__KEY_one__ID_00None00__C__"].value == ("one", None)
+        assert ret_val.value.pf.namespace["__C__KEY_two__ID_00None00__C__"].value == ("two", None)
+

tests/evaluators/test_error.py

@@ -0,0 +1,22 @@
+import pytest
+
+from core.error import ErrorContext, ErrorObj, MethodAccessError
+
+
+class DummyErrorObj(ErrorObj):
+    def __init__(self, msg):
+        self.msg = msg
+
+    def get_error_msg(self) -> str:
+        return self.msg
+
+
+@pytest.mark.parametrize("error_hint, expected", [
+    ("some value", "some value"),
+    (["value a", "value b"], "value a, value b"),
+    (MethodAccessError("a"), "Cannot access method 'a'"),
+    (DummyErrorObj("error msg"), "error msg")
+])
+def test_i_can_get_error_msg(context, error_hint, expected):
+    error = ErrorContext("Test", context, error_hint)
+    assert error.get_error_msg() == expected

tests/helpers.py

@@ -2,7 +2,7 @@ from common.global_symbols import NotInit
 from core.ExecutionContext import ExecutionContext
 from core.ReturnValue import ReturnValue
 from core.concept import Concept, ConceptMetadata, DefinitionType
-from core.services.SheerkaConceptManager import ConceptManager
+from services.SheerkaConceptManager import ConceptManager
 
 
 class GetNextId:
@@ -294,8 +294,6 @@ def get_metadatas(*args, **kwargs):
 def get_concepts(context: ExecutionContext, *concepts, **kwargs) -> list[Concept]:
     """
     Simple and quick way to get initialize concepts for a test
-    :param sheerka:
-    :type sheerka:
     :param context:
     :type context:
     :param concepts:
@@ -322,7 +320,7 @@ def get_concepts(context: ExecutionContext, *concepts, **kwargs) -> list[Concept
     return res
 
 
-def define_new_concept(context: ExecutionContext, c: str | Concept) -> Concept:
+def define_new_concept(context: ExecutionContext, c: str | Concept | ConceptMetadata) -> Concept:
     sheerka = context.sheerka
     if isinstance(c, str):
         retval = sheerka.define_new_concept(context, c)

tests/non_reg/test_non_reg1.py

@@ -0,0 +1,18 @@
+from base import BaseTest
+
+
+class TestNonReg1(BaseTest):
+
+    def test_i_can_evaluate_python(self, sheerka, user):
+        res = sheerka.evaluate_user_input("1 + 1", user)
+        assert len(res) == 1
+        ret_val = res[0]
+        assert ret_val.status is True
+        assert ret_val.value == 2
+
+    def test_i_can_evaluate_variable_that_is_not_defined(self, sheerka, user):
+        res = sheerka.evaluate_user_input("a", user)
+        assert len(res) == 1
+        ret_val = res[0]
+
+        assert ret_val.status is False

tests/parsers/test_parser_input.py

@@ -1,5 +1,7 @@
+import pytest
+
 from parsers.ParserInput import ParserInput
-from parsers.tokenizer import LexerError
+from parsers.tokenizer import LexerError, TokenKind
 
 
 def test_i_can_parser_input():
@@ -12,3 +14,27 @@ def test_i_can_detect_errors():
     parser_input = ParserInput('def concept "a')
     assert parser_input.init() is False
     assert isinstance(parser_input.exception, LexerError)
+
+
+def test_can_as_text_and_track_tokens():
+    parser_input = ParserInput("execute(c:name1: if r:#id: else c:name2:)")
+    parser_input.init()
+
+    switcher = {TokenKind.CONCEPT: lambda t: f"__CONCEPT__{t.value[0]}",
+                TokenKind.RULE: lambda t: f"__RULE__{t.value[1]}"}
+    tracker = {}
+    text = parser_input.as_text(switcher, tracker)
+
+    assert text == "execute(__CONCEPT__name1 if __RULE__id else __CONCEPT__name2)"
+    assert len(tracker) == 3
+    assert tracker["__CONCEPT__name1"] == parser_input.all_tokens[2]
+    assert tracker["__RULE__id"] == parser_input.all_tokens[6]
+    assert tracker["__CONCEPT__name2"] == parser_input.all_tokens[10]
+
+
+def test_i_must_call_init_before_call_as_text():
+    parser_input = ParserInput("execute(c:name1: if r:#id: else c:name2:)")
+    with pytest.raises(Exception) as ex:
+        parser_input.as_text()
+
+    assert ex.value.args[0] == "You must call init() first !"

tests/services/test_ConceptEvaluator.py

@@ -0,0 +1,101 @@
+import pytest
+
+from base import BaseTest
+from common.global_symbols import NotInit
+from conftest import NewOntology
+from core.concept import ConceptDefaultProps
+from core.python_fragment import PythonFragment
+from helpers import define_new_concept, get_metadata
+from services.SheerkaConceptEvaluator import ConceptEvaluator
+from services.SheerkaPython import EvaluationRef
+
+
+class TestConceptManager(BaseTest):
+
+    @pytest.fixture()
+    def service(self, sheerka) -> ConceptEvaluator:
+        return sheerka.services[ConceptEvaluator.NAME]
+
+    def test_i_can_build_concept(self, context, service):
+        metadata = get_metadata(
+            name="foo",
+            where="isinstance(x, Concept)",
+            pre="in_context(IS_QUESTION)",
+            body="one + a",
+            post="'post parameter'",
+            ret="self",
+            variables=(("a", "1"), ("b", "NotInit"))
+        )
+
+        compiled = service.build(context, metadata)
+        pf = getattr(compiled, ConceptDefaultProps.WHERE)
+        assert isinstance(pf, PythonFragment)
+        assert pf.source_code == metadata.where
+
+        pf = getattr(compiled, ConceptDefaultProps.PRE)
+        assert isinstance(pf, PythonFragment)
+        assert pf.source_code == metadata.pre
+
+        pf = getattr(compiled, ConceptDefaultProps.BODY)
+        assert isinstance(pf, PythonFragment)
+        assert pf.source_code == metadata.body
+
+        pf = getattr(compiled, ConceptDefaultProps.POST)
+        assert isinstance(pf, PythonFragment)
+        assert pf.source_code == metadata.post
+
+        pf = getattr(compiled, ConceptDefaultProps.RET)
+        assert isinstance(pf, PythonFragment)
+        assert pf.source_code == metadata.ret
+
+        pf = getattr(compiled, "a")
+        assert isinstance(pf, PythonFragment)
+        assert pf.source_code == metadata.variables[0][1]
+
+        pf = getattr(compiled, "b")
+        assert isinstance(pf, PythonFragment)
+        assert pf.source_code == metadata.variables[1][1]
+
+    def test_i_can_manage_when_no_source_code(self, context, service):
+        metadata = get_metadata(name="foo")
+
+        compiled = service.build(context, metadata)
+        assert getattr(compiled, ConceptDefaultProps.WHERE) is None
+        assert getattr(compiled, ConceptDefaultProps.PRE) is None
+        assert getattr(compiled, ConceptDefaultProps.BODY) is None
+        assert getattr(compiled, ConceptDefaultProps.POST) is None
+        assert getattr(compiled, ConceptDefaultProps.RET) is None
+
+    def test_i_can_detect_when_requested_names_are_concept_variables(self, context, service):
+        metadata = get_metadata(
+            name="foo",
+            body="one + a",
+            variables=(("a", "1"), ("b", "NotInit")))
+
+        compiled = service.build(context, metadata)
+        pf = getattr(compiled, ConceptDefaultProps.BODY)
+        assert isinstance(pf, PythonFragment)
+        assert pf.namespace == {"a": EvaluationRef("self", "a"),
+                                "b": EvaluationRef("self", "b")}
+
+    def test_i_can_eval_concept_attributes(self, context, service):
+        with NewOntology(context, "test_i_can_eval_concept_attributes"):
+            foo_metadata = get_metadata(name="foo",
+                                        where="isinstance(a, int)",
+                                        pre="True",
+                                        body="2 + a",
+                                        post="'post parameter'",
+                                        ret="self",
+                                        variables=(("a", "1"), ("b", "NotInit")))
+            foo = define_new_concept(context, foo_metadata)
+
+            res = service.evaluate_concept(context, foo)
+
+            assert context.sheerka.isinstance(res, foo)
+            assert res.get_value("a") == 1
+            assert res.get_value("b") == NotInit
+            assert res.get_value(ConceptDefaultProps.WHERE) is True
+            assert res.get_value(ConceptDefaultProps.PRE) is True
+            assert res.get_value(ConceptDefaultProps.BODY) == 3
+            assert res.get_value(ConceptDefaultProps.POST) == "post parameter"
+            assert res.get_value(ConceptDefaultProps.RET) == res

tests/services/test_ConceptManager.py

@@ -4,9 +4,9 @@ from base import BaseTest
 from common.global_symbols import NotFound, NotInit
 from conftest import NewOntology
 from core.BuiltinConcepts import BuiltinConcepts
-from core.ErrorContext import ErrorContext
 from core.concept import ConceptMetadata
-from core.services.SheerkaConceptManager import ConceptAlreadyDefined, ConceptManager
+from core.error import ErrorContext
+from services.SheerkaConceptManager import ConceptAlreadyDefined, ConceptManager
 from helpers import get_metadata
 
 
@@ -24,7 +24,7 @@ class TestConceptManager(BaseTest):
         """
         metadata = get_metadata("foo", "body")
         digest = service.compute_metadata_digest(metadata)
-        assert digest == "21a1c2f420da62f4dc60f600c95b19dd9527b19dd28fd38e17f5c0e28963d176"
+        assert digest == "7c0f1708968e0312be622950d3f21d588f718f7ba568054ece64d077052a6476"
 
         another_metadata = get_metadata("foo", "body")
         other_digest = service.compute_metadata_digest(another_metadata)
@@ -86,7 +86,7 @@ class TestConceptManager(BaseTest):
             assert metadata.name == "name"
             assert metadata.key == "name"
             assert metadata.body == "body"
-            assert metadata.digest == "eb0620bd4a317af8a403c0ae1e185a528f9b58f8b0878d990e62278f89cf10d5"
+            assert metadata.digest == "c75faa4efbc9ef9dbc5174c52786d5b066e2ece41486b81c27336e292917fecb"
             assert metadata.all_attrs == ('#where#', '#pre#', '#post#', '#body#', '#ret#')
 
             # is sorted in db
@@ -117,6 +117,11 @@ class TestConceptManager(BaseTest):
             res = service.define_new_concept(context, "name", body="body")
             assert res.status is True
 
+    def test_i_cannot_get_by_if_concept_does_not_exist(self, service):
+        assert service.get_by_id("unresolved_id") == NotFound
+        assert service.get_by_name("unresolved name") == NotFound
+        assert service.get_by_key("unresolved_hash") == NotFound
+
     def test_i_can_get_a_newly_created_concept(self, context, service):
         with NewOntology(context, "test_i_can_get_a_newly_created_concept"):
             res = service.define_new_concept(context, "name", body="body")
@@ -141,6 +146,19 @@ class TestConceptManager(BaseTest):
             assert foo.var1 == "value1"
             assert foo.var2 == "value2"
 
+    def test_i_can_manage_when_concepts_with_the_same_name(self, context, service):
+        with NewOntology(context, "test_i_can_manage_when_concepts_with_the_same_name"):
+            service.define_new_concept(context, "foo", body="body1")
+            service.define_new_concept(context, "foo", body="body2")
+
+            concepts = service.newn("foo")
+
+            assert len(concepts) == 2
+            assert concepts[0].name == "foo"
+            assert concepts[0].get_metadata().body == "body1"
+            assert concepts[1].name == "foo"
+            assert concepts[1].get_metadata().body == "body2"
+
     def test_i_can_instantiate_a_new_concept_by_its_id(self, context, service):
         with NewOntology(context, "test_i_can_instantiate_a_new_concept_by_its_id"):
             res = service.define_new_concept(context, "foo", variables=[("var1", None), ("var2", None)])
@@ -184,3 +202,64 @@ class TestConceptManager(BaseTest):
 
         context.sheerka.om.pop_ontology(context)
         assert service.get_by_id(res.value.metadata.id) is NotFound
+
+    def test_i_can_new(self, context, service):
+        with NewOntology(context, "test_i_can_new"):
+            res = service.define_new_concept(context, "name", body="body", variables=[("my_var", None)])
+            assert res.status
+            metadata = res.value.metadata
+
+            # I can create a new concept
+            res = service.new(metadata, my_var="my_var_value")
+            assert res.id == metadata.id
+            assert res.my_var == "my_var_value"
+
+            res = service.new((metadata.name, None), my_var="my_var_value")
+            assert res.id == metadata.id
+            assert res.my_var == "my_var_value"
+
+            res = service.new((None, metadata.id), my_var="my_var_value")
+            assert res.id == metadata.id
+            assert res.my_var == "my_var_value"
+
+            res = service.new("c:name:", my_var="my_var_value")
+            assert res.id == metadata.id
+            assert res.my_var == "my_var_value"
+
+            res = service.new("c:#1001:", my_var="my_var_value")
+            assert res.id == metadata.id
+            assert res.my_var == "my_var_value"
+
+            res = service.new("c:name#1001:", my_var="my_var_value")
+            assert res.id == metadata.id
+            assert res.my_var == "my_var_value"
+
+            # cannot new using id
+            assert service.new(f"1001").name == BuiltinConcepts.UNKNOWN_CONCEPT
+
+    def test_id_is_used_when_name_and_id_are_provided(self, context, service):
+        with NewOntology(context, "test_id_is_used_when_name_and_id_are_provided"):
+            res = service.define_new_concept(context, "name", body="body1")
+            metadata = res.value.metadata
+            service.define_new_concept(context, "name", body="body2")
+
+            assert service.new((metadata.name, metadata.id)).id == metadata.id
+
+    def test_unknown_concept_is_return_if_the_identifier_is_not_found(self, service):
+        assert service.new("unknown").name == BuiltinConcepts.UNKNOWN_CONCEPT
+
+    def test_can_get_all_concepts(self, context, service):
+        with NewOntology(context, "test_i_can_new"):
+            service.define_new_concept(context, "foo")
+            service.define_new_concept(context, "bar")
+            context.sheerka.om.push_ontology("another ontology")
+            service.define_new_concept(context, "baz")
+            service.define_new_concept(context, "qux")
+
+            all_concepts = service.get_all_concepts()
+            assert [c.name for c in all_concepts if not c.is_builtin] == ["foo", "bar", "baz", "qux"]
+
+            # sanity check. Concepts are discarded when ontology is popped
+            context.sheerka.om.pop_ontology(context)
+            all_concepts = service.get_all_concepts()
+            assert [c.name for c in all_concepts if not c.is_builtin] == ["foo", "bar"]

tests/services/test_SheerkaAdmin.py

@@ -0,0 +1,29 @@
+import pytest
+
+from base import BaseTest
+from services.SheerkaAdmin import SheerkaAdmin
+from helpers import get_concepts
+
+
+class TestConceptManager(BaseTest):
+
+    @pytest.fixture()
+    def service(self, sheerka):
+        return sheerka.services[SheerkaAdmin.NAME]
+
+    def test_i_can_test_extended_is_admin(self, context, service):
+        foo, bar = get_concepts(context, "foo", "bar", use_sheerka=True)
+
+        foo1 = context.sheerka.newn("foo")
+
+        assert service.extended_isinstance(1, int)
+        assert service.extended_isinstance(foo, "foo")
+        assert service.extended_isinstance(foo, foo1)
+        assert service.extended_isinstance(foo, foo1.get_metadata())
+        assert service.extended_isinstance(foo, "c:#1001:")
+
+        assert not service.extended_isinstance("1", int)
+        assert not service.extended_isinstance(foo, "bar")
+        assert not service.extended_isinstance(foo, bar)
+        assert not service.extended_isinstance(foo, bar.get_metadata())
+        assert not service.extended_isinstance(foo, "c:#1002:")

tests/services/test_SheerkaEngine.py

@@ -4,28 +4,27 @@ import pytest
 
 from base import BaseTest
 from core.BuiltinConcepts import BuiltinConcepts
-from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+from core.ExecutionContext import ExecutionContext, ContextActions
 from core.ReturnValue import ReturnValue
-from core.services.SheerkaEngine import SheerkaEngine
 from evaluators.CreateParserInput import CreateParserInput
 from evaluators.base_evaluator import AllReturnValuesEvaluator, BaseEvaluator, EvaluatorEvalResult, \
-    EvaluatorMatchResult, \
-    OneReturnValueEvaluator
+    EvaluatorMatchResult, OneReturnValueEvaluator
 from helpers import _rvc
+from services.SheerkaEngine import SheerkaEngine
 
 ALL_STEPS = [
-    ExecutionContextActions.BEFORE_PARSING,
-    ExecutionContextActions.PARSING,
-    ExecutionContextActions.AFTER_PARSING,
-    ExecutionContextActions.BEFORE_EVALUATION,
-    ExecutionContextActions.EVALUATION,
-    ExecutionContextActions.AFTER_EVALUATION
+    ContextActions.BEFORE_PARSING,
+    ContextActions.PARSING,
+    ContextActions.AFTER_PARSING,
+    ContextActions.BEFORE_EVALUATION,
+    ContextActions.EVALUATION,
+    ContextActions.AFTER_EVALUATION
 ]
 
 
 class OneReturnValueEvaluatorForTesting(OneReturnValueEvaluator):
     def __init__(self, name,
-                 step: ExecutionContextActions,
+                 step: ContextActions,
                  priority: int,
                  enabled=True,
                  match: bool | Callable = True,
@@ -56,12 +55,12 @@ class OneReturnValueEvaluatorForTesting(OneReturnValueEvaluator):
                 if ret_val != return_value:
                     ret_val.parents = [return_value]
 
-        return EvaluatorEvalResult(self.eval_result, self.eval_eaten or [return_value])
+        return EvaluatorEvalResult(self.eval_result, [return_value] if self.eval_eaten is None else self.eval_eaten)
 
 
 class AllReturnValuesEvaluatorForTesting(AllReturnValuesEvaluator):
     def __init__(self, name,
-                 step: ExecutionContextActions,
+                 step: ContextActions,
                  priority: int,
                  enabled=True,
                  match: bool | Callable = True,
@@ -91,31 +90,31 @@ class AllReturnValuesEvaluatorForTesting(AllReturnValuesEvaluator):
             for ret_val in self.eval_result:
                 ret_val.parents = return_values
 
-        return EvaluatorEvalResult(self.eval_result, self.eval_eaten or return_values)
+        return EvaluatorEvalResult(self.eval_result, return_values if self.eval_eaten is None else self.eval_eaten)
 
 
 class TestSheerkaEngine(BaseTest):
     @pytest.fixture()
     def service(self, sheerka):
-        return SheerkaEngine(sheerka)
+        return SheerkaEngine(sheerka)  # I want a new instance to keep Sheerka clean (when a change execution_plan)
 
     def test_i_can_compute_execution_plan(self, service):
         assert service.compute_execution_plan([]) == {}
 
-        e1 = BaseEvaluator("eval1", ExecutionContextActions.BEFORE_EVALUATION, 5)
-        e2 = BaseEvaluator("eval2", ExecutionContextActions.BEFORE_EVALUATION, 5)
-        e3 = BaseEvaluator("eval3", ExecutionContextActions.BEFORE_EVALUATION, 10)
-        e4 = BaseEvaluator("eval4", ExecutionContextActions.EVALUATION, 10)
-        e5 = BaseEvaluator("eval5", ExecutionContextActions.AFTER_EVALUATION, 10, enabled=False)
+        e1 = BaseEvaluator("eval1", ContextActions.BEFORE_EVALUATION, 5)
+        e2 = BaseEvaluator("eval2", ContextActions.BEFORE_EVALUATION, 5)
+        e3 = BaseEvaluator("eval3", ContextActions.BEFORE_EVALUATION, 10)
+        e4 = BaseEvaluator("eval4", ContextActions.EVALUATION, 10)
+        e5 = BaseEvaluator("eval5", ContextActions.AFTER_EVALUATION, 10, enabled=False)
         res = service.compute_execution_plan([e1, e2, e3, e4, e5])
-        assert res == {ExecutionContextActions.BEFORE_EVALUATION: {5: [e1, e2], 10: [e3]},
-                       ExecutionContextActions.EVALUATION: {10: [e4]}}
+        assert res == {ContextActions.BEFORE_EVALUATION: {5: [e1, e2], 10: [e3]},
+                       ContextActions.EVALUATION: {10: [e4]}}
 
     def test_i_can_call_execute(self, sheerka, context, service):
-        service.execution_plan = {ExecutionContextActions.BEFORE_EVALUATION: {50: [CreateParserInput()]}}
+        service.execution_plan = {ContextActions.BEFORE_EVALUATION: {50: [CreateParserInput()]}}
         start = [ReturnValue("TestSheerkaEngine", True, sheerka.newn(BuiltinConcepts.USER_INPUT, command="1 + 1"))]
 
-        ret = service.execute(context, start, [ExecutionContextActions.BEFORE_EVALUATION])
+        ret = service.execute(context, start, [ContextActions.BEFORE_EVALUATION])
         assert len(ret) == 1
         ret = ret[0]
         assert isinstance(ret, ReturnValue)
@@ -127,7 +126,7 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = {}
         start = [_rvc("foo")]
 
-        ret = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        ret = service.execute(context, start, [ContextActions.EVALUATION])
 
         assert ret == start
 
@@ -135,12 +134,12 @@ class TestSheerkaEngine(BaseTest):
         # properly init the service
         _ = OneReturnValueEvaluatorForTesting
         evaluators = [
-            _("eval1", ExecutionContextActions.AFTER_PARSING, 21, match=False),
-            _("eval2", ExecutionContextActions.BEFORE_EVALUATION, 5, match=False),
-            _("eval3", ExecutionContextActions.AFTER_EVALUATION, 12, match=False),
-            _("eval4", ExecutionContextActions.EVALUATION, 99, match=False),
-            _("eval5", ExecutionContextActions.BEFORE_PARSING, 5, match=False),
-            _("eval6", ExecutionContextActions.PARSING, 25, match=False),
+            _("eval1", ContextActions.AFTER_PARSING, 21, match=False),
+            _("eval2", ContextActions.BEFORE_EVALUATION, 5, match=False),
+            _("eval3", ContextActions.AFTER_EVALUATION, 12, match=False),
+            _("eval4", ContextActions.EVALUATION, 99, match=False),
+            _("eval5", ContextActions.BEFORE_PARSING, 5, match=False),
+            _("eval6", ContextActions.PARSING, 25, match=False),
         ]
         service.execution_plan = service.compute_execution_plan(evaluators)
 
@@ -155,15 +154,15 @@ class TestSheerkaEngine(BaseTest):
         # properly init the service
         _ = OneReturnValueEvaluatorForTesting
         evaluators = [
-            _("eval1", ExecutionContextActions.EVALUATION, 20, match=False),
-            _("eval2", ExecutionContextActions.EVALUATION, 5, match=False),
-            _("eval3", ExecutionContextActions.EVALUATION, 20, match=False),
-            _("eval4", ExecutionContextActions.EVALUATION, 99, match=False),
+            _("eval1", ContextActions.EVALUATION, 20, match=False),
+            _("eval2", ContextActions.EVALUATION, 5, match=False),
+            _("eval3", ContextActions.EVALUATION, 20, match=False),
+            _("eval4", ContextActions.EVALUATION, 99, match=False),
         ]
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [_rvc("foo")]
-        service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        service.execute(context, start, [ContextActions.EVALUATION])
 
         # to check what happened, look at the execution context children
         evaluators_executed = [ec.action_context["evaluator"] for ec in context.get_children() if
@@ -176,7 +175,7 @@ class TestSheerkaEngine(BaseTest):
         _ = OneReturnValueEvaluatorForTesting
         evaluators = [
             _("eval1",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda r: context.sheerka.isinstance(r.value, "foo"),
               eval_result=[rv_bar])
@@ -184,8 +183,8 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [rv_foo]
-        service.execute(context, start, [ExecutionContextActions.EVALUATION])
-        children = [ec for ec in context.get_children() if ec.action == ExecutionContextActions.EVALUATING_ITERATION]
+        service.execute(context, start, [ContextActions.EVALUATION])
+        children = [ec for ec in context.get_children() if ec.action == ContextActions.EVALUATING_ITERATION]
         assert len(children) == 2
 
     def test_eval_is_not_called_if_match_fails_for_one_return(self, context, service):
@@ -193,7 +192,7 @@ class TestSheerkaEngine(BaseTest):
         _ = OneReturnValueEvaluatorForTesting
         evaluators = [
             _("eval1",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda r: context.sheerka.isinstance(r.value, "foo"),
               eval_result=[_rvc("bar")])
@@ -201,7 +200,7 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [_rvc("baz")]
-        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        res = service.execute(context, start, [ContextActions.EVALUATION])
         assert res == start
 
         # check what happen in details
@@ -214,7 +213,7 @@ class TestSheerkaEngine(BaseTest):
         _ = OneReturnValueEvaluatorForTesting
         evaluators = [
             _("eval1",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda r: context.sheerka.isinstance(r.value, "foo"),
               eval_result=[_rvc("bar")])
@@ -222,7 +221,7 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [_rvc("foo")]
-        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        res = service.execute(context, start, [ContextActions.EVALUATION])
         assert res == [_rvc("bar")]
         assert res[0].parents == start
 
@@ -238,7 +237,7 @@ class TestSheerkaEngine(BaseTest):
         _ = OneReturnValueEvaluatorForTesting
         evaluators = [
             _("eval1",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda r: context.sheerka.isinstance(r.value, "foo"),
               eval_result=[rv_qux])
@@ -246,7 +245,7 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [rv_bar, rv_foo, rv_baz]
-        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        res = service.execute(context, start, [ContextActions.EVALUATION])
         assert res == [rv_bar, rv_qux, rv_baz]  # We must keep the order ! rv_qux replaces rv_foo
         assert res[0].parents is None
         assert res[1].parents == [rv_foo]
@@ -267,12 +266,12 @@ class TestSheerkaEngine(BaseTest):
         _ = OneReturnValueEvaluatorForTesting
         evaluators = [
             _("eval1",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda r: context.sheerka.isinstance(r.value, "foo"),
               eval_result=[rv_bar]),
             _("eval2",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda r: context.sheerka.isinstance(r.value, "foo"),
               eval_result=[rv_baz])
@@ -280,7 +279,7 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [rv_qux, rv_foo, rv_qux]
-        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        res = service.execute(context, start, [ContextActions.EVALUATION])
         assert res == [rv_qux, rv_bar, rv_baz, rv_qux]  # they both eat it !
         assert res[1].parents == [rv_foo]
         assert res[2].parents == [rv_foo]
@@ -293,12 +292,12 @@ class TestSheerkaEngine(BaseTest):
         _ = OneReturnValueEvaluatorForTesting
         evaluators = [
             _("eval1",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda r: context.sheerka.isinstance(r.value, "foo"),
               eval_result=[rv_bar]),
             _("eval2",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               30,
               match=lambda r: context.sheerka.isinstance(r.value, "foo"),
               eval_result=[rv_baz])
@@ -306,7 +305,7 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [rv_foo]
-        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        res = service.execute(context, start, [ContextActions.EVALUATION])
         assert res == [rv_baz]
         assert res[0].parents == start
 
@@ -315,7 +314,7 @@ class TestSheerkaEngine(BaseTest):
         _ = AllReturnValuesEvaluatorForTesting
         evaluators = [
             _("eval1",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda r: context.sheerka.isinstance(r[0].value, "foo"),
               eval_result=[_rvc("bar")])
@@ -323,11 +322,11 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [_rvc("baz")]
-        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        res = service.execute(context, start, [ContextActions.EVALUATION])
         assert res == start
 
         start = [_rvc("foo")]
-        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        res = service.execute(context, start, [ContextActions.EVALUATION])
         assert res == [_rvc("bar")]
         assert res[0].parents == start
 
@@ -338,7 +337,7 @@ class TestSheerkaEngine(BaseTest):
         _ = AllReturnValuesEvaluatorForTesting
         evaluators = [
             _("eval1",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda lst: context.sheerka.isinstance(lst[0].value, "foo"),
               eval_result=[rv_bar])
@@ -346,7 +345,7 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [rv_baz, rv_foo]  # foo is not the first in the list
-        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        res = service.execute(context, start, [ContextActions.EVALUATION])
         assert res == start
 
         # check what happen in details
@@ -360,7 +359,7 @@ class TestSheerkaEngine(BaseTest):
         _ = AllReturnValuesEvaluatorForTesting
         evaluators = [
             _("eval1",
-              ExecutionContextActions.EVALUATION,
+              ContextActions.EVALUATION,
               20,
               match=lambda lst: context.sheerka.isinstance(lst[0].value, "foo"),
               eval_result=[rv_bar])
@@ -368,7 +367,7 @@ class TestSheerkaEngine(BaseTest):
         service.execution_plan = service.compute_execution_plan(evaluators)
 
         start = [rv_foo, rv_baz]
-        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        res = service.execute(context, start, [ContextActions.EVALUATION])
         assert res == [rv_bar]
         assert res[0].parents == start
 
@@ -377,3 +376,28 @@ class TestSheerkaEngine(BaseTest):
         exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children()))
         evaluation_trace = exec_context.values["evaluation"]
         assert evaluation_trace == {"match": True, "new": res, "eaten": start}
+
+    def test_ret_val_not_removed_does_not_cause_infinite_recursion(self, context, service):
+        rv_foo, rv_bar = _rvc("foo"), _rvc("bar")  # rv => ReturnValue
+
+        # properly init the service
+        # both evaluator want to eat 'foo'
+        _ = OneReturnValueEvaluatorForTesting
+        evaluators = [
+            _("eval",
+              ContextActions.EVALUATION,
+              20,
+              match=lambda r: context.sheerka.isinstance(r.value, "foo"),
+              eval_result=[rv_bar], eval_eaten=[]),
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        # in the test, 'foo' produces 'bar', but is not removed
+        # during the second iteration, 'foo' still exists, so it will produce 'bar' again
+        # and so on...
+        # This test validate that the infinite loop is broken
+        start = [rv_foo]
+        res = service.execute(context, start, [ContextActions.EVALUATION])
+
+        assert res == [rv_bar]
+        assert res[0].parents == [rv_foo]

tests/services/test_SheerkaMemory.py

@@ -0,0 +1,75 @@
+import pytest
+
+from base import BaseTest, DummyObj
+from caching.FastCache import FastCache
+from core.ExecutionContext import ContextActions
+from services.SheerkaMemory import SheerkaMemory
+
+
+class TestSheerkaEngine(BaseTest):
+    @pytest.fixture()
+    def service(self, sheerka):
+        return SheerkaMemory(sheerka)  # I want a new instance to keep Sheerka clean (when I update stm)
+
+    def test_i_can_add_to_global_short_term_memory(self, service):
+        dummy = DummyObj()
+        service.add_to_short_term_memory(None, "a", dummy)
+
+        assert service.short_term_objects.copy() == {'global': {'a': dummy}}
+
+    def test_i_can_add_and_get_stm_data(self, context, service):
+        sub_context = context.push("TestSheerkaEngine", ContextActions.TESTING, None)
+
+        service.add_to_short_term_memory(None, "a", "global level")
+        service.add_to_short_term_memory(context, "a", "context level")
+        service.add_to_short_term_memory(sub_context, "a", "sub context level")
+
+        assert service.get_from_short_term_memory(sub_context, "a") == "sub context level"
+        assert service.get_from_short_term_memory(context, "a") == "context level"
+        assert service.get_from_short_term_memory(None, "a") == "global level"
+
+    def test_i_can_list_stm_data(self, context, service):
+        sub_context = context.push("TestSheerkaEngine", ContextActions.TESTING, None)
+
+        service.add_to_short_term_memory(None, "a", "global a")
+        service.add_to_short_term_memory(None, "b", "global b")
+        service.add_to_short_term_memory(context, "a", "context a")
+        service.add_to_short_term_memory(context, "c", "context c")
+        service.add_to_short_term_memory(sub_context, "d", "sub context d")
+        service.add_to_short_term_memory(sub_context, "a", "sub context a")
+
+        assert service.list_short_term_memory(sub_context) == {"a": "sub context a",
+                                                               "b": "global b",
+                                                               "c": "context c",
+                                                               "d": "sub context d"}
+
+        assert service.list_short_term_memory(context) == {"a": "context a",
+                                                           "b": "global b",
+                                                           "c": "context c"}
+
+        assert service.list_short_term_memory(None) == {"a": "global a",
+                                                        "b": "global b"}
+
+    def test_i_can_list_stm_data_when_context_have_no_entry(self, context, service):
+        sub_context = context.push("TestSheerkaEngine", ContextActions.TESTING, None)
+
+        service.add_to_short_term_memory(sub_context, "d", "sub context d")
+        service.add_to_short_term_memory(sub_context, "a", "sub context a")
+
+        assert service.list_short_term_memory(sub_context) == {"a": "sub context a", "d": "sub context d"}
+        assert service.list_short_term_memory(context) == {}
+        assert service.list_short_term_memory(None) == {}
+
+    def test_i_value_are_removed_when_cache_is_full(self, context, service):
+        service.short_term_objects = FastCache(3)
+        context1 = context.push("TestSheerkaEngine", ContextActions.TESTING, None)
+        context2 = context.push("TestSheerkaEngine", ContextActions.TESTING, None)
+        context3 = context.push("TestSheerkaEngine", ContextActions.TESTING, None)
+
+        service.add_to_short_term_memory(context, "a", "context")
+        service.add_to_short_term_memory(context1, "b", "context 1")
+        service.add_to_short_term_memory(context2, "c", "context 2")
+        assert context.id in service.short_term_objects
+
+        service.add_to_short_term_memory(context3, "d", "context 3")
+        assert context.id not in service.short_term_objects

tests/services/test_SheerkaPython.py

@@ -0,0 +1,127 @@
+import pytest
+
+from base import BaseTest, DummyObj
+from common.global_symbols import NoFirstToken, NotFound, NotInit, Removed
+from conftest import NewOntology
+from core.BuiltinConcepts import BuiltinConcepts
+from evaluators.PythonParser import PythonParser
+from helpers import _rv, define_new_concept, get_concepts, get_metadata
+from parsers.ParserInput import ParserInput
+from services.SheerkaPython import EvaluationRef, SheerkaPython
+
+
+def get_python_fragment(sheerka, context, command):
+    pi = ParserInput(command)
+    pi.init()
+    parser_start = _rv(sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=pi))
+    ret = PythonParser().eval(context, None, parser_start)
+    return ret.new[0].value.pf
+
+
+class TestSheerkaPython(BaseTest):
+    @pytest.fixture()
+    def service(self, sheerka) -> SheerkaPython:
+        return sheerka.services[SheerkaPython.NAME]
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1 + 1", 2),
+        ("echo('I have access to Sheerka !')", "I have access to Sheerka !"),
+        ("sheerka.echo('I have access to Sheerka !')", "I have access to Sheerka !"),
+        ("a=10\na", 10),
+        ("NotInit", NotInit),
+        ("NotFound", NotFound),
+        ("Removed", Removed),
+        ("NoFirstToken", NoFirstToken),
+    ])
+    def test_i_can_evaluate_simple_expression(self, sheerka, context, service, text, expected):
+        python_fragment = get_python_fragment(sheerka, context, text)
+        ret = service.evaluate_python(context, python_fragment)
+        assert ret == expected
+
+    def test_i_can_eval_isinstance_for_type(self, sheerka, context, service):
+        python_fragment = get_python_fragment(sheerka, context, "isinstance('some string', str)")
+        ret = service.evaluate_python(context, python_fragment)
+
+        assert ret is True
+
+    def test_i_can_eval_isinstance_for_concept(self, sheerka, context, service):
+        with NewOntology(context, "test_i_can_eval_isinstance_for_concept"):
+            get_concepts(context, "foo", use_sheerka=True)
+            python_fragment = get_python_fragment(sheerka, context, "isinstance(foo, 'foo')")
+            ret = service.evaluate_python(context, python_fragment)
+            assert ret is True
+
+            # 'foo' is also a Concept
+            python_fragment = get_python_fragment(sheerka, context, "isinstance(foo, Concept)")
+            ret = service.evaluate_python(context, python_fragment)
+            assert ret is True
+
+    def test_i_can_use_value_from_global_namespace(self, sheerka, context, service):
+        python_fragment = get_python_fragment(sheerka, context, "self.a")
+
+        ret = service.evaluate_python(context, python_fragment, {"self": DummyObj("my dummy value")})
+        assert ret == "my dummy value"
+
+    def test_i_can_eval_using_eval_ref(self, sheerka, context, service):
+        python_fragment = get_python_fragment(sheerka, context, "a")
+        python_fragment.namespace = {"a": EvaluationRef("self", "a")}
+
+        ret = service.evaluate_python(context, python_fragment, {"self": DummyObj("my dummy value")})
+        assert ret == "my dummy value"
+
+    @pytest.mark.skip("Concept evaluation is not implemented")
+    def test_i_can_eval_concept_properties(self, sheerka, context, service):
+        with NewOntology(context, "test_i_can_eval_concept_properties"):
+            foo_meta = get_metadata("foo", variables=[("a", "hello world")])
+            define_new_concept(context, foo_meta)
+
+            python_fragment = get_python_fragment(sheerka, context, "foo.a")
+            ret = service.evaluate_python(context, python_fragment)
+            assert ret == "hello world"
+
+    @pytest.mark.skip("Concept evaluation is not implemented")
+    def test_i_can_eval_python_mixed_with_concept(self, sheerka, context, service):
+        with NewOntology(context, "test_i_can_eval_python_mixed_with_concept"):
+            foo_meta = get_metadata("foo", variables=[("a", "1")])
+            bar_meta = get_metadata("bar", body="2")
+            get_concepts(context, foo_meta, bar_meta, use_sheerka=True)
+
+            python_fragment = get_python_fragment(sheerka, context, "bar + foo.a")
+            ret = service.evaluate_python(context, python_fragment)
+            assert ret == "3"
+
+    def test_i_can_remember_previous_results(self, sheerka, context, service):
+        python_fragment = get_python_fragment(sheerka, context, "a=10")
+        ret = service.evaluate_python(context, python_fragment)
+        assert ret is None
+
+        python_fragment = get_python_fragment(sheerka, context, "a")
+        ret = service.evaluate_python(context, python_fragment)
+        assert ret == 10
+
+    def test_i_can_import_module(self, sheerka, context, service):
+        python_fragment = get_python_fragment(sheerka, context, "import math")
+        ret = service.evaluate_python(context, python_fragment)
+        assert ret is None
+
+        python_fragment = get_python_fragment(sheerka, context, "math.sqrt(4)")
+        ret = service.evaluate_python(context, python_fragment)
+        assert ret == 2
+
+    def test_i_can_import_function_from_module(self, sheerka, context, service):
+        python_fragment = get_python_fragment(sheerka, context, "from math import sqrt")
+        ret = service.evaluate_python(context, python_fragment)
+        assert ret is None
+
+        python_fragment = get_python_fragment(sheerka, context, "sqrt(4)")
+        ret = service.evaluate_python(context, python_fragment)
+        assert ret == 2
+
+    def test_i_can_eval_when_context_is_needed(self, sheerka, context, service):
+        with NewOntology(context, "test_i_can_eval_when_context_is_needed"):
+            python_fragment = get_python_fragment(sheerka, context, "define_new_concept('foo')")
+            ret = service.evaluate_python(context, python_fragment)
+            assert sheerka.isinstance(ret.value, BuiltinConcepts.NEW_CONCEPT)
+            # for info, there are two level of value
+            # one for PythonEvaluator return value
+            # one for the ConceptManager return value

tests/sheerkapickle/test_sheerka_handlers.py

@@ -247,7 +247,7 @@ class TestSheerkaPickleHandler(BaseTest):
     def test_i_can_encode_decode_execution_context(self):
         sheerka = self.get_sheerka()
         c = Concept("foo").def_var("a")
-        context = ExecutionContext("who", Event("xxx"), sheerka, BuiltinConcepts.EVALUATE_CONCEPT, c, "my desc")
+        context = ExecutionContext("who", Event("xxx"), sheerka, BuiltinConcepts.EVALUATING_CONCEPT, c, "my desc")
         input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)]
         context.inputs = {"a": input_list, "b": set_full_serialization(Concept("foo"))}
         context.values = {"c": input_list, "d": set_full_serialization(Concept("bar"))}

tests/test_helpers.py

@@ -110,7 +110,7 @@ def test_i_can_auto_init():
     assert metadata.is_unique is False
     assert metadata.is_builtin is False
     assert metadata.definition_type is DefinitionType.DEFAULT
-    assert metadata.digest == '426d88b1b928a421366c12fb283267b89610cbfb9efb470813ea8b5ba37a2013'
+    assert metadata.digest == '9e058bc1261d1e2c785889147066ce89960fd6844db5bb6f1d1d809a8eb790b7'
 
 
 def test_sequences_are_incremented_when_multiples_call():