Fixed #20: I can parse simple concepts

tests/caching/test_DictionaryCache.py

@@ -142,7 +142,7 @@ class TestDictionaryCache(BaseTest):
         assert cache.get("key") is NotFound
         assert cache._cache == {}
 
-    def test_auto_configure_retrieves_the_whole_remote_repository(self, sdp, context):
+    def test_auto_configure_retrieves_the_whole_remote_repository(self, context, sdp):
         cache = DictionaryCache(sdp=sdp).auto_configure("test")
         with sdp.get_transaction(context.event) as transaction:
             transaction.add("test", "key1", "value1")
@@ -153,7 +153,7 @@ class TestDictionaryCache(BaseTest):
 
         assert cache.copy() == {'key1': 'value1', 'key2': 'value2'}
 
-    def test_we_do_no_go_twice_in_repo_when_not_found(self, sdp, context):
+    def test_we_do_no_go_twice_in_repo_when_not_found(self, context, sdp):
         cache = DictionaryCache(sdp=sdp).auto_configure("test")
 
         assert cache.get("key") is NotFound
@@ -163,3 +163,61 @@ class TestDictionaryCache(BaseTest):
             transaction.add("test", "key", "value")
 
         assert cache.get("key") is NotFound  # the key was previously requested
+
+    def test_i_can_add_path(self):
+        cache = DictionaryCache()
+
+        cache.add_path(["a", "b", "c"], "c_value")
+        cache.add_path(["a", "b", "d", "e"], "e_value")
+
+        assert cache.copy() == {'a': {'b': {'c': {"#values#": ['c_value']},
+                                            'd': {'e': {"#values#": ['e_value']}}}}}
+        assert len(cache) == 2
+
+    def test_i_can_get_multiple_values_in_the_same_path(self):
+        cache = DictionaryCache()
+
+        cache.add_path(["a", "b", "c"], "value1")
+        cache.add_path(["a", "b", "c"], "value2")
+        cache.add_path(["a", "b", "c", "d"], "value3")
+
+        assert cache.copy() == {'a': {'b': {'c': {'d': {'#values#': ['value3']},
+                                                  '#values#': ["value1", "value2"]}}}}
+        assert len(cache) == 3
+
+    def test_i_can_remove_path(self):
+        cache = DictionaryCache()
+
+        cache.add_path(["a", "b", "c"], "value1")
+        cache.add_path(["a", "b", "c"], "value2")
+
+        cache.remove_path(["a", "b", "c"], "value1")
+        assert cache.copy() == {'a': {'b': {'c': {"#values#": ['value2']}}}}
+        assert len(cache) == 1
+
+        cache.remove_path(["a", "b", "c"], "value2")
+        assert cache.copy() == {}
+        assert len(cache) == 0
+
+    def test_i_can_remove_when_not_exist(self):
+        # remove an entry that does not exist does not cause error
+
+        cache = DictionaryCache()
+
+        cache.add_path(["a", "b", "c"], "value1")
+        cache.add_path(["a", "b", "c"], "value2")
+
+        cache.remove_path(["a", "b", "c"], "value3")
+        cache.remove_path(["a", "b"], "value1")
+
+        assert cache.copy() == {'a': {'b': {'c': {"#values#": ['value1', 'value2']}}}}
+        assert len(cache) == 2
+
+    def test_i_can_get_from_path(self):
+        cache = DictionaryCache()
+
+        cache.add_path(["a", "b", "c"], "value1")
+        cache.add_path(["a", "b", "c"], "value2")
+
+        assert cache.get_from_path(["a", "b"]) is NotFound
+        assert cache.get_from_path(["a", "b", "c"]) == ["value1", "value2"]

tests/caching/test_FastCache.py

@@ -51,6 +51,15 @@ def test_not_found_is_returned_when_not_found():
     assert cache.get("foo") is NotFound
 
 
+def test_i_can_remove_an_item():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("to_keep1", "to_keep_value1")
+
+    cache.remove("key1")
+    assert cache.cache == {"to_keep1": "to_keep_value1"}
+
+
 def test_i_can_evict_by_key():
     cache = FastCache()
     cache.put("key1", "value1")
@@ -109,3 +118,33 @@ def test_i_can_copy():
     cache.put("key3", "value3")
 
     assert cache.copy() == {"key1": "value1", "key2": "value2", "key3": "value3"}
+
+
+def test_i_can_take_snapshots_and_revert():
+    # Test that I can create restoration points
+    # and come back later to them
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.snapshot()
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+    cache.snapshot()
+    cache.put("key4", "value4")
+    cache.put("key5", "value5")
+
+    assert cache.cache == {"key1": "value1",
+                           "key2": "value2",
+                           "key3": "value3",
+                           "key4": "value4",
+                           "key5": "value5"}
+
+    cache.revert_snapshot()
+    assert cache.cache == {"key1": "value1",
+                           "key2": "value2",
+                           "key3": "value3"}
+
+    cache.revert_snapshot()
+    assert cache.cache == {"key1": "value1"}
+
+    cache.revert_snapshot()  # no effect if nothing to revert
+    assert cache.cache == {"key1": "value1"}

tests/caching/test_ListCache.py

@@ -278,4 +278,261 @@ class TestListCache(BaseTest):
         cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else ["other new"])
         cache.update("old_key", "old_value", "new_key", "new_value")
         assert cache.get("old_key") == ["other old value"]
-        assert cache.get("new_key") == ["other new", "new_value"]
+        assert cache.get("new_key") == ["other new", "new_value"]
+
+    def test_i_can_delete_from_list_cache(self):
+        cache = ListCache()
+        cache.put("key", "value")
+        cache.put("key", "value2")  # we can append to this list
+
+        cache.delete("key", "value2")
+
+        assert len(cache) == 1
+        assert cache.get("key") == ["value"]
+
+        cache.delete("key", "value")
+
+        assert len(cache) == 0
+        assert cache.get("key") is NotFound
+
+    def test_delete_an_entry_that_does_not_exist_has_no_effect(self):
+        cache = ListCache()
+        cache.put("key", "value")
+
+        cache.delete("key", "value2")
+
+        assert len(cache) == 1
+        assert cache.get("key") == ["value"]
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_and_then_put_back(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)  # remove all values
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": ["value"]}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_one_value(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+        cache.put("key", "value2")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ["value2"]}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value3")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ['value2', 'value3']}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure(
+            "cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: ["value"])).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository_and_then_put_back(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure(
+            "cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["xxx"], extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value=None, alt_sdp=alt_sdp)  # remove all values
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": ["value"]}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_remaining_one_value(self):
+        # There is a value in alt_cache_manager,
+        # But this time, there are remaining values in current cache after deletion
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure(
+            "cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ["value2"]}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1, value2"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_and_then_put_back(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": ["value"]}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_one_value_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, one value remains in the cache
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ["value2"]}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_multiple_values_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, one value remains in the cache
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2", "value3"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ["value2", "value3"]}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_an_already_removed_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # But the alternate sdp returns Removed, which means that previous value was deleted
+        # It's like there is nothing to delete
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_deleting_an_entry_that_does_not_exist_is_not_an_error(self):
+        cache = ListCache()
+        cache.put("key", "value1")
+
+        cache.reset_events()
+        cache.delete("key3")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key3", "value")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value2")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value",
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.clear()
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()

tests/caching/test_ListIfNeededCache.py

@@ -610,6 +610,17 @@ class TestListIfNeededCache(BaseTest):
         assert cache.to_add == set()
         assert cache.to_remove == set()
 
+    def test_deleting_an_entry_that_does_not_exist_from_a_list_is_not_an_error(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+        cache.reset_events()
+
+        cache.delete("key", "value3")
+        assert len(cache) == 2
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
     def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self):
         cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
         alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value",
@@ -645,4 +656,4 @@ class TestListIfNeededCache(BaseTest):
 
         assert cache.copy() == {"key": "value"}
         assert cache.to_remove == set()
-        assert cache.to_add == {"key"}
+        assert cache.to_add == {"key"}

tests/conftest.py

@@ -1,8 +1,12 @@
+import inspect
+
 import pytest
 
 from helpers import GetNextId
 from server.authentication import User
 
+DEFAULT_ONTOLOGY_NAME = "current_test_"
+
 
 @pytest.fixture(scope="session")
 def sheerka():
@@ -75,12 +79,15 @@ class NewOntology:
     """
     from core.ExecutionContext import ExecutionContext
 
-    def __init__(self, context: ExecutionContext, name="current_test"):
+    def __init__(self, context: ExecutionContext, name=None):
         self.sheerka = context.sheerka
         self.context = context
         self.name = name
         self.ontology = None
 
+        if self.name is None:
+            self.name = inspect.stack()[1][3]
+
     def __enter__(self):
         self.ontology = self.sheerka.om.push_ontology(self.name)
         return self.ontology

tests/evaluators/test_DefConceptEvaluator.py

@@ -92,3 +92,16 @@ class TestDefConceptEvaluator(BaseTest):
             new_concept = res.new[0].value
             assert context.sheerka.isinstance(new_concept, BuiltinConcepts.NEW_CONCEPT)
             assert new_concept.body.variables == expected
+            assert new_concept.body.parameters == set(item[0] for item in expected)
+
+    def test_i_can_define_variables_that_are_not_parameters(self, context, evaluator):
+        with NewOntology(context, "test_i_can_define_variables_that_are_not_parameters"):
+            ret_val_input = get_ret_val_from(context, "def concept color def_var color_name")
+            res = evaluator.eval(context, None, ret_val_input)
+
+            assert len(res.new) == 1
+            assert res.new[0].status
+            new_concept = res.new[0].value
+            assert context.sheerka.isinstance(new_concept, BuiltinConcepts.NEW_CONCEPT)
+            assert new_concept.body.variables == [("color_name", NotInit)]
+            assert new_concept.body.parameters == set()

tests/evaluators/test_FilterSuccessful.py

@@ -0,0 +1,51 @@
+import pytest
+
+from base import BaseTest
+from evaluators.FilterSuccessful import FilterSuccessful
+from helpers import _rv, _rvf
+
+
+class TestFilterSuccessful(BaseTest):
+    @pytest.fixture()
+    def evaluator(self, sheerka):
+        return sheerka.evaluators[FilterSuccessful.NAME]
+
+    def test_i_can_match_and_eval(self, context, evaluator):
+        true1 = _rv("some_value1")
+        true2 = _rv("some_value2")
+        false1 = _rvf("some_value1")
+        false2 = _rvf("some_value2")
+
+        return_values = [true1]
+        m = evaluator.matches(context, return_values)
+        assert m.status is False
+
+        return_values = [true1, true2]
+        m = evaluator.matches(context, return_values)
+        assert m.status is False
+
+        return_values = [false1]
+        m = evaluator.matches(context, return_values)
+        assert m.status is False
+
+        return_values = [false1, false2]
+        m = evaluator.matches(context, return_values)
+        assert m.status is False
+
+        return_values = [true1, false1]
+        m = evaluator.matches(context, return_values)
+        assert m.status is True
+        assert m.obj == {'to_keep': [true1], 'to_drop': [false1]}
+
+        r = evaluator.eval(context, m.obj, return_values)
+        assert r.new == [true1]
+        assert r.eaten == [false1]
+
+        return_values = [true1, true2, false1, false2]
+        m = evaluator.matches(context, return_values)
+        assert m.status is True
+        assert m.obj == {'to_keep': [true1, true2], 'to_drop': [false1, false2]}
+
+        r = evaluator.eval(context, m.obj, return_values)
+        assert r.new == [true1, true2]
+        assert r.eaten == [false1, false2]

tests/evaluators/test_RecognizeSimpleConcept.py

@@ -0,0 +1,48 @@
+import pytest
+
+from base import BaseParserTest
+from conftest import NewOntology
+from core.BuiltinConcepts import BuiltinConcepts
+from evaluators.RecognizeSimpleConcept import RecognizeSimpleConcept
+from evaluators.base_evaluator import NotForMe
+from helpers import _rv, _rvf, get_concepts
+from parsers.ParserInput import ParserInput
+
+
+class TestRecognizeSimpleConcept(BaseParserTest):
+    @pytest.fixture()
+    def evaluator(self, sheerka):
+        return sheerka.evaluators[RecognizeSimpleConcept.NAME]
+
+    def test_i_can_match(self, sheerka, context, evaluator):
+        ret_val = _rv(sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=ParserInput("some text")))
+        assert evaluator.matches(context, ret_val).status is True
+
+        ret_val = _rv(sheerka.newn(BuiltinConcepts.UNKNOWN_CONCEPT))  # it responds to USER_INPUT only
+        assert evaluator.matches(context, ret_val).status is False
+
+        ret_val = _rvf(sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=ParserInput("some text")))  # status is false
+        assert evaluator.matches(context, ret_val).status is False
+
+    def test_i_can_recognize_a_concept(self, context, evaluator):
+        with NewOntology(context, "test_i_can_recognize_a_def_concept"):
+            concept, = get_concepts(context, "I am a new concept", use_sheerka=True)
+
+            ret_val_input = self.get_parser_input(context, "I am a new concept")
+            res = evaluator.eval(context, None, ret_val_input)
+
+            assert len(res.new) == 1
+            assert res.new[0].status
+            assert context.sheerka.isinstance(res.new[0].value, concept)
+
+            assert res.eaten == [ret_val_input]
+
+    def test_i_do_not_eat_when_not_for_me(self, context, evaluator):
+        with NewOntology(context, "test_i_can_recognize_a_def_concept"):
+            ret_val_input = self.get_parser_input(context, "unknown concept")
+            res = evaluator.eval(context, None, ret_val_input)
+
+            assert len(res.new) == 1
+            assert not res.new[0].status
+            assert isinstance(res.new[0].value, NotForMe)
+            assert len(res.eaten) == 0

tests/evaluators/test_ResolvePythonVsSimpleConcept.py

@@ -0,0 +1,57 @@
+import pytest
+
+from base import BaseTest
+from evaluators.PythonParser import PythonParser
+from evaluators.RecognizeDefConcept import RecognizeDefConcept
+from evaluators.RecognizeSimpleConcept import RecognizeSimpleConcept
+from evaluators.ResolvePythonVsSimpleConcept import ResolvePythonVsSimpleConcept
+from helpers import _rv, _rvf
+
+
+class TestResolvePythonVsSimpleConcept(BaseTest):
+    @pytest.fixture()
+    def evaluator(self, sheerka):
+        return sheerka.evaluators[ResolvePythonVsSimpleConcept.NAME]
+
+    def test_i_can_match_and_eval(self, context, evaluator):
+        python = _rv("some_value", who=PythonParser.NAME)
+        concept = _rv("some_value", who=RecognizeSimpleConcept.NAME)
+        other = _rv("some_value", who=RecognizeDefConcept.NAME)
+        python_nok = _rvf("some_value", who=PythonParser.NAME)
+        concept_nok = _rvf("some_value", who=RecognizeSimpleConcept.NAME)
+        other_nok = _rvf("some_value", who=RecognizeDefConcept.NAME)
+
+        # at least the two
+        return_values = [python, concept]
+        m = evaluator.matches(context, return_values)
+        assert m.status is True
+        assert m.obj == {'to_keep': concept, 'to_drop': python, 'others': []}
+
+        r = evaluator.eval(context, m.obj, return_values)
+        assert r.new == [concept]
+        assert r.eaten == [python]
+
+        # the two and other successful
+        return_values = [python, concept, other, other_nok]
+        m = evaluator.matches(context, return_values)
+        assert m.status is True
+        assert m.obj == {'to_keep': concept, 'to_drop': python, 'others': [other, other_nok]}
+
+        r = evaluator.eval(context, m.obj, return_values)
+        assert r.new == [concept, other, other_nok]
+        assert r.eaten == [python]
+
+        # python is not ok
+        return_values = [python_nok, concept]
+        m = evaluator.matches(context, return_values)
+        assert m.status is False
+
+        # concept is not ok
+        return_values = [python, concept_nok]
+        m = evaluator.matches(context, return_values)
+        assert m.status is False
+
+        # neither is not
+        return_values = [python_nok, concept_nok]
+        m = evaluator.matches(context, return_values)
+        assert m.status is False

tests/helpers.py

@@ -1,7 +1,11 @@
 from common.global_symbols import NotInit
+from common.utils import unstr_concept
 from core.ExecutionContext import ExecutionContext
 from core.ReturnValue import ReturnValue
 from core.concept import Concept, ConceptDefaultProps, ConceptMetadata, DefinitionType
+from parsers.ParserInput import ParserInput
+from parsers.state_machine import MetadataToken, UnrecognizedToken
+from parsers.tokenizer import Tokenizer
 from services.SheerkaConceptManager import ConceptManager
 
 ATTR_MAP = {
@@ -122,7 +126,34 @@ def get_evaluated_concept(blueprint: Concept | ConceptMetadata, **kwargs):
     :return:
     :rtype:
     """
+
+    def _isfloat(num):
+        try:
+            float(num)
+            return True
+        except ValueError:
+            return False
+
     res = Concept(blueprint.get_metadata())
+
+    for attr in ATTR_MAP:
+        source_code = getattr(res.get_metadata(), attr)
+        if source_code == "" or source_code is None:
+            value = NotInit
+        elif source_code[0] in ("'", '"'):
+            value = source_code[1:-1]
+        elif source_code in ("True", "False"):
+            value = source_code == "True"
+        elif source_code.isdecimal():
+            value = int(source_code)
+        elif _isfloat(source_code):
+            value = float(source_code)
+        else:
+            raise Exception(f"Cannot manage {attr=}, {source_code=}")
+
+        setattr(res, ATTR_MAP[attr], value)
+
+    # force values
     for k, v in kwargs.items():
         res.set_value(ATTR_MAP.get(k, k), v)
 
@@ -347,6 +378,13 @@ def get_concepts(context: ExecutionContext, *concepts, **kwargs) -> list[Concept
     return res
 
 
+def get_evaluated_concepts(context, *concepts, use_sheerka=False) -> list[Concept]:
+    if use_sheerka:
+        return [context.sheerka.evaluate_concept(context, Concept(c.get_metadata())) for c in concepts]
+    else:
+        return [get_evaluated_concept(concept) for concept in concepts]
+
+
 def define_new_concept(context: ExecutionContext, c: str | Concept | ConceptMetadata) -> Concept:
     sheerka = context.sheerka
     if isinstance(c, str):
@@ -381,6 +419,43 @@ def get_file_content(file_name):
         return f.read()
 
 
+def get_parser_input(text):
+    pi = ParserInput(text)
+    assert pi.init()
+
+    return pi
+
+
+def get_from(*args, **kwargs):
+    """
+    Convert the input to fix the positions
+    :param args:
+    :type args:
+    :return:
+    :rtype:
+    """
+    cache = {}  # I keep the name in cache to avoid having to remind it everytime
+    pos = 0
+    res = []
+    for item in args:
+        start = pos
+        if isinstance(item, MetadataToken):
+            if item.metadata.name:
+                cache[item.metadata.id] = item.metadata.name
+
+            tokens = list(Tokenizer(cache[item.metadata.id], yield_eof=False))
+            pos += len(tokens)
+            resolution_method = kwargs.get("resolution_method", item.resolution_method)
+            parser = kwargs.get("parser", item.parser)
+            res.append(MetadataToken(item.metadata, start, pos - 1, resolution_method, parser))
+        elif isinstance(item, UnrecognizedToken):
+            tokens = list(Tokenizer(item.buffer, yield_eof=False))
+            pos += len(tokens)
+            res.append(UnrecognizedToken(item.buffer, start, pos - 1))
+
+    return res
+
+
 def _rv(value, who="Test"):
     return ReturnValue(who=who, status=True, value=value)
 
@@ -400,3 +475,37 @@ def _rvf(value, who="Test"):
     :rtype:
     """
     return ReturnValue(who=who, status=False, value=value)
+
+
+def _ut(buffer, start=0, end=-1):
+    """
+    helper to UnrecognizedToken
+    :param buffer:
+    :type buffer:
+    :param start:
+    :type start:
+    :param end:
+    :type end:
+    :return:
+    :rtype:
+    """
+    return UnrecognizedToken(buffer, start, end)
+
+
+def _mt(concept_id, start=0, end=-1, resolution_method="id", parser="simple"):
+    """
+    helper to MetadataToken
+    :param concept_id:
+    :type concept_id:
+    :param start:
+    :type start:
+    :param end:
+    :type end:
+    :return:
+    :rtype:
+    """
+    name, _id = unstr_concept(concept_id)
+    if _id is None:
+        return MetadataToken(get_metadata(id=concept_id), start, end, resolution_method, parser)
+    else:
+        return MetadataToken(get_metadata(id=_id, name=name), start, end, resolution_method, parser)

tests/non_reg/test_non_reg1.py

@@ -1,4 +1,6 @@
 from base import BaseTest
+from conftest import NewOntology
+from core.BuiltinConcepts import BuiltinConcepts
 
 
 def get_ret_val(res):
@@ -17,9 +19,9 @@ class TestNonReg1(BaseTest):
 
     def test_i_cannot_evaluate_variable_that_is_not_defined(self, sheerka, user):
         res = sheerka.evaluate_user_input("a", user)
-        ret_val = get_ret_val(res)
 
-        assert ret_val.status is False
+        assert len(res) == 2
+        assert all([not ret_val.status for ret_val in res])
 
     def test_i_can_remember_variables(self, sheerka, user):
         sheerka.evaluate_user_input("a = 10", user)
@@ -28,3 +30,51 @@ class TestNonReg1(BaseTest):
         ret_val = get_ret_val(res)
 
         assert ret_val.value == 10
+
+    def test_i_can_define_a_new_concept(self, context, sheerka, user):
+        with NewOntology(context, "test_i_can_define_a_new_concept"):
+            res = sheerka.evaluate_user_input("def concept one as 1", user)
+
+            ret_val = get_ret_val(res)
+            assert ret_val.status
+            assert sheerka.isinstance(ret_val.value, BuiltinConcepts.NEW_CONCEPT)
+
+    def test_i_can_define_a_new_concept_and_use_it(self, context, sheerka, user):
+        with NewOntology(context, "test_i_can_define_a_new_concept_and_use_it"):
+            sheerka.evaluate_user_input("def concept one as 1", user)
+
+            res = sheerka.evaluate_user_input("one", user)
+            ret_val = get_ret_val(res)
+            assert ret_val.status
+            assert sheerka.isinstance(ret_val.value, "one")
+            assert not ret_val.value.get_runtime_info().is_evaluated
+
+    def test_i_can_get_i_concept_using_c_name_form(self, context, sheerka, user):
+        with NewOntology(context):
+            sheerka.evaluate_user_input("def concept one as 1", user)
+
+            res = sheerka.evaluate_user_input("c:one:", user)
+            ret_val = get_ret_val(res)
+            assert ret_val.status
+            assert sheerka.isinstance(ret_val.value, "one")
+            assert not ret_val.value.get_runtime_info().is_evaluated
+
+    def test_i_can_get_i_concept_using_c_id_form(self, context, sheerka, user):
+        with NewOntology(context):
+            sheerka.evaluate_user_input("def concept one as 1", user)
+
+            res = sheerka.evaluate_user_input("c:#1001:", user)
+            ret_val = get_ret_val(res)
+            assert ret_val.status
+            assert sheerka.isinstance(ret_val.value, "one")
+            assert not ret_val.value.get_runtime_info().is_evaluated
+
+    def test_i_can_recognize_concepts_with_long_name(self, context, sheerka, user):
+        with NewOntology(context):
+            sheerka.evaluate_user_input("def concept i am a concept", user)
+
+            res = sheerka.evaluate_user_input("i am a concept", user)
+            ret_val = get_ret_val(res)
+            assert ret_val.status
+            assert sheerka.isinstance(ret_val.value, "i am a concept")
+            assert not ret_val.value.get_runtime_info().is_evaluated

tests/parsers/test_ConceptDefinitionParser.py

@@ -2,22 +2,15 @@ import pytest
 
 from common.global_symbols import NotInit
 from core.concept import DefinitionType
+from helpers import get_parser_input
 from parsers.ConceptDefinitionParser import ConceptDefinition, ConceptDefinitionParser
-from parsers.ParserInput import ParserInput
 from parsers.parser_utils import ParsingError, UnexpectedEof, UnexpectedToken
 from parsers.tokenizer import Keywords, Token, TokenKind
 
 
-def get_parser_input(text):
-    pi = ParserInput(text)
-    assert pi.init()
-
-    return pi
-
-
-class TestRecognizeDefConcept:
+class TestConceptDefinitionParser:
     @pytest.fixture()
-    def parser(self, sheerka):
+    def parser(self):
         return ConceptDefinitionParser()
 
     @pytest.mark.parametrize("text", [

tests/parsers/test_SimpleConceptsParser.py

@@ -0,0 +1,142 @@
+import pytest
+
+from base import BaseTest
+from conftest import NewOntology
+from evaluators.base_evaluator import MultipleChoices
+from helpers import _mt, _ut, get_concepts, get_from, get_metadata, get_parser_input
+from parsers.SimpleParserParser import SimpleConceptsParser
+
+
+class TestSimpleConceptsParser(BaseTest):
+
+    @pytest.fixture()
+    def parser(self):
+        return SimpleConceptsParser()
+
+    @pytest.mark.parametrize("text, expected", [
+        ("I am a new concept", [_mt("1003", 0, 8)]),
+        ("xxx yyy I am a new concept", [_ut("xxx yyy ", 0, 3), _mt("1003", 4, 12)]),
+        ("I am a new concept xxx yyy", [_mt("1003", 0, 8), _ut(" xxx yyy", 9, 12)]),
+        ("xxx I am a new concept yyy", [_ut("xxx ", 0, 1), _mt("1003", 2, 10), _ut(" yyy", 11, 12)]),
+        ("c:#1003:", [_mt("1003", 0, 0)]),
+        ("xxx c:#1003: yyy", [_ut("xxx ", 0, 1), _mt("1003", 2, 2), _ut(" yyy", 3, 4)]),
+        ("xxx c:I am: yyy", [_ut("xxx ", 0, 1), _mt("1002", 2, 2), _ut(" yyy", 3, 4)]),
+        (" I am a new concept", [_ut(" ", 0, 0), _mt("1003", 1, 9)])
+    ])
+    def test_i_can_recognize_a_concept(self, context, parser, text, expected):
+        with NewOntology(context, "test_i_can_recognize_a_concept"):
+            get_concepts(context, "I", "I am", "I am a new concept", use_sheerka=True)
+
+            pi = get_parser_input(text)
+            res = parser.parse(context, pi)
+
+            assert res == MultipleChoices([expected])
+            assert not parser.error_sink
+
+    @pytest.mark.parametrize("text, expected", [
+        ("foo", [_mt("1001", 0, 0)]),
+        ("I am a new concept", [_mt("1001", 0, 8)])
+    ])
+    def test_i_can_recognize_a_concept_by_its_name_and_its_definition(self, context, parser, text, expected):
+        with NewOntology(context, "test_i_can_recognize_a_concept_by_its_name_and_its_definition"):
+            get_concepts(context, get_metadata(name="foo", definition="I am a new concept"), use_sheerka=True)
+
+            pi = get_parser_input(text)
+            res = parser.parse(context, pi)
+
+            assert res == MultipleChoices([expected])
+            assert not parser.error_sink
+
+    @pytest.mark.parametrize("text, expected", [
+        ("long concept name", [_mt("1001", 0, 4)]),
+        ("I am a new concept", [_mt("1001", 0, 8)])
+    ])
+    def test_i_can_recognize_a_concept_by_its_name_when_long_name(self, context, parser, text, expected):
+        with NewOntology(context, "test_i_can_recognize_a_concept_by_its_name_when_long_name"):
+            get_concepts(context, get_metadata(name="long concept name", definition="I am a new concept"),
+                         use_sheerka=True)
+
+            pi = get_parser_input(text)
+            res = parser.parse(context, pi)
+
+            assert res == MultipleChoices([expected])
+            assert not parser.error_sink
+
+    def test_i_can_parse_a_sequence_of_concept(self, context, parser):
+        with NewOntology(context, "test_i_can_parse_a_sequence_of_concept"):
+            get_concepts(context, "foo bar", "baz", "qux", use_sheerka=True)
+
+            pi = get_parser_input("foo bar baz foo, qux")
+            res = parser.parse(context, pi)
+
+            expected = [_mt("1001", 0, 2),
+                        _ut(" ", 3, 3),
+                        _mt("1002", 4, 4),
+                        _ut(" foo, ", 5, 8),
+                        _mt("1003", 9, 9)]
+
+            assert res == MultipleChoices([expected])
+            assert not parser.error_sink
+
+    def test_i_can_detect_multiple_choices(self, context, parser):
+        with NewOntology(context, "test_i_can_detect_multiple_choices"):
+            get_concepts(context, "foo bar", "bar baz", use_sheerka=True)
+
+            pi = get_parser_input("foo bar baz")
+            res = parser.parse(context, pi)
+
+            expected1 = [_mt("1001", 0, 2), _ut(" baz", 3, 4)]
+            expected2 = [_ut("foo ", 0, 1), _mt("1002", 2, 4)]
+
+            assert res == MultipleChoices([expected1, expected2])
+            assert not parser.error_sink
+
+    def test_i_can_detect_multiple_choices_2(self, context, parser):
+        with NewOntology(context, "test_i_can_detect_multiple_choices_2"):
+            get_concepts(context, "one two", "one", "two", use_sheerka=True)
+
+            pi = get_parser_input("one two")
+            res = parser.parse(context, pi)
+
+            expected1 = [_mt("1001", 0, 2)]
+            expected2 = [_mt("1002", 0, 0), _ut(" ", 1, 1), _mt("1003", 2, 2)]
+
+            assert res == MultipleChoices([expected1, expected2])
+            assert not parser.error_sink
+
+    def test_i_can_detect_multiple_choices_3(self, context, parser):
+        with NewOntology(context, "test_i_can_detect_multiple_choices_2"):
+            get_concepts(context, "one two", "one", "two", use_sheerka=True)
+
+            pi = get_parser_input("one two xxx one two")
+            res = parser.parse(context, pi)
+
+            e1 = get_from(_mt("c:one two#1001:"), _ut(" xxx "), _mt("c:#1001:"))
+            e2 = get_from(_mt("c:one#1002:"), _ut(" "), _mt("c:two#1003:"), _ut(" xxx "), _mt("c:one two#1001:"))
+            e3 = get_from(_mt("c:one two#1001:"), _ut(" xxx "), _mt("c:one#1002:"), _ut(" "), _mt("c:two#1003:"))
+            e4 = get_from(_mt("c:one#1002:"), _ut(" "), _mt("c:two#1003:"), _ut(" xxx "), _mt("c:#1002:"), _ut(" "),
+                          _mt("c:#1003:"))
+
+            assert res == MultipleChoices([e1, e2, e3, e4])
+            assert not parser.error_sink
+
+    def test_nothing_is_return_is_no_concept_is_recognized(self, context, parser):
+        pi = get_parser_input("one two three")
+        res = parser.parse(context, pi)
+
+        assert res == MultipleChoices([])
+
+    def test_i_can_manage_attribute_reference(self, context, parser):
+        with NewOntology(context, "test_i_can_detect_multiple_choices_2"):
+            get_concepts(context, "foo", "i am a concept", use_sheerka=True)
+
+            pi = get_parser_input("foo.attribute")
+            res = parser.parse(context, pi)
+            expected = [_mt("1001", 0, 0), _ut(".attribute", 1, 2)]
+            assert res == MultipleChoices([expected])
+
+            pi = get_parser_input("i am a concept.attribute")
+            res = parser.parse(context, pi)
+            expected = [_mt("1002", 0, 6), _ut(".attribute", 7, 8)]
+            assert res == MultipleChoices([expected])
+

tests/parsers/test_state_machine.py

@@ -0,0 +1,82 @@
+from dataclasses import dataclass
+
+from parsers.state_machine import End, Start, State, StateMachine, StateResult
+
+
+@dataclass
+class DummyExecutionContext:
+    count: int
+
+    def to_debug(self):
+        return {"count": self.count}
+
+
+class GenericTestState(State):
+    def __init__(self, name, next_state, fork=None):
+        super().__init__(name=name, next_states=[next_state])
+        self.next_state = next_state
+        self.fork = fork
+
+    def run(self, state_context) -> StateResult:
+        return StateResult(self.next_state, self.fork)
+
+    def __repr__(self):
+        return f"(GenericTestState {self.name} -> {self.next_state}, forks={len(self.fork) if self.fork else 0})"
+
+
+def test_i_can_execute_a_workflow():
+    wkf_as_list = [Start("start", ["a"]),
+                   GenericTestState("a", "b"),
+                   GenericTestState("b", "c"),
+                   GenericTestState("c", "end"),
+                   End("end", None)]
+
+    wkf = {state.name: state for state in wkf_as_list}
+
+    state_machine = StateMachine({"#wkf": wkf})
+    state_machine.run("#wkf", "start", DummyExecutionContext(0))
+
+    assert len(state_machine.paths) == 1
+    assert state_machine.paths[0].get_audit_trail() == ["#wkf:start", "#wkf:a", "#wkf:b", "#wkf:c", "#wkf:end"]
+
+
+def test_i_can_change_workflow():
+    wkf1_as_list = [Start("start", ["a"]),
+                    GenericTestState("a", "#wkf2")]
+
+    wkf2_as_list = [Start("start", ["c"]),
+                    GenericTestState("c", "end"),
+                    End("end", None)]
+
+    wkfs = {
+        "#wkf1": {state.name: state for state in wkf1_as_list},
+        "#wkf2": {state.name: state for state in wkf2_as_list}
+    }
+
+    state_machine = StateMachine(wkfs)
+    state_machine.run("#wkf1", "start", DummyExecutionContext(0))
+
+    assert len(state_machine.paths) == 1
+    assert state_machine.paths[0].get_audit_trail() == ["#wkf1:start", "#wkf1:a", "#wkf2:start", "#wkf2:c", "#wkf2:end"]
+
+
+def test_i_can_fork_path():
+    wkf_as_list = [Start("start", ["a"]),
+                   GenericTestState("a", "end", [("b", DummyExecutionContext(i)) for i in range(3)]),
+                   GenericTestState("b", "end"),
+                   End("end", None)]
+
+    wkf = {state.name: state for state in wkf_as_list}
+
+    state_machine = StateMachine({"#wkf": wkf})
+    state_machine.run("#wkf", "start", DummyExecutionContext(0))
+
+    assert len(state_machine.paths) == 4
+    assert state_machine.paths[0].get_audit_trail() == ["#wkf:start", "#wkf:a", "#wkf:end"]
+    assert state_machine.paths[0].history[1].forks == [1, 2, 3]
+    assert state_machine.paths[1].get_audit_trail() == ["#wkf:start", "#wkf:a", "#wkf:b", "#wkf:end"]
+    assert state_machine.paths[1].history[0].parents == [0]
+    assert state_machine.paths[2].get_audit_trail() == ["#wkf:start", "#wkf:a", "#wkf:b", "#wkf:end"]
+    assert state_machine.paths[2].history[0].parents == [0]
+    assert state_machine.paths[3].get_audit_trail() == ["#wkf:start", "#wkf:a", "#wkf:b", "#wkf:end"]
+    assert state_machine.paths[3].history[0].parents == [0]

tests/services/test_ConceptEvaluator.py

@@ -5,11 +5,10 @@ from common.global_symbols import NotInit
 from conftest import NewOntology
 from core.BuiltinConcepts import BuiltinConcepts
 from core.concept import ConceptDefaultProps
-from core.error import ErrorContext
 from core.python_fragment import PythonFragment
 from helpers import define_new_concept, get_concept, get_concepts, get_metadata
-from services.SheerkaConceptEvaluator import ConceptEvaluator
-from services.SheerkaPython import EvaluationRef
+from services.SheerkaConceptEvaluator import ConceptEvaluator, InfiniteRecursion, TooManyErrors
+from services.SheerkaPython import ObjectRef
 
 
 class TestConceptManager(BaseTest):
@@ -77,8 +76,8 @@ class TestConceptManager(BaseTest):
         compiled = service._build_attributes(context, metadata)
         pf = getattr(compiled, ConceptDefaultProps.BODY)
         assert isinstance(pf, PythonFragment)
-        assert pf.namespace == {"a": EvaluationRef("self", "a"),
-                                "b": EvaluationRef("self", "b")}
+        assert pf.namespace == {"a": ObjectRef("self", "a"),
+                                "b": ObjectRef("self", "b")}
 
     def test_i_can_manage_parsing_errors(self, context, service):
         metadata = get_metadata(
@@ -98,7 +97,7 @@ class TestConceptManager(BaseTest):
         assert pf.source_code == "NotInit"
 
         error = getattr(compiled, ConceptDefaultProps.BODY)
-        assert isinstance(error, ErrorContext)
+        assert isinstance(error, TooManyErrors)
 
     def test_i_can_eval_concept_attributes(self, context, service):
         with NewOntology(context, "test_i_can_eval_concept_attributes"):
@@ -225,6 +224,9 @@ class TestConceptManager(BaseTest):
             assert context.sheerka.objvalue(qux) == 1
 
     def test_concept_variables_precede_global_concepts(self, context, service):
+        # In this test, there is a variable named "foo"
+        # Its value is the concept 'bar'
+        # So when the body is evaluated, we expected Concept(bar), not Concept(foo)
         with NewOntology(context, "test_concept_variables_precede_global_concepts"):
             foo, bar, baz = get_concepts(context,
                                          get_concept("foo"),
@@ -237,6 +239,20 @@ class TestConceptManager(BaseTest):
             assert context.sheerka.isinstance(res, baz)
             assert context.sheerka.isinstance(res.body, bar)
 
+    def test_concept_variables_precede_global_concept_during_computation(self, context, service):
+        # In this test, there is a variable named "foo" and a concept also named "foo"
+        # When evaluated, foo + 1 must use the variable 'foo', not the Concept("foo")
+        with NewOntology(context, "test_concept_variables_precede_global_concepts"):
+            foo, bar = get_concepts(context,
+                                    get_concept("foo", body="2"),
+                                    get_concept("bar", body="foo + 1", variables=(("foo", "1"),)),
+                                    use_sheerka=True)
+
+            res = service.evaluate_concept(context, bar)
+
+            assert context.sheerka.isinstance(res, bar)
+            assert context.sheerka.objvalue(res) == 2
+
     def test_i_can_evaluate_concept_when_variables_reference_others_concepts_with_body(self, context, service):
         with NewOntology(context, "test_i_can_evaluate_concept_when_variables_reference_others_concepts_with_body"):
             foo, bar, baz = get_concepts(context,
@@ -463,6 +479,7 @@ class TestConceptManager(BaseTest):
             res = service.evaluate_concept(context, foo)
             assert context.sheerka.isinstance(res, BuiltinConcepts.EVALUATION_ERROR)
             assert context.sheerka.isinstance(res.concept, foo)
+            assert isinstance(res.reason, InfiniteRecursion)
             assert res.reason.ids == [foo.id, bar.id, baz.id]
 
     def test_i_can_detect_sub_infinite_loop(self, context, service):
@@ -476,6 +493,7 @@ class TestConceptManager(BaseTest):
             res = service.evaluate_concept(context, foo)
             assert context.sheerka.isinstance(res, BuiltinConcepts.EVALUATION_ERROR)
             assert context.sheerka.isinstance(res.concept, bar)
+            assert isinstance(res.reason, InfiniteRecursion)
             assert res.reason.ids == [bar.id, baz.id]
 
     def test_i_can_detect_auto_infinite_loop(self, context, service):
@@ -487,10 +505,11 @@ class TestConceptManager(BaseTest):
             res = service.evaluate_concept(context, foo)
             assert context.sheerka.isinstance(res, BuiltinConcepts.EVALUATION_ERROR)
             assert context.sheerka.isinstance(res.concept, foo)
+            assert isinstance(res.reason, InfiniteRecursion)
             assert res.reason.ids == [foo.id]
 
     def test_i_can_select_the_valid_result_when_multiple_choice_invalid_concept(self, context, service):
-        with NewOntology(context, "test_i_can_select_the_valid_result_when_multiple_choice"):
+        with NewOntology(context, "test_i_can_select_the_valid_result_when_multiple_choice_invalid_concept"):
             foo, two_ok, two_nok = get_concepts(context,
                                                 get_concept("foo", body="two"),
                                                 get_concept("two", body="1 +"),  # has to come before the other 'two'
@@ -502,7 +521,7 @@ class TestConceptManager(BaseTest):
             assert context.sheerka.objvalue(foo) == 2
 
     def test_i_can_select_the_valid_result_when_multiple_choice_evaluation_error(self, context, service):
-        with NewOntology(context, "test_i_can_select_the_valid_result_when_multiple_choice"):
+        with NewOntology(context, "test_i_can_select_the_valid_result_when_multiple_choice_evaluation_error"):
             foo, two_ok, two_nok = get_concepts(context,
                                                 get_concept("foo", body="two"),
                                                 get_concept("two", body="1 / 0"),  # has to come before the other 'two'
@@ -529,8 +548,35 @@ class TestConceptManager(BaseTest):
         with NewOntology(context, "test_i_do_not_use_ret_in_case_of_error"):
             foo, baz = get_concepts(context,
                                     get_concept("foo"),
-                                    get_concept("baz", body="foo", ret="bar"),
+                                    get_concept("baz", body="foo", ret="bar"),  # Concept("bar") is not defined
                                     use_sheerka=True)
 
             res = service.evaluate_concept(context, baz)
             assert context.sheerka.isinstance(res, BuiltinConcepts.EVALUATION_ERROR)
+
+    @pytest.mark.skip("Cannot remove concept")
+    def test_i_do_not_use_ret_in_case_of_error_when_concept_was_removed(self, context, service):
+        # Make sure that ret is not returned in case of UNKNOWN_CONCEPT error message
+        foo, bar, baz = get_concepts(context,
+                                     get_concept("foo"),
+                                     get_concept("bar"),
+                                     get_concept("baz", body="foo", ret="bar"),  # Concept("bar") is not defined
+                                     use_sheerka=True)
+        service.evaluate_concept(context, baz)  # creates the compiled for Concept("baz")
+        context.sheerka.remove_concept(bar)  # Concept("bar") no longer exists, but compiled for "baz" remains the same
+
+        res = service.evaluate_concept(context, baz)
+        assert context.sheerka.isinstance(res, BuiltinConcepts.EVALUATION_ERROR)
+        assert "#ret#" in res.reason
+        assert res.reason["#ret#"].value == context.sheerka.newn(BuiltinConcepts.UNKNOWN_CONCEPT, requested="bar")
+
+    def test_i_cannot_evaluate_when_error(self, context, service):
+        with NewOntology(context, "test_i_cannot_evaluate_when_error"):
+            foo, = get_concepts(context,
+                                get_concept("foo", body="I am a concept"),  # "one" does not exist
+                                use_sheerka=True)
+
+            res = service.evaluate_concept(context, foo)
+
+            assert context.sheerka.isinstance(res, BuiltinConcepts.INVALID_CONCEPT)
+

tests/services/test_ConceptManager.py

@@ -6,8 +6,8 @@ from conftest import NewOntology
 from core.BuiltinConcepts import BuiltinConcepts
 from core.concept import ConceptMetadata
 from core.error import ErrorContext
-from helpers import get_concepts, get_metadata
-from services.SheerkaConceptManager import ConceptAlreadyDefined, ConceptManager
+from helpers import get_concept, get_concepts, get_metadata
+from services.SheerkaConceptManager import ConceptAlreadyDefined, ConceptManager, ConceptRef
 
 
 class TestConceptManager(BaseTest):
@@ -86,7 +86,7 @@ class TestConceptManager(BaseTest):
             assert metadata.name == "name"
             assert metadata.key == "name"
             assert metadata.body == "body"
-            assert metadata.digest == "c75faa4efbc9ef9dbc5174c52786d5b066e2ece41486b81c27336e292917fecb"
+            assert metadata.digest == "f32363f42e698b1642c8f76f969d76d56f53f0e0732cb651e3360e3ede7b2b11"
             assert metadata.all_attrs == ('#where#', '#pre#', '#post#', '#body#', '#ret#')
 
             # is sorted in db
@@ -96,6 +96,60 @@ class TestConceptManager(BaseTest):
             assert om.get(ConceptManager.CONCEPTS_BY_KEY_ENTRY, metadata.key) == metadata
             assert om.get(ConceptManager.CONCEPTS_BY_HASH_ENTRY, metadata.digest) == metadata
 
+            # check first token
+            assert om.get(ConceptManager.CONCEPT_BY_FIRST_TOKEN_IN_KEY, "name") == ["1001"]
+
+    def test_i_can_define_a_new_concept_with_variables(self, context, service):
+        with NewOntology(context, "test_i_can_define_a_new_concept_with_variables"):
+            res = service.define_new_concept(context,
+                                             name="a multiplied by b",
+                                             variables=[("a", NotInit), ("b", NotInit)])
+
+            metadata = res.value.metadata
+            assert isinstance(metadata, ConceptMetadata)
+            assert metadata.id == "1001"
+            assert metadata.name == "a multiplied by b"
+            assert metadata.key == "__var__0 multiplied by __var__1"
+            assert metadata.digest == "17d2360d82fc4264e2bcb75e4aa30ee3de87531acee72f5d939e23bff246b2dd"
+            assert metadata.all_attrs == ('#where#', '#pre#', '#post#', '#body#', '#ret#', "a", "b")
+
+            # is sorted in db
+            om = context.sheerka.om
+            assert om.get(ConceptManager.CONCEPTS_BY_ID_ENTRY, metadata.id) == metadata
+            assert om.get(ConceptManager.CONCEPTS_BY_NAME_ENTRY, metadata.name) == metadata
+            assert om.get(ConceptManager.CONCEPTS_BY_KEY_ENTRY, metadata.key) == metadata
+            assert om.get(ConceptManager.CONCEPTS_BY_HASH_ENTRY, metadata.digest) == metadata
+
+            # check first token
+            assert om.get(ConceptManager.CONCEPT_BY_FIRST_TOKEN_IN_KEY, "multiplied") == ["1001"]
+
+    def test_i_can_define_a_new_concept_using_definition(self, context, service):
+        with NewOntology(context, "test_i_can_define_a_new_concept_using_definition"):
+            res = service.define_new_concept(context,
+                                             name="multiplication",
+                                             definition="a multiplied by b",
+                                             variables=[("a", NotInit), ("b", NotInit)])
+
+            metadata = res.value.metadata
+            assert isinstance(metadata, ConceptMetadata)
+            assert metadata.id == "1001"
+            assert metadata.name == "multiplication"
+            assert metadata.definition == "a multiplied by b"
+            assert metadata.key == "__var__0 multiplied by __var__1"
+            assert metadata.digest == "b29007ea67bddc48329a2ae0124a320e26c86fb6b106aad6581bc75dfdf5ebeb"
+            assert metadata.all_attrs == ('#where#', '#pre#', '#post#', '#body#', '#ret#', "a", "b")
+
+            # is sorted in db
+            om = context.sheerka.om
+            assert om.get(ConceptManager.CONCEPTS_BY_ID_ENTRY, metadata.id) == metadata
+            assert om.get(ConceptManager.CONCEPTS_BY_NAME_ENTRY, metadata.name) == metadata
+            assert om.get(ConceptManager.CONCEPTS_BY_KEY_ENTRY, metadata.key) == metadata
+            assert om.get(ConceptManager.CONCEPTS_BY_HASH_ENTRY, metadata.digest) == metadata
+
+            # check first token
+            assert om.get(ConceptManager.CONCEPT_BY_FIRST_TOKEN_IN_KEY, "multiplied") == ["1001"]
+            assert om.get(ConceptManager.CONCEPT_BY_FIRST_TOKEN_IN_NAME, "multiplication") == ["1001"]
+
     def test_i_cannot_create_the_same_concept_twice(self, context, service):
         with NewOntology(context, "test_i_cannot_create_the_same_concept_twice"):
             res = service.define_new_concept(context, "name", body="body")
@@ -176,11 +230,11 @@ class TestConceptManager(BaseTest):
     def test_i_cannot_instantiate_a_concept_which_does_not_exist(self, context, service):
         foo = service.newn("foo", var1="value1", var2="value2")
         assert foo.key == BuiltinConcepts.UNKNOWN_CONCEPT
-        assert foo.requested_name == "foo"
+        assert foo.requested == "foo"
 
         foo = service.newi("1001", var1="value1", var2="value2")
         assert foo.key == BuiltinConcepts.UNKNOWN_CONCEPT
-        assert foo.requested_id == "1001"
+        assert foo.requested == "#1001"
 
     def test_i_can_instantiate_by_name_when_multiple_results(self, context, service):
         with NewOntology(context, "test_i_can_instantiate_by_name_when_multiple_results"):
@@ -255,6 +309,48 @@ class TestConceptManager(BaseTest):
             assert context.sheerka.isinstance(res[0], foo)
             assert context.sheerka.isinstance(res[1], bar)
 
+    def test_i_can_new_using_concept_reference(self, context, service):
+        with NewOntology(context, "test_i_can_new_using_concept_reference"):
+            foo, bar, baz = get_concepts(context, "foo", "bar", "baz", use_sheerka=True)
+
+            foo.get_runtime_info().info["resolution_method"] = "id"
+            bar.get_runtime_info().info["resolution_method"] = "key"
+
+            foo_concept_ref = ConceptRef(foo)
+            res = service.new(foo_concept_ref)
+            assert context.sheerka.isinstance(res, foo)
+
+            bar_concept_ref = ConceptRef(bar)
+            res = service.new(bar_concept_ref)
+            assert context.sheerka.isinstance(res, bar)
+
+            baz_concept_ref = ConceptRef(baz)
+            res = service.new(baz_concept_ref)
+            assert context.sheerka.isinstance(res, baz)
+
+    def test_i_can_new_using_concept_reference_when_multiple_results(self, context, service):
+        with NewOntology(context, "test_i_can_new_using_concept_reference"):
+            foo1, foo2 = get_concepts(context,
+                                      get_concept("foo", body="1"),
+                                      get_concept("foo", body="2"),
+                                      use_sheerka=True)
+
+            foo = get_concept("foo")  # blueprint, no need to be known by Sheerka
+            foo.get_runtime_info().info["resolution_method"] = "name"
+            foo_concept_ref = ConceptRef(foo)
+
+            res = service.new(foo_concept_ref)
+            assert res == [foo1, foo2]
+
+    def test_i_cannot_new_using_concept_reference_when_unknown(self, context, service):
+        foo = get_concept("foo")  # not known by Sheerka
+        foo.get_runtime_info().info["resolution_method"] = "name"
+
+        foo_concept_ref = ConceptRef(foo)
+        res = service.new(foo_concept_ref)
+        assert context.sheerka.isinstance(res, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.requested == "foo"
+
     def test_unknown_concept_is_return_if_the_identifier_is_not_found(self, service):
         assert service.new("unknown").name == BuiltinConcepts.UNKNOWN_CONCEPT
 

tests/services/test_SheerkaDummyEventManager.py

@@ -0,0 +1,66 @@
+import pytest
+
+from base import BaseTest
+from services.SheerkaDummyEventManager import SheerkaDummyEventManager
+
+
+def example_of_function(context):
+    print(f"example_of_class_method. event={context.event.get_digest()}")
+
+
+def example_of_function_with_data(context, data):
+    print(f"example_of_class_method. event={context.event.get_digest()}, {data=}")
+
+
+class TestSheerkaEventManager(BaseTest):
+
+    @pytest.fixture()
+    def service(self, sheerka):
+        service = sheerka.services[SheerkaDummyEventManager.NAME]
+        yield service
+
+        service.test_only_reset_service()
+
+    def example_of_class_method(self, context):
+        print(f"example_of_class_method. event={context.event.get_digest()}")
+
+    @staticmethod
+    def example_of_static_method(context):
+        print(f"example_of_static_method. event={context.event.get_digest()}")
+
+    def example_of_class_method_with_data(self, context, data):
+        print(f"example_of_class_method. event={context.event.get_digest()}, {data=}")
+
+    @staticmethod
+    def example_of_static_method_with_data(context, data):
+        print(f"example_of_static_method. event={context.event.get_digest()}, {data=}")
+
+    def test_i_can_subscribe_and_publish(self, context, service, capsys):
+        topic = "my topic"
+
+        service.subscribe(topic, self.example_of_class_method)
+        service.subscribe(topic, self.example_of_static_method)
+        service.subscribe(topic, example_of_function)
+
+        service.publish(context, topic)
+
+        captured = capsys.readouterr()
+        assert captured.out == """example_of_class_method. event=xxx
+example_of_static_method. event=xxx
+example_of_class_method. event=xxx
+"""
+
+    def test_i_can_subscribe_and_publish_with_data(self, context, service, capsys):
+        topic = "my topic"
+
+        service.subscribe(topic, self.example_of_class_method_with_data)
+        service.subscribe(topic, self.example_of_static_method_with_data)
+        service.subscribe(topic, example_of_function_with_data)
+
+        service.publish(context, topic, "42")
+
+        captured = capsys.readouterr()
+        assert captured.out == """example_of_class_method. event=xxx, data='42'
+example_of_static_method. event=xxx, data='42'
+example_of_class_method. event=xxx, data='42'
+"""

tests/services/test_SheerkaPython.py

@@ -1,3 +1,5 @@
+import ast
+
 import pytest
 
 from base import BaseTest, DummyObj
@@ -7,11 +9,13 @@ from core.BuiltinConcepts import BuiltinConcepts
 from core.ExecutionContext import ContextActions
 from core.concept import ConceptDefaultProps
 from core.error import MethodAccessError
+from core.python_fragment import PythonFragment
 from evaluators.PythonParser import PythonParser
-from helpers import _rv, define_new_concept, get_concepts, get_evaluated_concept, get_metadata
+from helpers import _rv, define_new_concept, get_concepts, get_evaluated_concept, get_evaluated_concepts, get_metadata
 from parsers.ParserInput import ParserInput
 from parsers.tokenizer import Token, TokenKind
-from services.SheerkaPython import EvalMethod, EvaluationContext, EvaluationRef, Expando, MultipleResults, SheerkaPython
+from services.SheerkaConceptManager import ConceptRef
+from services.SheerkaPython import EvalMethod, EvaluationContext, Expando, MultipleResults, ObjectRef, SheerkaPython
 
 
 def get_python_fragment(sheerka, context, command):
@@ -70,7 +74,7 @@ class TestSheerkaPython(BaseTest):
 
     def test_i_can_eval_using_eval_ref(self, sheerka, context, service):
         python_fragment = get_python_fragment(sheerka, context, "a")
-        python_fragment.namespace = {"a": EvaluationRef("self", "a")}
+        python_fragment.namespace = {"a": ObjectRef("self", "a")}
 
         ret = service.evaluate_python(context, EvaluationContext(), python_fragment,
                                       {"self": DummyObj("my dummy value")})
@@ -95,6 +99,38 @@ class TestSheerkaPython(BaseTest):
             ret = service.evaluate_python(context, EvaluationContext(), python_fragment)
             assert ret == 3
 
+    def test_i_can_eval_when_multiple_concepts(self, sheerka, context, service):
+        with NewOntology(context, "test_i_can_eval_when_multiple_concepts"):
+            get_concepts(context,
+                         get_metadata("one", body="'one'"),
+                         get_metadata("one", body="1"),
+                         use_sheerka=True)
+            python_fragment = get_python_fragment(sheerka, context, "one + 1")
+
+            ret = service.evaluate_python(context, EvaluationContext(), python_fragment)
+
+            assert ret == 2
+
+    def test_i_can_eval_when_multiple_result_in_local_namespace(self, sheerka, context, service):
+        # In the test, the PythonFragment contains a MultipleResult in its namespace
+        # (normally, the MultipleResult is created inside the evaluate_python)
+        # We need to make sure that multiple results are created in the same way
+        with NewOntology(context, "test_i_can_eval_when_multiple_result_in_local_namespace"):
+            one1, one2 = get_concepts(context,
+                                      get_metadata("one", body="'one'"),
+                                      get_metadata("one", body="1"),
+                                      use_sheerka=True)
+
+            concept_ref = "__concept_id__"
+            ast_tree = ast.parse(concept_ref, "<user input>", 'eval')
+            ref = MultipleResults(ConceptRef(one1), ConceptRef(one2))
+            python_fragment = PythonFragment(concept_ref, ast_tree=ast_tree, namespace={concept_ref: ref})
+
+            ret = service.evaluate_python(context, EvaluationContext(eval_method=EvalMethod.All), python_fragment)
+
+            evaluated_one1, evaluated_one2 = get_evaluated_concepts(context, one1, one2, use_sheerka=True)
+            assert ret == MultipleResults(evaluated_one1, "one", evaluated_one2, 1)
+
     def test_i_can_remember_previous_results(self, sheerka, context, service):
         python_fragment = get_python_fragment(sheerka, context, "a=10")
         ret = service.evaluate_python(context, EvaluationContext(), python_fragment)
@@ -151,18 +187,6 @@ class TestSheerkaPython(BaseTest):
                                           get_evaluated_concept(foo_3, body='bar'),
                                           "bar")
 
-    def test_i_can_eval_when_multiple_concepts(self, sheerka, context, service):
-        with NewOntology(context, "test_i_can_eval_when_multiple_concepts"):
-            get_concepts(context,
-                         get_metadata("one", body="'one'"),
-                         get_metadata("one", body="1"),
-                         use_sheerka=True)
-            python_fragment = get_python_fragment(sheerka, context, "one + 1")
-
-            ret = service.evaluate_python(context, EvaluationContext(), python_fragment)
-
-            assert ret == 2
-
     def test_i_can_eval_until_a_successful_result_is_found(self, sheerka, context, service):
         with NewOntology(context, "test_i_can_eval_when_multiple_concepts"):
             get_concepts(context,
@@ -338,3 +362,10 @@ class TestSheerkaPython(BaseTest):
         foo, bar = get_concepts(context, "foo", "bar")
         assert MultipleResults(foo, "one", bar, 1).concepts_only() == MultipleResults(foo, bar)
         assert MultipleResults("one", 1).concepts_only() == MultipleResults()
+
+    def test_i_can_add_multiple_results_of_multiple_results(self, context):
+        foo, bar, baz, qux = get_concepts(context, "foo", "bar", "baz", "qux")
+        m1 = MultipleResults(foo, bar)
+        m2 = MultipleResults(bar, baz, m1)
+
+        assert m2.items == [bar, baz, foo, bar]

tests/test_helpers.py

@@ -2,7 +2,8 @@ import pytest
 
 from common.global_symbols import NotInit
 from core.concept import Concept, ConceptDefaultProps, ConceptMetadata, DefinitionType
-from helpers import GetNextId, get_concept, get_concepts, get_metadata, get_metadatas, get_evaluated_concept
+from helpers import GetNextId, _mt, _ut, get_concept, get_concepts, get_evaluated_concept, get_from, get_metadata, \
+    get_metadatas
 
 
 def test_i_can_get_default_value_when_get_metadata():
@@ -233,7 +234,7 @@ def test_i_can_get_multiple_concepts_when_same_name(sheerka, context):
     assert sheerka.isinstance(one_int, "one")
 
 
-def test_i_can_create_test_concept(sheerka, context):
+def test_i_can_create_test_concept():
     concept = get_concept("one", body="'one'")
 
     test_concept = get_evaluated_concept(concept, body='hello', a="value for a")
@@ -241,3 +242,39 @@ def test_i_can_create_test_concept(sheerka, context):
     assert test_concept.get_metadata() == concept.get_metadata()
     assert test_concept.get_value(ConceptDefaultProps.BODY) == "hello"
     assert test_concept.get_value("a") == "value for a"
+
+
+def test_i_can_dummy_evaluate_concept():
+    concept = get_concept("one", body="'one'", where="True", pre="False", ret="1", post="1.0")
+
+    evaluated = get_evaluated_concept(concept)
+    assert evaluated.get_value(ConceptDefaultProps.WHERE) is True
+    assert evaluated.get_value(ConceptDefaultProps.PRE) is False
+    assert evaluated.get_value(ConceptDefaultProps.BODY) == "one"
+    assert evaluated.get_value(ConceptDefaultProps.RET) == 1
+    assert evaluated.get_value(ConceptDefaultProps.POST) == 1.0
+
+    concept = get_concept("one", body='"one"', ret="'a value'")
+    evaluated = get_evaluated_concept(concept, ret='forced value')
+    assert evaluated.get_value(ConceptDefaultProps.WHERE) == NotInit
+    assert evaluated.get_value(ConceptDefaultProps.PRE) == NotInit
+    assert evaluated.get_value(ConceptDefaultProps.BODY) == "one"
+    assert evaluated.get_value(ConceptDefaultProps.RET) == "forced value"
+    assert evaluated.get_value(ConceptDefaultProps.POST) == NotInit
+
+
+def test_i_can_get_from():
+    res = get_from(_mt("c:i am a concept#1001:"))
+    assert res == [_mt("1001", 0, 6)]
+
+    res = get_from(_ut("some unrecognized stuff"))
+    assert res == [_ut("some unrecognized stuff", 0, 4)]
+
+    res = get_from(_mt("c:i am a concept#1001:"), _ut("some unrecognized stuff"))
+    assert res == [_mt("1001", 0, 6), _ut("some unrecognized stuff", 7, 11)]
+
+    res = get_from(_mt("c:i am a concept#1001:"), _ut("some unrecognized stuff"), parser="other")
+    assert res == [_mt("1001", 0, 6, parser="other"), _ut("some unrecognized stuff", 7, 11)]
+
+    res = get_from(_mt("c:i am a concept#1001:"), _mt("c:#1001:"))
+    assert res == [_mt("1001", 0, 6), _mt("1001", 7, 13)]