Reworked sdp

tests/test_sheerkaDataProvider.py

@@ -3,12 +3,12 @@ import hashlib
 import pytest
 import os
 from os import path
-from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
+from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderError
 from datetime import date, datetime
 import shutil
 import json
 
-from sdp.sheerkaSerializer import ObjectSerializer, BaseSerializer, Serializer
+from sdp.sheerkaSerializer import ObjectSerializer, BaseSerializer, Serializer, SerializerContext, PickleSerializer
 
 tests_root = path.abspath("../build/tests")
 
@@ -75,7 +75,7 @@ class ObjNoKey:
 
 
 class ObjDumpJson:
-    def __init__(self, key, value):
+    def __init__(self, key=None, value=None):
         self.key = key
         self.value = value
 
@@ -91,6 +91,10 @@ class ObjDumpJson:
         return self.key
 
     def get_digest(self):
+        """
+        Returns the digest of the event
+        :return: hexa form of the sha256
+        """
         return hashlib.sha256(f"Concept:{self.key}{self.value}".encode("utf-8")).hexdigest()
 
     def to_dict(self):
@@ -123,7 +127,7 @@ def test_i_can_init_the_data_provider():
     assert path.exists(path.join(tests_root, ".sheerka"))
 
 
-def test_i_can_add_and_retrieve_an_event():
+def test_i_can_save_and_load_an_event():
     sdp = SheerkaDataProvider(".sheerka")
     event = Event("hello world", date=date(year=2007, month=9, day=10), user="kodjo")
 
@@ -137,17 +141,23 @@ def test_i_can_add_and_retrieve_an_event():
     assert evt.message == "hello world"
 
 
-def test_i_can_add_an_object():
+def test_i_can_add_an_string():
     sdp = SheerkaDataProvider(".sheerka")
     event = Event("cmd add 'foo => bar'")
     event_digest = event.get_digest()
     obj = "foo => bar"
 
-    state_digest = sdp.add(event, "entry", obj)
-    state = sdp.load_state(state_digest)
+    entry, key = sdp.add(event, "entry", obj)
+    last_commit = sdp.get_snapshot()
+    state = sdp.load_state(last_commit)
+    loaded = sdp.get(entry, key)
+
+    assert entry == "entry"
+    assert key is None
+    assert loaded == obj
 
     assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest[0:24], event_digest))
-    assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, state_digest[0:24], state_digest))
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
     assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile))
 
     assert state.date is not None
@@ -155,121 +165,349 @@ def test_i_can_add_an_object():
     assert state.events == [event_digest]
     assert state.data == {"entry": "foo => bar"}
 
-    assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == state_digest
+    assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit
 
 
-def test_i_can_add_multiple_elements_in_an_entry():
+def test_i_can_add_several_strings_if_allow_multiple_is_true():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    sdp.add(Event("event"), "entry", "foo")
+    sdp.add(Event("event"), "entry", "foo")
+    entry, key = sdp.add(Event("event"), "entry", "bar")
+    loaded = sdp.get(entry, key)
+
+    assert entry == "entry"
+    assert key is None
+    assert loaded == ["foo", "foo", "bar"]
+
+
+def test_i_cannot_add_several_strings_if_allow_multiple_is_false():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    with pytest.raises(IndexError) as index_error:
+        sdp.add(Event("event"), "entry", "foo", False)
+        sdp.add(Event("event"), "entry", "foo", False)
+    assert index_error.value.args[0] == "entry"
+
+
+def test_i_can_add_an_object_with_no_key():
     sdp = SheerkaDataProvider(".sheerka")
     event = Event("cmd add 'foo => bar'")
+    event_digest = event.get_digest()
+    obj = ObjNoKey("a", "b")
 
-    state_digest1 = sdp.add(event, "entry", 1)
-    state1 = sdp.load_state(state_digest1)
+    entry, key = sdp.add(event, "entry", obj)
+    last_commit = sdp.get_snapshot()
+    state = sdp.load_state(last_commit)
+    loaded = sdp.get(entry, key)
 
-    state_digest2 = sdp.add(event, "entry", 2)
-    state2 = sdp.load_state(state_digest2)
+    assert entry == "entry"
+    assert key is None
+    assert loaded == obj
 
-    state_digest3 = sdp.add(event, "entry", 3)
-    state3 = sdp.load_state(state_digest3)
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest[0:24], event_digest))
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile))
 
-    assert state1.data == {"entry": 1}
-    assert state2.data == {"entry": [1, 2]}
-    assert state3.data == {"entry": [1, 2, 3]}
+    assert state.date is not None
+    assert state.parents == []
+    assert state.events == [event_digest]
+    assert state.data == {"entry": ObjNoKey("a", "b")}
+
+    assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit
 
 
-def test_i_can_add_element_using_auto_generated_key():
+def test_i_can_add_several_obj_no_key_if_allow_multiple_is_true():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    sdp.add(Event("event"), "entry", ObjNoKey("a", "b"))
+    sdp.add(Event("event"), "entry", ObjNoKey("a", "b"))
+    entry, key = sdp.add(Event("event"), "entry", ObjNoKey("c", "d"))
+    loaded = sdp.get(entry, key)
+
+    assert entry == "entry"
+    assert key is None
+    assert loaded == [ObjNoKey("a", "b"), ObjNoKey("a", "b"), ObjNoKey("c", "d")]
+
+
+def test_i_cannot_add_several_obj_no_key_if_allow_multiple_is_false():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    with pytest.raises(IndexError) as index_error:
+        sdp.add(Event("event"), "entry", ObjNoKey("a", "b"), False)
+        sdp.add(Event("event"), "entry", ObjNoKey("c", "d"), False)
+    assert index_error.value.args[0] == "entry"
+
+
+def test_i_can_add_a_dict():
+    sdp = SheerkaDataProvider(".sheerka")
+    event = Event("cmd add 'foo => bar'")
+    event_digest = event.get_digest()
+    obj = {"my_key": "my_value"}
+
+    entry, key = sdp.add(event, "entry", obj)
+    last_commit = sdp.get_snapshot()
+    state = sdp.load_state(last_commit)
+    loaded = sdp.get(entry, key)
+
+    loaded_value = sdp.get(entry, "my_key")  # we can retrieve by key
+
+    assert entry == "entry"
+    assert key is None  # we return None as dict may contains several entries
+    assert loaded == obj
+    assert loaded_value == "my_value"
+
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest[0:24], event_digest))
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile))
+
+    assert state.date is not None
+    assert state.parents == []
+    assert state.events == [event_digest]
+    assert state.data == {"entry": obj}
+
+    assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit
+
+
+def test_i_can_add_multiple_entries_at_once_with_dict():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    entry, key = sdp.add(Event("event"), "entry", {"my_key1": "value1", "my_key2": "value2"})
+    loaded = sdp.get(entry, key)
+    loaded_value1 = sdp.get(entry, "my_key1")
+    loaded_value2 = sdp.get(entry, "my_key2")
+
+    assert loaded == {"my_key1": "value1", "my_key2": "value2"}
+    assert loaded_value1 == "value1"
+    assert loaded_value2 == "value2"
+
+
+def test_i_can_add_same_key_with_dict_if_allow_multiple_is_true():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    sdp.add(Event("event"), "entry", {"my_key": "my_value"})
+    entry, key = sdp.add(Event("event"), "entry", {"my_key": "my_value"})
+    loaded1 = sdp.get(entry, key)
+
+    entry, key = sdp.add(Event("event"), "entry", {"my_key": "my_value2"})
+    loaded2 = sdp.get(entry, key)
+
+    assert entry == "entry"
+    assert key is None
+    assert loaded1 == {"my_key": ["my_value", "my_value"]}
+    assert loaded2 == {"my_key": ["my_value", "my_value", "my_value2"]}
+
+
+def test_i_cannot_add_same_key_with_dict_if_allow_multiple_is_false():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    with pytest.raises(IndexError) as index_error:
+        sdp.add(Event("event"), "entry", {"my_key": "my_value"}, False)
+        sdp.add(Event("event"), "entry", {"my_key": "my_value2"}, False)
+    assert index_error.value.args[0] == "entry.my_key"
+
+
+def test_i_can_add_object_with_different_key_if_allow_multiple_is_false():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    sdp.add(Event("event"), "entry", {"my_key": "a"}, False)
+    sdp.add(Event("event"), "entry", {"my_key2": "b"}, False)
+
+    assert sdp.get("entry", "my_key") == "a"
+    assert sdp.get("entry", "my_key2") == "b"
+
+
+def test_i_can_add_obj_with_key():
+    sdp = SheerkaDataProvider(".sheerka")
+    event = Event("cmd add 'foo => bar'")
+    event_digest = event.get_digest()
+    obj1 = ObjWithKey("key1", "b")
+    obj2 = ObjSetKey("c", key="key2")
+
+    entry1, key1 = sdp.add(event, "entry", obj1)  # test when key is taken from obj.get_key()
+    entry2, key2 = sdp.add(event, "entry2", obj2)  # test when key is taken from obj.key
+    last_commit = sdp.get_snapshot()
+    state = sdp.load_state(last_commit)
+
+    loaded1 = sdp.get(entry1, key1)
+    loaded2 = sdp.get(entry2, key2)
+
+    assert entry1 == "entry"
+    assert key1 == "key1"
+    assert loaded1 == ObjWithKey("key1", "b")
+    assert entry2 == "entry2"
+    assert key2 == "key2"
+    assert loaded2 == ObjSetKey("c", key="key2")
+
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest[0:24], event_digest))
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
+    assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile))
+
+    assert state.date is not None
+    assert len(state.parents) == 1
+    assert state.events == [event_digest]
+    assert state.data == {"entry": {"key1": obj1}, "entry2": {"key2": obj2}}
+
+    assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit
+
+
+def test_i_can_add_objects_with_same_key_if_allow_multiple_is_true():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    sdp.add(Event("event"), "entry", ObjWithKey("my_key", "b"))
+    entry, key = sdp.add(Event("event"), "entry", ObjSetKey("c", key="my_key"))
+    loaded1 = sdp.get(entry, key)
+
+    entry, key = sdp.add(Event("event"), "entry", ObjSetKey("c", key="my_key"))
+    sdp.add(Event("event"), "entry", ObjSetKey("c", key="my_key2"))  # to prove that it does not melt everything
+    loaded2 = sdp.get(entry, key)
+
+    assert entry == "entry"
+    assert key == "my_key"
+    assert loaded1 == [ObjWithKey("my_key", "b"), ObjSetKey("c", key="my_key")]
+    assert loaded2 == [ObjWithKey("my_key", "b"), ObjSetKey("c", key="my_key"), ObjSetKey("c", key="my_key")]
+
+
+def test_i_cannot_add_object_with_same_key_if_allow_multiple_is_false():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    with pytest.raises(IndexError) as index_error:
+        sdp.add(Event("event"), "entry", ObjWithKey("my_key", "b"), False)
+        sdp.add(Event("event"), "entry", ObjSetKey("c", key="my_key"), False)
+    assert index_error.value.args[0] == "entry.my_key"
+
+
+def test_i_can_add_obj_with_key_to_a_list():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    sdp.add(Event("event"), "entry", "foo")
+    sdp.add(Event("event"), "entry", "bar")  # entry is now a list
+    sdp.add(Event("event"), "entry", ObjWithKey("a", "b"))  # this entry must no be taken as a object with a key
+
+    loaded = sdp.get("entry")
+    assert loaded == ["foo", "bar", ObjWithKey("a", "b")]
+
+
+def test_i_cannot_add_obj_with_no_key_when_then_entry_has_keys():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    with pytest.raises(SheerkaDataProviderError) as error:
+        sdp.add(Event("event"), "entry", ObjWithKey("a", "b"))
+        sdp.add(Event("event"), "entry", "foo")
+
+    assert error.value.obj == "foo"
+
+
+def test_i_can_add_string_using_auto_generated_key():
     sdp = SheerkaDataProvider(".sheerka")
     event = Event("cmd add 'foo => bar'")
     key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile)
 
-    sdp.add_with_auto_key(event, "entry1", "foo")
-    sdp.add_with_auto_key(event, "entry1", "bar")
-    sdp.add_with_auto_key(event, "entry2", "baz")
+    entry1, key1 = sdp.add_with_auto_key(event, "entry1", "foo")
+    entry2, key2 = sdp.add_with_auto_key(event, "entry1", "bar")
+    entry3, key3 = sdp.add_with_auto_key(event, "entry2", "baz")
 
     state = sdp.load_state(sdp.get_snapshot())
 
     assert path.exists(key_file)
     assert read_json_file(key_file) == {"entry1": 2, "entry2": 1}
     assert state.data == {"entry1": {"1": "foo", "2": "bar"}, "entry2": {"1": "baz"}}
+    assert entry1 == "entry1"
+    assert entry2 == "entry1"
+    assert entry3 == "entry2"
+    assert key1 == "1"
+    assert key2 == "2"
+    assert key3 == "1"
 
 
-def test_i_can_add_and_auto_set_the_key():
+def test_i_can_add_object_using_auto_generated_key():
     sdp = SheerkaDataProvider(".sheerka")
     event = Event("cmd add 'foo => bar'")
     key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile)
 
-    sdp.add_with_auto_key(event, "entry1", ObjSetKey("foo"))
-    sdp.add_with_auto_key(event, "entry1", ObjSetKey("bar"))
+    entry1, key1 = sdp.add_with_auto_key(event, "entry1", ObjNoKey("a", "b"))
+    entry2, key2 = sdp.add_with_auto_key(event, "entry1", ObjNoKey("a", "b"))
 
     state = sdp.load_state(sdp.get_snapshot())
 
     assert path.exists(key_file)
     assert read_json_file(key_file) == {"entry1": 2}
-    assert state.data == {"entry1": {"1": ObjSetKey("foo", "1"), "2": ObjSetKey("bar", "2")}}
+    assert state.data == {"entry1": {"1": ObjNoKey("a", "b"), "2": ObjNoKey("a", "b")}}
+    assert entry1 == "entry1"
+    assert entry2 == "entry1"
+    assert key1 == "1"
+    assert key2 == "2"
 
 
-def test_i_can_add_an_object_with_its_own_key():
+def test_object_key_is_updated_when_possible_using_auto_generated_key():
     sdp = SheerkaDataProvider(".sheerka")
     event = Event("cmd add 'foo => bar'")
+    key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile)
 
-    sdp.add(event, "entry", ObjWithKey(1, "foo"))
-    sdp.add(event, "entry", ObjWithKey(2, "bar"))
+    entry1, key1 = sdp.add_with_auto_key(event, "entry1", ObjSetKey("foo"))
+    entry2, key2 = sdp.add_with_auto_key(event, "entry1", ObjSetKey("foo"))
 
     state = sdp.load_state(sdp.get_snapshot())
-    assert state.data == {"entry": {"1": ObjWithKey(1, "foo"), "2": ObjWithKey(2, "bar")}}
 
-
-def test_i_can_add_dictionary():
-    sdp = SheerkaDataProvider(".sheerka")
-    event = Event("cmd add 'foo => bar'")
-
-    sdp.add(event, "entry", {"1": "foo"})
-    sdp.add(event, "entry", {"2": "bar"})
-
-    state = sdp.load_state(sdp.get_snapshot())
-    assert state.data == {"entry": {"1": "foo", "2": "bar"}}
-
-
-def test_i_cannot_add_the_same_key_twice():
-    sdp = SheerkaDataProvider(".sheerka")
-    sdp.add(Event("event"), "entry", {"1": "foo"})
-
-    with pytest.raises(IndexError):
-        sdp.add(Event("event"), "entry", {"1": "foo"})
-
-
-def test_i_cannot_add_the_same_element_twice():
-    sdp = SheerkaDataProvider(".sheerka")
-    sdp.add(Event("event"), "entry", ObjWithKey(1, "foo"))
-
-    with pytest.raises(IndexError):
-        sdp.add(Event("event"), "entry", ObjWithKey(1, "foo"))
+    assert path.exists(key_file)
+    assert read_json_file(key_file) == {"entry1": 2}
+    assert state.data == {"entry1": {"1": ObjSetKey("foo", "1"), "2": ObjSetKey("foo", "2")}}
+    assert entry1 == "entry1"
+    assert entry2 == "entry1"
+    assert key1 == "1"
+    assert key2 == "2"
 
 
 def test_i_can_set_objects_with_key():
     sdp = SheerkaDataProvider(".sheerka")
-    sdp.set(Event("event"), "entry", ObjWithKey(1, "foo"))
-    sdp.set(Event("event"), "entry", ObjWithKey(2, "foo"))
+    sdp.add(Event("event"), "entry", ObjWithKey(1, "foo"))
+    entry, key = sdp.set(Event("event"), "entry", ObjWithKey(2, "foo"))
 
     state = sdp.load_state(sdp.get_snapshot())
     assert state.data == {"entry": {"2": ObjWithKey(2, "foo")}}
+    assert entry == "entry"
+    assert key == "2"
 
 
 def test_i_can_set_objects_with_no_key():
     sdp = SheerkaDataProvider(".sheerka")
-    sdp.set(Event("event"), "entry", ObjNoKey(1, "foo"))
-    sdp.set(Event("event"), "entry", ObjNoKey(2, "foo"))
+    sdp.add(Event("event"), "entry", ObjNoKey(1, "foo"))
+    entry, key = sdp.set(Event("event"), "entry", ObjNoKey(2, "foo"))
 
     state = sdp.load_state(sdp.get_snapshot())
     assert state.data == {"entry": ObjNoKey(2, "foo")}
+    assert entry == "entry"
+    assert key is None
 
 
 def test_i_can_set_from_list_to_dict():
     sdp = SheerkaDataProvider(".sheerka")
     sdp.set(Event("event"), "entry", [ObjNoKey(1, "foo"), ObjNoKey(2, "foo")])
-    sdp.set(Event("event"), "entry", {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")})
+    entry, key = sdp.set(Event("event"), "entry", {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")})
 
     state = sdp.load_state(sdp.get_snapshot())
     assert state.data == {"entry": {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")}}
+    assert entry == "entry"
+    assert key is None
+
+
+def test_i_can_set_using_reference():
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey)))
+    sdp.add(Event("event"), "entry", ObjWithKey(1, "foo"))
+    entry, key = sdp.set(Event("event"), "entry", ObjWithKey(2, "foo"), use_ref=True)
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {"2": '##REF##:9b14e03847d73c640f54ea9b46ba62b19e5451ecd300428a225be012ad9f25f9'}}
+    assert entry == "entry"
+    assert key == "2"
+
+    assert path.exists(sdp.get_obj_path(SheerkaDataProvider.ObjectsFolder,
+                                        "9b14e03847d73c640f54ea9b46ba62b19e5451ecd300428a225be012ad9f25f9"))
+
+    # sanity check, make sure that I can load back
+    loaded = sdp.get(entry, key)
+    assert loaded == ObjWithKey(2, "foo")
 
 
 def test_i_can_add_unique():
@@ -277,10 +515,28 @@ def test_i_can_add_unique():
     sdp.add_unique(Event("event"), "entry", ObjNoKey(1, "foo"))
     sdp.add_unique(Event("event"), "entry", ObjNoKey(1, "foo"))
     sdp.add_unique(Event("event"), "entry", ObjNoKey(2, "bar"))
-    sdp.add_unique(Event("event"), "entry", ObjNoKey(2, "bar"))
+    entry, key = sdp.add_unique(Event("event"), "entry", ObjNoKey(2, "bar"))
 
     state = sdp.load_state(sdp.get_snapshot())
     assert state.data == {"entry": {ObjNoKey(1, "foo"), ObjNoKey(2, "bar")}}
+    assert entry == "entry"
+    assert key is None
+
+
+def test_i_can_add_reference_of_an_object_with_a_key():
+    sdp = SheerkaDataProvider(".sheerka")
+    obj = ObjDumpJson("my_key", "value1")
+    obj_serializer = ObjectSerializer(BaseSerializer.get_full_qualified_name(obj))
+    sdp.serializer.register(obj_serializer)
+
+    entry, key = sdp.add(Event("event"), "entry", obj, use_ref=True)
+    state = sdp.load_state(sdp.get_snapshot())
+    digest = state.data["entry"]["my_key"][len(SheerkaDataProvider.REF_PREFIX):]
+
+    assert key == obj.key
+    assert entry == "entry"
+    assert state.data == {'entry': {'my_key': f"{SheerkaDataProvider.REF_PREFIX}{digest}"}}
+    assert sdp.load_obj(digest) == obj
 
 
 def test_i_can_keep_state_history():
@@ -289,12 +545,14 @@ def test_i_can_keep_state_history():
     event1 = Event("cmd add 'foo => bar'")
     event_digest1 = event1.get_digest()
     obj1 = "foo => bar"
-    state_digest1 = sdp.add(event1, "entry1", obj1)
+    sdp.add(event1, "entry1", obj1)
+    state_digest1 = sdp.get_snapshot()
 
     event2 = Event("cmd add 'foo => baz'")
     event_digest2 = event2.get_digest()
     obj2 = "foo => baz"
-    state_digest2 = sdp.add(event2, "entry2", obj2)
+    sdp.add(event2, "entry2", obj2)
+    state_digest2 = sdp.get_snapshot()
 
     state2 = sdp.load_state(state_digest2)
 
@@ -318,24 +576,30 @@ def test_i_can_list_elements_when_there_is_nothing_to_list():
 
 def test_i_can_list_when_no_key():
     sdp = SheerkaDataProvider(".sheerka")
+    sdp.serializer.register(PickleSerializer(lambda obj: isinstance(obj, str)))
+
     sdp.add(Event("event"), "entry1", "foo")
     sdp.add(Event("event"), "entry1", "bar")
-    sdp.add(Event("event"), "entry2", "baz")
+    sdp.add(Event("event"), "entry1", "baz", use_ref=True)
+    sdp.add(Event("event"), "entry2", "xyz")
 
     result = sdp.list("entry1")
 
-    assert list(result) == ["foo", "bar"]
+    assert list(result) == ["foo", "bar", "baz"]
 
 
 def test_i_can_list_when_key():
     sdp = SheerkaDataProvider(".sheerka")
+    sdp.serializer.register(PickleSerializer(lambda obj: isinstance(obj, ObjWithKey)))
+
     sdp.add(Event("event"), "entry1", {"1": "foo"})
     sdp.add(Event("event"), "entry1", {"2": "bar"})
-    sdp.add(Event("event"), "entry2", {"3": "baz"})
+    sdp.add(Event("event"), "entry1", ObjWithKey("3", "value"), use_ref=True)
+    sdp.add(Event("event"), "entry2", {"4": "xxx"})
 
     result = sdp.list("entry1")
 
-    assert list(result) == ["foo", "bar"]
+    assert list(result) == ["foo", "bar", ObjWithKey("3", "value")]
 
 
 def test_i_can_list_when_one_element():
@@ -459,32 +723,234 @@ def test_i_cannot_remove_if_entry_does_not_exist():
         assert str(e) == "entry"
 
 
-def test_i_can_replace_an_entry():
+def test_i_cannot_modify_an_entry_without_a_key():
     sdp = SheerkaDataProvider(".sheerka")
-    sdp.add(Event("event"), "entry1", "foo")
-    sdp.add(Event("event"), "entry1", "bar")
 
-    sdp.modify(Event("event"), "entry1", None, "baz")
-    result = sdp.list("entry1")
+    with pytest.raises(SheerkaDataProviderError) as error:
+        sdp.modify(Event("event"), "entry", None, "baz")
 
-    assert list(result) == ["baz"]
+    assert error.value.args[0] == "Key is mandatory."
 
 
-def test_i_cannot_update_an_entry_that_does_not_exist():
+def test_i_can_modify_dict_with_a_key():
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.add(Event("event"), "entry", {"key1": "foo"})
+    sdp.add(Event("event"), "entry", {"key2": "bar"})
+
+    entry, key = sdp.modify(Event("event"), "entry", "key1", "baz")
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {"key1": "baz", "key2": "bar"}}
+    assert entry == "entry"
+    assert key == "key1"
+
+
+def test_i_can_modify_an_object_with_a_key():
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.add(Event("event"), "entry", ObjWithKey("key1", "foo"))
+    sdp.add(Event("event"), "entry", ObjWithKey("key2", "bar"))
+
+    entry, key = sdp.modify(Event("event"), "entry", "key1", ObjWithKey("key1", "baz"))
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {"key1": ObjWithKey("key1", "baz"), "key2": ObjWithKey("key2", "bar")}}
+    assert entry == "entry"
+    assert key == "key1"
+
+
+def test_i_can_modify_an_object_while_changing_the_key():
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.add(Event("event"), "entry", ObjWithKey("key1", "foo"))
+    sdp.add(Event("event"), "entry", ObjWithKey("key2", "bar"))
+
+    entry, key = sdp.modify(Event("event"), "entry", "key1", ObjWithKey("key3", "baz"))
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {"key2": ObjWithKey("key2", "bar"), "key3": ObjWithKey("key3", "baz")}}
+    assert entry == "entry"
+    assert key == "key3"
+
+
+def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_key():
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.add(Event("event"), "entry", ObjWithKey("key1", "foo"))
+    sdp.add(Event("event"), "entry", ObjWithKey("key2", "bar"))
+
+    entry, key = sdp.modify(Event("event"), "entry", "key2", ObjWithKey("key1", "bar"))
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {"key1": [ObjWithKey("key1", "foo"), ObjWithKey("key1", "bar")]}}
+    assert entry == "entry"
+    assert key == "key1"
+
+
+def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_list():
+    """
+    In this example, the item to modify is within a list, and its key has changed
+    and in the new key, there is already a list
+    :return:
+    """
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
+
+    sdp.add(Event("event"), "entry", ObjDumpJson("key1", "value11"))
+    sdp.add(Event("event"), "entry", ObjDumpJson("key1", "value12"))
+    sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value21"))
+    sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value22"))
+
+    new_value = ObjDumpJson("key1", "value13")
+    setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest())
+    entry, key = sdp.modify(Event("event"), "entry", "key2", new_value)
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {
+        "key1": [ObjDumpJson("key1", "value11"), ObjDumpJson("key1", "value12"), ObjDumpJson("key1", "value13")],
+        "key2": [ObjDumpJson("key2", "value22")]
+    }}
+    assert entry == "entry"
+    assert key == "key1"
+
+
+def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_nothing():
+    """
+    In this example, the item to modify is within a list, and its key has changed
+    and in the new key, there is nothing (the new key does not exist)
+    :return:
+    """
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
+
+    sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value21"))
+    sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value22"))
+
+    new_value = ObjDumpJson("key1", "value13")
+    setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest())
+    entry, key = sdp.modify(Event("event"), "entry", "key2", new_value)
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {
+        "key1": ObjDumpJson("key1", "value13"),
+        "key2": [ObjDumpJson("key2", "value22")]
+    }}
+    assert entry == "entry"
+    assert key == "key1"
+
+
+def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_one_item():
+    """
+    In this example, the item to modify is within a list, and its key has changed
+    and in the new key, there is only one element
+    :return:
+    """
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
+
+    sdp.add(Event("event"), "entry", ObjDumpJson("key1", "value11"))
+    sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value21"))
+    sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value22"))
+
+    new_value = ObjDumpJson("key1", "value13")
+    setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest())
+    entry, key = sdp.modify(Event("event"), "entry", "key2", new_value)
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {
+        "key1": [ObjDumpJson("key1", "value11"), ObjDumpJson("key1", "value13")],
+        "key2": [ObjDumpJson("key2", "value22")]
+    }}
+    assert entry == "entry"
+    assert key == "key1"
+
+
+def test_i_can_modify_a_ref():
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey)))
+    sdp.add(Event("event"), "entry", ObjWithKey("key1", "foo"))
+    entry, key = sdp.add(Event("event"), "entry", ObjWithKey("key2", "bar"), use_ref=True)
+
+    sdp.modify(Event("event"), "entry", "key2", ObjWithKey("key2", "baz"))
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {
+        "key1": ObjWithKey("key1", "foo"),
+        "key2": "##REF##:eb297e98710dd17244bb0e38eb9f1bf72cba692a8f8d94e9eb2d898e130cac8b"}}
+    assert entry == "entry"
+    assert key == "key2"
+
+
+def test_i_cannot_modify_an_entry_that_does_not_exist():
     sdp = SheerkaDataProvider(".sheerka")
 
     with pytest.raises(IndexError) as e:
         sdp.modify(Event("event"), "entry", "key", "foo")
-        assert str(e) == "entry"
+
+    assert str(e.value) == "entry"
 
 
-def test_i_cannot_update_a_key_that_does_not_exist():
+def test_i_cannot_modify_a_key_that_does_not_exist():
     sdp = SheerkaDataProvider(".sheerka")
     sdp.add(Event("event"), "entry1", {"1": "foo"})
 
     with pytest.raises(IndexError) as e:
         sdp.modify(Event("event"), "entry1", "2", "bar")
-        assert str(e) == "entry.1"
+        assert str(e) == "entry1.2"
+
+
+def test_i_cannot_modify_a_list_when_origin_is_unknown():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    sdp.add(Event("event"), "entry", ObjWithKey("key", "value1"))
+    sdp.add(Event("event"), "entry", ObjWithKey("key", "value2"))  # same they
+
+    state = sdp.load_state(sdp.get_snapshot())
+
+    with pytest.raises(SheerkaDataProviderError) as error:
+        sdp.modify(Event("event"), "entry", "key", ObjWithKey("key", "value2"))
+
+    assert error.value.obj == ObjWithKey("key", "value2")
+    assert error.value.args[0] == "Multiple entries under 'entry.key'"
+
+
+def test_i_can_modify_a_list_when_the_origin_is_known():
+    sdp = SheerkaDataProvider(".sheerka")
+
+    sdp.add(Event("event"), "entry", ObjDumpJson("key", "value1"))
+    sdp.add(Event("event"), "entry", ObjDumpJson("key", "value2"))  # same they
+
+    new_value = ObjDumpJson("key", "value3")
+    setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key", "value1").get_digest())
+
+    sdp.modify(Event("event"), "entry", "key", new_value)
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {"key": [ObjDumpJson("key", "value3"), ObjDumpJson("key", "value2")]}}
+
+
+def test_i_can_modify_a_list_when_the_origin_is_known_2():
+    """
+    This time, we check that the origin is automatically set when the object was saved as a reference
+    We also check that all objects are still persisted as reference
+    :return:
+    """
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
+
+    sdp.add(Event("event"), "entry", ObjDumpJson("key", "value1"), use_ref=True)
+    sdp.add(Event("event"), "entry", ObjDumpJson("key", "value2"), use_ref=True)  # same they
+
+    objs = sdp.get("entry", "key")  # origin is automatically set to the loaded objects
+    objs[0].value = "value3"
+
+    sdp.modify(Event("event"), "entry", "key", objs[0])
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {"key": [
+        "##REF##:621771a3af6a331e9abb3a63fb25e0cac4b13df0b292dfa30db6bd89031bfad0",
+        "##REF##:5fe085e8366d35c5f04a18b2d3dada376128b246e07c66de5872830b00f5f517"]}}
+
+    # checks that all objects are (still) persisted
+    path.exists(sdp.get_obj_path(sdp.ObjectsFolder, "621771a3af6a331e9abb3a63fb25e0cac4b13df0b292dfa30db6bd89031bfad0"))
+    path.exists(sdp.get_obj_path(sdp.ObjectsFolder, "5fe085e8366d35c5f04a18b2d3dada376128b246e07c66de5872830b00f5f517"))
+    path.exists(sdp.get_obj_path(sdp.ObjectsFolder, "1aac9e0d5c74c3bb989fd0f9def792bba36c5595d32f61be7cbb1a38dcf75327"))
 
 
 def test_i_can_get_the_entire_entry():
@@ -522,6 +988,30 @@ def test_i_can_get_an_entry_by_key():
     assert result_safe == "bar"
 
 
+def test_i_can_get_object_save_by_reference():
+    sdp = SheerkaDataProvider(".sheerka")
+    obj = ObjDumpJson("my_key", "value1")
+    sdp.serializer.register(ObjectSerializer(BaseSerializer.get_full_qualified_name(obj)))
+
+    entry, key = sdp.add(Event("event"), "entry", obj, use_ref=True)
+    loaded = sdp.get(entry, key)
+
+    assert loaded == obj
+
+
+def test_i_can_get_objects_from_list_when_saved_by_reference():
+    sdp = SheerkaDataProvider(".sheerka")
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
+
+    sdp.add(Event("event"), "entry", ObjDumpJson("key", "value1"), use_ref=True)
+    sdp.add(Event("event"), "entry", ObjDumpJson("key", "value2"), use_ref=True)  # same they
+
+    objs = sdp.get("entry", "key")
+
+    assert objs[0] == ObjDumpJson("key", "value1")
+    assert objs[1] == ObjDumpJson("key", "value2")
+
+
 def test_i_cannot_get_an_entry_that_does_not_exist():
     sdp = SheerkaDataProvider(".sheerka")
 
@@ -604,53 +1094,52 @@ def test_i_can_test_than_an_entry_exits():
     assert sdp.exists("entry")
 
 
-def test_i_can_save_and_load_object_with_history():
+def test_i_can_save_and_load_object_ref_with_history():
     sdp = SheerkaDataProvider(".sheerka")
     obj = ObjDumpJson("my_key", "value1")
     sdp.serializer.register(ObjectSerializer(BaseSerializer.get_full_qualified_name(obj)))
 
-    entry, key = sdp.add_ref("Obj", obj)
+    entry, key = sdp.add(Event("event"), "entry", obj, use_ref=True)
     loaded = sdp.get(entry, key)
     history = getattr(loaded, Serializer.HISTORY)
 
     assert key == obj.key
-    assert entry == "Obj"
+    assert entry == "entry"
     assert loaded.key == obj.key
     assert loaded.value == obj.value
 
-    assert getattr(history, Serializer.USERNAME) == "kodjo"
-    assert getattr(history, Serializer.MODIFICATION_DATE) != ""
-    assert getattr(history, Serializer.PARENTS) == []
+    assert history[Serializer.USERNAME] == "kodjo"
+    assert history[Serializer.MODIFICATION_DATE] != ""
+    assert history[Serializer.PARENTS] == []
 
     assert os.path.exists(sdp.get_obj_path(sdp.ObjectsFolder, obj.get_digest()))
 
     # save a second type with no modification
-    previous_modification_time = getattr(history, Serializer.MODIFICATION_DATE)
-    previous_parents = getattr(history, Serializer.PARENTS)
+    previous_modification_time = history[Serializer.MODIFICATION_DATE]
+    previous_parents = history[Serializer.PARENTS]
 
-    sdp.add_ref("Obj", loaded)
+    sdp.modify(Event("event"), "entry", key, loaded)
     loaded = sdp.get(entry, key)
     history = getattr(loaded, Serializer.HISTORY)
 
-    assert getattr(history, Serializer.MODIFICATION_DATE) == previous_modification_time
-    assert getattr(history, Serializer.PARENTS) == previous_parents
+    assert history[Serializer.MODIFICATION_DATE] == previous_modification_time
+    assert history[Serializer.PARENTS] == previous_parents
 
     # save again, but with a modification
     previous_digest = loaded.get_digest()
     loaded.value = "value2"
 
-    sdp.add_ref("Obj", loaded)
+    sdp.modify(Event("event"), "entry", key, loaded)
     loaded2 = sdp.get(entry, key)
-    history2 = getattr(loaded, Serializer.HISTORY)
+    history2 = getattr(loaded2, Serializer.HISTORY)
 
     assert loaded2.key == loaded.key
     assert loaded2.value == loaded.value
 
-    assert getattr(history2, Serializer.USERNAME) == "kodjo"
-    assert getattr(history2, Serializer.MODIFICATION_DATE) != ""
-    assert getattr(history2, Serializer.PARENTS) == [previous_digest]
-
-
-
-
+    assert history2[Serializer.USERNAME] == "kodjo"
+    assert history2[Serializer.MODIFICATION_DATE] != ""
+    assert history2[Serializer.PARENTS] == [previous_digest]
 
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {"entry": {
+        "my_key": '##REF##:e6bf5b56428cfce0f08c94f2c3625dc3b3a8180d7229eaa9f8aa967fb16e5256'}}