Added ModifyConcept function, and fixed 'isa' not working

src/core/concept.py

@@ -1,6 +1,6 @@
 import hashlib
 from collections import namedtuple
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from enum import Enum
 from core.sheerka_logger import get_logger
 
@@ -15,6 +15,7 @@ PROPERTIES_FOR_DIGEST = ("name", "key",
 PROPERTIES_TO_SERIALIZE = PROPERTIES_FOR_DIGEST + tuple(["id"])
 PROPERTIES_FOR_NEW = ("where", "pre", "post", "body", "desc")
 VARIABLE_PREFIX = "__var__"
+ORIGIN = "##origin##"  # same as Serializer.ORIGIN but I don't want to include the reference
 
 
 class ConceptParts(Enum):
@@ -97,6 +98,7 @@ class Concept:
         self.bnf = None
         self.log = get_logger("core." + self.__class__.__name__)
         self.init_log = get_logger("init.core." + self.__class__.__name__)
+        self.original_definition_hash = None
 
     def __repr__(self):
         return f"({self.metadata.id}){self.metadata.name}"
@@ -243,7 +245,19 @@ class Concept:
     def body(self):
         return self.values[ConceptParts.BODY] if ConceptParts.BODY in self.values else None
 
-    def get_digest(self):
+    def get_origin(self):
+        """
+        Return the digest used to save the concept if it exists
+        :return:
+        """
+        if hasattr(self, ORIGIN):
+            return getattr(self, ORIGIN)
+        return None
+
+    def set_origin(self, origin):
+        setattr(self, ORIGIN, origin)
+
+    def get_definition_hash(self):
         """
         Returns the digest of the event
         :return: hexa form of the sha256
@@ -301,6 +315,11 @@ class Concept:
         for k, v in other.props.items():
             self.set_prop(k, v.value)
 
+        # origin
+        from sdp.sheerkaSerializer import Serializer
+        if hasattr(other, Serializer.ORIGIN):
+            setattr(self, Serializer.ORIGIN, getattr(other, Serializer.ORIGIN))
+
         return self
 
     def set_prop(self, prop_name, prop_value):
@@ -362,6 +381,12 @@ class Concept:
         self.metadata.is_evaluated = True
         return self
 
+    def freeze_definition_hash(self):
+        self.original_definition_hash = self.get_definition_hash()
+
+    def get_original_definition_hash(self):
+        return self.original_definition_hash
+
 
 class Property:
     """

src/core/sheerka/Services/SheerkaCreateNewConcept.py

@@ -1,6 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept
 from core.concept import Concept
-from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
+from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError, SheerkaDataProviderRef
 
 CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
 
@@ -30,7 +30,7 @@ class SheerkaCreateNewConcept:
         # checks for duplicate concepts
         # TODO checks if it exists in cache first
 
-        if self.sheerka.sdp.exists(self.sheerka.CONCEPTS_ENTRY, concept.key, concept.get_digest()):
+        if self.sheerka.sdp.exists(self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()):
             error = SheerkaDataProviderDuplicateKeyError(self.sheerka.CONCEPTS_ENTRY + "." + concept.key, concept)
             return self.sheerka.ret(
                 self.logger_name,
@@ -55,22 +55,35 @@ class SheerkaCreateNewConcept:
                 if not init_ret_value.status:
                     return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
 
+        concept.freeze_definition_hash()
+
         # save the new concept in sdp
-        concept.metadata.full_serialization = True
         try:
             # TODO : needs to make these calls atomic (or at least one single call)
             # save the new concept
-            self.sheerka.sdp.add(
+            concept.metadata.full_serialization = True
+            result = self.sheerka.sdp.add(
                 context.event.get_digest(),
                 self.sheerka.CONCEPTS_ENTRY,
                 concept,
                 use_ref=True)
+            concept.metadata.full_serialization = False
+
+            # update the concept (I hope that it's enough)
+            concept.set_origin(result.digest)
+
             # save it by id
             self.sheerka.sdp.add(
                 context.event.get_digest(),
                 self.sheerka.CONCEPTS_BY_ID_ENTRY,
-                {concept.id: concept.get_digest()},
+                SheerkaDataProviderRef(concept.id, result.digest))
-                is_ref=True)
+
+            # records the hash
+            self.sheerka.sdp.add(
+                context.event.get_digest(),
+                self.sheerka.CONCEPTS_BY_HASH_ENTRY,
+                SheerkaDataProviderRef(concept.get_definition_hash(), result.digest))
+
             # update the definition table
             if concepts_definitions is not None:
                 self.sheerka.sdp.set(
@@ -88,7 +101,7 @@ class SheerkaCreateNewConcept:
                 error.args[0])
 
         # Updates the caches
-        concept.metadata.full_serialization = False
+
         self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
         self.sheerka.cache_by_id[concept.id] = concept
         if init_ret_value is not None and init_ret_value.status:

src/core/sheerka/Services/SheerkaDump.py

@@ -63,7 +63,7 @@ class SheerkaDump:
                 else:
                     self.sheerka.log.info("No property")
 
-                self.sheerka.log.info(f"digest : {c.get_digest()}")
+                self.sheerka.log.info(f"digest : {c.get_origin()}")
 
                 if self.sheerka.isaset(context, c):
                     items = self.sheerka.get_set_elements(context, c)

src/core/sheerka/Services/SheerkaModifyConcept.py

@@ -1,4 +1,5 @@
 from core.builtin_concepts import BuiltinConcepts
+from sdp.sheerkaDataProvider import SheerkaDataProviderRef
 
 
 class SheerkaModifyConcept:
@@ -8,7 +9,41 @@ class SheerkaModifyConcept:
 
     def modify_concept(self, context, concept):
 
-        self.sheerka.sdp.modify(context.event.get_digest(), self.sheerka.CONCEPTS_ENTRY, concept.key, concept)
+        try:
+            # modify the entry
+            concept.metadata.full_serialization = True
+            result = self.sheerka.sdp.modify(
+                context.event.get_digest(),
+                self.sheerka.CONCEPTS_ENTRY,
+                concept.key,
+                concept)
+            concept.metadata.full_serialization = False
+
+            # update its reference
+            self.sheerka.sdp.modify(
+                context.event.get_digest(),
+                self.sheerka.CONCEPTS_BY_ID_ENTRY,
+                concept.id,
+                SheerkaDataProviderRef(concept.id, result.digest, concept.get_origin()))
+
+            # update the hash entry
+            self.sheerka.sdp.modify(
+                context.event.get_digest(),
+                self.sheerka.CONCEPTS_BY_HASH_ENTRY,
+                concept.get_original_definition_hash(),
+                SheerkaDataProviderRef(concept.get_definition_hash(), result.digest, concept.get_origin()))
+
+        except IndexError as error:
+            context.log_error(f"Failed to update concept '{concept}'.", who=self.logger_name)
+            return self.sheerka.ret(
+                self.logger_name,
+                False,
+                self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept),
+                error.args[0])
+
+        # update cache
+        self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
+        self.sheerka.cache_by_id[concept.id] = concept
 
         ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
         return ret

src/core/sheerka/Services/SheerkaSetsManager.py

@@ -48,8 +48,8 @@ class SheerkaSetsManager:
         assert concept_set.id
 
         try:
-            ret = self.sheerka.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id)
+            result = self.sheerka.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id)
-            if ret == (None, None):  # concept already in set
+            if result.already_exists:  # concept already in set
                 return self.sheerka.ret(
                     self.logger_name,
                     False,

src/core/sheerka/Sheerka.py

@@ -34,6 +34,7 @@ class Sheerka(Concept):
 
     CONCEPTS_ENTRY = "All_Concepts"  # to store all the concepts
     CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"
+    CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash" # store hash of concepts definitions (not values)
     CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions"  # to store definitions (bnf) of concepts
     BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts"  # sequential key for builtin concepts
     USER_CONCEPTS_KEYS = "User_Concepts"  # sequential key for user defined concepts
@@ -482,6 +483,7 @@ class Sheerka(Concept):
         # otherwise, create another instance
         concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
         concept.update_from(template)
+        concept.freeze_definition_hash()
 
         if len(kwargs) == 0:
             return concept

src/sdp/sheerkaDataProvider.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from datetime import datetime, date
 import hashlib
 import json
@@ -206,12 +207,32 @@ class State:
     def modify_in_list(self, entry, key, obj, obj_key, obj_origin, load_ref_if_needed, save_ref_if_needed):
         found = False
         to_remove = None
+        new_digest = None
+
+        def _get_item_origin(o):
+            if hasattr(o, Serializer.ORIGIN):
+                return getattr(o, Serializer.ORIGIN)
+
+            if isinstance(o, dict) and Serializer.ORIGIN in o:
+                return o[Serializer.ORIGIN]
+
+            if hasattr(o, "get_digest"):
+                return o.get_digest()
+
+            if isinstance(o, str):
+                return o
+
+            return None
+
         for i in range(len(self.data[entry][key])):
             item, is_ref = load_ref_if_needed(self.data[entry][key][i])
-            if not hasattr(item, "get_digest"):
+            item_origin = _get_item_origin(item)
+            if item_origin is None:
                 continue
-            if item.get_digest() == obj_origin:
+            if item_origin == obj_origin:
                 obj = save_ref_if_needed(is_ref, obj)
+                if is_ref:
+                    new_digest = obj[len(SheerkaDataProvider.REF_PREFIX):]
                 if obj_key == key:
                     self.data[entry][key][i] = obj
                 else:
@@ -226,6 +247,8 @@ class State:
         if to_remove is not None:
             del self.data[entry][key][to_remove]
 
+        return new_digest
+
     def remove(self, entry, filter):
         if filter is None:
             del (self.data[entry])
@@ -282,6 +305,28 @@ class SheerkaDataProviderDuplicateKeyError(Exception):
         self.obj = obj
 
 
+@dataclass
+class SheerkaDataProviderResult:
+    obj: object
+    entry: str
+    key: str
+    digest: str
+    already_exists: bool = False
+
+
+@dataclass
+class SheerkaDataProviderRef:
+    key: str
+    target: str
+    original_target: str = None
+
+    def get_digest(self):
+        return self.original_target
+
+    def get_key(self):
+        return self.key
+
+
 class SheerkaDataProvider:
     """Manages the state of the system"""
 
@@ -305,7 +350,6 @@ class SheerkaDataProvider:
 
         self.serializer = Serializer()
 
-
     @staticmethod
     def get_obj_key(obj):
         """
@@ -344,6 +388,9 @@ class SheerkaDataProvider:
         if hasattr(obj, Serializer.ORIGIN):
             return getattr(obj, Serializer.ORIGIN)
 
+        if isinstance(obj, SheerkaDataProviderRef):
+            return  obj.original_target
+
         return None
 
     @staticmethod
@@ -359,7 +406,7 @@ class SheerkaDataProvider:
     def is_reference(obj):
         return isinstance(obj, str) and obj.startswith(SheerkaDataProvider.REF_PREFIX)
 
-    def add(self, event_digest: str, entry, obj, allow_multiple=True, use_ref=False, is_ref=False):
+    def add(self, event_digest: str, entry, obj, allow_multiple=True, use_ref=False):
         """
         Adds obj to the entry 'entry'
         :param event_digest: digest of the event that triggers the modification of the state
@@ -372,11 +419,7 @@ class SheerkaDataProvider:
         :return: (entry, key) to retrieve the object
         """
 
-        if use_ref and is_ref:
+        original_obj = obj.copy() if isinstance(obj, dict) else obj
-            raise SheerkaDataProviderError("Cannot use use_ref and is_ref at the same time", None)
-
-        if is_ref and not isinstance(obj, dict):
-            raise SheerkaDataProviderError("is_ref can only be used with dictionaries", obj)
 
         snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile)
         state = self.load_state(snapshot)
@@ -406,15 +449,11 @@ class SheerkaDataProvider:
             obj.set_digest(self.save_obj(obj.obj))
             obj.obj = self.REF_PREFIX + obj.get_digest()
 
-        if is_ref:
-            for k, v in obj.obj.items():
-                obj.obj[k] = self.REF_PREFIX + v
-
         state.update(entry, obj)
 
         new_snapshot = self.save_state(state)
         self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot)
-        return entry, key
+        return SheerkaDataProviderResult(original_obj, entry, obj.get_key(), obj.get_digest())
 
     def add_with_auto_key(self, event_digest: str, entry, obj):
         """
@@ -424,14 +463,20 @@ class SheerkaDataProvider:
         :param obj:
         :return:
         """
+
+        original_obj = obj.copy() if isinstance(obj, dict) else obj
+
         next_key = self.get_next_key(entry)
         if hasattr(obj, "set_key"):
             obj.set_key(next_key)
-        self.add(event_digest, entry, ObjToUpdate(obj, next_key))
+        res = self.add(event_digest, entry, ObjToUpdate(obj, next_key))
-        return entry, next_key
+        return SheerkaDataProviderResult(original_obj, res.entry, res.key, res.digest)
 
     def add_unique(self, event_digest: str, entry, obj):
         """Add an entry and make sure it's unique"""
+
+        original_obj = obj.copy() if isinstance(obj, dict) else obj
+
         snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile)
         state = self.load_state(snapshot)
 
@@ -448,7 +493,12 @@ class SheerkaDataProvider:
 
         new_snapshot = self.save_state(state)
         self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot)
-        return (None if already_exist else entry), None
+        return SheerkaDataProviderResult(
+            original_obj,
+            entry,
+            None,
+            None,
+            already_exist)
 
     def set(self, event_digest, entry, obj, use_ref=False, is_ref=False):
         """
@@ -462,6 +512,8 @@ class SheerkaDataProvider:
         :return:
         """
 
+        original_obj = obj.copy() if isinstance(obj, dict) else obj
+
         if use_ref and is_ref:
             raise SheerkaDataProviderError("Cannot use use_ref and is_ref at the same time", None)
 
@@ -486,7 +538,7 @@ class SheerkaDataProvider:
 
         new_snapshot = self.save_state(state)
         self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot)
-        return entry, key
+        return SheerkaDataProviderResult(original_obj, entry, key, self.get_obj_digest(obj))
 
     def modify(self, event_digest, entry, key, obj):
         """
@@ -499,6 +551,8 @@ class SheerkaDataProvider:
         :return:
         """
 
+        original_obj = obj.copy() if isinstance(obj, dict) else obj
+
         if key is None:
             raise SheerkaDataProviderError("Key is mandatory.", None)
 
@@ -517,21 +571,33 @@ class SheerkaDataProvider:
 
         # Gets obj original key, it will help to know if the key has changed
         obj_key = self.get_obj_key(obj) or key
+        digest = None
 
         if isinstance(state.data[entry][key], list):
             obj_origin = self.get_obj_origin(obj)
             if obj_origin is None:
                 raise (SheerkaDataProviderError(f"Multiple entries under '{entry}.{key}'", obj))
 
-            state.modify_in_list(entry, key, obj, obj_key, obj_origin, self.load_ref_if_needed, self.save_ref_if_needed)
+            digest = state.modify_in_list(
+                entry,
+                key,
+                obj,
+                obj_key,
+                obj_origin,
+                self.load_ref_if_needed,
+                self.save_ref_if_needed)
 
         else:
-            obj = self.save_ref_if_needed(self.is_reference(state.data[entry][key]), obj)
+            was_saved_as_reference = self.is_reference(state.data[entry][key])
+            if was_saved_as_reference:
+                obj = self.save_ref_if_needed(True, obj)
+                digest = self.get_obj_digest(obj)
+
             state.modify(entry, key, obj, obj_key)
 
         new_snapshot = self.save_state(state)
         self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot)
-        return entry, obj_key
+        return SheerkaDataProviderResult(original_obj, entry, obj_key, digest)
 
     def list(self, entry, filter=None):
         """
@@ -814,16 +880,15 @@ class SheerkaDataProvider:
         return obj
 
     def load_ref_if_needed(self, obj, load_origin=True):
-        if not isinstance(obj, str):
+        if isinstance(obj, SheerkaDataProviderRef):
-            return obj, False
+            resolved = self.load_obj(obj.target, load_origin)
-        if not obj.startswith(SheerkaDataProvider.REF_PREFIX):
+            return resolved, False
+
+        if not isinstance(obj, str) or not obj.startswith(SheerkaDataProvider.REF_PREFIX):
             return obj, False
 
         resolved = self.load_obj(obj[len(SheerkaDataProvider.REF_PREFIX):], load_origin)
-        if resolved is None:
+        return (obj, False) if resolved is None else (resolved, True)
-            return obj, False
-
-        return resolved, True
 
     def save_ref_if_needed(self, save_ref, obj):
         if not save_ref:

src/sheerkapickle/SheerkaUnpickler.py

@@ -5,7 +5,8 @@ from sheerkapickle import tags, utils, handlers
 
 
 def decode(sheerka, obj):
-    return SheerkaUnpickler(sheerka).restore(json.loads(obj))
+    decoded = SheerkaUnpickler(sheerka).restore(json.loads(obj))
+    return decoded
 
 
 class SheerkaUnpickler:

src/sheerkapickle/sheerka_handlers.py

@@ -74,6 +74,7 @@ class ConceptHandler(BaseHandler):
                 # get value
                 instance.set_metadata_value(ConceptParts(key), resolved_value)
 
+        instance.freeze_definition_hash()
         return instance
 
 
@@ -92,6 +93,7 @@ class UserInputHandler(ConceptHandler):
         instance.__init__(data["text"], data["user_name"])
         instance.metadata.key = data[CONCEPT_ID][0]
         instance.metadata.id = data[CONCEPT_ID][1]
+        instance.freeze_definition_hash()
         return instance
 
 
@@ -122,6 +124,7 @@ class ReturnValueHandler(BaseHandler):
 
         instance.metadata.key = data[CONCEPT_ID][0]
         instance.metadata.id = data[CONCEPT_ID][1]
+        instance.freeze_definition_hash()
         return instance
 
 

tests/BaseTest.py

@@ -36,23 +36,26 @@ class BaseTest:
     def init_concepts(self, *concepts, **kwargs):
         sheerka = self.get_sheerka(**kwargs)
         context = self.get_context(sheerka)
+        create_new = kwargs.get("create_new", False)
 
         result = []
         for c in concepts:
             if isinstance(c, str):
                 c = Concept(c)
-            c.init_key()
-            sheerka.set_id_if_needed(c, False)
 
-            # manage concepts with bnf definitions
             if c.metadata.definition:
                 bnf_parser = BnfParser()
                 res = bnf_parser.parse(context, c.metadata.definition)
                 if res.status:
                     c.bnf = res.value.value
                 sheerka.create_new_concept(context, c)
+            elif create_new:
+                sheerka.create_new_concept(context, c)
+            else:
+                c.init_key()
+                sheerka.set_id_if_needed(c, False)
+                sheerka.add_in_cache(c)
 
-            sheerka.add_in_cache(c)
             result.append(c)
 
         return sheerka, context, *result

tests/core/test_SheerkaCreateNewConcept.py

@@ -1,5 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import PROPERTIES_TO_SERIALIZE, Concept
+from core.sheerka.Sheerka import Sheerka
 from sdp.sheerkaDataProvider import SheerkaDataProvider
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -25,7 +26,10 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
         assert concept.key in sheerka.cache_by_key
         assert concept.id in sheerka.cache_by_id
         assert sheerka.sdp.io.exists(
-            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest()))
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_origin()))
+        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
+        assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert sheerka.sdp.exists(Sheerka.CONCEPTS_ENTRY, concept.key)
 
     def test_i_cannot_add_the_same_concept_twice(self):
         """
@@ -77,7 +81,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
         concept = self.get_default_concept()
         sheerka.create_new_concept(self.get_context(sheerka), concept)
 
-        sheerka.cache_by_key = {}  # reset the cache
+        sheerka.reset_cache()
         loaded = sheerka.get(concept.key)
 
         assert loaded == concept
@@ -86,6 +90,16 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
         loaded = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
         assert loaded == concept
 
+    def test_i_can_instantiate_a_concept_from_sdp(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo")
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        sheerka.reset_cache()
+        loaded = sheerka.new("foo")
+
+        assert loaded == concept
+
     def test_i_can_get_a_concept_by_its_id(self):
         sheerka = self.get_sheerka()
         concept = self.get_default_concept()
@@ -105,6 +119,8 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
         res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
         res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
 
+        assert res1.status
+        assert res2.status
         assert res1.value.body.key == res2.value.body.key  # same key
 
         sheerka.cache_by_key = {}  # reset the cache

tests/core/test_SheerkaModifyConcept.py

@@ -0,0 +1,104 @@
+import pytest
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from core.sheerka.Sheerka import Sheerka
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka):
+    def test_i_can_modify_a_concept(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
+
+        foo_instance = sheerka.new("foo")
+        foo_instance.metadata.body = "value"
+        foo_instance.set_prop(BuiltinConcepts.ISA, bar)
+        foo_instance.set_metadata_value(ConceptParts.BODY, "body value")
+        res = sheerka.modify_concept(context, foo_instance)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.metadata.body == "value"
+        assert res.body.body.get_prop(BuiltinConcepts.ISA) == bar
+        assert res.body.body.body == "body value"
+
+        # test that object
+        sheerka.reset_cache()
+        foo_from_sheerka = sheerka.new("foo")
+        assert foo_from_sheerka.metadata.body == "value"
+        assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar
+        assert foo_from_sheerka.body == "body value"
+
+        # test that ref by id is updated
+        sheerka.reset_cache()
+        foo_from_sheerka = sheerka.get_by_id(foo.id)
+        assert foo_from_sheerka.metadata.body == "value"
+        assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar
+        assert foo_from_sheerka.body == "body value"
+
+        # test that ref by hash is updated
+        foo_from_sdp = sheerka.sdp.get(Sheerka.CONCEPTS_BY_HASH_ENTRY, foo_instance.get_definition_hash())
+        assert foo_from_sdp.metadata.body == "value"
+        assert foo_from_sdp.get_prop(BuiltinConcepts.ISA) == bar
+        assert foo_from_sdp.body == "body value"
+
+        # previous ref by hash is removed (since that definition hash has changed)
+        with pytest.raises(IndexError):
+            sheerka.sdp.get(Sheerka.CONCEPTS_BY_HASH_ENTRY, foo_instance.get_original_definition_hash())
+
+    def test_i_can_modify_concept_modifying_only_properties_and_body(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
+
+        foo_instance = sheerka.new("foo")
+        foo_instance.set_prop(BuiltinConcepts.ISA, bar)
+        foo_instance.set_metadata_value(ConceptParts.BODY, "body value")
+        res = sheerka.modify_concept(context, foo_instance)
+
+        assert res.status
+
+        foo_from_sheerka = sheerka.new("foo")
+        assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar
+        assert foo_from_sheerka.body == "body value"
+
+    def test_cache_is_updated_when_a_concept_is_modified(self):
+        sheerka, context, foo = self.init_concepts("foo", create_new=True)
+
+        foo_instance = sheerka.new("foo")
+        foo_instance.metadata.body = "value"
+        res = sheerka.modify_concept(context, foo_instance)
+        assert res.status
+
+        foo_from_sheerka = sheerka.get("foo")
+        assert foo_from_sheerka.metadata.body == "value"
+
+        foo_by_id_from_sheerka = sheerka.get_by_id(foo.id)
+        assert foo_by_id_from_sheerka.metadata.body == "value"
+
+    def test_i_cannot_modify_a_concept_that_does_not_exists(self):
+        sheerka, context, foo = self.init_concepts("foo", create_new=False)
+
+        foo_instance = sheerka.new("foo")
+        foo_instance.metadata.body = "value"
+        res = sheerka.modify_concept(context, foo_instance)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.body.body.key == foo.key
+
+    def test_i_can_modify_a_concept_that_is_in_a_list(self):
+        sheerka, context, foo1, foo2 = self.init_concepts(
+            Concept("foo", body="1"),
+            Concept("foo", body="2"), create_new=True)
+
+        foo2_instance = sheerka.new("foo")[1]
+        foo2_instance.metadata.body = "value"
+
+        res = sheerka.modify_concept(context, foo2_instance)
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
+        assert res.body.body.metadata.body == "value"
+
+        sheerka.reset_cache()
+        foo_from_sheerka = sheerka.new("foo")
+        assert foo_from_sheerka[0].metadata.body == "1"
+        assert foo_from_sheerka[1].metadata.body == "value"

tests/core/test_SheerkaSetsManager.py

@@ -200,8 +200,10 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
         assert sheerka.isinset(twenty_one, number)
 
     def test_i_can_set_isa(self):
-        sheerka, context, foo, all_foos = self.init_concepts(Concept("foo"), Concept("all_foo"), use_dict=False)
+        sheerka, context, foo, all_foos = self.init_concepts(
-        sheerka.create_new_concept(context, foo)
+            "foo", "all_foo",
+            create_new=True,
+            use_dict=False)
 
         assert BuiltinConcepts.ISA not in foo.props
 

tests/core/test_concept.py

@@ -198,6 +198,9 @@ def test_i_can_update_from():
         id="123456"
     ).def_prop("a", "10").def_prop("b", None)
 
+    # make sure origin is preserved
+    setattr(template, "##origin##", "digest")
+
     template.values[ConceptParts.BODY] = "value in body"
     template.values[ConceptParts.WHERE] = "value in where"
     template.values[ConceptParts.PRE] = "value in pre"
@@ -208,3 +211,4 @@ def test_i_can_update_from():
     concept = Concept().update_from(template)
 
     assert concept == template
+    assert getattr(concept, "##origin##") == "digest"

tests/evaluators/test_AddConceptInSetEvaluator.py

@@ -51,18 +51,20 @@ class TestAddConceptInSetEvaluator(TestUsingMemoryBasedSheerka):
         assert res.value.body == "bar"
 
     def test_i_can_add_concept_to_a_set_of_concept(self):
-        context = self.get_context()
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
-        foo = Concept("foo")
-        context.sheerka.create_new_concept(context, foo)
-
-        bar = Concept("bar")
-        context.sheerka.create_new_concept(context, bar)
-
         ret_val = get_ret_val("foo", "bar")
+
         res = AddConceptInSetEvaluator().eval(context, ret_val)
+        foo = sheerka.new("foo")  # reload it
 
         assert res.status
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
+        assert context.sheerka.isaset(context, bar)
+        assert context.sheerka.isinset(foo, bar)
+        assert context.sheerka.isa(foo, bar)
+
+        foo_from_sheerka = context.sheerka.get("foo")
+        assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == [bar]
 
     def test_i_can_add_concept_with_a_body_to_a_set_of_concept(self):
         context = self.get_context()

tests/non_reg/test_sheerka_non_reg.py

@@ -105,7 +105,7 @@ as:
         assert concept_saved.key in sheerka.cache_by_key
         assert concept_saved.id in sheerka.cache_by_id
         assert sheerka.sdp.io.exists(
-            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_origin()))
 
     def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept(self):
         """
@@ -135,7 +135,7 @@ as:
         assert concept_saved.key in sheerka.cache_by_key
         assert concept_saved.id in sheerka.cache_by_id
         assert sheerka.sdp.io.exists(
-            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_origin()))
 
     def test_i_cannot_eval_the_same_def_concept_twice(self):
         text = """

tests/sdp/test_sheerkaDataProvider.py

@@ -4,7 +4,7 @@ import pytest
 import os
 from os import path
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderError, \
-    SheerkaDataProviderDuplicateKeyError
+    SheerkaDataProviderDuplicateKeyError, SheerkaDataProviderResult, SheerkaDataProviderRef
 from datetime import date, datetime
 import shutil
 import json
@@ -123,6 +123,36 @@ class ObjDumpJson:
         self.key = as_dict["key"]
 
 
+class ObjDumpJsonNoDigest:
+    """
+    Object where the key can be resolved using get_key()
+    that can be used to dump as Json, 
+    But with no builtin digest computation
+    """
+
+    def __init__(self, key=None, value=None):
+        self.key = key
+        self.value = value
+
+    def __eq__(self, obj):
+        return isinstance(obj, ObjDumpJsonNoDigest) and \
+               self.key == obj.key and \
+               self.value == obj.value
+
+    def __repr__(self):
+        return f"ObjDumpJsonNoDigest({self.key}, {self.value})"
+
+    def get_key(self):
+        return self.key
+
+    def to_dict(self):
+        return self.__dict__
+
+    def from_dict(self, as_dict):
+        self.value = as_dict["value"]
+        self.key = as_dict["key"]
+
+
 class ObjWithDigestNoKey:
     """
     Object that can compute its digest.
@@ -301,13 +331,15 @@ def test_i_can_add_an_string(root):
     sdp = SheerkaDataProvider(root)
     obj = "foo => bar"
 
-    entry, key = sdp.add(evt_digest, "entry", obj)
+    result = sdp.add(evt_digest, "entry", obj)
     last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile)
     state = sdp.load_state(last_commit)
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
 
-    assert entry == "entry"
+    assert result.obj == obj
-    assert key is None
+    assert result.entry == "entry"
+    assert result.key is None
+    assert result.digest is None
     assert loaded == obj
 
     assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
@@ -330,11 +362,13 @@ def test_i_can_add_several_strings_if_allow_multiple_is_true(root):
 
     sdp.add(evt_digest, "entry", "foo")
     sdp.add(evt_digest, "entry", "foo")
-    entry, key = sdp.add(evt_digest, "entry", "bar")
+    result = sdp.add(evt_digest, "entry", "bar")
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
 
-    assert entry == "entry"
+    assert result.obj == "bar"
-    assert key is None
+    assert result.entry == "entry"
+    assert result.key is None
+    assert result.digest is None
     assert loaded == ["foo", "foo", "bar"]
 
 
@@ -359,14 +393,15 @@ def test_i_can_add_an_object_with_no_key(root):
     sdp = SheerkaDataProvider(root)
     obj = ObjNoKey("a", "b")
 
-    entry, key = sdp.add(evt_digest, "entry", obj)
+    result = sdp.add(evt_digest, "entry", obj)
     last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile)
     state = sdp.load_state(last_commit)
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
 
-    assert entry == "entry"
+    assert result.obj == obj
-    assert key is None
+    assert result.entry == "entry"
-    assert loaded == obj
+    assert result.key is None
+    assert result.digest is None
 
     assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
     assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.HeadFile))
@@ -388,11 +423,13 @@ def test_i_can_add_several_obj_no_key_if_allow_multiple_is_true(root):
 
     sdp.add(evt_digest, "entry", ObjNoKey("a", "b"))
     sdp.add(evt_digest, "entry", ObjNoKey("a", "b"))
-    entry, key = sdp.add(evt_digest, "entry", ObjNoKey("c", "d"))
+    result = sdp.add(evt_digest, "entry", ObjNoKey("c", "d"))
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
 
-    assert entry == "entry"
+    assert result.obj == ObjNoKey("c", "d")
-    assert key is None
+    assert result.entry == "entry"
+    assert result.key is None
+    assert result.digest is None
     assert loaded == [ObjNoKey("a", "b"), ObjNoKey("a", "b"), ObjNoKey("c", "d")]
 
 
@@ -429,15 +466,18 @@ def test_i_can_add_a_dict(root):
     sdp = SheerkaDataProvider(root)
     obj = {"my_key": "my_value"}
 
-    entry, key = sdp.add(evt_digest, "entry", obj)
+    result = sdp.add(evt_digest, "entry", obj)
     last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile)
     state = sdp.load_state(last_commit)
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
 
-    loaded_value = sdp.get(entry, "my_key")  # we can retrieve by key
+    loaded_value = sdp.get(result.entry, "my_key")  # we can retrieve by key
+
+    assert result.obj == obj
+    assert result.entry == "entry"
+    assert result.key is None  # we return None as dict may contains several entries
+    assert result.digest is None
 
-    assert entry == "entry"
-    assert key is None  # we return None as dict may contains several entries
     assert loaded == obj
     assert loaded_value == "my_value"
 
@@ -458,11 +498,17 @@ def test_i_can_add_a_dict(root):
 ])
 def test_i_can_add_multiple_entries_at_once_with_dict(root):
     sdp = SheerkaDataProvider(root)
+    obj = {"my_key1": "value1", "my_key2": "value2"}
 
-    entry, key = sdp.add(evt_digest, "entry", {"my_key1": "value1", "my_key2": "value2"})
+    result = sdp.add(evt_digest, "entry", obj)
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
-    loaded_value1 = sdp.get(entry, "my_key1")
+    loaded_value1 = sdp.get(result.entry, "my_key1")
-    loaded_value2 = sdp.get(entry, "my_key2")
+    loaded_value2 = sdp.get(result.entry, "my_key2")
+
+    assert result.obj == obj
+    assert result.entry == "entry"
+    assert result.key is None  # we return None as dict may contains several entries
+    assert result.digest is None
 
     assert loaded == {"my_key1": "value1", "my_key2": "value2"}
     assert loaded_value1 == "value1"
@@ -477,14 +523,14 @@ def test_i_can_add_same_key_with_dict_if_allow_multiple_is_true(root):
     sdp = SheerkaDataProvider(root)
 
     sdp.add(evt_digest, "entry", {"my_key": "my_value"})
-    entry, key = sdp.add(evt_digest, "entry", {"my_key": "my_value"})
+    result = sdp.add(evt_digest, "entry", {"my_key": "my_value"})
-    loaded1 = sdp.get(entry, key)
+    loaded1 = sdp.get(result.entry, result.key)
 
-    entry, key = sdp.add(evt_digest, "entry", {"my_key": "my_value2"})
+    result = sdp.add(evt_digest, "entry", {"my_key": "my_value2"})
-    loaded2 = sdp.get(entry, key)
+    loaded2 = sdp.get(result.entry, result.key)
 
-    assert entry == "entry"
+    assert result.entry == "entry"
-    assert key is None
+    assert result.key is None
     assert loaded1 == {"my_key": ["my_value", "my_value"]}
     assert loaded2 == {"my_key": ["my_value", "my_value", "my_value2"]}
 
@@ -525,19 +571,25 @@ def test_i_can_add_obj_with_key(root):
     obj1 = ObjWithKey("key1", "b")
     obj2 = ObjSetKey("c", key="key2")
 
-    entry1, key1 = sdp.add(evt_digest, "entry", obj1)  # test when key is taken from obj.get_key()
+    result1 = sdp.add(evt_digest, "entry", obj1)  # test when key is taken from obj.get_key()
-    entry2, key2 = sdp.add(evt_digest, "entry2", obj2)  # test when key is taken from obj.key
+    result2 = sdp.add(evt_digest, "entry2", obj2)  # test when key is taken from obj.key
     last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile)
     state = sdp.load_state(last_commit)
 
-    loaded1 = sdp.get(entry1, key1)
+    loaded1 = sdp.get(result1.entry, result1.key)
-    loaded2 = sdp.get(entry2, key2)
+    loaded2 = sdp.get(result2.entry, result2.key)
+
+    assert result1.obj == obj1
+    assert result1.entry == "entry"
+    assert result1.key == "key1"
+    assert result1.digest is None
+
+    assert result2.obj == obj2
+    assert result2.entry == "entry2"
+    assert result2.key == "key2"
+    assert result2.digest is None
 
-    assert entry1 == "entry"
-    assert key1 == "key1"
     assert loaded1 == ObjWithKey("key1", "b")
-    assert entry2 == "entry2"
-    assert key2 == "key2"
     assert loaded2 == ObjSetKey("c", key="key2")
 
     assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
@@ -559,15 +611,13 @@ def test_i_can_add_objects_with_same_key_if_allow_multiple_is_true(root):
     sdp = SheerkaDataProvider(root)
 
     sdp.add(evt_digest, "entry", ObjWithKey("my_key", "b"))
-    entry, key = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key"))
+    result = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key"))
-    loaded1 = sdp.get(entry, key)
+    loaded1 = sdp.get(result.entry, result.key)
 
-    entry, key = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key"))
+    result = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key"))
     sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key2"))  # to prove that it does not melt everything
-    loaded2 = sdp.get(entry, key)
+    loaded2 = sdp.get(result.entry, result.key)
 
-    assert entry == "entry"
-    assert key == "my_key"
     assert loaded1 == [ObjWithKey("my_key", "b"), ObjSetKey("c", key="my_key")]
     assert loaded2 == [ObjWithKey("my_key", "b"), ObjSetKey("c", key="my_key"), ObjSetKey("c", key="my_key")]
 
@@ -608,13 +658,23 @@ def test_i_can_add_a_reference(root):
     sdp = SheerkaDataProvider(root)
     sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey)))
     obj1 = ObjWithDigestWithKey(1, "foo")
-    sdp.add(evt_digest, "entry", obj1, use_ref=True)
+    result1 = sdp.add(evt_digest, "entry", obj1, use_ref=True)
-    sdp.add(evt_digest, "entry_by_value", {obj1.b: obj1.get_digest()}, is_ref=True)
+    result3 = sdp.add(evt_digest, "entry_by_ref", SheerkaDataProviderRef(obj1.b, obj1.get_digest()))
 
     # another object
     obj2 = ObjWithDigestWithKey(2, "bar")
     sdp.add(evt_digest, "entry", obj2, use_ref=True)
-    sdp.add(evt_digest, "entry_by_value", {obj2.b: obj2.get_digest()}, is_ref=True)
+    sdp.add(evt_digest, "entry_by_ref", SheerkaDataProviderRef(obj2.b, obj2.get_digest()))
+
+    assert result1.obj == obj1
+    assert result1.entry == "entry"
+    assert result1.key == str(obj1.get_key())
+    assert result1.digest == obj1.get_digest()
+
+    assert result3.obj == SheerkaDataProviderRef(obj1.b, obj1.get_digest())
+    assert result3.entry == "entry_by_ref"
+    assert result3.key == "foo"
+    assert result3.digest is None
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {
@@ -622,22 +682,47 @@ def test_i_can_add_a_reference(root):
             "1": '##REF##:' + obj1.get_digest(),
             "2": '##REF##:' + obj2.get_digest(),
         },
-        "entry_by_value": {
+        "entry_by_ref": {
-            "foo": '##REF##:' + obj1.get_digest(),
+            "foo": SheerkaDataProviderRef(obj1.b, obj1.get_digest()),
-            "bar": '##REF##:' + obj2.get_digest()
+            "bar": SheerkaDataProviderRef(obj2.b, obj2.get_digest())
         },
     }
 
-    # sanity check, make sure that I can load back
+    # make sure that I can load back
-    loaded1 = sdp.get("entry_by_value", "foo")
+    loaded1 = sdp.get("entry_by_ref", "foo")
     assert loaded1 == ObjWithDigestWithKey(1, "foo")
     assert getattr(loaded1, Serializer.ORIGIN) == obj1.get_digest()
 
-    loaded2 = sdp.get("entry_by_value", "bar")
+    loaded2 = sdp.get("entry_by_ref", "bar")
     assert loaded2 == ObjWithDigestWithKey(2, "bar")
     assert getattr(loaded2, Serializer.ORIGIN) == obj2.get_digest()
 
 
+@pytest.mark.parametrize("root", [
+    ".sheerka",
+    "mem://"
+])
+def test_i_can_have_multiple_is_ref_to_the_same_key(root):
+    sdp = SheerkaDataProvider(root)
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey)))
+    ref_result1 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(1, "foo"), use_ref=True)
+    ref_result2 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(2, "bar"), use_ref=True)
+
+    sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef("1", ref_result1.digest))
+    sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef("1", ref_result2.digest))
+
+    state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
+    assert state.data == {'entry': {'1': '##REF##:1foo', '2': '##REF##:2bar'},
+                          'entry_ref': {'1': [SheerkaDataProviderRef("1", ref_result1.digest),
+                                              SheerkaDataProviderRef("1", ref_result2.digest)]},
+                          }
+
+    loaded = sdp.get("entry_ref", "1")
+    assert len(loaded) == 2
+    assert loaded[0] == ObjWithDigestWithKey(1, "foo")
+    assert loaded[1] == ObjWithDigestWithKey(2, "bar")
+
+
 @pytest.mark.parametrize("root", [
     ".sheerka",
     "mem://"
@@ -660,21 +745,24 @@ def test_i_can_add_string_using_auto_generated_key(root):
     sdp = SheerkaDataProvider(root)
     key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile)
 
-    entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", "foo")
+    result1 = sdp.add_with_auto_key(evt_digest, "entry1", "foo")
-    entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", "bar")
+    result2 = sdp.add_with_auto_key(evt_digest, "entry1", "bar")
-    entry3, key3 = sdp.add_with_auto_key(evt_digest, "entry2", "baz")
+    result3 = sdp.add_with_auto_key(evt_digest, "entry2", "baz")
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
 
     assert sdp.io.exists(key_file)
     assert read_json_file(sdp, key_file) == {"entry1": 2, "entry2": 1}
     assert state.data == {"entry1": {"1": "foo", "2": "bar"}, "entry2": {"1": "baz"}}
-    assert entry1 == "entry1"
+    assert result1.obj == "foo"
-    assert entry2 == "entry1"
+    assert result2.obj == "bar"
-    assert entry3 == "entry2"
+    assert result3.obj == "baz"
-    assert key1 == "1"
+    assert result1.entry == "entry1"
-    assert key2 == "2"
+    assert result2.entry == "entry1"
-    assert key3 == "1"
+    assert result3.entry == "entry2"
+    assert result1.digest is None
+    assert result2.digest is None
+    assert result3.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -759,20 +847,6 @@ def test_i_cannot_add_the_same_digest_twice_in_the_same_entry4(root):
     assert error.value.args[0] == "Duplicate object."
 
 
-def test_i_cannot_add_using_use_ref_and_is_ref():
-    sdp = SheerkaDataProvider("mem://")
-
-    with pytest.raises(SheerkaDataProviderError) as error:
-        sdp.add(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), use_ref=True, is_ref=True)
-
-
-def test_i_cannot_add_using_is_ref_if_obj_is_not_a_dictionary():
-    sdp = SheerkaDataProvider("mem://")
-
-    with pytest.raises(SheerkaDataProviderError) as error:
-        sdp.add(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), is_ref=True)
-
-
 @pytest.mark.parametrize("root", [
     ".sheerka",
     "mem://"
@@ -800,18 +874,23 @@ def test_i_can_add_object_using_auto_generated_key(root):
     sdp = SheerkaDataProvider(root)
     key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile)
 
-    entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b"))
+    result1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b"))
-    entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b"))
+    result2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b"))
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
 
     assert sdp.io.exists(key_file)
     assert read_json_file(sdp, key_file) == {"entry1": 2}
     assert state.data == {"entry1": {"1": ObjNoKey("a", "b"), "2": ObjNoKey("a", "b")}}
-    assert entry1 == "entry1"
+
-    assert entry2 == "entry1"
+    assert result1.obj == ObjNoKey("a", "b")
-    assert key1 == "1"
+    assert result2.obj == ObjNoKey("a", "b")
-    assert key2 == "2"
+    assert result1.entry == "entry1"
+    assert result2.entry == "entry1"
+    assert result1.key == "1"
+    assert result2.key == "2"
+    assert result1.digest is None
+    assert result2.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -822,18 +901,23 @@ def test_object_key_is_updated_when_possible_using_auto_generated_key(root):
     sdp = SheerkaDataProvider(root)
     key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile)
 
-    entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo"))
+    result1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo"))
-    entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo"))
+    result2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo"))
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
 
     assert sdp.io.exists(key_file)
     assert read_json_file(sdp, key_file) == {"entry1": 2}
     assert state.data == {"entry1": {"1": ObjSetKey("foo", "1"), "2": ObjSetKey("foo", "2")}}
-    assert entry1 == "entry1"
+
-    assert entry2 == "entry1"
+    assert result1.obj == ObjSetKey("foo", "1")
-    assert key1 == "1"
+    assert result2.obj == ObjSetKey("foo", "2")
-    assert key2 == "2"
+    assert result1.entry == "entry1"
+    assert result2.entry == "entry1"
+    assert result1.key == "1"
+    assert result2.key == "2"
+    assert result1.digest is None
+    assert result2.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -843,12 +927,13 @@ def test_object_key_is_updated_when_possible_using_auto_generated_key(root):
 def test_i_can_set_objects_with_key(root):
     sdp = SheerkaDataProvider(root)
     sdp.add(evt_digest, "entry", ObjWithKey(1, "foo"))
-    entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"))
+    result = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"))
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {"2": ObjWithKey(2, "foo")}}
-    assert entry == "entry"
+    assert result.entry == "entry"
-    assert key == "2"
+    assert result.key == "2"
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -858,12 +943,13 @@ def test_i_can_set_objects_with_key(root):
 def test_i_can_set_objects_with_no_key(root):
     sdp = SheerkaDataProvider(root)
     sdp.add(evt_digest, "entry", ObjNoKey(1, "foo"))
-    entry, key = sdp.set(evt_digest, "entry", ObjNoKey(2, "foo"))
+    result = sdp.set(evt_digest, "entry", ObjNoKey(2, "foo"))
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": ObjNoKey(2, "foo")}
-    assert entry == "entry"
+    assert result.entry == "entry"
-    assert key is None
+    assert result.key is None
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -873,12 +959,13 @@ def test_i_can_set_objects_with_no_key(root):
 def test_i_can_set_from_list_to_dict(root):
     sdp = SheerkaDataProvider(root)
     sdp.set(evt_digest, "entry", [ObjNoKey(1, "foo"), ObjNoKey(2, "foo")])
-    entry, key = sdp.set(evt_digest, "entry", {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")})
+    result = sdp.set(evt_digest, "entry", {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")})
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")}}
-    assert entry == "entry"
+    assert result.entry == "entry"
-    assert key is None
+    assert result.key is None
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -889,18 +976,21 @@ def test_i_can_set_using_reference(root):
     sdp = SheerkaDataProvider(root)
     sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey)))
     sdp.add(evt_digest, "entry", ObjWithKey(1, "foo"))
-    entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"), use_ref=True)
+    result = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"), use_ref=True)
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {"2": '##REF##:43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9'}}
-    assert entry == "entry"
+
-    assert key == "2"
+    assert result.obj == ObjWithKey(2, "foo")
+    assert result.entry == "entry"
+    assert result.key == "2"
+    assert result.digest == "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9"
 
     assert sdp.io.exists(sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder,
                                              "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9"))
 
     # sanity check, make sure that I can load back
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
     assert loaded == ObjWithKey(2, "foo")
     assert getattr(loaded, Serializer.ORIGIN) == "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9"
 
@@ -952,12 +1042,15 @@ def test_i_can_add_an_object_with_a_key_as_a_reference(root):
     obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
     sdp.serializer.register(obj_serializer)
 
-    entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
+    result = sdp.add(evt_digest, "entry", obj, use_ref=True)
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     digest = state.data["entry"]["my_key"][len(SheerkaDataProvider.REF_PREFIX):]
 
-    assert key == obj.key
+    assert result.obj == obj
-    assert entry == "entry"
+    assert result.entry == "entry"
+    assert result.key == obj.key
+    assert result.digest == obj.get_digest()
+    assert digest == result.digest
     assert state.data == {'entry': {'my_key': f"{SheerkaDataProvider.REF_PREFIX}{digest}"}}
 
     loaded = sdp.load_obj(digest)
@@ -973,15 +1066,18 @@ def test_i_can_add_a_dictionary_as_a_reference(root):
     sdp = SheerkaDataProvider(root)
     obj = {"my_key": "value1"}
 
-    obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
+    # No need to register a serializer for dictionaries
-    sdp.serializer.register(obj_serializer)
 
-    entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
+    result = sdp.add(evt_digest, "entry", obj, use_ref=True)
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     digest = state.data["entry"][len(SheerkaDataProvider.REF_PREFIX):]
 
-    assert key is None
+    assert result.obj == obj
-    assert entry == "entry"
+    assert result.entry == "entry"
+    assert result.key is None  # we return None as dict may contains several entries
+    assert result.digest == "1790cae3f354ecb6b419faaa2ee2c374ff33efb8cddafda9960924036ac04c1f"  # a digest is created
+    assert digest == result.digest
+
     assert state.data == {'entry': f"{SheerkaDataProvider.REF_PREFIX}{digest}"}
 
     loaded = sdp.load_obj(digest)
@@ -990,23 +1086,50 @@ def test_i_can_add_a_dictionary_as_a_reference(root):
     assert len(loaded) == 2
 
 
+@pytest.mark.parametrize("root", [
+    ".sheerka",
+    "mem://"
+])
+def test_i_can_add_an_object_with_no_builtin_digest_as_a_reference(root):
+    sdp = SheerkaDataProvider(root)
+    obj = ObjDumpJsonNoDigest("a", "b")
+
+    obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
+    sdp.serializer.register(obj_serializer)
+
+    result = sdp.add(evt_digest, "entry", obj, use_ref=True)
+    state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
+    digest = state.data["entry"][obj.get_key()][len(SheerkaDataProvider.REF_PREFIX):]
+
+    assert result.obj == obj
+    assert result.entry == "entry"
+    assert result.key == obj.get_key()
+    assert result.digest is not None
+    assert digest == result.digest
+
+    assert state.data == {'entry': {obj.key: f"{SheerkaDataProvider.REF_PREFIX}{result.digest}"}}
+
+    loaded = sdp.load_obj(digest)
+    assert getattr(loaded, Serializer.ORIGIN) == digest
+
+
 @pytest.mark.parametrize("root", [
     ".sheerka",
     "mem://"
 ])
 def test_i_can_add_unique(root):
     sdp = SheerkaDataProvider(root)
-    entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo"))
+    result = sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo"))
-    assert (entry, key) == ("entry", None)
+    assert result == SheerkaDataProviderResult(ObjNoKey(1, "foo"), "entry", None, None, False)
 
-    entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo"))
+    result = sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo"))
-    assert (entry, key) == (None, None)
+    assert result == SheerkaDataProviderResult(ObjNoKey(1, "foo"), "entry", None, None, True)
 
-    entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar"))
+    result = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar"))
-    assert (entry, key) == ("entry", None)
+    assert result == SheerkaDataProviderResult(ObjNoKey(2, "bar"), "entry", None, None, False)
 
-    entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar"))
+    result = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar"))
-    assert (entry, key) == (None, None)
+    assert result == SheerkaDataProviderResult(ObjNoKey(2, "bar"), "entry", None, None, True)
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {ObjNoKey(1, "foo"), ObjNoKey(2, "bar")}}
@@ -1338,12 +1461,14 @@ def test_i_can_modify_dict_with_a_key(root):
     sdp.add(evt_digest, "entry", {"key1": "foo"})
     sdp.add(evt_digest, "entry", {"key2": "bar"})
 
-    entry, key = sdp.modify(evt_digest, "entry", "key1", "baz")
+    result = sdp.modify(evt_digest, "entry", "key1", "baz")
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {"key1": "baz", "key2": "bar"}}
-    assert entry == "entry"
+    assert result.obj == "baz"
-    assert key == "key1"
+    assert result.entry == "entry"
+    assert result.key == "key1"
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -1355,12 +1480,15 @@ def test_i_can_modify_an_object_with_a_key(root):
     sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo"))
     sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"))
 
-    entry, key = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key1", "baz"))
+    result = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key1", "baz"))
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
+
     assert state.data == {"entry": {"key1": ObjWithKey("key1", "baz"), "key2": ObjWithKey("key2", "bar")}}
-    assert entry == "entry"
+    assert result.obj == ObjWithKey("key1", "baz")
-    assert key == "key1"
+    assert result.entry == "entry"
+    assert result.key == "key1"
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -1372,12 +1500,14 @@ def test_i_can_modify_an_object_while_changing_the_key(root):
     sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo"))
     sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"))
 
-    entry, key = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key3", "baz"))
+    result = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key3", "baz"))
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {"key2": ObjWithKey("key2", "bar"), "key3": ObjWithKey("key3", "baz")}}
-    assert entry == "entry"
+    assert result.obj == ObjWithKey("key3", "baz")
-    assert key == "key3"
+    assert result.entry == "entry"
+    assert result.key == "key3"
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -1389,12 +1519,14 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_key(root):
     sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo"))
     sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"))
 
-    entry, key = sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key1", "bar"))
+    result = sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key1", "bar"))
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {"key1": [ObjWithKey("key1", "foo"), ObjWithKey("key1", "bar")]}}
-    assert entry == "entry"
+    assert result.obj == ObjWithKey("key1", "bar")
-    assert key == "key1"
+    assert result.entry == "entry"
+    assert result.key == "key1"
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -1408,7 +1540,6 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_list(
     :return:
     """
     sdp = SheerkaDataProvider(root)
-    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
 
     sdp.add(evt_digest, "entry", ObjDumpJson("key1", "value11"))
     sdp.add(evt_digest, "entry", ObjDumpJson("key1", "value12"))
@@ -1417,15 +1548,17 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_list(
 
     new_value = ObjDumpJson("key1", "value13")
     setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest())
-    entry, key = sdp.modify(evt_digest, "entry", "key2", new_value)
+    result = sdp.modify(evt_digest, "entry", "key2", new_value)
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {
         "key1": [ObjDumpJson("key1", "value11"), ObjDumpJson("key1", "value12"), ObjDumpJson("key1", "value13")],
         "key2": [ObjDumpJson("key2", "value22")]
     }}
-    assert entry == "entry"
+    assert result.obj == new_value
-    assert key == "key1"
+    assert result.entry == "entry"
+    assert result.key == "key1"
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -1446,15 +1579,17 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_nothi
 
     new_value = ObjDumpJson("key1", "value13")
     setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest())
-    entry, key = sdp.modify(evt_digest, "entry", "key2", new_value)
+    result = sdp.modify(evt_digest, "entry", "key2", new_value)
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {
         "key1": ObjDumpJson("key1", "value13"),
         "key2": [ObjDumpJson("key2", "value22")]
     }}
-    assert entry == "entry"
+    assert result.obj == new_value
-    assert key == "key1"
+    assert result.entry == "entry"
+    assert result.key == "key1"
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
@@ -1476,34 +1611,143 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_one_i
 
     new_value = ObjDumpJson("key1", "value13")
     setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest())
-    entry, key = sdp.modify(evt_digest, "entry", "key2", new_value)
+    result = sdp.modify(evt_digest, "entry", "key2", new_value)
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {
         "key1": [ObjDumpJson("key1", "value11"), ObjDumpJson("key1", "value13")],
         "key2": [ObjDumpJson("key2", "value22")]
     }}
-    assert entry == "entry"
+    assert result.obj == new_value
-    assert key == "key1"
+    assert result.entry == "entry"
+    assert result.key == "key1"
+    assert result.digest is None
 
 
 @pytest.mark.parametrize("root", [
     ".sheerka",
     "mem://"
 ])
-def test_i_can_modify_a_ref(root):
+def test_i_can_modify_a_object_saved_by_ref(root):
     sdp = SheerkaDataProvider(root)
     sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey)))
     sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo"))
-    entry, key = sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"), use_ref=True)
+    sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"), use_ref=True)
 
-    sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key2", "baz"))
+    result = sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key2", "baz"))
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {"entry": {
         "key1": ObjWithKey("key1", "foo"),
         "key2": "##REF##:041d3cca905b51bc2c66251e73e56b836aae7b9435ee3d7eb05d44bb67ff575e"}}
-    assert entry == "entry"
+    assert result.obj == ObjWithKey("key2", "baz")
-    assert key == "key2"
+    assert result.entry == "entry"
+    assert result.key == "key2"
+    assert result.digest == "041d3cca905b51bc2c66251e73e56b836aae7b9435ee3d7eb05d44bb67ff575e"
+
+
+@pytest.mark.parametrize("root", [
+    ".sheerka",
+    "mem://"
+])
+def test_i_can_modify_an_object_saved_by_ref_in_a_list(root):
+    sdp = SheerkaDataProvider(root)
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJsonNoDigest)))
+
+    sdp.add(evt_digest, "entry", ObjDumpJsonNoDigest("key1", "value11"), use_ref=True)
+    sdp.add(evt_digest, "entry", ObjDumpJsonNoDigest("key1", "value12"), use_ref=True)
+    result = sdp.add(evt_digest, "entry", ObjDumpJsonNoDigest("key2", "value21"), use_ref=True)
+    sdp.add(evt_digest, "entry", ObjDumpJsonNoDigest("key2", "value22"), use_ref=True)
+
+    new_value = ObjDumpJsonNoDigest("key1", "value13")
+    setattr(new_value, Serializer.ORIGIN, result.digest)
+    result = sdp.modify(evt_digest, "entry", "key2", new_value)
+
+    state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
+    assert state.data == {"entry": {
+        'key1': ['##REF##:f80a0c0aceb1a7a3d238c0cff2d86d6bd3a62e0c1a65c5b505f43b10c4604bd8',
+                 '##REF##:239a8238d188c37afa10b1bcc312ca8a0e78f6e75d688ca65d08e16717ff68b0',
+                 '##REF##:9d0a2bf9d4081de0b14837ea46bc7a1cfb6b7562f7ae86255ea9bd0ac53a6437'],
+        'key2': ['##REF##:df8a38b07f469f2ff8001ea6a70f77f4f9ce85d69c530091fcaf4b380f1500d3']
+    }}
+    assert result.obj == new_value
+    assert result.entry == "entry"
+    assert result.key == "key1"
+    assert result.digest is not None
+
+
+@pytest.mark.parametrize("root", [
+    ".sheerka",
+    "mem://"
+])
+def test_i_can_modify_a_data_provider_ref(root):
+    # first, create a valid entry
+    sdp = SheerkaDataProvider(root)
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey)))
+    obj = ObjWithDigestWithKey("1", "foo")
+    sdp.add(evt_digest, "entry", obj, use_ref=True)
+    sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef(obj.b, obj.get_digest()))
+
+    state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
+    assert state.data == {
+        "entry": {"1": "##REF##:1foo"},
+        "entry_ref": {"foo": SheerkaDataProviderRef(obj.b, obj.get_digest())}}
+
+    # modify this entry
+    obj_new = ObjWithDigestWithKey("1", "bar")
+    sdp.modify(evt_digest, "entry", obj_new.a, obj_new)
+    result = sdp.modify(evt_digest, "entry_ref", "foo", SheerkaDataProviderRef(obj.b, obj_new.get_digest()))
+
+    state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
+    assert state.data == {
+        "entry": {"1": "##REF##:1bar"},
+        "entry_ref": {"foo": SheerkaDataProviderRef(obj.b, obj_new.get_digest())}}
+
+    assert result.obj == SheerkaDataProviderRef(obj.b, obj_new.get_digest())
+    assert result.entry == "entry_ref"
+    assert result.key == "foo"
+    assert result.digest is None  # digest is not set as what is saved (the digest) is not saved by ref
+
+    # sanity check, I can load the modified entry
+    loaded = sdp.get("entry_ref", "foo")
+    assert loaded == ObjWithDigestWithKey("1", "bar")
+
+
+@pytest.mark.parametrize("root", [
+    ".sheerka",
+    "mem://"
+])
+def test_i_can_modify_is_ref_when_in_list(root):
+    sdp = SheerkaDataProvider(root)
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey)))
+    ref_result1 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(1, "foo"), use_ref=True)
+    ref_result2 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(2, "bar"), use_ref=True)
+
+    sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef("1", ref_result1.digest))
+    sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef("1", ref_result2.digest))
+
+    ref_result3 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(3, "baz"), use_ref=True)
+
+    result = sdp.modify(
+        evt_digest,
+        "entry_ref",
+        "1",
+        SheerkaDataProviderRef("1", ref_result3.digest, ref_result2.digest))
+
+    state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
+    assert state.data == {'entry': {'1': '##REF##:1foo', '2': '##REF##:2bar', '3': '##REF##:3baz'},
+                          'entry_ref': {'1': [
+                              SheerkaDataProviderRef("1", ref_result1.digest),
+                              SheerkaDataProviderRef("1", ref_result3.digest, ref_result2.digest)]}}
+
+    loaded = sdp.get("entry_ref", "1")
+    assert len(loaded) == 2
+    assert loaded[0] == ObjWithDigestWithKey(1, "foo")
+    assert loaded[1] == ObjWithDigestWithKey(3, "baz")
+
+    assert result.obj == SheerkaDataProviderRef("1", ref_result3.digest, ref_result2.digest)
+    assert result.entry == "entry_ref"
+    assert result.key == "1"
+    assert result.digest is None  # digest is not set as what is saved (the digest) is not saved by ref
 
 
 @pytest.mark.parametrize("root", [
@@ -1661,8 +1905,8 @@ def test_i_can_get_object_saved_by_reference(root):
     obj = ObjDumpJson("my_key", "value1")
     sdp.serializer.register(JsonSerializer(core.utils.get_full_qualified_name(obj)))
 
-    entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
+    result = sdp.add(evt_digest, "entry", obj, use_ref=True)
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
 
     assert loaded == obj
 
@@ -1876,12 +2120,15 @@ def test_i_can_save_and_load_object_ref_with_history(root):
     obj = ObjDumpJson("my_key", "value1")
     sdp.serializer.register(JsonSerializer(core.utils.get_full_qualified_name(obj)))
 
-    entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
+    result = sdp.add(evt_digest, "entry", obj, use_ref=True)
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
     history = getattr(loaded, Serializer.HISTORY)
 
-    assert key == obj.key
+    assert result.obj == obj
-    assert entry == "entry"
+    assert result.entry == "entry"
+    assert result.key == obj.key
+    assert result.digest == obj.get_digest()
+
     assert loaded.key == obj.key
     assert loaded.value == obj.value
 
@@ -1895,8 +2142,8 @@ def test_i_can_save_and_load_object_ref_with_history(root):
     previous_modification_time = history[Serializer.MODIFICATION_DATE]
     previous_parents = history[Serializer.PARENTS]
 
-    sdp.modify(evt_digest, "entry", key, loaded)
+    sdp.modify(evt_digest, "entry", result.key, loaded)
-    loaded = sdp.get(entry, key)
+    loaded = sdp.get(result.entry, result.key)
     history = getattr(loaded, Serializer.HISTORY)
 
     assert history[Serializer.MODIFICATION_DATE] == previous_modification_time
@@ -1906,8 +2153,8 @@ def test_i_can_save_and_load_object_ref_with_history(root):
     previous_digest = loaded.get_digest()
     loaded.value = "value2"
 
-    sdp.modify(evt_digest, "entry", key, loaded)
+    sdp.modify(evt_digest, "entry", result.key, loaded)
-    loaded2 = sdp.get(entry, key)
+    loaded2 = sdp.get(result.entry, result.key)
     history2 = getattr(loaded2, Serializer.HISTORY)
 
     assert loaded2.key == loaded.key
@@ -1932,10 +2179,10 @@ def test_i_can_add_obj_with_same_key_and_get_them_back(root):
     obj2 = ObjDumpJson("key", "value2")
     sdp.serializer.register(JsonSerializer(core.utils.get_full_qualified_name(obj1)))
 
-    entry1, key1 = sdp.add(evt_digest, "entry", obj1, use_ref=True)
+    result = sdp.add(evt_digest, "entry", obj1, use_ref=True)
-    entry2, key2 = sdp.add(evt_digest, "entry", obj2, use_ref=True)
+    sdp.add(evt_digest, "entry", obj2, use_ref=True)
 
-    loaded = sdp.get_safe(entry1, key1)
+    loaded = sdp.get(result.entry, result.key)
 
     assert len(loaded) == 2
     assert loaded[0] == obj1
@@ -1953,14 +2200,14 @@ def test_i_get_safe_dictionary_without_origin(root):
     obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
     sdp.serializer.register(obj_serializer)
 
-    entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
+    result = sdp.add(evt_digest, "entry", obj, use_ref=True)
-    from_db = sdp.get_safe(entry, key)
+    from_db = sdp.get(result.entry, result.key)
 
     assert len(from_db) == 2
     assert from_db["my_key"] == obj["my_key"]
     assert Serializer.ORIGIN in from_db
 
-    from_db_no_origin = sdp.get_safe(entry, key, load_origin=False)
+    from_db_no_origin = sdp.get_safe(result.entry, result.key, load_origin=False)
     assert len(from_db_no_origin) == 1
     assert from_db_no_origin["my_key"] == obj["my_key"]
     assert Serializer.ORIGIN not in from_db_no_origin
@@ -1977,14 +2224,14 @@ def test_i_get_dictionary_without_origin(root):
     obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
     sdp.serializer.register(obj_serializer)
 
-    entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
+    result = sdp.add(evt_digest, "entry", obj, use_ref=True)
-    from_db = sdp.get(entry, key)
+    from_db = sdp.get(result.entry, result.key)
 
     assert len(from_db) == 2
     assert from_db["my_key"] == obj["my_key"]
     assert Serializer.ORIGIN in from_db
 
-    from_db_no_origin = sdp.get(entry, key, load_origin=False)
+    from_db_no_origin = sdp.get(result.entry, result.key, load_origin=False)
     assert len(from_db_no_origin) == 1
     assert from_db_no_origin["my_key"] == obj["my_key"]
     assert Serializer.ORIGIN not in from_db_no_origin
@@ -2001,12 +2248,12 @@ def test_i_get_safe_object_without_origin(root):
     obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj))
     sdp.serializer.register(obj_serializer)
 
-    entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
+    result = sdp.add(evt_digest, "entry", obj, use_ref=True)
-    from_db = sdp.get_safe(entry, key)
+    from_db = sdp.get(result.entry, result.key)
 
     assert from_db == obj
     assert hasattr(from_db, Serializer.ORIGIN)
 
-    from_db_no_origin = sdp.get_safe(entry, key, load_origin=False)
+    from_db_no_origin = sdp.get_safe(result.entry, result.key, load_origin=False)
     assert from_db_no_origin == obj
     assert not hasattr(from_db_no_origin, Serializer.ORIGIN)