From 7cd94e888f277eb8c880ba45c228bb3e66a882e6 Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Thu, 20 Feb 2020 11:30:53 +0100 Subject: [PATCH] Added ModifyConcept function, and fixed 'isa' not working --- src/core/concept.py | 29 +- .../Services/SheerkaCreateNewConcept.py | 27 +- src/core/sheerka/Services/SheerkaDump.py | 2 +- .../sheerka/Services/SheerkaModifyConcept.py | 37 +- .../sheerka/Services/SheerkaSetsManager.py | 4 +- src/core/sheerka/Sheerka.py | 2 + src/sdp/sheerkaDataProvider.py | 121 +++- src/sheerkapickle/SheerkaUnpickler.py | 3 +- src/sheerkapickle/sheerka_handlers.py | 3 + tests/BaseTest.py | 11 +- tests/core/test_SheerkaCreateNewConcept.py | 20 +- tests/core/test_SheerkaModifyConcept.py | 104 ++++ tests/core/test_SheerkaSetsManager.py | 6 +- tests/core/test_concept.py | 4 + .../test_AddConceptInSetEvaluator.py | 16 +- tests/non_reg/test_sheerka_non_reg.py | 4 +- tests/sdp/test_sheerkaDataProvider.py | 585 +++++++++++++----- 17 files changed, 750 insertions(+), 228 deletions(-) create mode 100644 tests/core/test_SheerkaModifyConcept.py diff --git a/src/core/concept.py b/src/core/concept.py index 68e6668..921310c 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -1,6 +1,6 @@ import hashlib from collections import namedtuple -from dataclasses import dataclass, field +from dataclasses import dataclass from enum import Enum from core.sheerka_logger import get_logger @@ -15,6 +15,7 @@ PROPERTIES_FOR_DIGEST = ("name", "key", PROPERTIES_TO_SERIALIZE = PROPERTIES_FOR_DIGEST + tuple(["id"]) PROPERTIES_FOR_NEW = ("where", "pre", "post", "body", "desc") VARIABLE_PREFIX = "__var__" +ORIGIN = "##origin##" # same as Serializer.ORIGIN but I don't want to include the reference class ConceptParts(Enum): @@ -97,6 +98,7 @@ class Concept: self.bnf = None self.log = get_logger("core." + self.__class__.__name__) self.init_log = get_logger("init.core." + self.__class__.__name__) + self.original_definition_hash = None def __repr__(self): return f"({self.metadata.id}){self.metadata.name}" @@ -243,7 +245,19 @@ class Concept: def body(self): return self.values[ConceptParts.BODY] if ConceptParts.BODY in self.values else None - def get_digest(self): + def get_origin(self): + """ + Return the digest used to save the concept if it exists + :return: + """ + if hasattr(self, ORIGIN): + return getattr(self, ORIGIN) + return None + + def set_origin(self, origin): + setattr(self, ORIGIN, origin) + + def get_definition_hash(self): """ Returns the digest of the event :return: hexa form of the sha256 @@ -301,6 +315,11 @@ class Concept: for k, v in other.props.items(): self.set_prop(k, v.value) + # origin + from sdp.sheerkaSerializer import Serializer + if hasattr(other, Serializer.ORIGIN): + setattr(self, Serializer.ORIGIN, getattr(other, Serializer.ORIGIN)) + return self def set_prop(self, prop_name, prop_value): @@ -362,6 +381,12 @@ class Concept: self.metadata.is_evaluated = True return self + def freeze_definition_hash(self): + self.original_definition_hash = self.get_definition_hash() + + def get_original_definition_hash(self): + return self.original_definition_hash + class Property: """ diff --git a/src/core/sheerka/Services/SheerkaCreateNewConcept.py b/src/core/sheerka/Services/SheerkaCreateNewConcept.py index eb9e148..a6a0c7e 100644 --- a/src/core/sheerka/Services/SheerkaCreateNewConcept.py +++ b/src/core/sheerka/Services/SheerkaCreateNewConcept.py @@ -1,6 +1,6 @@ from core.builtin_concepts import BuiltinConcepts, ErrorConcept from core.concept import Concept -from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError +from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError, SheerkaDataProviderRef CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser" @@ -30,7 +30,7 @@ class SheerkaCreateNewConcept: # checks for duplicate concepts # TODO checks if it exists in cache first - if self.sheerka.sdp.exists(self.sheerka.CONCEPTS_ENTRY, concept.key, concept.get_digest()): + if self.sheerka.sdp.exists(self.sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()): error = SheerkaDataProviderDuplicateKeyError(self.sheerka.CONCEPTS_ENTRY + "." + concept.key, concept) return self.sheerka.ret( self.logger_name, @@ -55,22 +55,35 @@ class SheerkaCreateNewConcept: if not init_ret_value.status: return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value)) + concept.freeze_definition_hash() + # save the new concept in sdp - concept.metadata.full_serialization = True try: # TODO : needs to make these calls atomic (or at least one single call) # save the new concept - self.sheerka.sdp.add( + concept.metadata.full_serialization = True + result = self.sheerka.sdp.add( context.event.get_digest(), self.sheerka.CONCEPTS_ENTRY, concept, use_ref=True) + concept.metadata.full_serialization = False + + # update the concept (I hope that it's enough) + concept.set_origin(result.digest) + # save it by id self.sheerka.sdp.add( context.event.get_digest(), self.sheerka.CONCEPTS_BY_ID_ENTRY, - {concept.id: concept.get_digest()}, - is_ref=True) + SheerkaDataProviderRef(concept.id, result.digest)) + + # records the hash + self.sheerka.sdp.add( + context.event.get_digest(), + self.sheerka.CONCEPTS_BY_HASH_ENTRY, + SheerkaDataProviderRef(concept.get_definition_hash(), result.digest)) + # update the definition table if concepts_definitions is not None: self.sheerka.sdp.set( @@ -88,7 +101,7 @@ class SheerkaCreateNewConcept: error.args[0]) # Updates the caches - concept.metadata.full_serialization = False + self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key) self.sheerka.cache_by_id[concept.id] = concept if init_ret_value is not None and init_ret_value.status: diff --git a/src/core/sheerka/Services/SheerkaDump.py b/src/core/sheerka/Services/SheerkaDump.py index 11454d7..43f51ca 100644 --- a/src/core/sheerka/Services/SheerkaDump.py +++ b/src/core/sheerka/Services/SheerkaDump.py @@ -63,7 +63,7 @@ class SheerkaDump: else: self.sheerka.log.info("No property") - self.sheerka.log.info(f"digest : {c.get_digest()}") + self.sheerka.log.info(f"digest : {c.get_origin()}") if self.sheerka.isaset(context, c): items = self.sheerka.get_set_elements(context, c) diff --git a/src/core/sheerka/Services/SheerkaModifyConcept.py b/src/core/sheerka/Services/SheerkaModifyConcept.py index 227cdbb..ea47603 100644 --- a/src/core/sheerka/Services/SheerkaModifyConcept.py +++ b/src/core/sheerka/Services/SheerkaModifyConcept.py @@ -1,4 +1,5 @@ from core.builtin_concepts import BuiltinConcepts +from sdp.sheerkaDataProvider import SheerkaDataProviderRef class SheerkaModifyConcept: @@ -8,7 +9,41 @@ class SheerkaModifyConcept: def modify_concept(self, context, concept): - self.sheerka.sdp.modify(context.event.get_digest(), self.sheerka.CONCEPTS_ENTRY, concept.key, concept) + try: + # modify the entry + concept.metadata.full_serialization = True + result = self.sheerka.sdp.modify( + context.event.get_digest(), + self.sheerka.CONCEPTS_ENTRY, + concept.key, + concept) + concept.metadata.full_serialization = False + + # update its reference + self.sheerka.sdp.modify( + context.event.get_digest(), + self.sheerka.CONCEPTS_BY_ID_ENTRY, + concept.id, + SheerkaDataProviderRef(concept.id, result.digest, concept.get_origin())) + + # update the hash entry + self.sheerka.sdp.modify( + context.event.get_digest(), + self.sheerka.CONCEPTS_BY_HASH_ENTRY, + concept.get_original_definition_hash(), + SheerkaDataProviderRef(concept.get_definition_hash(), result.digest, concept.get_origin())) + + except IndexError as error: + context.log_error(f"Failed to update concept '{concept}'.", who=self.logger_name) + return self.sheerka.ret( + self.logger_name, + False, + self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept), + error.args[0]) + + # update cache + self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key) + self.sheerka.cache_by_id[concept.id] = concept ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) return ret diff --git a/src/core/sheerka/Services/SheerkaSetsManager.py b/src/core/sheerka/Services/SheerkaSetsManager.py index 88b24ed..948c638 100644 --- a/src/core/sheerka/Services/SheerkaSetsManager.py +++ b/src/core/sheerka/Services/SheerkaSetsManager.py @@ -48,8 +48,8 @@ class SheerkaSetsManager: assert concept_set.id try: - ret = self.sheerka.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id) - if ret == (None, None): # concept already in set + result = self.sheerka.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id) + if result.already_exists: # concept already in set return self.sheerka.ret( self.logger_name, False, diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 7b1e5bd..cd5204a 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -34,6 +34,7 @@ class Sheerka(Concept): CONCEPTS_ENTRY = "All_Concepts" # to store all the concepts CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID" + CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash" # store hash of concepts definitions (not values) CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions" # to store definitions (bnf) of concepts BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts" # sequential key for builtin concepts USER_CONCEPTS_KEYS = "User_Concepts" # sequential key for user defined concepts @@ -482,6 +483,7 @@ class Sheerka(Concept): # otherwise, create another instance concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept() concept.update_from(template) + concept.freeze_definition_hash() if len(kwargs) == 0: return concept diff --git a/src/sdp/sheerkaDataProvider.py b/src/sdp/sheerkaDataProvider.py index 4666889..919bf80 100644 --- a/src/sdp/sheerkaDataProvider.py +++ b/src/sdp/sheerkaDataProvider.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from datetime import datetime, date import hashlib import json @@ -206,12 +207,32 @@ class State: def modify_in_list(self, entry, key, obj, obj_key, obj_origin, load_ref_if_needed, save_ref_if_needed): found = False to_remove = None + new_digest = None + + def _get_item_origin(o): + if hasattr(o, Serializer.ORIGIN): + return getattr(o, Serializer.ORIGIN) + + if isinstance(o, dict) and Serializer.ORIGIN in o: + return o[Serializer.ORIGIN] + + if hasattr(o, "get_digest"): + return o.get_digest() + + if isinstance(o, str): + return o + + return None + for i in range(len(self.data[entry][key])): item, is_ref = load_ref_if_needed(self.data[entry][key][i]) - if not hasattr(item, "get_digest"): + item_origin = _get_item_origin(item) + if item_origin is None: continue - if item.get_digest() == obj_origin: + if item_origin == obj_origin: obj = save_ref_if_needed(is_ref, obj) + if is_ref: + new_digest = obj[len(SheerkaDataProvider.REF_PREFIX):] if obj_key == key: self.data[entry][key][i] = obj else: @@ -226,6 +247,8 @@ class State: if to_remove is not None: del self.data[entry][key][to_remove] + return new_digest + def remove(self, entry, filter): if filter is None: del (self.data[entry]) @@ -282,6 +305,28 @@ class SheerkaDataProviderDuplicateKeyError(Exception): self.obj = obj +@dataclass +class SheerkaDataProviderResult: + obj: object + entry: str + key: str + digest: str + already_exists: bool = False + + +@dataclass +class SheerkaDataProviderRef: + key: str + target: str + original_target: str = None + + def get_digest(self): + return self.original_target + + def get_key(self): + return self.key + + class SheerkaDataProvider: """Manages the state of the system""" @@ -305,7 +350,6 @@ class SheerkaDataProvider: self.serializer = Serializer() - @staticmethod def get_obj_key(obj): """ @@ -344,6 +388,9 @@ class SheerkaDataProvider: if hasattr(obj, Serializer.ORIGIN): return getattr(obj, Serializer.ORIGIN) + if isinstance(obj, SheerkaDataProviderRef): + return obj.original_target + return None @staticmethod @@ -359,7 +406,7 @@ class SheerkaDataProvider: def is_reference(obj): return isinstance(obj, str) and obj.startswith(SheerkaDataProvider.REF_PREFIX) - def add(self, event_digest: str, entry, obj, allow_multiple=True, use_ref=False, is_ref=False): + def add(self, event_digest: str, entry, obj, allow_multiple=True, use_ref=False): """ Adds obj to the entry 'entry' :param event_digest: digest of the event that triggers the modification of the state @@ -372,11 +419,7 @@ class SheerkaDataProvider: :return: (entry, key) to retrieve the object """ - if use_ref and is_ref: - raise SheerkaDataProviderError("Cannot use use_ref and is_ref at the same time", None) - - if is_ref and not isinstance(obj, dict): - raise SheerkaDataProviderError("is_ref can only be used with dictionaries", obj) + original_obj = obj.copy() if isinstance(obj, dict) else obj snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) state = self.load_state(snapshot) @@ -406,15 +449,11 @@ class SheerkaDataProvider: obj.set_digest(self.save_obj(obj.obj)) obj.obj = self.REF_PREFIX + obj.get_digest() - if is_ref: - for k, v in obj.obj.items(): - obj.obj[k] = self.REF_PREFIX + v - state.update(entry, obj) new_snapshot = self.save_state(state) self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot) - return entry, key + return SheerkaDataProviderResult(original_obj, entry, obj.get_key(), obj.get_digest()) def add_with_auto_key(self, event_digest: str, entry, obj): """ @@ -424,14 +463,20 @@ class SheerkaDataProvider: :param obj: :return: """ + + original_obj = obj.copy() if isinstance(obj, dict) else obj + next_key = self.get_next_key(entry) if hasattr(obj, "set_key"): obj.set_key(next_key) - self.add(event_digest, entry, ObjToUpdate(obj, next_key)) - return entry, next_key + res = self.add(event_digest, entry, ObjToUpdate(obj, next_key)) + return SheerkaDataProviderResult(original_obj, res.entry, res.key, res.digest) def add_unique(self, event_digest: str, entry, obj): """Add an entry and make sure it's unique""" + + original_obj = obj.copy() if isinstance(obj, dict) else obj + snapshot = self.get_snapshot(SheerkaDataProvider.HeadFile) state = self.load_state(snapshot) @@ -448,7 +493,12 @@ class SheerkaDataProvider: new_snapshot = self.save_state(state) self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot) - return (None if already_exist else entry), None + return SheerkaDataProviderResult( + original_obj, + entry, + None, + None, + already_exist) def set(self, event_digest, entry, obj, use_ref=False, is_ref=False): """ @@ -462,6 +512,8 @@ class SheerkaDataProvider: :return: """ + original_obj = obj.copy() if isinstance(obj, dict) else obj + if use_ref and is_ref: raise SheerkaDataProviderError("Cannot use use_ref and is_ref at the same time", None) @@ -486,7 +538,7 @@ class SheerkaDataProvider: new_snapshot = self.save_state(state) self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot) - return entry, key + return SheerkaDataProviderResult(original_obj, entry, key, self.get_obj_digest(obj)) def modify(self, event_digest, entry, key, obj): """ @@ -499,6 +551,8 @@ class SheerkaDataProvider: :return: """ + original_obj = obj.copy() if isinstance(obj, dict) else obj + if key is None: raise SheerkaDataProviderError("Key is mandatory.", None) @@ -517,21 +571,33 @@ class SheerkaDataProvider: # Gets obj original key, it will help to know if the key has changed obj_key = self.get_obj_key(obj) or key + digest = None if isinstance(state.data[entry][key], list): obj_origin = self.get_obj_origin(obj) if obj_origin is None: raise (SheerkaDataProviderError(f"Multiple entries under '{entry}.{key}'", obj)) - state.modify_in_list(entry, key, obj, obj_key, obj_origin, self.load_ref_if_needed, self.save_ref_if_needed) + digest = state.modify_in_list( + entry, + key, + obj, + obj_key, + obj_origin, + self.load_ref_if_needed, + self.save_ref_if_needed) else: - obj = self.save_ref_if_needed(self.is_reference(state.data[entry][key]), obj) + was_saved_as_reference = self.is_reference(state.data[entry][key]) + if was_saved_as_reference: + obj = self.save_ref_if_needed(True, obj) + digest = self.get_obj_digest(obj) + state.modify(entry, key, obj, obj_key) new_snapshot = self.save_state(state) self.set_snapshot(SheerkaDataProvider.HeadFile, new_snapshot) - return entry, obj_key + return SheerkaDataProviderResult(original_obj, entry, obj_key, digest) def list(self, entry, filter=None): """ @@ -814,16 +880,15 @@ class SheerkaDataProvider: return obj def load_ref_if_needed(self, obj, load_origin=True): - if not isinstance(obj, str): - return obj, False - if not obj.startswith(SheerkaDataProvider.REF_PREFIX): + if isinstance(obj, SheerkaDataProviderRef): + resolved = self.load_obj(obj.target, load_origin) + return resolved, False + + if not isinstance(obj, str) or not obj.startswith(SheerkaDataProvider.REF_PREFIX): return obj, False resolved = self.load_obj(obj[len(SheerkaDataProvider.REF_PREFIX):], load_origin) - if resolved is None: - return obj, False - - return resolved, True + return (obj, False) if resolved is None else (resolved, True) def save_ref_if_needed(self, save_ref, obj): if not save_ref: diff --git a/src/sheerkapickle/SheerkaUnpickler.py b/src/sheerkapickle/SheerkaUnpickler.py index 1bdb06a..c504f78 100644 --- a/src/sheerkapickle/SheerkaUnpickler.py +++ b/src/sheerkapickle/SheerkaUnpickler.py @@ -5,7 +5,8 @@ from sheerkapickle import tags, utils, handlers def decode(sheerka, obj): - return SheerkaUnpickler(sheerka).restore(json.loads(obj)) + decoded = SheerkaUnpickler(sheerka).restore(json.loads(obj)) + return decoded class SheerkaUnpickler: diff --git a/src/sheerkapickle/sheerka_handlers.py b/src/sheerkapickle/sheerka_handlers.py index 0d23c6e..dafc6ef 100644 --- a/src/sheerkapickle/sheerka_handlers.py +++ b/src/sheerkapickle/sheerka_handlers.py @@ -74,6 +74,7 @@ class ConceptHandler(BaseHandler): # get value instance.set_metadata_value(ConceptParts(key), resolved_value) + instance.freeze_definition_hash() return instance @@ -92,6 +93,7 @@ class UserInputHandler(ConceptHandler): instance.__init__(data["text"], data["user_name"]) instance.metadata.key = data[CONCEPT_ID][0] instance.metadata.id = data[CONCEPT_ID][1] + instance.freeze_definition_hash() return instance @@ -122,6 +124,7 @@ class ReturnValueHandler(BaseHandler): instance.metadata.key = data[CONCEPT_ID][0] instance.metadata.id = data[CONCEPT_ID][1] + instance.freeze_definition_hash() return instance diff --git a/tests/BaseTest.py b/tests/BaseTest.py index 5896084..59ec4d9 100644 --- a/tests/BaseTest.py +++ b/tests/BaseTest.py @@ -36,23 +36,26 @@ class BaseTest: def init_concepts(self, *concepts, **kwargs): sheerka = self.get_sheerka(**kwargs) context = self.get_context(sheerka) + create_new = kwargs.get("create_new", False) result = [] for c in concepts: if isinstance(c, str): c = Concept(c) - c.init_key() - sheerka.set_id_if_needed(c, False) - # manage concepts with bnf definitions if c.metadata.definition: bnf_parser = BnfParser() res = bnf_parser.parse(context, c.metadata.definition) if res.status: c.bnf = res.value.value sheerka.create_new_concept(context, c) + elif create_new: + sheerka.create_new_concept(context, c) + else: + c.init_key() + sheerka.set_id_if_needed(c, False) + sheerka.add_in_cache(c) - sheerka.add_in_cache(c) result.append(c) return sheerka, context, *result diff --git a/tests/core/test_SheerkaCreateNewConcept.py b/tests/core/test_SheerkaCreateNewConcept.py index bbe22b6..2655711 100644 --- a/tests/core/test_SheerkaCreateNewConcept.py +++ b/tests/core/test_SheerkaCreateNewConcept.py @@ -1,5 +1,6 @@ from core.builtin_concepts import BuiltinConcepts from core.concept import PROPERTIES_TO_SERIALIZE, Concept +from core.sheerka.Sheerka import Sheerka from sdp.sheerkaDataProvider import SheerkaDataProvider from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -25,7 +26,10 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): assert concept.key in sheerka.cache_by_key assert concept.id in sheerka.cache_by_id assert sheerka.sdp.io.exists( - sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest())) + sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_origin())) + assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash()) + assert sheerka.sdp.exists(Sheerka.CONCEPTS_BY_ID_ENTRY, concept.id) + assert sheerka.sdp.exists(Sheerka.CONCEPTS_ENTRY, concept.key) def test_i_cannot_add_the_same_concept_twice(self): """ @@ -77,7 +81,7 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): concept = self.get_default_concept() sheerka.create_new_concept(self.get_context(sheerka), concept) - sheerka.cache_by_key = {} # reset the cache + sheerka.reset_cache() loaded = sheerka.get(concept.key) assert loaded == concept @@ -86,6 +90,16 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): loaded = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, concept.id) assert loaded == concept + def test_i_can_instantiate_a_concept_from_sdp(self): + sheerka = self.get_sheerka() + concept = Concept("foo") + sheerka.create_new_concept(self.get_context(sheerka), concept) + + sheerka.reset_cache() + loaded = sheerka.new("foo") + + assert loaded == concept + def test_i_can_get_a_concept_by_its_id(self): sheerka = self.get_sheerka() concept = self.get_default_concept() @@ -105,6 +119,8 @@ class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka): res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1) res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2) + assert res1.status + assert res2.status assert res1.value.body.key == res2.value.body.key # same key sheerka.cache_by_key = {} # reset the cache diff --git a/tests/core/test_SheerkaModifyConcept.py b/tests/core/test_SheerkaModifyConcept.py new file mode 100644 index 0000000..f29ef94 --- /dev/null +++ b/tests/core/test_SheerkaModifyConcept.py @@ -0,0 +1,104 @@ +import pytest +from core.builtin_concepts import BuiltinConcepts +from core.concept import Concept, ConceptParts +from core.sheerka.Sheerka import Sheerka + +from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka + + +class TestSheerkaModifyConcept(TestUsingMemoryBasedSheerka): + def test_i_can_modify_a_concept(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True) + + foo_instance = sheerka.new("foo") + foo_instance.metadata.body = "value" + foo_instance.set_prop(BuiltinConcepts.ISA, bar) + foo_instance.set_metadata_value(ConceptParts.BODY, "body value") + res = sheerka.modify_concept(context, foo_instance) + + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT) + assert res.body.body.metadata.body == "value" + assert res.body.body.get_prop(BuiltinConcepts.ISA) == bar + assert res.body.body.body == "body value" + + # test that object + sheerka.reset_cache() + foo_from_sheerka = sheerka.new("foo") + assert foo_from_sheerka.metadata.body == "value" + assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar + assert foo_from_sheerka.body == "body value" + + # test that ref by id is updated + sheerka.reset_cache() + foo_from_sheerka = sheerka.get_by_id(foo.id) + assert foo_from_sheerka.metadata.body == "value" + assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar + assert foo_from_sheerka.body == "body value" + + # test that ref by hash is updated + foo_from_sdp = sheerka.sdp.get(Sheerka.CONCEPTS_BY_HASH_ENTRY, foo_instance.get_definition_hash()) + assert foo_from_sdp.metadata.body == "value" + assert foo_from_sdp.get_prop(BuiltinConcepts.ISA) == bar + assert foo_from_sdp.body == "body value" + + # previous ref by hash is removed (since that definition hash has changed) + with pytest.raises(IndexError): + sheerka.sdp.get(Sheerka.CONCEPTS_BY_HASH_ENTRY, foo_instance.get_original_definition_hash()) + + def test_i_can_modify_concept_modifying_only_properties_and_body(self): + sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True) + + foo_instance = sheerka.new("foo") + foo_instance.set_prop(BuiltinConcepts.ISA, bar) + foo_instance.set_metadata_value(ConceptParts.BODY, "body value") + res = sheerka.modify_concept(context, foo_instance) + + assert res.status + + foo_from_sheerka = sheerka.new("foo") + assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == bar + assert foo_from_sheerka.body == "body value" + + def test_cache_is_updated_when_a_concept_is_modified(self): + sheerka, context, foo = self.init_concepts("foo", create_new=True) + + foo_instance = sheerka.new("foo") + foo_instance.metadata.body = "value" + res = sheerka.modify_concept(context, foo_instance) + assert res.status + + foo_from_sheerka = sheerka.get("foo") + assert foo_from_sheerka.metadata.body == "value" + + foo_by_id_from_sheerka = sheerka.get_by_id(foo.id) + assert foo_by_id_from_sheerka.metadata.body == "value" + + def test_i_cannot_modify_a_concept_that_does_not_exists(self): + sheerka, context, foo = self.init_concepts("foo", create_new=False) + + foo_instance = sheerka.new("foo") + foo_instance.metadata.body = "value" + res = sheerka.modify_concept(context, foo_instance) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT) + assert res.body.body.key == foo.key + + def test_i_can_modify_a_concept_that_is_in_a_list(self): + sheerka, context, foo1, foo2 = self.init_concepts( + Concept("foo", body="1"), + Concept("foo", body="2"), create_new=True) + + foo2_instance = sheerka.new("foo")[1] + foo2_instance.metadata.body = "value" + + res = sheerka.modify_concept(context, foo2_instance) + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT) + assert res.body.body.metadata.body == "value" + + sheerka.reset_cache() + foo_from_sheerka = sheerka.new("foo") + assert foo_from_sheerka[0].metadata.body == "1" + assert foo_from_sheerka[1].metadata.body == "value" diff --git a/tests/core/test_SheerkaSetsManager.py b/tests/core/test_SheerkaSetsManager.py index 56a5f83..ca23356 100644 --- a/tests/core/test_SheerkaSetsManager.py +++ b/tests/core/test_SheerkaSetsManager.py @@ -200,8 +200,10 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka): assert sheerka.isinset(twenty_one, number) def test_i_can_set_isa(self): - sheerka, context, foo, all_foos = self.init_concepts(Concept("foo"), Concept("all_foo"), use_dict=False) - sheerka.create_new_concept(context, foo) + sheerka, context, foo, all_foos = self.init_concepts( + "foo", "all_foo", + create_new=True, + use_dict=False) assert BuiltinConcepts.ISA not in foo.props diff --git a/tests/core/test_concept.py b/tests/core/test_concept.py index a4428e2..5b27fa9 100644 --- a/tests/core/test_concept.py +++ b/tests/core/test_concept.py @@ -198,6 +198,9 @@ def test_i_can_update_from(): id="123456" ).def_prop("a", "10").def_prop("b", None) + # make sure origin is preserved + setattr(template, "##origin##", "digest") + template.values[ConceptParts.BODY] = "value in body" template.values[ConceptParts.WHERE] = "value in where" template.values[ConceptParts.PRE] = "value in pre" @@ -208,3 +211,4 @@ def test_i_can_update_from(): concept = Concept().update_from(template) assert concept == template + assert getattr(concept, "##origin##") == "digest" diff --git a/tests/evaluators/test_AddConceptInSetEvaluator.py b/tests/evaluators/test_AddConceptInSetEvaluator.py index 9b0d0e0..1f9b810 100644 --- a/tests/evaluators/test_AddConceptInSetEvaluator.py +++ b/tests/evaluators/test_AddConceptInSetEvaluator.py @@ -51,18 +51,20 @@ class TestAddConceptInSetEvaluator(TestUsingMemoryBasedSheerka): assert res.value.body == "bar" def test_i_can_add_concept_to_a_set_of_concept(self): - context = self.get_context() - foo = Concept("foo") - context.sheerka.create_new_concept(context, foo) - - bar = Concept("bar") - context.sheerka.create_new_concept(context, bar) - + sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True) ret_val = get_ret_val("foo", "bar") + res = AddConceptInSetEvaluator().eval(context, ret_val) + foo = sheerka.new("foo") # reload it assert res.status assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS) + assert context.sheerka.isaset(context, bar) + assert context.sheerka.isinset(foo, bar) + assert context.sheerka.isa(foo, bar) + + foo_from_sheerka = context.sheerka.get("foo") + assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == [bar] def test_i_can_add_concept_with_a_body_to_a_set_of_concept(self): context = self.get_context() diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index 92c6a54..a834ad2 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -105,7 +105,7 @@ as: assert concept_saved.key in sheerka.cache_by_key assert concept_saved.id in sheerka.cache_by_id assert sheerka.sdp.io.exists( - sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) + sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_origin())) def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept(self): """ @@ -135,7 +135,7 @@ as: assert concept_saved.key in sheerka.cache_by_key assert concept_saved.id in sheerka.cache_by_id assert sheerka.sdp.io.exists( - sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) + sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_origin())) def test_i_cannot_eval_the_same_def_concept_twice(self): text = """ diff --git a/tests/sdp/test_sheerkaDataProvider.py b/tests/sdp/test_sheerkaDataProvider.py index 97f72f0..fe76e95 100644 --- a/tests/sdp/test_sheerkaDataProvider.py +++ b/tests/sdp/test_sheerkaDataProvider.py @@ -4,7 +4,7 @@ import pytest import os from os import path from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderError, \ - SheerkaDataProviderDuplicateKeyError + SheerkaDataProviderDuplicateKeyError, SheerkaDataProviderResult, SheerkaDataProviderRef from datetime import date, datetime import shutil import json @@ -123,6 +123,36 @@ class ObjDumpJson: self.key = as_dict["key"] +class ObjDumpJsonNoDigest: + """ + Object where the key can be resolved using get_key() + that can be used to dump as Json, + But with no builtin digest computation + """ + + def __init__(self, key=None, value=None): + self.key = key + self.value = value + + def __eq__(self, obj): + return isinstance(obj, ObjDumpJsonNoDigest) and \ + self.key == obj.key and \ + self.value == obj.value + + def __repr__(self): + return f"ObjDumpJsonNoDigest({self.key}, {self.value})" + + def get_key(self): + return self.key + + def to_dict(self): + return self.__dict__ + + def from_dict(self, as_dict): + self.value = as_dict["value"] + self.key = as_dict["key"] + + class ObjWithDigestNoKey: """ Object that can compute its digest. @@ -301,13 +331,15 @@ def test_i_can_add_an_string(root): sdp = SheerkaDataProvider(root) obj = "foo => bar" - entry, key = sdp.add(evt_digest, "entry", obj) + result = sdp.add(evt_digest, "entry", obj) last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile) state = sdp.load_state(last_commit) - loaded = sdp.get(entry, key) + loaded = sdp.get(result.entry, result.key) - assert entry == "entry" - assert key is None + assert result.obj == obj + assert result.entry == "entry" + assert result.key is None + assert result.digest is None assert loaded == obj assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) @@ -330,11 +362,13 @@ def test_i_can_add_several_strings_if_allow_multiple_is_true(root): sdp.add(evt_digest, "entry", "foo") sdp.add(evt_digest, "entry", "foo") - entry, key = sdp.add(evt_digest, "entry", "bar") - loaded = sdp.get(entry, key) + result = sdp.add(evt_digest, "entry", "bar") + loaded = sdp.get(result.entry, result.key) - assert entry == "entry" - assert key is None + assert result.obj == "bar" + assert result.entry == "entry" + assert result.key is None + assert result.digest is None assert loaded == ["foo", "foo", "bar"] @@ -359,14 +393,15 @@ def test_i_can_add_an_object_with_no_key(root): sdp = SheerkaDataProvider(root) obj = ObjNoKey("a", "b") - entry, key = sdp.add(evt_digest, "entry", obj) + result = sdp.add(evt_digest, "entry", obj) last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile) state = sdp.load_state(last_commit) - loaded = sdp.get(entry, key) + loaded = sdp.get(result.entry, result.key) - assert entry == "entry" - assert key is None - assert loaded == obj + assert result.obj == obj + assert result.entry == "entry" + assert result.key is None + assert result.digest is None assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) @@ -388,11 +423,13 @@ def test_i_can_add_several_obj_no_key_if_allow_multiple_is_true(root): sdp.add(evt_digest, "entry", ObjNoKey("a", "b")) sdp.add(evt_digest, "entry", ObjNoKey("a", "b")) - entry, key = sdp.add(evt_digest, "entry", ObjNoKey("c", "d")) - loaded = sdp.get(entry, key) + result = sdp.add(evt_digest, "entry", ObjNoKey("c", "d")) + loaded = sdp.get(result.entry, result.key) - assert entry == "entry" - assert key is None + assert result.obj == ObjNoKey("c", "d") + assert result.entry == "entry" + assert result.key is None + assert result.digest is None assert loaded == [ObjNoKey("a", "b"), ObjNoKey("a", "b"), ObjNoKey("c", "d")] @@ -429,15 +466,18 @@ def test_i_can_add_a_dict(root): sdp = SheerkaDataProvider(root) obj = {"my_key": "my_value"} - entry, key = sdp.add(evt_digest, "entry", obj) + result = sdp.add(evt_digest, "entry", obj) last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile) state = sdp.load_state(last_commit) - loaded = sdp.get(entry, key) + loaded = sdp.get(result.entry, result.key) - loaded_value = sdp.get(entry, "my_key") # we can retrieve by key + loaded_value = sdp.get(result.entry, "my_key") # we can retrieve by key + + assert result.obj == obj + assert result.entry == "entry" + assert result.key is None # we return None as dict may contains several entries + assert result.digest is None - assert entry == "entry" - assert key is None # we return None as dict may contains several entries assert loaded == obj assert loaded_value == "my_value" @@ -458,11 +498,17 @@ def test_i_can_add_a_dict(root): ]) def test_i_can_add_multiple_entries_at_once_with_dict(root): sdp = SheerkaDataProvider(root) + obj = {"my_key1": "value1", "my_key2": "value2"} - entry, key = sdp.add(evt_digest, "entry", {"my_key1": "value1", "my_key2": "value2"}) - loaded = sdp.get(entry, key) - loaded_value1 = sdp.get(entry, "my_key1") - loaded_value2 = sdp.get(entry, "my_key2") + result = sdp.add(evt_digest, "entry", obj) + loaded = sdp.get(result.entry, result.key) + loaded_value1 = sdp.get(result.entry, "my_key1") + loaded_value2 = sdp.get(result.entry, "my_key2") + + assert result.obj == obj + assert result.entry == "entry" + assert result.key is None # we return None as dict may contains several entries + assert result.digest is None assert loaded == {"my_key1": "value1", "my_key2": "value2"} assert loaded_value1 == "value1" @@ -477,14 +523,14 @@ def test_i_can_add_same_key_with_dict_if_allow_multiple_is_true(root): sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", {"my_key": "my_value"}) - entry, key = sdp.add(evt_digest, "entry", {"my_key": "my_value"}) - loaded1 = sdp.get(entry, key) + result = sdp.add(evt_digest, "entry", {"my_key": "my_value"}) + loaded1 = sdp.get(result.entry, result.key) - entry, key = sdp.add(evt_digest, "entry", {"my_key": "my_value2"}) - loaded2 = sdp.get(entry, key) + result = sdp.add(evt_digest, "entry", {"my_key": "my_value2"}) + loaded2 = sdp.get(result.entry, result.key) - assert entry == "entry" - assert key is None + assert result.entry == "entry" + assert result.key is None assert loaded1 == {"my_key": ["my_value", "my_value"]} assert loaded2 == {"my_key": ["my_value", "my_value", "my_value2"]} @@ -525,19 +571,25 @@ def test_i_can_add_obj_with_key(root): obj1 = ObjWithKey("key1", "b") obj2 = ObjSetKey("c", key="key2") - entry1, key1 = sdp.add(evt_digest, "entry", obj1) # test when key is taken from obj.get_key() - entry2, key2 = sdp.add(evt_digest, "entry2", obj2) # test when key is taken from obj.key + result1 = sdp.add(evt_digest, "entry", obj1) # test when key is taken from obj.get_key() + result2 = sdp.add(evt_digest, "entry2", obj2) # test when key is taken from obj.key last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile) state = sdp.load_state(last_commit) - loaded1 = sdp.get(entry1, key1) - loaded2 = sdp.get(entry2, key2) + loaded1 = sdp.get(result1.entry, result1.key) + loaded2 = sdp.get(result2.entry, result2.key) + + assert result1.obj == obj1 + assert result1.entry == "entry" + assert result1.key == "key1" + assert result1.digest is None + + assert result2.obj == obj2 + assert result2.entry == "entry2" + assert result2.key == "key2" + assert result2.digest is None - assert entry1 == "entry" - assert key1 == "key1" assert loaded1 == ObjWithKey("key1", "b") - assert entry2 == "entry2" - assert key2 == "key2" assert loaded2 == ObjSetKey("c", key="key2") assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) @@ -559,15 +611,13 @@ def test_i_can_add_objects_with_same_key_if_allow_multiple_is_true(root): sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjWithKey("my_key", "b")) - entry, key = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key")) - loaded1 = sdp.get(entry, key) + result = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key")) + loaded1 = sdp.get(result.entry, result.key) - entry, key = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key")) + result = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key")) sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key2")) # to prove that it does not melt everything - loaded2 = sdp.get(entry, key) + loaded2 = sdp.get(result.entry, result.key) - assert entry == "entry" - assert key == "my_key" assert loaded1 == [ObjWithKey("my_key", "b"), ObjSetKey("c", key="my_key")] assert loaded2 == [ObjWithKey("my_key", "b"), ObjSetKey("c", key="my_key"), ObjSetKey("c", key="my_key")] @@ -608,13 +658,23 @@ def test_i_can_add_a_reference(root): sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey))) obj1 = ObjWithDigestWithKey(1, "foo") - sdp.add(evt_digest, "entry", obj1, use_ref=True) - sdp.add(evt_digest, "entry_by_value", {obj1.b: obj1.get_digest()}, is_ref=True) + result1 = sdp.add(evt_digest, "entry", obj1, use_ref=True) + result3 = sdp.add(evt_digest, "entry_by_ref", SheerkaDataProviderRef(obj1.b, obj1.get_digest())) # another object obj2 = ObjWithDigestWithKey(2, "bar") sdp.add(evt_digest, "entry", obj2, use_ref=True) - sdp.add(evt_digest, "entry_by_value", {obj2.b: obj2.get_digest()}, is_ref=True) + sdp.add(evt_digest, "entry_by_ref", SheerkaDataProviderRef(obj2.b, obj2.get_digest())) + + assert result1.obj == obj1 + assert result1.entry == "entry" + assert result1.key == str(obj1.get_key()) + assert result1.digest == obj1.get_digest() + + assert result3.obj == SheerkaDataProviderRef(obj1.b, obj1.get_digest()) + assert result3.entry == "entry_by_ref" + assert result3.key == "foo" + assert result3.digest is None state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == { @@ -622,22 +682,47 @@ def test_i_can_add_a_reference(root): "1": '##REF##:' + obj1.get_digest(), "2": '##REF##:' + obj2.get_digest(), }, - "entry_by_value": { - "foo": '##REF##:' + obj1.get_digest(), - "bar": '##REF##:' + obj2.get_digest() + "entry_by_ref": { + "foo": SheerkaDataProviderRef(obj1.b, obj1.get_digest()), + "bar": SheerkaDataProviderRef(obj2.b, obj2.get_digest()) }, } - # sanity check, make sure that I can load back - loaded1 = sdp.get("entry_by_value", "foo") + # make sure that I can load back + loaded1 = sdp.get("entry_by_ref", "foo") assert loaded1 == ObjWithDigestWithKey(1, "foo") assert getattr(loaded1, Serializer.ORIGIN) == obj1.get_digest() - loaded2 = sdp.get("entry_by_value", "bar") + loaded2 = sdp.get("entry_by_ref", "bar") assert loaded2 == ObjWithDigestWithKey(2, "bar") assert getattr(loaded2, Serializer.ORIGIN) == obj2.get_digest() +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_have_multiple_is_ref_to_the_same_key(root): + sdp = SheerkaDataProvider(root) + sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey))) + ref_result1 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(1, "foo"), use_ref=True) + ref_result2 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(2, "bar"), use_ref=True) + + sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef("1", ref_result1.digest)) + sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef("1", ref_result2.digest)) + + state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) + assert state.data == {'entry': {'1': '##REF##:1foo', '2': '##REF##:2bar'}, + 'entry_ref': {'1': [SheerkaDataProviderRef("1", ref_result1.digest), + SheerkaDataProviderRef("1", ref_result2.digest)]}, + } + + loaded = sdp.get("entry_ref", "1") + assert len(loaded) == 2 + assert loaded[0] == ObjWithDigestWithKey(1, "foo") + assert loaded[1] == ObjWithDigestWithKey(2, "bar") + + @pytest.mark.parametrize("root", [ ".sheerka", "mem://" @@ -660,21 +745,24 @@ def test_i_can_add_string_using_auto_generated_key(root): sdp = SheerkaDataProvider(root) key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile) - entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", "foo") - entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", "bar") - entry3, key3 = sdp.add_with_auto_key(evt_digest, "entry2", "baz") + result1 = sdp.add_with_auto_key(evt_digest, "entry1", "foo") + result2 = sdp.add_with_auto_key(evt_digest, "entry1", "bar") + result3 = sdp.add_with_auto_key(evt_digest, "entry2", "baz") state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert sdp.io.exists(key_file) assert read_json_file(sdp, key_file) == {"entry1": 2, "entry2": 1} assert state.data == {"entry1": {"1": "foo", "2": "bar"}, "entry2": {"1": "baz"}} - assert entry1 == "entry1" - assert entry2 == "entry1" - assert entry3 == "entry2" - assert key1 == "1" - assert key2 == "2" - assert key3 == "1" + assert result1.obj == "foo" + assert result2.obj == "bar" + assert result3.obj == "baz" + assert result1.entry == "entry1" + assert result2.entry == "entry1" + assert result3.entry == "entry2" + assert result1.digest is None + assert result2.digest is None + assert result3.digest is None @pytest.mark.parametrize("root", [ @@ -759,20 +847,6 @@ def test_i_cannot_add_the_same_digest_twice_in_the_same_entry4(root): assert error.value.args[0] == "Duplicate object." -def test_i_cannot_add_using_use_ref_and_is_ref(): - sdp = SheerkaDataProvider("mem://") - - with pytest.raises(SheerkaDataProviderError) as error: - sdp.add(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), use_ref=True, is_ref=True) - - -def test_i_cannot_add_using_is_ref_if_obj_is_not_a_dictionary(): - sdp = SheerkaDataProvider("mem://") - - with pytest.raises(SheerkaDataProviderError) as error: - sdp.add(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), is_ref=True) - - @pytest.mark.parametrize("root", [ ".sheerka", "mem://" @@ -800,18 +874,23 @@ def test_i_can_add_object_using_auto_generated_key(root): sdp = SheerkaDataProvider(root) key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile) - entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b")) - entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b")) + result1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b")) + result2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b")) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert sdp.io.exists(key_file) assert read_json_file(sdp, key_file) == {"entry1": 2} assert state.data == {"entry1": {"1": ObjNoKey("a", "b"), "2": ObjNoKey("a", "b")}} - assert entry1 == "entry1" - assert entry2 == "entry1" - assert key1 == "1" - assert key2 == "2" + + assert result1.obj == ObjNoKey("a", "b") + assert result2.obj == ObjNoKey("a", "b") + assert result1.entry == "entry1" + assert result2.entry == "entry1" + assert result1.key == "1" + assert result2.key == "2" + assert result1.digest is None + assert result2.digest is None @pytest.mark.parametrize("root", [ @@ -822,18 +901,23 @@ def test_object_key_is_updated_when_possible_using_auto_generated_key(root): sdp = SheerkaDataProvider(root) key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile) - entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo")) - entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo")) + result1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo")) + result2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo")) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert sdp.io.exists(key_file) assert read_json_file(sdp, key_file) == {"entry1": 2} assert state.data == {"entry1": {"1": ObjSetKey("foo", "1"), "2": ObjSetKey("foo", "2")}} - assert entry1 == "entry1" - assert entry2 == "entry1" - assert key1 == "1" - assert key2 == "2" + + assert result1.obj == ObjSetKey("foo", "1") + assert result2.obj == ObjSetKey("foo", "2") + assert result1.entry == "entry1" + assert result2.entry == "entry1" + assert result1.key == "1" + assert result2.key == "2" + assert result1.digest is None + assert result2.digest is None @pytest.mark.parametrize("root", [ @@ -843,12 +927,13 @@ def test_object_key_is_updated_when_possible_using_auto_generated_key(root): def test_i_can_set_objects_with_key(root): sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjWithKey(1, "foo")) - entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo")) + result = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo")) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": {"2": ObjWithKey(2, "foo")}} - assert entry == "entry" - assert key == "2" + assert result.entry == "entry" + assert result.key == "2" + assert result.digest is None @pytest.mark.parametrize("root", [ @@ -858,12 +943,13 @@ def test_i_can_set_objects_with_key(root): def test_i_can_set_objects_with_no_key(root): sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjNoKey(1, "foo")) - entry, key = sdp.set(evt_digest, "entry", ObjNoKey(2, "foo")) + result = sdp.set(evt_digest, "entry", ObjNoKey(2, "foo")) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": ObjNoKey(2, "foo")} - assert entry == "entry" - assert key is None + assert result.entry == "entry" + assert result.key is None + assert result.digest is None @pytest.mark.parametrize("root", [ @@ -873,12 +959,13 @@ def test_i_can_set_objects_with_no_key(root): def test_i_can_set_from_list_to_dict(root): sdp = SheerkaDataProvider(root) sdp.set(evt_digest, "entry", [ObjNoKey(1, "foo"), ObjNoKey(2, "foo")]) - entry, key = sdp.set(evt_digest, "entry", {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")}) + result = sdp.set(evt_digest, "entry", {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")}) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")}} - assert entry == "entry" - assert key is None + assert result.entry == "entry" + assert result.key is None + assert result.digest is None @pytest.mark.parametrize("root", [ @@ -889,18 +976,21 @@ def test_i_can_set_using_reference(root): sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey))) sdp.add(evt_digest, "entry", ObjWithKey(1, "foo")) - entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"), use_ref=True) + result = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"), use_ref=True) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": {"2": '##REF##:43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9'}} - assert entry == "entry" - assert key == "2" + + assert result.obj == ObjWithKey(2, "foo") + assert result.entry == "entry" + assert result.key == "2" + assert result.digest == "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9" assert sdp.io.exists(sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9")) # sanity check, make sure that I can load back - loaded = sdp.get(entry, key) + loaded = sdp.get(result.entry, result.key) assert loaded == ObjWithKey(2, "foo") assert getattr(loaded, Serializer.ORIGIN) == "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9" @@ -952,12 +1042,15 @@ def test_i_can_add_an_object_with_a_key_as_a_reference(root): obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj)) sdp.serializer.register(obj_serializer) - entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True) + result = sdp.add(evt_digest, "entry", obj, use_ref=True) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) digest = state.data["entry"]["my_key"][len(SheerkaDataProvider.REF_PREFIX):] - assert key == obj.key - assert entry == "entry" + assert result.obj == obj + assert result.entry == "entry" + assert result.key == obj.key + assert result.digest == obj.get_digest() + assert digest == result.digest assert state.data == {'entry': {'my_key': f"{SheerkaDataProvider.REF_PREFIX}{digest}"}} loaded = sdp.load_obj(digest) @@ -973,15 +1066,18 @@ def test_i_can_add_a_dictionary_as_a_reference(root): sdp = SheerkaDataProvider(root) obj = {"my_key": "value1"} - obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj)) - sdp.serializer.register(obj_serializer) + # No need to register a serializer for dictionaries - entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True) + result = sdp.add(evt_digest, "entry", obj, use_ref=True) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) digest = state.data["entry"][len(SheerkaDataProvider.REF_PREFIX):] - assert key is None - assert entry == "entry" + assert result.obj == obj + assert result.entry == "entry" + assert result.key is None # we return None as dict may contains several entries + assert result.digest == "1790cae3f354ecb6b419faaa2ee2c374ff33efb8cddafda9960924036ac04c1f" # a digest is created + assert digest == result.digest + assert state.data == {'entry': f"{SheerkaDataProvider.REF_PREFIX}{digest}"} loaded = sdp.load_obj(digest) @@ -990,23 +1086,50 @@ def test_i_can_add_a_dictionary_as_a_reference(root): assert len(loaded) == 2 +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_an_object_with_no_builtin_digest_as_a_reference(root): + sdp = SheerkaDataProvider(root) + obj = ObjDumpJsonNoDigest("a", "b") + + obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj)) + sdp.serializer.register(obj_serializer) + + result = sdp.add(evt_digest, "entry", obj, use_ref=True) + state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) + digest = state.data["entry"][obj.get_key()][len(SheerkaDataProvider.REF_PREFIX):] + + assert result.obj == obj + assert result.entry == "entry" + assert result.key == obj.get_key() + assert result.digest is not None + assert digest == result.digest + + assert state.data == {'entry': {obj.key: f"{SheerkaDataProvider.REF_PREFIX}{result.digest}"}} + + loaded = sdp.load_obj(digest) + assert getattr(loaded, Serializer.ORIGIN) == digest + + @pytest.mark.parametrize("root", [ ".sheerka", "mem://" ]) def test_i_can_add_unique(root): sdp = SheerkaDataProvider(root) - entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo")) - assert (entry, key) == ("entry", None) + result = sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo")) + assert result == SheerkaDataProviderResult(ObjNoKey(1, "foo"), "entry", None, None, False) - entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo")) - assert (entry, key) == (None, None) + result = sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo")) + assert result == SheerkaDataProviderResult(ObjNoKey(1, "foo"), "entry", None, None, True) - entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar")) - assert (entry, key) == ("entry", None) + result = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar")) + assert result == SheerkaDataProviderResult(ObjNoKey(2, "bar"), "entry", None, None, False) - entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar")) - assert (entry, key) == (None, None) + result = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar")) + assert result == SheerkaDataProviderResult(ObjNoKey(2, "bar"), "entry", None, None, True) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": {ObjNoKey(1, "foo"), ObjNoKey(2, "bar")}} @@ -1338,12 +1461,14 @@ def test_i_can_modify_dict_with_a_key(root): sdp.add(evt_digest, "entry", {"key1": "foo"}) sdp.add(evt_digest, "entry", {"key2": "bar"}) - entry, key = sdp.modify(evt_digest, "entry", "key1", "baz") + result = sdp.modify(evt_digest, "entry", "key1", "baz") state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": {"key1": "baz", "key2": "bar"}} - assert entry == "entry" - assert key == "key1" + assert result.obj == "baz" + assert result.entry == "entry" + assert result.key == "key1" + assert result.digest is None @pytest.mark.parametrize("root", [ @@ -1355,12 +1480,15 @@ def test_i_can_modify_an_object_with_a_key(root): sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo")) sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar")) - entry, key = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key1", "baz")) + result = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key1", "baz")) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) + assert state.data == {"entry": {"key1": ObjWithKey("key1", "baz"), "key2": ObjWithKey("key2", "bar")}} - assert entry == "entry" - assert key == "key1" + assert result.obj == ObjWithKey("key1", "baz") + assert result.entry == "entry" + assert result.key == "key1" + assert result.digest is None @pytest.mark.parametrize("root", [ @@ -1372,12 +1500,14 @@ def test_i_can_modify_an_object_while_changing_the_key(root): sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo")) sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar")) - entry, key = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key3", "baz")) + result = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key3", "baz")) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": {"key2": ObjWithKey("key2", "bar"), "key3": ObjWithKey("key3", "baz")}} - assert entry == "entry" - assert key == "key3" + assert result.obj == ObjWithKey("key3", "baz") + assert result.entry == "entry" + assert result.key == "key3" + assert result.digest is None @pytest.mark.parametrize("root", [ @@ -1389,12 +1519,14 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_key(root): sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo")) sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar")) - entry, key = sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key1", "bar")) + result = sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key1", "bar")) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": {"key1": [ObjWithKey("key1", "foo"), ObjWithKey("key1", "bar")]}} - assert entry == "entry" - assert key == "key1" + assert result.obj == ObjWithKey("key1", "bar") + assert result.entry == "entry" + assert result.key == "key1" + assert result.digest is None @pytest.mark.parametrize("root", [ @@ -1408,7 +1540,6 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_list( :return: """ sdp = SheerkaDataProvider(root) - sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson))) sdp.add(evt_digest, "entry", ObjDumpJson("key1", "value11")) sdp.add(evt_digest, "entry", ObjDumpJson("key1", "value12")) @@ -1417,15 +1548,17 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_list( new_value = ObjDumpJson("key1", "value13") setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest()) - entry, key = sdp.modify(evt_digest, "entry", "key2", new_value) + result = sdp.modify(evt_digest, "entry", "key2", new_value) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": { "key1": [ObjDumpJson("key1", "value11"), ObjDumpJson("key1", "value12"), ObjDumpJson("key1", "value13")], "key2": [ObjDumpJson("key2", "value22")] }} - assert entry == "entry" - assert key == "key1" + assert result.obj == new_value + assert result.entry == "entry" + assert result.key == "key1" + assert result.digest is None @pytest.mark.parametrize("root", [ @@ -1446,15 +1579,17 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_nothi new_value = ObjDumpJson("key1", "value13") setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest()) - entry, key = sdp.modify(evt_digest, "entry", "key2", new_value) + result = sdp.modify(evt_digest, "entry", "key2", new_value) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": { "key1": ObjDumpJson("key1", "value13"), "key2": [ObjDumpJson("key2", "value22")] }} - assert entry == "entry" - assert key == "key1" + assert result.obj == new_value + assert result.entry == "entry" + assert result.key == "key1" + assert result.digest is None @pytest.mark.parametrize("root", [ @@ -1476,34 +1611,143 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_one_i new_value = ObjDumpJson("key1", "value13") setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest()) - entry, key = sdp.modify(evt_digest, "entry", "key2", new_value) + result = sdp.modify(evt_digest, "entry", "key2", new_value) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": { "key1": [ObjDumpJson("key1", "value11"), ObjDumpJson("key1", "value13")], "key2": [ObjDumpJson("key2", "value22")] }} - assert entry == "entry" - assert key == "key1" + assert result.obj == new_value + assert result.entry == "entry" + assert result.key == "key1" + assert result.digest is None @pytest.mark.parametrize("root", [ ".sheerka", "mem://" ]) -def test_i_can_modify_a_ref(root): +def test_i_can_modify_a_object_saved_by_ref(root): sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey))) sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo")) - entry, key = sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"), use_ref=True) + sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"), use_ref=True) - sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key2", "baz")) + result = sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key2", "baz")) state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == {"entry": { "key1": ObjWithKey("key1", "foo"), "key2": "##REF##:041d3cca905b51bc2c66251e73e56b836aae7b9435ee3d7eb05d44bb67ff575e"}} - assert entry == "entry" - assert key == "key2" + assert result.obj == ObjWithKey("key2", "baz") + assert result.entry == "entry" + assert result.key == "key2" + assert result.digest == "041d3cca905b51bc2c66251e73e56b836aae7b9435ee3d7eb05d44bb67ff575e" + + +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_an_object_saved_by_ref_in_a_list(root): + sdp = SheerkaDataProvider(root) + sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJsonNoDigest))) + + sdp.add(evt_digest, "entry", ObjDumpJsonNoDigest("key1", "value11"), use_ref=True) + sdp.add(evt_digest, "entry", ObjDumpJsonNoDigest("key1", "value12"), use_ref=True) + result = sdp.add(evt_digest, "entry", ObjDumpJsonNoDigest("key2", "value21"), use_ref=True) + sdp.add(evt_digest, "entry", ObjDumpJsonNoDigest("key2", "value22"), use_ref=True) + + new_value = ObjDumpJsonNoDigest("key1", "value13") + setattr(new_value, Serializer.ORIGIN, result.digest) + result = sdp.modify(evt_digest, "entry", "key2", new_value) + + state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) + assert state.data == {"entry": { + 'key1': ['##REF##:f80a0c0aceb1a7a3d238c0cff2d86d6bd3a62e0c1a65c5b505f43b10c4604bd8', + '##REF##:239a8238d188c37afa10b1bcc312ca8a0e78f6e75d688ca65d08e16717ff68b0', + '##REF##:9d0a2bf9d4081de0b14837ea46bc7a1cfb6b7562f7ae86255ea9bd0ac53a6437'], + 'key2': ['##REF##:df8a38b07f469f2ff8001ea6a70f77f4f9ce85d69c530091fcaf4b380f1500d3'] + }} + assert result.obj == new_value + assert result.entry == "entry" + assert result.key == "key1" + assert result.digest is not None + + +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_a_data_provider_ref(root): + # first, create a valid entry + sdp = SheerkaDataProvider(root) + sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey))) + obj = ObjWithDigestWithKey("1", "foo") + sdp.add(evt_digest, "entry", obj, use_ref=True) + sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef(obj.b, obj.get_digest())) + + state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) + assert state.data == { + "entry": {"1": "##REF##:1foo"}, + "entry_ref": {"foo": SheerkaDataProviderRef(obj.b, obj.get_digest())}} + + # modify this entry + obj_new = ObjWithDigestWithKey("1", "bar") + sdp.modify(evt_digest, "entry", obj_new.a, obj_new) + result = sdp.modify(evt_digest, "entry_ref", "foo", SheerkaDataProviderRef(obj.b, obj_new.get_digest())) + + state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) + assert state.data == { + "entry": {"1": "##REF##:1bar"}, + "entry_ref": {"foo": SheerkaDataProviderRef(obj.b, obj_new.get_digest())}} + + assert result.obj == SheerkaDataProviderRef(obj.b, obj_new.get_digest()) + assert result.entry == "entry_ref" + assert result.key == "foo" + assert result.digest is None # digest is not set as what is saved (the digest) is not saved by ref + + # sanity check, I can load the modified entry + loaded = sdp.get("entry_ref", "foo") + assert loaded == ObjWithDigestWithKey("1", "bar") + + +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_is_ref_when_in_list(root): + sdp = SheerkaDataProvider(root) + sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey))) + ref_result1 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(1, "foo"), use_ref=True) + ref_result2 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(2, "bar"), use_ref=True) + + sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef("1", ref_result1.digest)) + sdp.add(evt_digest, "entry_ref", SheerkaDataProviderRef("1", ref_result2.digest)) + + ref_result3 = sdp.add(evt_digest, "entry", ObjWithDigestWithKey(3, "baz"), use_ref=True) + + result = sdp.modify( + evt_digest, + "entry_ref", + "1", + SheerkaDataProviderRef("1", ref_result3.digest, ref_result2.digest)) + + state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) + assert state.data == {'entry': {'1': '##REF##:1foo', '2': '##REF##:2bar', '3': '##REF##:3baz'}, + 'entry_ref': {'1': [ + SheerkaDataProviderRef("1", ref_result1.digest), + SheerkaDataProviderRef("1", ref_result3.digest, ref_result2.digest)]}} + + loaded = sdp.get("entry_ref", "1") + assert len(loaded) == 2 + assert loaded[0] == ObjWithDigestWithKey(1, "foo") + assert loaded[1] == ObjWithDigestWithKey(3, "baz") + + assert result.obj == SheerkaDataProviderRef("1", ref_result3.digest, ref_result2.digest) + assert result.entry == "entry_ref" + assert result.key == "1" + assert result.digest is None # digest is not set as what is saved (the digest) is not saved by ref @pytest.mark.parametrize("root", [ @@ -1661,8 +1905,8 @@ def test_i_can_get_object_saved_by_reference(root): obj = ObjDumpJson("my_key", "value1") sdp.serializer.register(JsonSerializer(core.utils.get_full_qualified_name(obj))) - entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True) - loaded = sdp.get(entry, key) + result = sdp.add(evt_digest, "entry", obj, use_ref=True) + loaded = sdp.get(result.entry, result.key) assert loaded == obj @@ -1876,12 +2120,15 @@ def test_i_can_save_and_load_object_ref_with_history(root): obj = ObjDumpJson("my_key", "value1") sdp.serializer.register(JsonSerializer(core.utils.get_full_qualified_name(obj))) - entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True) - loaded = sdp.get(entry, key) + result = sdp.add(evt_digest, "entry", obj, use_ref=True) + loaded = sdp.get(result.entry, result.key) history = getattr(loaded, Serializer.HISTORY) - assert key == obj.key - assert entry == "entry" + assert result.obj == obj + assert result.entry == "entry" + assert result.key == obj.key + assert result.digest == obj.get_digest() + assert loaded.key == obj.key assert loaded.value == obj.value @@ -1895,8 +2142,8 @@ def test_i_can_save_and_load_object_ref_with_history(root): previous_modification_time = history[Serializer.MODIFICATION_DATE] previous_parents = history[Serializer.PARENTS] - sdp.modify(evt_digest, "entry", key, loaded) - loaded = sdp.get(entry, key) + sdp.modify(evt_digest, "entry", result.key, loaded) + loaded = sdp.get(result.entry, result.key) history = getattr(loaded, Serializer.HISTORY) assert history[Serializer.MODIFICATION_DATE] == previous_modification_time @@ -1906,8 +2153,8 @@ def test_i_can_save_and_load_object_ref_with_history(root): previous_digest = loaded.get_digest() loaded.value = "value2" - sdp.modify(evt_digest, "entry", key, loaded) - loaded2 = sdp.get(entry, key) + sdp.modify(evt_digest, "entry", result.key, loaded) + loaded2 = sdp.get(result.entry, result.key) history2 = getattr(loaded2, Serializer.HISTORY) assert loaded2.key == loaded.key @@ -1932,10 +2179,10 @@ def test_i_can_add_obj_with_same_key_and_get_them_back(root): obj2 = ObjDumpJson("key", "value2") sdp.serializer.register(JsonSerializer(core.utils.get_full_qualified_name(obj1))) - entry1, key1 = sdp.add(evt_digest, "entry", obj1, use_ref=True) - entry2, key2 = sdp.add(evt_digest, "entry", obj2, use_ref=True) + result = sdp.add(evt_digest, "entry", obj1, use_ref=True) + sdp.add(evt_digest, "entry", obj2, use_ref=True) - loaded = sdp.get_safe(entry1, key1) + loaded = sdp.get(result.entry, result.key) assert len(loaded) == 2 assert loaded[0] == obj1 @@ -1953,14 +2200,14 @@ def test_i_get_safe_dictionary_without_origin(root): obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj)) sdp.serializer.register(obj_serializer) - entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True) - from_db = sdp.get_safe(entry, key) + result = sdp.add(evt_digest, "entry", obj, use_ref=True) + from_db = sdp.get(result.entry, result.key) assert len(from_db) == 2 assert from_db["my_key"] == obj["my_key"] assert Serializer.ORIGIN in from_db - from_db_no_origin = sdp.get_safe(entry, key, load_origin=False) + from_db_no_origin = sdp.get_safe(result.entry, result.key, load_origin=False) assert len(from_db_no_origin) == 1 assert from_db_no_origin["my_key"] == obj["my_key"] assert Serializer.ORIGIN not in from_db_no_origin @@ -1977,14 +2224,14 @@ def test_i_get_dictionary_without_origin(root): obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj)) sdp.serializer.register(obj_serializer) - entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True) - from_db = sdp.get(entry, key) + result = sdp.add(evt_digest, "entry", obj, use_ref=True) + from_db = sdp.get(result.entry, result.key) assert len(from_db) == 2 assert from_db["my_key"] == obj["my_key"] assert Serializer.ORIGIN in from_db - from_db_no_origin = sdp.get(entry, key, load_origin=False) + from_db_no_origin = sdp.get(result.entry, result.key, load_origin=False) assert len(from_db_no_origin) == 1 assert from_db_no_origin["my_key"] == obj["my_key"] assert Serializer.ORIGIN not in from_db_no_origin @@ -2001,12 +2248,12 @@ def test_i_get_safe_object_without_origin(root): obj_serializer = JsonSerializer(core.utils.get_full_qualified_name(obj)) sdp.serializer.register(obj_serializer) - entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True) - from_db = sdp.get_safe(entry, key) + result = sdp.add(evt_digest, "entry", obj, use_ref=True) + from_db = sdp.get(result.entry, result.key) assert from_db == obj assert hasattr(from_db, Serializer.ORIGIN) - from_db_no_origin = sdp.get_safe(entry, key, load_origin=False) + from_db_no_origin = sdp.get_safe(result.entry, result.key, load_origin=False) assert from_db_no_origin == obj assert not hasattr(from_db_no_origin, Serializer.ORIGIN)