diff --git a/.gitignore b/.gitignore index dff3700..eb57703 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ venv .pytest_cache .idea __pycache__ -build \ No newline at end of file +build +prof \ No newline at end of file diff --git a/Makefile b/Makefile index 4f4180b..a9a1b7f 100644 --- a/Makefile +++ b/Makefile @@ -6,5 +6,6 @@ test: clean clean: rm -rf build + rm -rf prof find . -name '.pytest_cache' -exec rm -rf {} + find . -name '__pycache__' -exec rm -rf {} + diff --git a/docs/blog.rst b/docs/blog.rst index c02cc0a..3f328bd 100644 --- a/docs/blog.rst +++ b/docs/blog.rst @@ -470,3 +470,27 @@ MUST be different ? How I handle when the explicitly have to be the same ? I seems that the purpose of the **ExactConceptParser** is to find exact match. I need another way to express that 'a' and 'b' can be the same. +2019-11-21 +********** + +MemoryFS, is it a joke ? +""""""""""""""""""""""""""""" + +I spent this day working on a improving the test performances. By default Sheerka +persists its data on the file system (even if I said that where the data is saved) +is not important for the sdp module. + +For each test, a folder in initialized to hold concepts information. And this folder +is destroyed after usage. For almost every single test ! + +So I decided to implement fs.MemoryFS. Information in memory is supposed to be +faster than on the disk ! + +I was very disappointed, after a afternoon of refactoring that it is actually slower +than the native io implementation. + +Even now that I am writing it, I just can't believe it. I must I have implemented +it wrong. But the profiling shows that the time is lost in the under layers of the +FS library. + +It's a shame ! \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 5f3ed85..ced1479 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,14 @@ +appdirs==1.4.3 atomicwrites==1.3.0 attrs==19.3.0 +fs==2.4.11 more-itertools==7.2.0 packaging==19.2 pluggy==0.13.0 py==1.8.0 +Pygments==2.4.2 pyparsing==2.4.4 pytest==5.2.2 +pytz==2019.3 six==1.13.0 wcwidth==0.1.7 diff --git a/sdp/sheerkaDataProvider.py b/sdp/sheerkaDataProvider.py index dcd54fe..3e51e05 100644 --- a/sdp/sheerkaDataProvider.py +++ b/sdp/sheerkaDataProvider.py @@ -1,9 +1,11 @@ -from os import path -import os +# from os import path +# import os from datetime import datetime, date import hashlib import json import zlib + +from sdp.sheerkaDataProviderIO import SheerkaDataProviderIO from sdp.sheerkaSerializer import Serializer, SerializerContext import logging @@ -278,23 +280,11 @@ class SheerkaDataProvider: def __init__(self, root=None): log.debug("Initializing sdp.") - self.root = path.abspath(path.join(path.expanduser("~"), ".sheerka")) \ - if root is None \ - else path.abspath(root) - log.debug("root is set to '" + self.root + "'") - - if not path.exists(self.root): - log.debug("root folder not found. Creating it.") - os.makedirs(self.root) - self.first_time = True - else: - self.first_time = False + self.io = SheerkaDataProviderIO.get(root) + self.first_time = self.io.first_time self.serializer = Serializer() - def get_obj_path(self, object_type, digest): - return path.join(self.root, object_type, digest[:24], digest) - @staticmethod def get_obj_key(obj): """ @@ -620,15 +610,11 @@ class SheerkaDataProvider: :return: digest of the event """ digest = event.get_digest() - target_path = path.join(self.root, SheerkaDataProvider.EventFolder, digest[:24], digest) - if path.exists(target_path): + target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + if self.io.exists(target_path): return digest - if not path.exists(path.dirname(target_path)): - os.makedirs(path.dirname(target_path)) - - with open(target_path, "wb") as f: - f.write(self.serializer.serialize(event, None).read()) + self.io.write_binary(target_path, self.serializer.serialize(event, None).read()) return digest @@ -638,31 +624,27 @@ class SheerkaDataProvider: :param digest: :return: """ - target_path = path.join(self.root, SheerkaDataProvider.EventFolder, digest[:24], digest) - with open(target_path, "rb") as f: + target_path = self.io.get_obj_path(SheerkaDataProvider.EventFolder, digest) + + with self.io.open(target_path, "rb") as f: return self.serializer.deserialize(f, None) def save_state(self, state: State): digest = state.get_digest() log.debug(f"Saving new state. digest={digest}") - target_path = path.join(self.root, SheerkaDataProvider.StateFolder, digest[:24], digest) - if path.exists(target_path): + target_path = self.io.get_obj_path(SheerkaDataProvider.StateFolder, digest) + if self.io.exists(target_path): return digest - if not path.exists(path.dirname(target_path)): - os.makedirs(path.dirname(target_path)) - - with open(target_path, "wb") as f: - f.write(self.serializer.serialize(state, None).read()) - + self.io.write_binary(target_path, self.serializer.serialize(state, None).read()) return digest def load_state(self, digest): if digest is None: return State() - target_path = path.join(self.root, SheerkaDataProvider.StateFolder, digest[:24], digest) - with open(target_path, "rb") as f: + target_path = self.io.get_obj_path(SheerkaDataProvider.StateFolder, digest) + with self.io.open(target_path, "rb") as f: return self.serializer.deserialize(f, None) def save_obj(self, obj): @@ -670,16 +652,12 @@ class SheerkaDataProvider: stream = self.serializer.serialize(obj, SerializerContext(user_name="kodjo")) digest = obj.get_digest() if hasattr(obj, "get_digest") else self.get_stream_digest(stream) - target_path = path.join(self.root, SheerkaDataProvider.ObjectsFolder, digest[:24], digest) - if path.exists(target_path): + target_path = self.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, digest) + if self.io.exists(target_path): log.debug(f"...already saved. digest is {digest}") return digest - if not path.exists(path.dirname(target_path)): - os.makedirs(path.dirname(target_path)) - - with open(target_path, "wb") as f: - f.write(stream.read()) + self.io.write_binary(target_path, stream.read()) log.debug(f"...digest={digest}.") return digest @@ -688,11 +666,11 @@ class SheerkaDataProvider: if digest is None: return None - target_path = path.join(self.root, SheerkaDataProvider.ObjectsFolder, digest[:24], digest) - if not path.exists(target_path): + target_path = self.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, digest) + if not self.io.exists(target_path): return None - with open(target_path, "rb") as f: + with self.io.open(target_path, "rb") as f: obj = self.serializer.deserialize(f, SerializerContext(origin=digest)) # set the origin of the object @@ -721,7 +699,7 @@ class SheerkaDataProvider: def get_cache_params(self, category, key): digest = hashlib.sha3_256(f"{category}:{key}".encode("utf-8")).hexdigest() - cache_path = path.join(self.root, SheerkaDataProvider.CacheFolder, digest[:24], digest) + cache_path = self.io.get_obj_path(SheerkaDataProvider.CacheFolder, digest) return digest, cache_path def add_to_cache(self, category, key, obj, update=False): @@ -735,15 +713,10 @@ class SheerkaDataProvider: """ digest, cache_path = self.get_cache_params(category, key) - if path.exists(cache_path) and not update: + if self.io.exists(cache_path) and not update: return digest - if not path.exists(path.dirname(cache_path)): - os.makedirs(path.dirname(cache_path)) - - with open(cache_path, "wb") as f: - f.write(zlib.compress(obj.encode("utf-8"), 9)) - + self.io.write_binary(cache_path, zlib.compress(obj.encode("utf-8"), 9)) return digest def load_from_cache(self, category, key): @@ -755,10 +728,10 @@ class SheerkaDataProvider: """ digest, cache_path = self.get_cache_params(category, key) - if not path.exists(cache_path): + if not self.io.exists(cache_path): raise IndexError(f"{category}.{key}") - with open(cache_path, "rb") as f: + with self.io.open(cache_path, "rb") as f: return zlib.decompress(f.read()).decode("utf-8") def remove_from_cache(self, category, key): @@ -769,8 +742,8 @@ class SheerkaDataProvider: :return: """ digest, cache_path = self.get_cache_params(category, key) - if path.exists(cache_path): - os.remove(cache_path) + if self.io.exists(cache_path): + self.io.remove(cache_path) return digest @@ -782,32 +755,34 @@ class SheerkaDataProvider: :return: """ digest, cache_path = self.get_cache_params(category, key) - return path.exists(cache_path) + return self.io.exists(cache_path) def get_snapshot(self): - head_file = path.join(self.root, SheerkaDataProvider.HeadFile) - if not path.exists(head_file): + head_file = self.io.path_join(SheerkaDataProvider.HeadFile) + if not self.io.exists(head_file): return None - with open(head_file, "r") as f: - return f.read() + return self.io.read_text(head_file) + # with open(head_file, "r") as f: + # return f.read() def set_snapshot(self, digest): - head_file = path.join(self.root, SheerkaDataProvider.HeadFile) - with open(head_file, "w") as f: - return f.write(digest) + head_file = self.io.path_join(SheerkaDataProvider.HeadFile) + return self.io.write_text(head_file, digest) + # with open(head_file, "w") as f: + # return f.write(digest) def load_keys(self): - keys_file = path.join(self.root, SheerkaDataProvider.KeysFile) - if not path.exists(keys_file): + keys_file = self.io.path_join(SheerkaDataProvider.KeysFile) + if not self.io.exists(keys_file): keys = {} else: - with open(keys_file, "r") as f: + with self.io.open(keys_file, "r") as f: keys = json.load(f) return keys def save_keys(self, keys): - keys_file = path.join(self.root, SheerkaDataProvider.KeysFile) - with open(keys_file, "w") as f: + keys_file = self.io.path_join(SheerkaDataProvider.KeysFile) + with self.io.open(keys_file, "w") as f: json.dump(keys, f) def get_next_key(self, entry): diff --git a/sdp/sheerkaDataProviderIO.py b/sdp/sheerkaDataProviderIO.py new file mode 100644 index 0000000..70b49c5 --- /dev/null +++ b/sdp/sheerkaDataProviderIO.py @@ -0,0 +1,189 @@ +import io +from os import path +import os +from fs.memoryfs import MemoryFS +import logging + + +class SheerkaDataProviderIO: + + def __init__(self, root): + self.root = root + + def exists(self, file_path): + pass + + def open(self, file_path, mode): + pass + + def read_text(self, file_path): + pass + + def read_binary(self, file_path): + pass + + def write_text(self, file_path, content): + pass + + def write_binary(self, file_path, content): + pass + + def remove(self, file_path): + pass + + @staticmethod + def get(root): + if root == "mem://": + return SheerkaDataProviderDictionaryIO() + else: + return SheerkaDataProviderFileIO(root) + + def get_obj_path(self, object_type, digest): + return path.join(self.root, object_type, digest[:24], digest) + + def path_join(self, *paths): + return path.join(self.root, *paths) + + +class SheerkaDataProviderFileIO(SheerkaDataProviderIO): + log = logging.getLogger("FileIO") + + def __init__(self, root): + root = path.abspath(path.join(path.expanduser("~"), ".sheerka")) \ + if root is None \ + else path.abspath(root) + super().__init__(root) + + self.log.debug("root is set to '" + self.root + "'") + + if not path.exists(self.root): + self.log.debug("root folder not found. Creating it.") + os.makedirs(self.root) + self.first_time = True + else: + self.first_time = False + + def open(self, file_path, mode): + return open(file_path, mode) + + def read_text(self, file_path): + with open(file_path) as f: + return f.read() + + def read_binary(self, file_path): + with open(file_path, "rb") as f: + return f.read() + + def write_text(self, file_path, content): + self._write(file_path, content, "w") + + def write_binary(self, file_path, content): + self._write(file_path, content, "wb") + + def exists(self, file_path): + return path.exists(file_path) + + def remove(self, file_path): + os.remove(file_path) + + @staticmethod + def _write(file_path, content, mode): + if not path.exists(path.dirname(file_path)): + os.makedirs(path.dirname(file_path)) + + with open(file_path, mode) as f: + f.write(content) + + +class SheerkaDataProviderMemoryIO(SheerkaDataProviderIO): + log = logging.getLogger("MemoryIO") + + def __init__(self): + super().__init__("") + self.mem_fs = MemoryFS() + self.log.debug("Initializing memory file.") + self.first_time = True + + def open(self, file_path, mode): + return self.mem_fs.open(file_path, mode) + + def exists(self, file_path): + return self.mem_fs.exists(file_path) + + def read_text(self, file_path): + return self.mem_fs.readtext(file_path) + + def read_binary(self, file_path): + return self.mem_fs.readbytes(file_path) + + def write_binary(self, file_path, content): + self._ensure_parent_folder(file_path) + self.mem_fs.writebytes(file_path, content) + + def write_text(self, file_path, content): + self._ensure_parent_folder(file_path) + self.mem_fs.writetext(file_path, content) + + def remove(self, file_path): + self.mem_fs.remove(file_path) + + def _ensure_parent_folder(self, file_path): + if not self.mem_fs.exists(path.dirname(file_path)): + self.mem_fs.makedirs(path.dirname(file_path)) + + +class SheerkaDataProviderDictionaryIO(SheerkaDataProviderIO): + log = logging.getLogger("DictionaryIO") + + def __init__(self): + super().__init__("") + self.cache = {} + self.log.debug("Initializing dictionary file.") + self.first_time = True + + def exists(self, file_path): + if file_path == "": + return True + + return file_path in self.cache + + def read_text(self, file_path): + return self.cache[file_path] + + def read_binary(self, file_path): + return self.cache[file_path] + + def write_binary(self, file_path, content): + self.cache[file_path] = content + + def write_text(self, file_path, content): + self.cache[file_path] = content + + def remove(self, file_path): + del (self.cache[file_path]) + + def open(self, file_path, mode): + if "w" in mode: + stream = io.BytesIO() if "b" in mode else io.StringIO() + stream.close = on_close(self, file_path, stream)(stream.close) + return stream + + return io.BytesIO(self.cache[file_path]) if "b" in mode else io.StringIO(self.cache[file_path]) + + +def on_close(dictionary_io, file_path, stream): + """ + Decorator to intercept the close. + I guess that there are solution that are more elegant + :param dictionary_io: + :param file_path: + :param stream: + :return: + """ + def decorator(func): + def wrapper(*args, **kwargs): + stream.seek(0) + dictionary_io.cache[file_path] = stream.read() + func(*args, **kwargs) + return wrapper + return decorator diff --git a/tests/test_ConceptEvaluator.py b/tests/test_ConceptEvaluator.py index 8ad3712..ed32a7c 100644 --- a/tests/test_ConceptEvaluator.py +++ b/tests/test_ConceptEvaluator.py @@ -1,7 +1,3 @@ -import os -import shutil -from os import path - import pytest from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts @@ -10,27 +6,10 @@ from core.sheerka import Sheerka, ExecutionContext from evaluators.ConceptEvaluator import ConceptEvaluator from parsers.BaseParser import BaseParser -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - os.chdir(current_pwd) - def get_context(): sheerka = Sheerka() - sheerka.initialize(root_folder) + sheerka.initialize("mem://") return ExecutionContext("test", "xxx", sheerka) @@ -218,4 +197,3 @@ def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(): assert result.value.property_name == "b" assert context.sheerka.isinstance(result.value.error, BuiltinConcepts.TOO_MANY_ERRORS) assert result.value.concept == concept_plus - diff --git a/tests/test_DefaultParser.py b/tests/test_DefaultParser.py index 2967e62..860fbe8 100644 --- a/tests/test_DefaultParser.py +++ b/tests/test_DefaultParser.py @@ -1,36 +1,12 @@ -import os -import shutil - import pytest -from os import path import ast from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept from core.sheerka import Sheerka, ExecutionContext -from parsers.BaseParser import BaseParser -from parsers.PythonParser import PythonParser, PythonNode, PythonErrorNode +from parsers.PythonParser import PythonParser, PythonNode from core.tokenizer import Keywords, Tokenizer from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode -# from parsers.DefaultParser import NumberNode, StringNode, VariableNode, TrueNode, FalseNode, NullNode, BinaryNode -from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode, NopNode - -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - os.chdir(current_pwd) - +from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode # def nop(): # return NopNode() @@ -92,7 +68,7 @@ def get_concept(name, where=None, pre=None, post=None, body=None): def get_context(): sheerka = Sheerka() - sheerka.initialize(root_folder) + sheerka.initialize("mem://") return ExecutionContext("test", "xxx", sheerka) diff --git a/tests/test_ExactConceptParser.py b/tests/test_ExactConceptParser.py index 443ee38..3dd875b 100644 --- a/tests/test_ExactConceptParser.py +++ b/tests/test_ExactConceptParser.py @@ -1,33 +1,9 @@ -import pytest -from os import path -import shutil -import os - -from core.builtin_concepts import ParserResultConcept, BuiltinConcepts +from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, Property from core.sheerka import Sheerka, ExecutionContext from core.tokenizer import Tokenizer -from parsers.DefaultParser import DefaultParser from parsers.ExactConceptParser import ExactConceptParser -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - - os.chdir(current_pwd) - def test_i_can_compute_combinations(): parser = ExactConceptParser() @@ -150,7 +126,7 @@ def test_i_can_detect_concept_from_tokens(): def get_context(): sheerka = Sheerka() - sheerka.initialize(root_folder) + sheerka.initialize("mem://") return ExecutionContext("sheerka", "xxxx", sheerka) diff --git a/tests/test_MultipleSameSuccessEvaluator.py b/tests/test_MultipleSameSuccessEvaluator.py index e6243b5..006bfca 100644 --- a/tests/test_MultipleSameSuccessEvaluator.py +++ b/tests/test_MultipleSameSuccessEvaluator.py @@ -1,9 +1,3 @@ -import os -import shutil -from os import path - -import pytest - from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts from core.concept import Concept from core.sheerka import Sheerka, ExecutionContext @@ -11,27 +5,10 @@ from evaluators.BaseEvaluator import BaseEvaluator from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator from parsers.BaseParser import BaseParser -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - os.chdir(current_pwd) - def get_context(): sheerka = Sheerka() - sheerka.initialize(root_folder) + sheerka.initialize("mem://") return ExecutionContext("test", "xxx", sheerka) diff --git a/tests/test_PyhtonEvaluator.py b/tests/test_PyhtonEvaluator.py index 2c724c2..e919af6 100644 --- a/tests/test_PyhtonEvaluator.py +++ b/tests/test_PyhtonEvaluator.py @@ -1,7 +1,4 @@ import pytest -import shutil -from os import path -import os from core.builtin_concepts import ReturnValueConcept, ParserResultConcept from core.sheerka import Sheerka, ExecutionContext @@ -9,27 +6,10 @@ from core.concept import Concept from evaluators.PythonEvaluator import PythonEvaluator from parsers.PythonParser import PythonNode, PythonParser -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - os.chdir(current_pwd) - def get_context(): sheerka = Sheerka() - sheerka.initialize(root_folder) + sheerka.initialize("mem://") return ExecutionContext("test", "xxx", sheerka) diff --git a/tests/test_PythonParser.py b/tests/test_PythonParser.py index a016c5d..e6b0850 100644 --- a/tests/test_PythonParser.py +++ b/tests/test_PythonParser.py @@ -1,37 +1,16 @@ import ast -import os -import shutil -from os import path import pytest from core.builtin_concepts import ParserResultConcept from core.sheerka import Sheerka, ExecutionContext from core.tokenizer import Tokenizer -from parsers.BaseParser import BaseParser from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - os.chdir(current_pwd) - def get_context(): sheerka = Sheerka() - sheerka.initialize(root_folder) + sheerka.initialize("mem://") return ExecutionContext("test", "xxx", sheerka) diff --git a/tests/test_ast.py b/tests/test_ast.py index 309e343..5e7afaf 100644 --- a/tests/test_ast.py +++ b/tests/test_ast.py @@ -1,8 +1,3 @@ -import os -import shutil -from os import path - -import pytest import ast from core.ast.nodes import NodeParent, GenericNodeConcept @@ -11,28 +6,10 @@ from core.ast.visitors import ConceptNodeVisitor, UnreferencedNamesVisitor from core.builtin_concepts import BuiltinConcepts from core.sheerka import Sheerka -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - - os.chdir(current_pwd) - def get_sheerka(): sheerka = Sheerka() - sheerka.initialize(root_folder) + sheerka.initialize("mem://") return sheerka diff --git a/tests/test_builtin_helpers.py b/tests/test_builtin_helpers.py index 6e0ecc4..10856b0 100644 --- a/tests/test_builtin_helpers.py +++ b/tests/test_builtin_helpers.py @@ -1,31 +1,7 @@ -import shutil -from os import path -import os - -import pytest - from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts from core.sheerka import Sheerka, ExecutionContext import core.builtin_helpers -tests_root = path.abspath("../build/tests") -root_folder = "init_folder" - - -@pytest.fixture(autouse=True) -def init_test(): - if path.exists(tests_root): - shutil.rmtree(tests_root) - - if not path.exists(tests_root): - os.makedirs(tests_root) - current_pwd = os.getcwd() - os.chdir(tests_root) - - yield None - - os.chdir(current_pwd) - def test_i_can_use_expect_one_when_empty(): sheerka = get_sheerka() @@ -136,7 +112,7 @@ def test_i_can_use_expect_one_when_not_a_list_false(): def get_sheerka(): sheerka = Sheerka() - sheerka.initialize(root_folder) + sheerka.initialize("mem://") return sheerka diff --git a/tests/test_sheerka.py b/tests/test_sheerka.py index dd6bc47..4993e76 100644 --- a/tests/test_sheerka.py +++ b/tests/test_sheerka.py @@ -5,14 +5,11 @@ import os from os import path import shutil -from core import utils from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept -from core.concept import Concept, ConceptParts +from core.concept import Concept from core.sheerka import Sheerka, ExecutionContext from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator -from parsers.DefaultParser import DefaultParser -from parsers.PythonParser import PythonParser -from sdp.sheerkaDataProvider import SheerkaDataProvider, SheerkaDataProviderDuplicateKeyError +from sdp.sheerkaDataProvider import SheerkaDataProvider tests_root = path.abspath("../build/tests") root_folder = "init_folder" @@ -64,12 +61,12 @@ def test_builtin_concepts_are_initialized(): def test_builtin_concepts_can_be_updated(): - sheerka = get_sheerka() + sheerka = get_sheerka(root_folder) loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA) loaded_sheerka.desc = "I have a description" sheerka.sdp.modify("Test", sheerka.CONCEPTS_ENTRY, loaded_sheerka.key, loaded_sheerka) - sheerka = get_sheerka() + sheerka = get_sheerka(root_folder) loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA) assert loaded_sheerka.desc == "I have a description" @@ -92,7 +89,7 @@ def test_i_can_add_a_concept(): assert concept_found.id == "1001" assert concept.key in sheerka.concepts_cache - assert path.exists(sheerka.sdp.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest())) + assert sheerka.sdp.io.exists(sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest())) def test_i_cannot_add_the_same_concept_twice(): @@ -417,7 +414,7 @@ as: assert getattr(concept_saved, prop) == getattr(expected, prop) assert concept_saved.key in sheerka.concepts_cache - assert path.exists(sheerka.sdp.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) + assert sheerka.sdp.io.exists(sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept(): @@ -446,7 +443,7 @@ def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept() assert getattr(concept_saved, prop) == getattr(expected, prop) assert concept_saved.key in sheerka.concepts_cache - assert path.exists(sheerka.sdp.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) + assert sheerka.sdp.io.exists(sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest())) def test_i_cannot_eval_the_same_def_concept_twice(): @@ -558,9 +555,9 @@ def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(): assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME) -def get_sheerka(): +def get_sheerka(root="mem://"): sheerka = Sheerka() - sheerka.initialize(root_folder) + sheerka.initialize(root) return sheerka diff --git a/tests/test_sheerkaDataProvider.py b/tests/test_sheerkaDataProvider.py index b35c2a6..57a47a2 100644 --- a/tests/test_sheerkaDataProvider.py +++ b/tests/test_sheerkaDataProvider.py @@ -9,20 +9,15 @@ from datetime import date, datetime import shutil import json -from sdp.sheerkaSerializer import ObjectSerializer, BaseSerializer, Serializer, SerializerContext, PickleSerializer +from sdp.sheerkaSerializer import ObjectSerializer, Serializer, PickleSerializer import core.utils tests_root = path.abspath("../build/tests") evt_digest = "3a571cb6034ef6fc8d7fe91948d0d29728eed74de02bac7968b0e9facca2c2d7" -def read_text_file(file_name): - with open(file_name, "r") as f: - return f.read() - - -def read_json_file(file_name): - with open(file_name, "r") as f: +def read_json_file(sdp, file_name): + with sdp.io.open(file_name, "r") as f: return json.load(f) @@ -166,29 +161,41 @@ def init_test(): os.chdir(current_pwd) -def test_i_can_init_the_data_provider(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root, expected", [ + (".sheerka", path.abspath(path.join(tests_root, ".sheerka"))), + ("mem://", "") +]) +def test_i_can_init_the_data_provider(root, expected): + sdp = SheerkaDataProvider(root) - assert sdp.root == path.abspath(path.join(tests_root, ".sheerka")) - assert path.exists(path.join(tests_root, ".sheerka")) + assert sdp.io.root == expected + assert sdp.io.exists(sdp.io.root) -def test_i_can_save_and_load_an_event(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_save_and_load_an_event(root): + sdp = SheerkaDataProvider(root) event = Event("hello world", date=date(year=2007, month=9, day=10), user="kodjo") evt_digest = sdp.save_event(event) evt = sdp.load_event(evt_digest) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, evt_digest[0:24], evt_digest)) assert evt.version == 1 assert evt.date == datetime(year=2007, month=9, day=10) assert evt.user == "kodjo" assert evt.message == "hello world" + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.EventFolder, evt_digest[0:24], evt_digest)) -def test_i_can_add_an_string(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_an_string(root): + sdp = SheerkaDataProvider(root) obj = "foo => bar" entry, key = sdp.add(evt_digest, "entry", obj) @@ -200,19 +207,23 @@ def test_i_can_add_an_string(): assert key is None assert loaded == obj - assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) assert state.date is not None assert state.parents == [] assert state.events == [evt_digest] assert state.data == {"entry": "foo => bar"} - assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit + assert sdp.io.read_text(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) == last_commit -def test_i_can_add_several_strings_if_allow_multiple_is_true(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_several_strings_if_allow_multiple_is_true(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", "foo") sdp.add(evt_digest, "entry", "foo") @@ -224,8 +235,12 @@ def test_i_can_add_several_strings_if_allow_multiple_is_true(): assert loaded == ["foo", "foo", "bar"] -def test_i_cannot_add_several_strings_if_allow_multiple_is_false(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_add_several_strings_if_allow_multiple_is_false(root): + sdp = SheerkaDataProvider(root) with pytest.raises(IndexError) as index_error: sdp.add(evt_digest, "entry", "foo", False) @@ -233,8 +248,12 @@ def test_i_cannot_add_several_strings_if_allow_multiple_is_false(): assert index_error.value.args[0] == "entry" -def test_i_can_add_an_object_with_no_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_an_object_with_no_key(root): + sdp = SheerkaDataProvider(root) obj = ObjNoKey("a", "b") entry, key = sdp.add(evt_digest, "entry", obj) @@ -246,19 +265,23 @@ def test_i_can_add_an_object_with_no_key(): assert key is None assert loaded == obj - assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) assert state.date is not None assert state.parents == [] assert state.events == [evt_digest] assert state.data == {"entry": ObjNoKey("a", "b")} - assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit + assert sdp.io.read_text(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) == last_commit -def test_i_can_add_several_obj_no_key_if_allow_multiple_is_true(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_several_obj_no_key_if_allow_multiple_is_true(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjNoKey("a", "b")) sdp.add(evt_digest, "entry", ObjNoKey("a", "b")) @@ -270,8 +293,12 @@ def test_i_can_add_several_obj_no_key_if_allow_multiple_is_true(): assert loaded == [ObjNoKey("a", "b"), ObjNoKey("a", "b"), ObjNoKey("c", "d")] -def test_i_cannot_add_several_obj_no_key_if_allow_multiple_is_false(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_add_several_obj_no_key_if_allow_multiple_is_false(root): + sdp = SheerkaDataProvider(root) with pytest.raises(IndexError) as index_error: sdp.add(evt_digest, "entry", ObjNoKey("a", "b"), False) @@ -279,8 +306,12 @@ def test_i_cannot_add_several_obj_no_key_if_allow_multiple_is_false(): assert index_error.value.args[0] == "entry" -def test_i_can_add_a_dict(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_a_dict(root): + sdp = SheerkaDataProvider(root) obj = {"my_key": "my_value"} entry, key = sdp.add(evt_digest, "entry", obj) @@ -295,19 +326,23 @@ def test_i_can_add_a_dict(): assert loaded == obj assert loaded_value == "my_value" - assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) assert state.date is not None assert state.parents == [] assert state.events == [evt_digest] assert state.data == {"entry": obj} - assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit + assert sdp.io.read_text(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) == last_commit -def test_i_can_add_multiple_entries_at_once_with_dict(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_multiple_entries_at_once_with_dict(root): + sdp = SheerkaDataProvider(root) entry, key = sdp.add(evt_digest, "entry", {"my_key1": "value1", "my_key2": "value2"}) loaded = sdp.get(entry, key) @@ -319,8 +354,12 @@ def test_i_can_add_multiple_entries_at_once_with_dict(): assert loaded_value2 == "value2" -def test_i_can_add_same_key_with_dict_if_allow_multiple_is_true(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_same_key_with_dict_if_allow_multiple_is_true(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", {"my_key": "my_value"}) entry, key = sdp.add(evt_digest, "entry", {"my_key": "my_value"}) @@ -335,8 +374,12 @@ def test_i_can_add_same_key_with_dict_if_allow_multiple_is_true(): assert loaded2 == {"my_key": ["my_value", "my_value", "my_value2"]} -def test_i_cannot_add_same_key_with_dict_if_allow_multiple_is_false(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_add_same_key_with_dict_if_allow_multiple_is_false(root): + sdp = SheerkaDataProvider(root) with pytest.raises(IndexError) as index_error: sdp.add(evt_digest, "entry", {"my_key": "my_value"}, False) @@ -344,8 +387,12 @@ def test_i_cannot_add_same_key_with_dict_if_allow_multiple_is_false(): assert index_error.value.args[0] == "entry.my_key" -def test_i_can_add_object_with_different_key_if_allow_multiple_is_false(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_object_with_different_key_if_allow_multiple_is_false(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", {"my_key": "a"}, False) sdp.add(evt_digest, "entry", {"my_key2": "b"}, False) @@ -354,8 +401,12 @@ def test_i_can_add_object_with_different_key_if_allow_multiple_is_false(): assert sdp.get("entry", "my_key2") == "b" -def test_i_can_add_obj_with_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_obj_with_key(root): + sdp = SheerkaDataProvider(root) obj1 = ObjWithKey("key1", "b") obj2 = ObjSetKey("c", key="key2") @@ -374,19 +425,23 @@ def test_i_can_add_obj_with_key(): assert key2 == "key2" assert loaded2 == ObjSetKey("c", key="key2") - assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) assert state.date is not None assert len(state.parents) == 1 assert state.events == [evt_digest] assert state.data == {"entry": {"key1": obj1}, "entry2": {"key2": obj2}} - assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit + assert sdp.io.read_text(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) == last_commit -def test_i_can_add_objects_with_same_key_if_allow_multiple_is_true(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_objects_with_same_key_if_allow_multiple_is_true(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjWithKey("my_key", "b")) entry, key = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key")) @@ -402,8 +457,12 @@ def test_i_can_add_objects_with_same_key_if_allow_multiple_is_true(): assert loaded2 == [ObjWithKey("my_key", "b"), ObjSetKey("c", key="my_key"), ObjSetKey("c", key="my_key")] -def test_i_cannot_add_object_with_same_key_if_allow_multiple_is_false(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_add_object_with_same_key_if_allow_multiple_is_false(root): + sdp = SheerkaDataProvider(root) with pytest.raises(IndexError) as index_error: sdp.add(evt_digest, "entry", ObjWithKey("my_key", "b"), False) @@ -411,8 +470,12 @@ def test_i_cannot_add_object_with_same_key_if_allow_multiple_is_false(): assert index_error.value.args[0] == "entry.my_key" -def test_i_can_add_obj_with_key_to_a_list(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_obj_with_key_to_a_list(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", "foo") sdp.add(evt_digest, "entry", "bar") # entry is now a list @@ -422,8 +485,12 @@ def test_i_can_add_obj_with_key_to_a_list(): assert loaded == ["foo", "bar", ObjWithKey("a", "b")] -def test_i_cannot_add_obj_with_no_key_when_then_entry_has_keys(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_add_obj_with_no_key_when_then_entry_has_keys(root): + sdp = SheerkaDataProvider(root) with pytest.raises(SheerkaDataProviderError) as error: sdp.add(evt_digest, "entry", ObjWithKey("a", "b")) @@ -432,9 +499,13 @@ def test_i_cannot_add_obj_with_no_key_when_then_entry_has_keys(): assert error.value.obj == "foo" -def test_i_can_add_string_using_auto_generated_key(): - sdp = SheerkaDataProvider(".sheerka") - key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile) +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_string_using_auto_generated_key(root): + sdp = SheerkaDataProvider(root) + key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile) entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", "foo") entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", "bar") @@ -442,8 +513,8 @@ def test_i_can_add_string_using_auto_generated_key(): state = sdp.load_state(sdp.get_snapshot()) - assert path.exists(key_file) - assert read_json_file(key_file) == {"entry1": 2, "entry2": 1} + assert sdp.io.exists(key_file) + assert read_json_file(sdp, key_file) == {"entry1": 2, "entry2": 1} assert state.data == {"entry1": {"1": "foo", "2": "bar"}, "entry2": {"1": "baz"}} assert entry1 == "entry1" assert entry2 == "entry1" @@ -453,12 +524,16 @@ def test_i_can_add_string_using_auto_generated_key(): assert key3 == "1" -def test_i_cannot_add_the_same_digest_twice_in_the_same_entry(): +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_add_the_same_digest_twice_in_the_same_entry(root): """ If get_digest() is implemented, checks for duplicates :return: """ - sdp = SheerkaDataProvider(".sheerka") + sdp = SheerkaDataProvider(root) with pytest.raises(SheerkaDataProviderDuplicateKeyError) as error: sdp.add(evt_digest, "entry", ObjWithDigestNoKey("a", "b")) @@ -469,12 +544,16 @@ def test_i_cannot_add_the_same_digest_twice_in_the_same_entry(): assert error.value.args[0] == "Duplicate object." -def test_i_cannot_add_the_same_digest_twice_in_the_same_entry2(): +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_add_the_same_digest_twice_in_the_same_entry2(root): """ If get_digest() is implemented, checks for duplicates in list when no key :return: """ - sdp = SheerkaDataProvider(".sheerka") + sdp = SheerkaDataProvider(root) with pytest.raises(SheerkaDataProviderDuplicateKeyError) as error: sdp.add(evt_digest, "entry", ObjWithDigestNoKey("a", "b")) @@ -486,12 +565,16 @@ def test_i_cannot_add_the_same_digest_twice_in_the_same_entry2(): assert error.value.args[0] == "Duplicate object." -def test_i_cannot_add_the_same_digest_twice_in_the_same_entry3(): +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_add_the_same_digest_twice_in_the_same_entry3(root): """ If get_digest() is implemented, checks for duplicates when the key is provided :return: """ - sdp = SheerkaDataProvider(".sheerka") + sdp = SheerkaDataProvider(root) with pytest.raises(SheerkaDataProviderDuplicateKeyError) as error: sdp.add(evt_digest, "entry", ObjWithDigestWithKey("a", "b")) @@ -502,12 +585,16 @@ def test_i_cannot_add_the_same_digest_twice_in_the_same_entry3(): assert error.value.args[0] == "Duplicate object." -def test_i_cannot_add_the_same_digest_twice_in_the_same_entry4(): +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_add_the_same_digest_twice_in_the_same_entry4(root): """ If get_digest() is implemented, checks for duplicates in list when the key is provided :return: """ - sdp = SheerkaDataProvider(".sheerka") + sdp = SheerkaDataProvider(root) with pytest.raises(SheerkaDataProviderDuplicateKeyError) as error: sdp.add(evt_digest, "entry", ObjWithDigestWithKey("a", "b")) @@ -519,9 +606,13 @@ def test_i_cannot_add_the_same_digest_twice_in_the_same_entry4(): assert error.value.args[0] == "Duplicate object." -def test_i_can_get_and_set_key(): - sdp = SheerkaDataProvider(".sheerka") - key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile) +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_get_and_set_key(root): + sdp = SheerkaDataProvider(root) + key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile) sdp.set_key("entry1", 1000) sdp.get_next_key("entry1") @@ -530,21 +621,25 @@ def test_i_can_get_and_set_key(): sdp.get_next_key("entry2") sdp.get_next_key("entry2") - assert path.exists(key_file) - assert read_json_file(key_file) == {"entry1": 1003, "entry2": 2} + assert sdp.io.exists(key_file) + assert read_json_file(sdp, key_file) == {"entry1": 1003, "entry2": 2} -def test_i_can_add_object_using_auto_generated_key(): - sdp = SheerkaDataProvider(".sheerka") - key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile) +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_object_using_auto_generated_key(root): + sdp = SheerkaDataProvider(root) + key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile) entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b")) entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b")) state = sdp.load_state(sdp.get_snapshot()) - assert path.exists(key_file) - assert read_json_file(key_file) == {"entry1": 2} + assert sdp.io.exists(key_file) + assert read_json_file(sdp, key_file) == {"entry1": 2} assert state.data == {"entry1": {"1": ObjNoKey("a", "b"), "2": ObjNoKey("a", "b")}} assert entry1 == "entry1" assert entry2 == "entry1" @@ -552,17 +647,21 @@ def test_i_can_add_object_using_auto_generated_key(): assert key2 == "2" -def test_object_key_is_updated_when_possible_using_auto_generated_key(): - sdp = SheerkaDataProvider(".sheerka") - key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile) +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_object_key_is_updated_when_possible_using_auto_generated_key(root): + sdp = SheerkaDataProvider(root) + key_file = path.join(sdp.io.root, SheerkaDataProvider.KeysFile) entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo")) entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo")) state = sdp.load_state(sdp.get_snapshot()) - assert path.exists(key_file) - assert read_json_file(key_file) == {"entry1": 2} + assert sdp.io.exists(key_file) + assert read_json_file(sdp, key_file) == {"entry1": 2} assert state.data == {"entry1": {"1": ObjSetKey("foo", "1"), "2": ObjSetKey("foo", "2")}} assert entry1 == "entry1" assert entry2 == "entry1" @@ -570,8 +669,12 @@ def test_object_key_is_updated_when_possible_using_auto_generated_key(): assert key2 == "2" -def test_i_can_set_objects_with_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_set_objects_with_key(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjWithKey(1, "foo")) entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo")) @@ -581,8 +684,12 @@ def test_i_can_set_objects_with_key(): assert key == "2" -def test_i_can_set_objects_with_no_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_set_objects_with_no_key(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjNoKey(1, "foo")) entry, key = sdp.set(evt_digest, "entry", ObjNoKey(2, "foo")) @@ -592,8 +699,12 @@ def test_i_can_set_objects_with_no_key(): assert key is None -def test_i_can_set_from_list_to_dict(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_set_from_list_to_dict(root): + sdp = SheerkaDataProvider(root) sdp.set(evt_digest, "entry", [ObjNoKey(1, "foo"), ObjNoKey(2, "foo")]) entry, key = sdp.set(evt_digest, "entry", {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")}) @@ -603,8 +714,12 @@ def test_i_can_set_from_list_to_dict(): assert key is None -def test_i_can_set_using_reference(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_set_using_reference(root): + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey))) sdp.add(evt_digest, "entry", ObjWithKey(1, "foo")) entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"), use_ref=True) @@ -614,29 +729,20 @@ def test_i_can_set_using_reference(): assert entry == "entry" assert key == "2" - assert path.exists(sdp.get_obj_path(SheerkaDataProvider.ObjectsFolder, - "95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268")) + assert sdp.io.exists(sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, + "95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268")) # sanity check, make sure that I can load back loaded = sdp.get(entry, key) assert loaded == ObjWithKey(2, "foo") -def test_i_can_add_unique(): - sdp = SheerkaDataProvider(".sheerka") - sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo")) - sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo")) - sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar")) - entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar")) - - state = sdp.load_state(sdp.get_snapshot()) - assert state.data == {"entry": {ObjNoKey(1, "foo"), ObjNoKey(2, "bar")}} - assert entry == "entry" - assert key is None - - -def test_i_can_add_reference_of_an_object_with_a_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_reference_of_an_object_with_a_key(root): + sdp = SheerkaDataProvider(root) obj = ObjDumpJson("my_key", "value1") obj_serializer = ObjectSerializer(core.utils.get_full_qualified_name(obj)) sdp.serializer.register(obj_serializer) @@ -651,8 +757,29 @@ def test_i_can_add_reference_of_an_object_with_a_key(): assert sdp.load_obj(digest) == obj -def test_i_can_keep_state_history(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_unique(root): + sdp = SheerkaDataProvider(root) + sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo")) + sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo")) + sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar")) + entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar")) + + state = sdp.load_state(sdp.get_snapshot()) + assert state.data == {"entry": {ObjNoKey(1, "foo"), ObjNoKey(2, "bar")}} + assert entry == "entry" + assert key is None + + +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_keep_state_history(root): + sdp = SheerkaDataProvider(root) event1 = Event("cmd add 'foo => bar'") event_digest1 = sdp.save_event(event1) @@ -668,26 +795,34 @@ def test_i_can_keep_state_history(): state2 = sdp.load_state(state_digest2) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest1[0:24], event_digest1)) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, state_digest1[0:24], state_digest1)) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest2[0:24], event_digest2)) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, state_digest2[0:24], state_digest2)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.EventFolder, event_digest1[0:24], event_digest1)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, state_digest1[0:24], state_digest1)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.EventFolder, event_digest2[0:24], event_digest2)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, state_digest2[0:24], state_digest2)) assert state2.date is not None assert state2.parents == [state_digest1] assert state2.events == [event_digest2] assert state2.data == {"entry1": "foo => bar", "entry2": "foo => baz"} -def test_i_can_list_elements_when_there_is_nothing_to_list(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_list_elements_when_there_is_nothing_to_list(root): + sdp = SheerkaDataProvider(root) result = sdp.list("entry") assert list(result) == [] -def test_i_can_list_when_no_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_list_when_no_key(root): + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda obj: isinstance(obj, str))) sdp.add(evt_digest, "entry1", "foo") @@ -700,8 +835,12 @@ def test_i_can_list_when_no_key(): assert list(result) == ["foo", "bar", "baz"] -def test_i_can_list_when_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_list_when_key(root): + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda obj: isinstance(obj, ObjWithKey))) sdp.add(evt_digest, "entry1", {"1": "foo"}) @@ -714,8 +853,12 @@ def test_i_can_list_when_key(): assert list(result) == ["foo", "bar", ObjWithKey("3", "value")] -def test_i_can_list_when_one_element(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_list_when_one_element(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", "foo") sdp.add(evt_digest, "entry2", "baz") @@ -724,8 +867,12 @@ def test_i_can_list_when_one_element(): assert list(result) == ["foo"] -def test_i_can_filter_on_key_for_dict(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_filter_on_key_for_dict(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", {"1": "foo"}) sdp.add(evt_digest, "entry1", {"2": "bar"}) @@ -734,8 +881,12 @@ def test_i_can_filter_on_key_for_dict(): assert list(result) == ["foo"] -def test_i_can_filter_on_key_for_objects(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_filter_on_key_for_objects(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", ObjWithKey("a1", "b1")) sdp.add(evt_digest, "entry1", ObjWithKey("a2", "b2")) @@ -744,8 +895,12 @@ def test_i_can_filter_on_key_for_objects(): assert list(result) == [ObjWithKey("a1", "b1")] -def test_i_can_filter_on_attribute_for_dict(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_filter_on_attribute_for_dict(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", {"1": {"a": "a1", "b": "b1"}}) sdp.add(evt_digest, "entry1", {"2": {"a": "a2", "b": "b2"}}) @@ -754,8 +909,12 @@ def test_i_can_filter_on_attribute_for_dict(): assert list(result) == [{"a": "a2", "b": "b2"}] -def test_i_can_filter_on_attribute_for_object(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_filter_on_attribute_for_object(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", ObjWithKey("a1", "b1")) sdp.add(evt_digest, "entry1", ObjWithKey("a2", "b2")) @@ -764,8 +923,12 @@ def test_i_can_filter_on_attribute_for_object(): assert list(result) == [ObjWithKey("a2", "b2")] -def test_i_can_filter_a_list(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_filter_a_list(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", "foo") sdp.add(evt_digest, "entry1", "bar") @@ -774,8 +937,12 @@ def test_i_can_filter_a_list(): assert list(result) == ["bar"] -def test_i_can_filter_a_list_of_object(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_filter_a_list_of_object(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", ObjNoKey("a1", "b1")) sdp.add(evt_digest, "entry1", ObjNoKey("a2", "b2")) @@ -784,20 +951,28 @@ def test_i_can_filter_a_list_of_object(): assert list(result) == [ObjNoKey("a1", "b1")] -def test_i_can_remove_all_elements(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_remove_all_elements(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", "foo") sdp.add(evt_digest, "entry1", "bar") state_digest = sdp.remove(evt_digest, "entry1") result = sdp.list("entry1") - assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == state_digest + assert sdp.io.read_text(path.join(sdp.io.root, SheerkaDataProvider.HeadFile)) == state_digest assert list(result) == [] -def test_i_can_remove_a_element(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_remove_a_element(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", "foo") sdp.add(evt_digest, "entry1", "bar") @@ -807,8 +982,12 @@ def test_i_can_remove_a_element(): assert list(result) == ["bar"] -def test_i_can_remove_dict_by_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_remove_dict_by_key(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", {"1": ObjNoKey("a1", "b1")}) sdp.add(evt_digest, "entry1", {"2": ObjNoKey("a2", "b2")}) @@ -818,8 +997,12 @@ def test_i_can_remove_dict_by_key(): assert list(result) == [ObjNoKey("a1", "b1")] -def test_i_can_remove_when_only_one_element(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_remove_when_only_one_element(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", "foo") sdp.remove(evt_digest, "entry1", lambda o: o == "foo") @@ -828,15 +1011,23 @@ def test_i_can_remove_when_only_one_element(): assert list(result) == [] -def test_i_cannot_remove_if_entry_does_not_exist(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_remove_if_entry_does_not_exist(root): + sdp = SheerkaDataProvider(root) with pytest.raises(IndexError) as e: sdp.remove(evt_digest, "entry") assert str(e) == "entry" -def test_i_cannot_modify_an_entry_without_a_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_modify_an_entry_without_a_key(root): + sdp = SheerkaDataProvider(root) with pytest.raises(SheerkaDataProviderError) as error: sdp.modify(evt_digest, "entry", None, "baz") @@ -844,8 +1035,12 @@ def test_i_cannot_modify_an_entry_without_a_key(): assert error.value.args[0] == "Key is mandatory." -def test_i_can_modify_dict_with_a_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_dict_with_a_key(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", {"key1": "foo"}) sdp.add(evt_digest, "entry", {"key2": "bar"}) @@ -857,8 +1052,12 @@ def test_i_can_modify_dict_with_a_key(): assert key == "key1" -def test_i_can_modify_an_object_with_a_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_an_object_with_a_key(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo")) sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar")) @@ -870,8 +1069,12 @@ def test_i_can_modify_an_object_with_a_key(): assert key == "key1" -def test_i_can_modify_an_object_while_changing_the_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_an_object_while_changing_the_key(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo")) sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar")) @@ -883,8 +1086,12 @@ def test_i_can_modify_an_object_while_changing_the_key(): assert key == "key3" -def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_key(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo")) sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar")) @@ -896,13 +1103,17 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_key(): assert key == "key1" -def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_list(): +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_list(root): """ In this example, the item to modify is within a list, and its key has changed and in the new key, there is already a list :return: """ - sdp = SheerkaDataProvider(".sheerka") + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson))) sdp.add(evt_digest, "entry", ObjDumpJson("key1", "value11")) @@ -923,13 +1134,17 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_list( assert key == "key1" -def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_nothing(): +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_nothing(root): """ In this example, the item to modify is within a list, and its key has changed and in the new key, there is nothing (the new key does not exist) :return: """ - sdp = SheerkaDataProvider(".sheerka") + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson))) sdp.add(evt_digest, "entry", ObjDumpJson("key2", "value21")) @@ -948,13 +1163,17 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_nothi assert key == "key1" -def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_one_item(): +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_one_item(root): """ In this example, the item to modify is within a list, and its key has changed and in the new key, there is only one element :return: """ - sdp = SheerkaDataProvider(".sheerka") + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson))) sdp.add(evt_digest, "entry", ObjDumpJson("key1", "value11")) @@ -974,8 +1193,12 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_one_i assert key == "key1" -def test_i_can_modify_a_ref(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_a_ref(root): + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey))) sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo")) entry, key = sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"), use_ref=True) @@ -989,8 +1212,12 @@ def test_i_can_modify_a_ref(): assert key == "key2" -def test_i_cannot_modify_an_entry_that_does_not_exist(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_modify_an_entry_that_does_not_exist(root): + sdp = SheerkaDataProvider(root) with pytest.raises(IndexError) as e: sdp.modify(evt_digest, "entry", "key", "foo") @@ -998,8 +1225,12 @@ def test_i_cannot_modify_an_entry_that_does_not_exist(): assert str(e.value) == "entry" -def test_i_cannot_modify_a_key_that_does_not_exist(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_modify_a_key_that_does_not_exist(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", {"1": "foo"}) with pytest.raises(IndexError) as e: @@ -1007,8 +1238,12 @@ def test_i_cannot_modify_a_key_that_does_not_exist(): assert str(e) == "entry1.2" -def test_i_cannot_modify_a_list_when_origin_is_unknown(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_modify_a_list_when_origin_is_unknown(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjWithKey("key", "value1")) sdp.add(evt_digest, "entry", ObjWithKey("key", "value2")) # same they @@ -1022,8 +1257,12 @@ def test_i_cannot_modify_a_list_when_origin_is_unknown(): assert error.value.args[0] == "Multiple entries under 'entry.key'" -def test_i_can_modify_a_list_when_the_origin_is_known(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_a_list_when_the_origin_is_known(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry", ObjDumpJson("key", "value1")) sdp.add(evt_digest, "entry", ObjDumpJson("key", "value2")) # same they @@ -1037,13 +1276,17 @@ def test_i_can_modify_a_list_when_the_origin_is_known(): assert state.data == {"entry": {"key": [ObjDumpJson("key", "value3"), ObjDumpJson("key", "value2")]}} -def test_i_can_modify_a_list_when_the_origin_is_known_2(): +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_modify_a_list_when_the_origin_is_known_2(root): """ This time, we check that the origin is automatically set when the object was saved as a reference We also check that all objects are still persisted as reference :return: """ - sdp = SheerkaDataProvider(".sheerka") + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson))) sdp.add(evt_digest, "entry", ObjDumpJson("key", "value1"), use_ref=True) @@ -1060,13 +1303,20 @@ def test_i_can_modify_a_list_when_the_origin_is_known_2(): "##REF##:5fe085e8366d35c5f04a18b2d3dada376128b246e07c66de5872830b00f5f517"]}} # checks that all objects are (still) persisted - path.exists(sdp.get_obj_path(sdp.ObjectsFolder, "621771a3af6a331e9abb3a63fb25e0cac4b13df0b292dfa30db6bd89031bfad0")) - path.exists(sdp.get_obj_path(sdp.ObjectsFolder, "5fe085e8366d35c5f04a18b2d3dada376128b246e07c66de5872830b00f5f517")) - path.exists(sdp.get_obj_path(sdp.ObjectsFolder, "1aac9e0d5c74c3bb989fd0f9def792bba36c5595d32f61be7cbb1a38dcf75327")) + sdp.io.exists( + sdp.io.get_obj_path(sdp.ObjectsFolder, "621771a3af6a331e9abb3a63fb25e0cac4b13df0b292dfa30db6bd89031bfad0")) + sdp.io.exists( + sdp.io.get_obj_path(sdp.ObjectsFolder, "5fe085e8366d35c5f04a18b2d3dada376128b246e07c66de5872830b00f5f517")) + sdp.io.exists( + sdp.io.get_obj_path(sdp.ObjectsFolder, "1aac9e0d5c74c3bb989fd0f9def792bba36c5595d32f61be7cbb1a38dcf75327")) -def test_i_can_get_the_entire_entry(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_get_the_entire_entry(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", "foo") sdp.add(evt_digest, "entry1", "bar") @@ -1077,8 +1327,12 @@ def test_i_can_get_the_entire_entry(): assert result_safe == ["foo", "bar"] -def test_i_can_get_an_entry_with_on_object(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_get_an_entry_with_on_object(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", "foo") result = sdp.get("entry1") @@ -1088,8 +1342,12 @@ def test_i_can_get_an_entry_with_on_object(): assert result_safe == "foo" -def test_i_can_get_an_entry_by_key(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_get_an_entry_by_key(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", {"1": "foo"}) sdp.add(evt_digest, "entry1", {"2": "bar"}) @@ -1100,8 +1358,12 @@ def test_i_can_get_an_entry_by_key(): assert result_safe == "bar" -def test_i_can_get_object_save_by_reference(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_get_object_save_by_reference(root): + sdp = SheerkaDataProvider(root) obj = ObjDumpJson("my_key", "value1") sdp.serializer.register(ObjectSerializer(core.utils.get_full_qualified_name(obj))) @@ -1111,8 +1373,12 @@ def test_i_can_get_object_save_by_reference(): assert loaded == obj -def test_i_can_get_objects_from_list_when_saved_by_reference(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_get_objects_from_list_when_saved_by_reference(root): + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson))) sdp.add(evt_digest, "entry", ObjDumpJson("key", "value1"), use_ref=True) @@ -1124,8 +1390,12 @@ def test_i_can_get_objects_from_list_when_saved_by_reference(): assert objs[1] == ObjDumpJson("key", "value2") -def test_i_cannot_get_an_entry_that_does_not_exist(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_get_an_entry_that_does_not_exist(root): + sdp = SheerkaDataProvider(root) assert sdp.get_safe("entry") is None with pytest.raises(IndexError) as e: @@ -1133,8 +1403,12 @@ def test_i_cannot_get_an_entry_that_does_not_exist(): assert str(e) == "entry" -def test_i_cannot_get_a_key_that_does_not_exist(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_cannot_get_a_key_that_does_not_exist(root): + sdp = SheerkaDataProvider(root) sdp.add(evt_digest, "entry1", {"1": "foo"}) assert sdp.get_safe("entry1", "2") is None @@ -1143,23 +1417,31 @@ def test_i_cannot_get_a_key_that_does_not_exist(): assert str(e) == "entry.1" -def test_i_can_save_and_retrieve_cache(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_save_and_retrieve_cache(root): + sdp = SheerkaDataProvider(root) txt = "foo bar baz foo bar baz foo bar baz" key = "key_to_use" category = "cache_category" assert not sdp.in_cache(category, key) digest = sdp.add_to_cache(category, key, txt) - assert path.exists(path.join(sdp.root, SheerkaDataProvider.CacheFolder, digest[0:24], digest)) + assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.CacheFolder, digest[0:24], digest)) assert sdp.in_cache(category, key) from_cache = sdp.load_from_cache(category, key) assert from_cache == txt -def test_cache_is_not_updated_by_default(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_cache_is_not_updated_by_default(root): + sdp = SheerkaDataProvider(root) txt = "foo bar baz foo bar baz foo bar baz" txt2 = "foo foo foo foo foo foo foo foo foo" key = "key_to_use" @@ -1172,8 +1454,12 @@ def test_cache_is_not_updated_by_default(): assert from_cache == txt -def test_i_can_update_cache(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_update_cache(root): + sdp = SheerkaDataProvider(root) txt = "foo bar baz foo bar baz foo bar baz" txt2 = "foo foo foo foo foo foo foo foo foo" key = "key_to_use" @@ -1186,28 +1472,40 @@ def test_i_can_update_cache(): assert from_cache == txt2 -def test_i_can_remove_from_cache(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_remove_from_cache(root): + sdp = SheerkaDataProvider(root) txt = "foo bar baz foo bar baz foo bar baz" key = "key_to_use" category = "cache_category" sdp.add_to_cache(category, key, txt) digest = sdp.remove_from_cache(category, key) - assert not path.exists(path.join(sdp.root, SheerkaDataProvider.CacheFolder, digest[0:24], digest)) + assert not sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.CacheFolder, digest[0:24], digest)) assert not sdp.in_cache(category, key) -def test_i_can_test_than_an_entry_exists(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_test_than_an_entry_exists(root): + sdp = SheerkaDataProvider(root) assert not sdp.exists("entry") sdp.add(evt_digest, "entry", "value") assert sdp.exists("entry") -def test_i_can_test_if_a_key_exists(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_test_if_a_key_exists(root): + sdp = SheerkaDataProvider(root) obj = ObjWithDigestWithKey("key", "value") assert not sdp.exists("entry") @@ -1218,8 +1516,12 @@ def test_i_can_test_if_a_key_exists(): assert sdp.exists("entry", obj.get_key()) -def test_i_can_test_that_the_object_exists(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_test_that_the_object_exists(root): + sdp = SheerkaDataProvider(root) obj = ObjWithDigestWithKey("key", "value") assert not sdp.exists("entry") @@ -1242,8 +1544,12 @@ def test_i_can_test_that_the_object_exists(): assert sdp.exists("entry2", obj.get_key(), obj.get_digest()) -def test_i_can_test_than_the_object_exists_when_using_references(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_test_than_the_object_exists_when_using_references(root): + sdp = SheerkaDataProvider(root) sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey))) obj = ObjWithDigestWithKey("key", "value") @@ -1267,8 +1573,12 @@ def test_i_can_test_than_the_object_exists_when_using_references(): assert sdp.exists("entry2", obj.get_key(), obj.get_digest()) -def test_i_can_save_and_load_object_ref_with_history(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_save_and_load_object_ref_with_history(root): + sdp = SheerkaDataProvider(root) obj = ObjDumpJson("my_key", "value1") sdp.serializer.register(ObjectSerializer(core.utils.get_full_qualified_name(obj))) @@ -1285,7 +1595,7 @@ def test_i_can_save_and_load_object_ref_with_history(): assert history[Serializer.MODIFICATION_DATE] != "" assert history[Serializer.PARENTS] == [] - assert os.path.exists(sdp.get_obj_path(sdp.ObjectsFolder, obj.get_digest())) + assert sdp.io.exists(sdp.io.get_obj_path(sdp.ObjectsFolder, obj.get_digest())) # save a second type with no modification previous_modification_time = history[Serializer.MODIFICATION_DATE] @@ -1318,8 +1628,12 @@ def test_i_can_save_and_load_object_ref_with_history(): "my_key": '##REF##:e6bf5b56428cfce0f08c94f2c3625dc3b3a8180d7229eaa9f8aa967fb16e5256'}} -def test_i_can_add_obj_with_same_key_and_get_them_back(): - sdp = SheerkaDataProvider(".sheerka") +@pytest.mark.parametrize("root", [ + ".sheerka", + "mem://" +]) +def test_i_can_add_obj_with_same_key_and_get_them_back(root): + sdp = SheerkaDataProvider(root) obj1 = ObjDumpJson("key", "value1") obj2 = ObjDumpJson("key", "value2") sdp.serializer.register(ObjectSerializer(core.utils.get_full_qualified_name(obj1))) @@ -1332,4 +1646,3 @@ def test_i_can_add_obj_with_same_key_and_get_them_back(): assert len(loaded) == 2 assert loaded[0] == obj1 assert loaded[1] == obj2 - diff --git a/tests/test_sheerkaSerializer.py b/tests/test_sheerkaSerializer.py index ab2bac3..f0121f1 100644 --- a/tests/test_sheerkaSerializer.py +++ b/tests/test_sheerkaSerializer.py @@ -2,7 +2,7 @@ import pytest from dataclasses import dataclass from sdp.sheerkaDataProvider import Event -from sdp.sheerkaSerializer import Serializer, ObjectSerializer, SerializerContext, BaseSerializer +from sdp.sheerkaSerializer import Serializer, ObjectSerializer, SerializerContext from datetime import datetime import core.utils