kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initialized logging

Browse Source
This commit is contained in:
Kodjo Sossouvi
2019-11-05 19:56:00 +01:00
parent b12204360e
commit 0d2adf1b6c
10 changed files with 448 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/concept.py
+24 -11
View File
@@ -1,5 +1,8 @@
import hashlib
from enum import Enum
import logging
log = logging.getLogger(__name__)
class ConceptParts(Enum):
@@ -15,11 +18,9 @@ class Concept:
A concept is a the base object of our universe
Everything is a concept
"""
props_to_serialize = ("id", "name", "where", "pre", "post", "body", "desc")
props_to_serialize = ("id", "is_builtin", "name", "where", "pre", "post", "body", "desc")
key_name = "concepts"
def __init__(self, name=None, is_builtin=False, where=None, pre=None, post=None, body=None, desc=None):
def __init__(self, name=None, is_builtin=False, where=None, pre=None, post=None, body=None, desc=None, key=None):
self.name = name
self.is_builtin = is_builtin
self.where = where # condition to recognize variables in name
@@ -27,16 +28,16 @@ class Concept:
self.post = post # list of post conditions after calling the main function
self.body = body # main method, can also be the value of the concept
self.desc = desc
self.key = None
self.parent = None
self.id = None
self.key = key
self.props = [] # list of Property for this concept
self.functions = {} # list of helper functions
self.codes = {}
self.codes = {} # cached ast for the where, pre, post and body parts
def __repr__(self):
return f"({self.key}){self.name}"
return f"({self.id}){self.name}"
def __eq__(self, other):
if not isinstance(other, Concept):
@@ -50,6 +51,9 @@ class Concept:
def __hash__(self):
return hash(self.name)
def get_key(self):
return self.key
def add_codes(self, codes):
"""
From a dict of <ConceptParts, AST>
@@ -73,13 +77,22 @@ class Concept:
def to_dict(self):
props_as_dict = dict((prop, getattr(self, prop)) for prop in self.props_to_serialize)
props_as_dict["props"] = [(p.name, p.value) for p in self.props]
return props_as_dict
def from_dict(self, as_dict):
for prop in self.props_to_serialize:
setattr(self, prop, as_dict[prop])
if prop in as_dict:
setattr(self, prop, as_dict[prop])
if "props" in as_dict:
for n, v in as_dict["props"]:
self.props.append(Property(n, v))
return self
def update_from(self, other):
for prop in self.props_to_serialize:
setattr(self, prop, getattr(other, prop))
class ErrorConcept(Concept):
def __init__(self, where=None, pre=None, post=None, body=None, desc=None):
@@ -94,6 +107,6 @@ class Property:
Defines a behaviour of Concept
"""
def __init__(self, concept, value):
self.concept = concept
def __init__(self, name, value):
self.name = name
self.value = value
core/sheerka.py
+93 -27
View File
@@ -1,10 +1,14 @@
from dataclasses import dataclass
from core.concept import Concept, ErrorConcept
from parsers.PythonParser import PythonParser
from core.concept import Concept, ErrorConcept, Property
from parsers.PythonParser import PythonParser, PythonGetNamesVisitor, PythonNode
from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
from parsers.DefaultParser import DefaultParser, DefConceptNode
import logging
log = logging.getLogger(__name__)
class Singleton(type):
_instances = {}
@@ -27,6 +31,14 @@ class ReturnValue:
message: str = None
@dataclass
class ExecutionContext:
"""
To keep track of the execution of a request
"""
event_digest: str
class Sheerka(Concept, metaclass=Singleton):
"""
Main controller for the project
@@ -37,11 +49,16 @@ class Sheerka(Concept, metaclass=Singleton):
ERROR_CONCEPT_NAME = "Error"
SUCCESS_CONCEPT_NAME = "Success"
CONCEPTS_ENTRY = "Concepts"
BUILTIN_CONCEPTS_KEYS = "Builtins"
USER_CONCEPTS_KEYS = "Concepts"
def __init__(self):
log.debug("Starting Sheerka.")
super().__init__(Sheerka.NAME)
# list of all concepts known be the system
self.concepts = []
# cache of the most used concepts
self.concepts_cache = []
# a concept can be instantiated
# ex: File is a concept, but File('foo.txt') is an instance
@@ -52,20 +69,10 @@ class Sheerka(Concept, metaclass=Singleton):
# ex: hello => say('hello')
self.rules = []
self.create_builtin_concepts()
self.sdp = None
self.parsers = []
def create_builtin_concepts(self):
"""
Initializes the builtin concepts
:return: None
"""
self.concepts.append(self)
self.concepts.append(Concept(Sheerka.UNKNOWN_CONCEPT_NAME))
self.concepts.append(Concept(Sheerka.SUCCESS_CONCEPT_NAME))
self.concepts.append(Concept(Sheerka.ERROR_CONCEPT_NAME))
self.key = self.NAME
def initialize(self, root_folder=None):
"""
@@ -79,13 +86,54 @@ class Sheerka(Concept, metaclass=Singleton):
try:
self.sdp = SheerkaDataProvider(root_folder)
self.parsers.append(lambda text: DefaultParser(text, PythonParser))
except IOError as e:
return ReturnValue(False, self.get_concept(Sheerka.ERROR_CONCEPT_NAME, True), e)
return ReturnValue(True, self.get_concept(Sheerka.SUCCESS_CONCEPT_NAME, True))
if self.sdp.first_time:
self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
self.create_builtin_concepts()
except IOError as e:
return ReturnValue(False, self.get_concept(Sheerka.ERROR_CONCEPT_NAME), e)
return ReturnValue(True, self.get_concept(Sheerka.SUCCESS_CONCEPT_NAME))
def set_id_if_needed(self, obj, is_builtin):
"""
Set the key for the concept if needed
:param obj:
:param is_builtin:
:return:
"""
if obj.id is not None:
return
obj.id = self.sdp.get_next_key(self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS)
log.debug(f"Setting id '{obj.id}' to concept '{obj.name}'.")
def create_builtin_concepts(self):
"""
Initializes the builtin concepts
:return: None
"""
log.debug("Initializing builtin concepts")
builtins = [
self,
Concept(Sheerka.UNKNOWN_CONCEPT_NAME, key=Sheerka.UNKNOWN_CONCEPT_NAME),
Concept(Sheerka.SUCCESS_CONCEPT_NAME, key=Sheerka.SUCCESS_CONCEPT_NAME),
Concept(Sheerka.ERROR_CONCEPT_NAME, key=Sheerka.ERROR_CONCEPT_NAME),
]
for concept in builtins:
from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.name)
if from_db is None:
log.debug(f"'{concept.name}' concept is not found. Adding.")
self.set_id_if_needed(concept, True)
self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
else:
log.debug(f"Found concept '{from_db}'. Updating.")
concept.update_from(from_db)
def eval(self, text):
# evt_digest = self.sdp.save_event(Event(text))
evt_digest = self.sdp.save_event(Event(text))
exec_context = ExecutionContext(evt_digest)
result = self.try_parse(text)
return_values = []
@@ -93,12 +141,13 @@ class Sheerka(Concept, metaclass=Singleton):
if not status:
return_values.append(ReturnValue(False, ErrorConcept(body=node)))
elif status and isinstance(node, DefConceptNode):
return_values.append(self.add_concept(node))
return_values.append(self.add_concept(exec_context, node))
return return_values
def try_parse(self, text):
result = []
log.debug(f"Parsing '{text}'")
for parser in self.parsers:
p = parser(text)
# try:
@@ -110,32 +159,49 @@ class Sheerka(Concept, metaclass=Singleton):
result.append((p.name, not p.has_error, p.error_sink if p.has_error else tree))
return result
def get_concept(self, name, is_builtin=False):
def get_concept(self, name):
"""
Given a concept name, tries to find it
:param name: name of the concept to look for
:param is_builtin: is it a builtin concept ?
:return: concept if found, UNKNOWN_CONCEPT otherwise
"""
for concept in self.concepts:
if concept.name == name and concept.is_builtin == is_builtin:
for concept in self.concepts_cache:
if concept.name == name:
return concept
return self.concepts[1]
return ErrorConcept()
def add_concept(self, def_concept_node: DefConceptNode):
def add_concept(self, exec_context, def_concept_node: DefConceptNode):
"""
Adds a new concept to the system
:param exec_context:
:param def_concept_node: DefConceptNode
:return: digest of the new concept
"""
# validate the node
get_names_visitor = PythonGetNamesVisitor()
concept = Concept(def_concept_node.name)
for prop in ("where", "pre", "post", "body"):
# put back the sources
concept_part_node = getattr(def_concept_node, prop)
value = concept_part_node.source if hasattr(concept_part_node, "source") else ""
setattr(concept, prop, value)
if isinstance(concept_part_node, PythonNode):
get_names_visitor.visit(concept_part_node.ast)
source = concept_part_node.source if hasattr(concept_part_node, "source") else ""
setattr(concept, prop, source)
# try to find variables (eg props)
for token in def_concept_node.tokens["name"]:
if token.value in get_names_visitor.names:
concept.props.append(Property(token.value, None))
concept.key = DefaultParser.get_concept_name(def_concept_node.tokens["name"], [p.name for p in concept.props])
concept.add_codes(def_concept_node.get_codes())
self.set_id_if_needed(concept, False)
self.sdp.add(exec_context.event_digest, self.CONCEPTS_ENTRY, concept, use_ref=True)
return ReturnValue(True, concept)
@staticmethod
main.py
+61 -11
View File
@@ -1,22 +1,72 @@
import getopt
import sys
from core.utils import sysarg_to_string
from core.sheerka import Sheerka
import logging
def usage():
print("Sheerka v0.1\n")
print("usage:")
print(sys.argv[0] + "[-hd] command ")
def main():
sheerka = Sheerka()
sheerka.initialize()
# first, record the event
event_as_string = sysarg_to_string(sys.argv[1:])
result = sheerka.eval(event_as_string)
def sysarg_to_string(argv):
"""
Transform a list of strings into a single string
Add quotes if needed
:return:
"""
if argv is None or not argv:
return ""
# execute the concepts
print(result)
return True
result = ""
first = True
for s in argv:
if not first:
result += " "
result += '"' + s + '"' if " " in s else s
first = False
return result
def init_logging(debug):
if debug:
log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s"
log_level = logging.DEBUG
else:
log_format = "%(message)s"
log_level = logging.INFO
logging.basicConfig(format=log_format, level=log_level)
def main(argv):
try:
opts, args = getopt.getopt(argv, "hd", ["help", "debug"])
debug = False
for o, a in opts:
if o in ('-h', "--help"):
usage()
return True
if o in ('-d', "--debug"):
debug = True
init_logging(debug)
sheerka = Sheerka()
sheerka.initialize()
_in = sysarg_to_string(args)
result = sheerka.eval(_in)
logging.info(result)
return result[-1].status
except getopt.GetoptError:
usage()
sys.exit(2)
if __name__ == '__main__':
res = main()
res = main(sys.argv[1:])
exit(0 if res else 1)
parsers/DefaultParser.py
+28 -8
View File
@@ -1,6 +1,9 @@
from parsers.BaseParser import BaseParser, Node, NopNode, ErrorNode
from parsers.tokenizer import Tokenizer, TokenKind, Token, Keywords
from dataclasses import dataclass, field
import logging
log = logging.getLogger(__name__)
@dataclass()
@@ -27,11 +30,16 @@ class UnexpectedTokenErrorNode(DefaultParserErrorNode):
message: str
expected_tokens: list
def __post_init__(self):
log.debug("UnexpectedToken : " + self.message)
@dataclass()
class SyntaxErrorNode(DefaultParserErrorNode):
message: str
pass
def __post_init__(self):
log.debug("SyntaxError : " + self.message)
@dataclass()
@@ -164,7 +172,7 @@ class DefaultParser(BaseParser):
return
@staticmethod
def get_concept_name(tokens):
def get_concept_name(tokens, variables=None):
name = ""
first = True
for token in tokens:
@@ -172,7 +180,10 @@ class DefaultParser(BaseParser):
break
if not first:
name += " "
name += token.value[1:-1] if token.type == TokenKind.STRING else token.value
if variables is not None and token.value in variables:
name += "__var__" + str(variables.index(token.value))
else:
name += token.value[1:-1] if token.type == TokenKind.STRING else token.value
first = False
return name
@@ -235,6 +246,8 @@ class DefaultParser(BaseParser):
def_concept_parts = [Keywords.AS, Keywords.WHERE, Keywords.PRE, Keywords.POST]
tokens_found = {} # Node token is supposed to be a list, but here, it will be a dict
token = self.get_token()
if token.value != Keywords.CONCEPT:
return self.add_error(UnexpectedTokenErrorNode([token], "Syntax error.", [Keywords.CONCEPT]))
@@ -251,6 +264,7 @@ class DefaultParser(BaseParser):
self.next_token()
token = self.get_token()
name = self.get_concept_name(name_as_tokens)
tokens_found["name"] = name_as_tokens
# try to parse as, where, pre and post declarations
tokens = {
@@ -276,6 +290,8 @@ class DefaultParser(BaseParser):
self.next_token(False)
token = self.get_token()
for t in tokens:
tokens_found[t.value] = tokens[t]
asts = {
Keywords.AS: NopNode(),
@@ -304,11 +320,15 @@ class DefaultParser(BaseParser):
self.add_error(sub_tree, False)
asts[keyword] = sub_tree
return DefConceptNode([], name,
asts[Keywords.WHERE],
asts[Keywords.PRE],
asts[Keywords.POST],
asts[Keywords.AS])
def_concept_node = DefConceptNode(tokens_found, # dict instead of list is wanted.
name,
asts[Keywords.WHERE],
asts[Keywords.PRE],
asts[Keywords.POST],
asts[Keywords.AS])
log.debug(f"Found DefConcept node '{def_concept_node}'")
return def_concept_node
def parse_expression(self):
return self.parse_addition()
parsers/PythonParser.py
+18 -1
View File
@@ -2,6 +2,9 @@ from parsers.BaseParser import BaseParser, Node, ErrorNode
from dataclasses import dataclass
import ast
import copy
import logging
log = logging.getLogger(__name__)
@dataclass()
@@ -17,7 +20,7 @@ class PythonNode(Node):
def __repr__(self):
return "PythonNode(" + ast.dump(self.ast) + ")"
#return "PythonNode(" + self.source + ")"
# return "PythonNode(" + self.source + ")"
class PythonParser(BaseParser):
@@ -73,3 +76,17 @@ class PythonParser(BaseParser):
return eval(compile(self.expr_to_expression(last_ast.body[0]), "<ast>", "eval"), globals())
else:
exec(compile(last_ast, "<ast>", "exec"), globals())
class PythonGetNamesVisitor(ast.NodeVisitor):
"""
This visitor will find all the name declared in the ast
"""
def __init__(self):
self.names = set()
log.debug("Searching for names.")
def visit_Name(self, node):
log.debug(f"Found name : {node.id}")
self.names.add(node.id)
parsers/tokenizer.py
+1 -1
View File
@@ -179,7 +179,7 @@ class Tokenizer:
self.column += len(number)
elif c == "'" or c == '"':
string, newlines = self.eat_string(self.i, self.line, self.column)
yield Token(TokenKind.STRING, string, self.i, self.line, self.column)
yield Token(TokenKind.STRING, string, self.i, self.line, self.column) # quotes are kept
self.i += len(string)
self.column = 1 if newlines > 0 else self.column + len(string)
self.line += newlines
sdp/sheerkaDataProvider.py
+29 -18
View File
@@ -5,6 +5,9 @@ import hashlib
import json
import zlib
from sdp.sheerkaSerializer import Serializer, SerializerContext
import logging
log = logging.getLogger(__name__)
def json_default_converter(o):
@@ -203,13 +206,19 @@ class SheerkaDataProvider:
REF_PREFIX = "##REF##:"
def __init__(self, root=None):
log.debug("Initializing sdp.")
self.root = path.abspath(path.join(path.expanduser("~"), ".sheerka")) \
if root is None \
else path.abspath(root)
log.debug("root is set to '" + self.root + "'")
if not path.exists(self.root):
log.debug("root folder not found. Creating it.")
os.makedirs(self.root)
self.first_time = True
else:
self.first_time = False
self.serializer = Serializer()
@@ -239,10 +248,10 @@ class SheerkaDataProvider:
def is_reference(obj):
return isinstance(obj, str) and obj.startswith(SheerkaDataProvider.REF_PREFIX)
def add(self, event: Event, entry, obj, allow_multiple=True, use_ref=False):
def add(self, event_digest: str, entry, obj, allow_multiple=True, use_ref=False):
"""
Adds obj to the entry 'entry'
:param event: events that triggers the update of the state
:param event_digest: digest of the event that triggers the modification of the state
:param entry: entry of the state to update
:param obj: obj to insert or add
:param allow_multiple: if set to true, the same key can be added several times.
@@ -252,12 +261,14 @@ class SheerkaDataProvider:
:return: (entry, key) to retrieve the object
"""
event_digest = self.save_event(event)
snapshot = self.get_snapshot()
state = self.load_state(snapshot)
log.debug(f"Adding obj '{obj}' in entry '{entry}' (allow_multiple={allow_multiple}, use_ref={use_ref})")
# check uniqueness, cannot add the same key twice if allow_multiple == False
key = self.get_obj_key(obj)
log.debug(f"key found : '{key}'") if key else log.debug("No key found")
if not allow_multiple:
if isinstance(obj, dict):
for k in obj:
@@ -270,6 +281,7 @@ class SheerkaDataProvider:
state.parents = [] if snapshot is None else [snapshot]
state.events = [event_digest]
state.date = datetime.now()
log.debug(state.data)
if use_ref:
digest = self.save_obj(obj)
@@ -281,10 +293,10 @@ class SheerkaDataProvider:
self.set_snapshot(new_snapshot)
return entry, key
def add_with_auto_key(self, event: Event, entry, obj):
def add_with_auto_key(self, event_digest: str, entry, obj):
"""
Add obj to entry. An autogenerated key created for obj
:param event:
:param event_digest:
:param entry:
:param obj:
:return:
@@ -292,12 +304,11 @@ class SheerkaDataProvider:
next_key = self.get_next_key(entry)
if hasattr(obj, "set_key"):
obj.set_key(next_key)
self.add(event, entry, ObjWithKey(next_key, obj))
self.add(event_digest, entry, ObjWithKey(next_key, obj))
return entry, next_key
def add_unique(self, event: Event, entry, obj):
def add_unique(self, event_digest: str, entry, obj):
"""Add an entry and make sure it's unique"""
event_digest = self.save_event(event)
snapshot = self.get_snapshot()
state = self.load_state(snapshot)
@@ -313,17 +324,16 @@ class SheerkaDataProvider:
self.set_snapshot(new_snapshot)
return entry, None
def set(self, event: Event, entry, obj, use_ref=False):
def set(self, event_digest, entry, obj, use_ref=False):
"""
Add or replace an entry. The entry is reinitialized.
If the previous value was dict, all keys are lost
:param event:
:param event_digest:
:param entry:
:param obj:
:param use_ref:
:return:
"""
event_digest = self.save_event(event)
snapshot = self.get_snapshot()
state = self.load_state(snapshot)
@@ -340,11 +350,11 @@ class SheerkaDataProvider:
self.set_snapshot(new_snapshot)
return entry, key
def modify(self, event: Event, entry, key, obj):
def modify(self, event_digest, entry, key, obj):
"""
Replace an element
If the key is not provided, has the same effect than set eg, the entry is reset
:param event:
:param event_digest:
:param entry:
:param key: key of the object to update
:param obj: new data
@@ -354,7 +364,6 @@ class SheerkaDataProvider:
if key is None:
raise SheerkaDataProviderError("Key is mandatory.", None)
event_digest = self.save_event(event)
snapshot = self.get_snapshot()
state = self.load_state(snapshot)
@@ -416,10 +425,10 @@ class SheerkaDataProvider:
if filter_to_use(element):
yield self.load_ref_if_needed(element)[0]
def remove(self, event: Event, entry, filter=None):
def remove(self, event_digest, entry, filter=None):
"""
Removes elements under the entry 'entry'
:param event: event that triggers the deletion
:param event_digest: event that triggers the deletion
:param entry:
:param filter: filter to use
:return: new sha256 of the state
@@ -431,8 +440,6 @@ class SheerkaDataProvider:
if entry not in state.data:
raise IndexError(entry)
event_digest = self.save_event(event)
state.parents = [] if snapshot is None else [snapshot]
state.events = [event_digest]
state.date = datetime.now()
@@ -523,6 +530,7 @@ class SheerkaDataProvider:
def save_state(self, state: State):
digest = state.get_digest()
log.debug(f"Saving new state. digest={digest}")
target_path = path.join(self.root, SheerkaDataProvider.StateFolder, digest[:24], digest)
if path.exists(target_path):
return digest
@@ -544,11 +552,13 @@ class SheerkaDataProvider:
return self.serializer.deserialize(f, None)
def save_obj(self, obj):
log.debug(f"Saving '{obj}' as reference...")
stream = self.serializer.serialize(obj, SerializerContext(user_name="kodjo"))
digest = obj.get_digest() if hasattr(obj, "get_digest") else self.get_stream_digest(stream)
target_path = path.join(self.root, SheerkaDataProvider.ObjectsFolder, digest[:24], digest)
if path.exists(target_path):
log.debug(f"...already saved. digest is {digest}")
return digest
if not path.exists(path.dirname(target_path)):
@@ -557,6 +567,7 @@ class SheerkaDataProvider:
with open(target_path, "wb") as f:
f.write(stream.read())
log.debug(f"...digest is {digest}.")
return digest
def load_obj(self, digest):
sdp/sheerkaSerializer.py
+16 -1
View File
@@ -4,6 +4,9 @@ import datetime
import struct
import io
from dataclasses import dataclass
import logging
log = logging.getLogger(__name__)
def json_default_converter(o):
@@ -33,12 +36,15 @@ class Serializer:
HISTORY = "##history##"
def __init__(self):
log.debug("Initializing serializers")
self._cache = []
# add builtin serializers
self.register(EventSerializer())
self.register(StateSerializer())
self.register(ConceptSerializer())
self.register(SheerkaSerializer())
def register(self, serializer):
"""
@@ -46,6 +52,7 @@ class Serializer:
:param serializer:
:return:
"""
log.debug(f"Adding serializer {serializer}")
self._cache.append(serializer)
def serialize(self, obj, context):
@@ -151,6 +158,9 @@ class BaseSerializer:
else:
return module + '.' + obj.__class__.__name__
def __repr__(self):
return self.__class__.__name__ + ' (' + self.name + ", version=" + str(self.version) + ")"
class EventSerializer(BaseSerializer):
def __init__(self):
@@ -224,9 +234,14 @@ class PickleSerializer(BaseSerializer):
class StateSerializer(PickleSerializer):
def __init__(self, ):
PickleSerializer.__init__(self, lambda obj: BaseSerializer.get_full_qualified_name(
obj) == "sdp.sheerkaDataProvider.State", "S", 1)
obj) == "sdp.sheerkaDataProvider.State", "S", 1)
class ConceptSerializer(ObjectSerializer):
def __init__(self):
ObjectSerializer.__init__(self, "core.concept.Concept", "C", 1)
class SheerkaSerializer(ObjectSerializer):
def __init__(self):
ObjectSerializer.__init__(self, "core.sheerka.Sheerka", "C", 1)
tests/test_sheerka.py
+13 -7
View File
@@ -32,13 +32,12 @@ def init_test():
def test_root_folder_is_created_after_initialization():
return_value = Sheerka().initialize(root_folder)
assert return_value.status, "initialisation should be successful"
assert Sheerka().concept_equals(return_value.value, Sheerka().get_concept("success"))
assert os.path.exists(root_folder), "init folder should be created"
def test_lists_of_concepts_is_initialized():
Sheerka().initialize(root_folder)
assert len(Sheerka().concepts) > 1
assert len(Sheerka().concepts_cache) > 1
# def test_null_concept_are_equals():
@@ -74,18 +73,25 @@ def test_i_can_add_a_concept():
sheerka = Sheerka()
sheerka.initialize(root_folder)
res = sheerka.add_concept(concept)
concept_found = res.value
assert res.status
assert res.value == Concept(
assert concept_found == Concept(
name="a + b",
where="isinstance(a, int) and isinstance(b, int)",
pre="isinstance(a, int) and isinstance(b, int)",
post="isinstance(res, int)",
body="def func(x,y):\n return x+y\nfunc(a,b)")
assert isinstance(res.value.codes[ConceptParts.WHERE], ast.Expression)
assert isinstance(res.value.codes[ConceptParts.PRE], ast.Expression)
assert isinstance(res.value.codes[ConceptParts.POST], ast.Expression)
assert isinstance(res.value.codes[ConceptParts.BODY], ast.Module)
assert isinstance(concept_found.codes[ConceptParts.WHERE], ast.Expression)
assert isinstance(concept_found.codes[ConceptParts.PRE], ast.Expression)
assert isinstance(concept_found.codes[ConceptParts.POST], ast.Expression)
assert isinstance(concept_found.codes[ConceptParts.BODY], ast.Module)
all_props = [p.name for p in concept_found.props]
assert all_props == ["a", "b"]
assert concept_found.key == "__var__0 + __var__1"
assert concept_found.id == "100"
# def test_i_cannot_add_the_same_concept_twice():
# concept1 = DefConceptNode(name="concept")
tests/test_sheerkaDataProvider.py
+165 -164
View File
@@ -11,6 +11,7 @@ import json
from sdp.sheerkaSerializer import ObjectSerializer, BaseSerializer, Serializer, SerializerContext, PickleSerializer
tests_root = path.abspath("../build/tests")
evt_digest = "3a571cb6034ef6fc8d7fe91948d0d29728eed74de02bac7968b0e9facca2c2d7"
def read_text_file(file_name):
@@ -143,11 +144,9 @@ def test_i_can_save_and_load_an_event():
def test_i_can_add_an_string():
sdp = SheerkaDataProvider(".sheerka")
event = Event("cmd add 'foo => bar'")
event_digest = event.get_digest()
obj = "foo => bar"
entry, key = sdp.add(event, "entry", obj)
entry, key = sdp.add(evt_digest, "entry", obj)
last_commit = sdp.get_snapshot()
state = sdp.load_state(last_commit)
loaded = sdp.get(entry, key)
@@ -156,13 +155,12 @@ def test_i_can_add_an_string():
assert key is None
assert loaded == obj
assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest[0:24], event_digest))
assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile))
assert state.date is not None
assert state.parents == []
assert state.events == [event_digest]
assert state.events == [evt_digest]
assert state.data == {"entry": "foo => bar"}
assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit
@@ -171,9 +169,9 @@ def test_i_can_add_an_string():
def test_i_can_add_several_strings_if_allow_multiple_is_true():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", "foo")
sdp.add(Event("event"), "entry", "foo")
entry, key = sdp.add(Event("event"), "entry", "bar")
sdp.add(evt_digest, "entry", "foo")
sdp.add(evt_digest, "entry", "foo")
entry, key = sdp.add(evt_digest, "entry", "bar")
loaded = sdp.get(entry, key)
assert entry == "entry"
@@ -185,18 +183,16 @@ def test_i_cannot_add_several_strings_if_allow_multiple_is_false():
sdp = SheerkaDataProvider(".sheerka")
with pytest.raises(IndexError) as index_error:
sdp.add(Event("event"), "entry", "foo", False)
sdp.add(Event("event"), "entry", "foo", False)
sdp.add(evt_digest, "entry", "foo", False)
sdp.add(evt_digest, "entry", "foo", False)
assert index_error.value.args[0] == "entry"
def test_i_can_add_an_object_with_no_key():
sdp = SheerkaDataProvider(".sheerka")
event = Event("cmd add 'foo => bar'")
event_digest = event.get_digest()
obj = ObjNoKey("a", "b")
entry, key = sdp.add(event, "entry", obj)
entry, key = sdp.add(evt_digest, "entry", obj)
last_commit = sdp.get_snapshot()
state = sdp.load_state(last_commit)
loaded = sdp.get(entry, key)
@@ -205,13 +201,12 @@ def test_i_can_add_an_object_with_no_key():
assert key is None
assert loaded == obj
assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest[0:24], event_digest))
assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile))
assert state.date is not None
assert state.parents == []
assert state.events == [event_digest]
assert state.events == [evt_digest]
assert state.data == {"entry": ObjNoKey("a", "b")}
assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit
@@ -220,9 +215,9 @@ def test_i_can_add_an_object_with_no_key():
def test_i_can_add_several_obj_no_key_if_allow_multiple_is_true():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", ObjNoKey("a", "b"))
sdp.add(Event("event"), "entry", ObjNoKey("a", "b"))
entry, key = sdp.add(Event("event"), "entry", ObjNoKey("c", "d"))
sdp.add(evt_digest, "entry", ObjNoKey("a", "b"))
sdp.add(evt_digest, "entry", ObjNoKey("a", "b"))
entry, key = sdp.add(evt_digest, "entry", ObjNoKey("c", "d"))
loaded = sdp.get(entry, key)
assert entry == "entry"
@@ -234,18 +229,16 @@ def test_i_cannot_add_several_obj_no_key_if_allow_multiple_is_false():
sdp = SheerkaDataProvider(".sheerka")
with pytest.raises(IndexError) as index_error:
sdp.add(Event("event"), "entry", ObjNoKey("a", "b"), False)
sdp.add(Event("event"), "entry", ObjNoKey("c", "d"), False)
sdp.add(evt_digest, "entry", ObjNoKey("a", "b"), False)
sdp.add(evt_digest, "entry", ObjNoKey("c", "d"), False)
assert index_error.value.args[0] == "entry"
def test_i_can_add_a_dict():
sdp = SheerkaDataProvider(".sheerka")
event = Event("cmd add 'foo => bar'")
event_digest = event.get_digest()
obj = {"my_key": "my_value"}
entry, key = sdp.add(event, "entry", obj)
entry, key = sdp.add(evt_digest, "entry", obj)
last_commit = sdp.get_snapshot()
state = sdp.load_state(last_commit)
loaded = sdp.get(entry, key)
@@ -257,13 +250,12 @@ def test_i_can_add_a_dict():
assert loaded == obj
assert loaded_value == "my_value"
assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest[0:24], event_digest))
assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile))
assert state.date is not None
assert state.parents == []
assert state.events == [event_digest]
assert state.events == [evt_digest]
assert state.data == {"entry": obj}
assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit
@@ -272,7 +264,7 @@ def test_i_can_add_a_dict():
def test_i_can_add_multiple_entries_at_once_with_dict():
sdp = SheerkaDataProvider(".sheerka")
entry, key = sdp.add(Event("event"), "entry", {"my_key1": "value1", "my_key2": "value2"})
entry, key = sdp.add(evt_digest, "entry", {"my_key1": "value1", "my_key2": "value2"})
loaded = sdp.get(entry, key)
loaded_value1 = sdp.get(entry, "my_key1")
loaded_value2 = sdp.get(entry, "my_key2")
@@ -285,11 +277,11 @@ def test_i_can_add_multiple_entries_at_once_with_dict():
def test_i_can_add_same_key_with_dict_if_allow_multiple_is_true():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", {"my_key": "my_value"})
entry, key = sdp.add(Event("event"), "entry", {"my_key": "my_value"})
sdp.add(evt_digest, "entry", {"my_key": "my_value"})
entry, key = sdp.add(evt_digest, "entry", {"my_key": "my_value"})
loaded1 = sdp.get(entry, key)
entry, key = sdp.add(Event("event"), "entry", {"my_key": "my_value2"})
entry, key = sdp.add(evt_digest, "entry", {"my_key": "my_value2"})
loaded2 = sdp.get(entry, key)
assert entry == "entry"
@@ -302,16 +294,16 @@ def test_i_cannot_add_same_key_with_dict_if_allow_multiple_is_false():
sdp = SheerkaDataProvider(".sheerka")
with pytest.raises(IndexError) as index_error:
sdp.add(Event("event"), "entry", {"my_key": "my_value"}, False)
sdp.add(Event("event"), "entry", {"my_key": "my_value2"}, False)
sdp.add(evt_digest, "entry", {"my_key": "my_value"}, False)
sdp.add(evt_digest, "entry", {"my_key": "my_value2"}, False)
assert index_error.value.args[0] == "entry.my_key"
def test_i_can_add_object_with_different_key_if_allow_multiple_is_false():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", {"my_key": "a"}, False)
sdp.add(Event("event"), "entry", {"my_key2": "b"}, False)
sdp.add(evt_digest, "entry", {"my_key": "a"}, False)
sdp.add(evt_digest, "entry", {"my_key2": "b"}, False)
assert sdp.get("entry", "my_key") == "a"
assert sdp.get("entry", "my_key2") == "b"
@@ -319,13 +311,11 @@ def test_i_can_add_object_with_different_key_if_allow_multiple_is_false():
def test_i_can_add_obj_with_key():
sdp = SheerkaDataProvider(".sheerka")
event = Event("cmd add 'foo => bar'")
event_digest = event.get_digest()
obj1 = ObjWithKey("key1", "b")
obj2 = ObjSetKey("c", key="key2")
entry1, key1 = sdp.add(event, "entry", obj1) # test when key is taken from obj.get_key()
entry2, key2 = sdp.add(event, "entry2", obj2) # test when key is taken from obj.key
entry1, key1 = sdp.add(evt_digest, "entry", obj1) # test when key is taken from obj.get_key()
entry2, key2 = sdp.add(evt_digest, "entry2", obj2) # test when key is taken from obj.key
last_commit = sdp.get_snapshot()
state = sdp.load_state(last_commit)
@@ -339,13 +329,12 @@ def test_i_can_add_obj_with_key():
assert key2 == "key2"
assert loaded2 == ObjSetKey("c", key="key2")
assert path.exists(path.join(sdp.root, SheerkaDataProvider.EventFolder, event_digest[0:24], event_digest))
assert path.exists(path.join(sdp.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
assert path.exists(path.join(sdp.root, SheerkaDataProvider.HeadFile))
assert state.date is not None
assert len(state.parents) == 1
assert state.events == [event_digest]
assert state.events == [evt_digest]
assert state.data == {"entry": {"key1": obj1}, "entry2": {"key2": obj2}}
assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == last_commit
@@ -354,12 +343,12 @@ def test_i_can_add_obj_with_key():
def test_i_can_add_objects_with_same_key_if_allow_multiple_is_true():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", ObjWithKey("my_key", "b"))
entry, key = sdp.add(Event("event"), "entry", ObjSetKey("c", key="my_key"))
sdp.add(evt_digest, "entry", ObjWithKey("my_key", "b"))
entry, key = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key"))
loaded1 = sdp.get(entry, key)
entry, key = sdp.add(Event("event"), "entry", ObjSetKey("c", key="my_key"))
sdp.add(Event("event"), "entry", ObjSetKey("c", key="my_key2")) # to prove that it does not melt everything
entry, key = sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key"))
sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key2")) # to prove that it does not melt everything
loaded2 = sdp.get(entry, key)
assert entry == "entry"
@@ -372,17 +361,17 @@ def test_i_cannot_add_object_with_same_key_if_allow_multiple_is_false():
sdp = SheerkaDataProvider(".sheerka")
with pytest.raises(IndexError) as index_error:
sdp.add(Event("event"), "entry", ObjWithKey("my_key", "b"), False)
sdp.add(Event("event"), "entry", ObjSetKey("c", key="my_key"), False)
sdp.add(evt_digest, "entry", ObjWithKey("my_key", "b"), False)
sdp.add(evt_digest, "entry", ObjSetKey("c", key="my_key"), False)
assert index_error.value.args[0] == "entry.my_key"
def test_i_can_add_obj_with_key_to_a_list():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", "foo")
sdp.add(Event("event"), "entry", "bar") # entry is now a list
sdp.add(Event("event"), "entry", ObjWithKey("a", "b")) # this entry must no be taken as a object with a key
sdp.add(evt_digest, "entry", "foo")
sdp.add(evt_digest, "entry", "bar") # entry is now a list
sdp.add(evt_digest, "entry", ObjWithKey("a", "b")) # this entry must no be taken as a object with a key
loaded = sdp.get("entry")
assert loaded == ["foo", "bar", ObjWithKey("a", "b")]
@@ -392,20 +381,19 @@ def test_i_cannot_add_obj_with_no_key_when_then_entry_has_keys():
sdp = SheerkaDataProvider(".sheerka")
with pytest.raises(SheerkaDataProviderError) as error:
sdp.add(Event("event"), "entry", ObjWithKey("a", "b"))
sdp.add(Event("event"), "entry", "foo")
sdp.add(evt_digest, "entry", ObjWithKey("a", "b"))
sdp.add(evt_digest, "entry", "foo")
assert error.value.obj == "foo"
def test_i_can_add_string_using_auto_generated_key():
sdp = SheerkaDataProvider(".sheerka")
event = Event("cmd add 'foo => bar'")
key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile)
entry1, key1 = sdp.add_with_auto_key(event, "entry1", "foo")
entry2, key2 = sdp.add_with_auto_key(event, "entry1", "bar")
entry3, key3 = sdp.add_with_auto_key(event, "entry2", "baz")
entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", "foo")
entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", "bar")
entry3, key3 = sdp.add_with_auto_key(evt_digest, "entry2", "baz")
state = sdp.load_state(sdp.get_snapshot())
@@ -420,13 +408,27 @@ def test_i_can_add_string_using_auto_generated_key():
assert key3 == "1"
def test_i_can_get_and_set_key():
sdp = SheerkaDataProvider(".sheerka")
key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile)
sdp.set_key("entry1", 1000)
sdp.get_next_key("entry1")
sdp.get_next_key("entry1")
sdp.get_next_key("entry1")
sdp.get_next_key("entry2")
sdp.get_next_key("entry2")
assert path.exists(key_file)
assert read_json_file(key_file) == {"entry1": 1003, "entry2": 2}
def test_i_can_add_object_using_auto_generated_key():
sdp = SheerkaDataProvider(".sheerka")
event = Event("cmd add 'foo => bar'")
key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile)
entry1, key1 = sdp.add_with_auto_key(event, "entry1", ObjNoKey("a", "b"))
entry2, key2 = sdp.add_with_auto_key(event, "entry1", ObjNoKey("a", "b"))
entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b"))
entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjNoKey("a", "b"))
state = sdp.load_state(sdp.get_snapshot())
@@ -441,11 +443,10 @@ def test_i_can_add_object_using_auto_generated_key():
def test_object_key_is_updated_when_possible_using_auto_generated_key():
sdp = SheerkaDataProvider(".sheerka")
event = Event("cmd add 'foo => bar'")
key_file = path.join(sdp.root, SheerkaDataProvider.KeysFile)
entry1, key1 = sdp.add_with_auto_key(event, "entry1", ObjSetKey("foo"))
entry2, key2 = sdp.add_with_auto_key(event, "entry1", ObjSetKey("foo"))
entry1, key1 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo"))
entry2, key2 = sdp.add_with_auto_key(evt_digest, "entry1", ObjSetKey("foo"))
state = sdp.load_state(sdp.get_snapshot())
@@ -460,8 +461,8 @@ def test_object_key_is_updated_when_possible_using_auto_generated_key():
def test_i_can_set_objects_with_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", ObjWithKey(1, "foo"))
entry, key = sdp.set(Event("event"), "entry", ObjWithKey(2, "foo"))
sdp.add(evt_digest, "entry", ObjWithKey(1, "foo"))
entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"))
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {"2": ObjWithKey(2, "foo")}}
@@ -471,8 +472,8 @@ def test_i_can_set_objects_with_key():
def test_i_can_set_objects_with_no_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", ObjNoKey(1, "foo"))
entry, key = sdp.set(Event("event"), "entry", ObjNoKey(2, "foo"))
sdp.add(evt_digest, "entry", ObjNoKey(1, "foo"))
entry, key = sdp.set(evt_digest, "entry", ObjNoKey(2, "foo"))
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": ObjNoKey(2, "foo")}
@@ -482,8 +483,8 @@ def test_i_can_set_objects_with_no_key():
def test_i_can_set_from_list_to_dict():
sdp = SheerkaDataProvider(".sheerka")
sdp.set(Event("event"), "entry", [ObjNoKey(1, "foo"), ObjNoKey(2, "foo")])
entry, key = sdp.set(Event("event"), "entry", {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")})
sdp.set(evt_digest, "entry", [ObjNoKey(1, "foo"), ObjNoKey(2, "foo")])
entry, key = sdp.set(evt_digest, "entry", {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")})
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {"1": ObjNoKey(1, "foo"), "2": ObjNoKey(2, "foo")}}
@@ -494,8 +495,8 @@ def test_i_can_set_from_list_to_dict():
def test_i_can_set_using_reference():
sdp = SheerkaDataProvider(".sheerka")
sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey)))
sdp.add(Event("event"), "entry", ObjWithKey(1, "foo"))
entry, key = sdp.set(Event("event"), "entry", ObjWithKey(2, "foo"), use_ref=True)
sdp.add(evt_digest, "entry", ObjWithKey(1, "foo"))
entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"), use_ref=True)
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {"2": '##REF##:9b14e03847d73c640f54ea9b46ba62b19e5451ecd300428a225be012ad9f25f9'}}
@@ -512,10 +513,10 @@ def test_i_can_set_using_reference():
def test_i_can_add_unique():
sdp = SheerkaDataProvider(".sheerka")
sdp.add_unique(Event("event"), "entry", ObjNoKey(1, "foo"))
sdp.add_unique(Event("event"), "entry", ObjNoKey(1, "foo"))
sdp.add_unique(Event("event"), "entry", ObjNoKey(2, "bar"))
entry, key = sdp.add_unique(Event("event"), "entry", ObjNoKey(2, "bar"))
sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo"))
sdp.add_unique(evt_digest, "entry", ObjNoKey(1, "foo"))
sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar"))
entry, key = sdp.add_unique(evt_digest, "entry", ObjNoKey(2, "bar"))
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {ObjNoKey(1, "foo"), ObjNoKey(2, "bar")}}
@@ -529,7 +530,7 @@ def test_i_can_add_reference_of_an_object_with_a_key():
obj_serializer = ObjectSerializer(BaseSerializer.get_full_qualified_name(obj))
sdp.serializer.register(obj_serializer)
entry, key = sdp.add(Event("event"), "entry", obj, use_ref=True)
entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
state = sdp.load_state(sdp.get_snapshot())
digest = state.data["entry"]["my_key"][len(SheerkaDataProvider.REF_PREFIX):]
@@ -543,15 +544,15 @@ def test_i_can_keep_state_history():
sdp = SheerkaDataProvider(".sheerka")
event1 = Event("cmd add 'foo => bar'")
event_digest1 = event1.get_digest()
event_digest1 = sdp.save_event(event1)
obj1 = "foo => bar"
sdp.add(event1, "entry1", obj1)
sdp.add(event_digest1, "entry1", obj1)
state_digest1 = sdp.get_snapshot()
event2 = Event("cmd add 'foo => baz'")
event_digest2 = event2.get_digest()
event_digest2 = sdp.save_event(event2)
obj2 = "foo => baz"
sdp.add(event2, "entry2", obj2)
sdp.add(event_digest2, "entry2", obj2)
state_digest2 = sdp.get_snapshot()
state2 = sdp.load_state(state_digest2)
@@ -578,10 +579,10 @@ def test_i_can_list_when_no_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.serializer.register(PickleSerializer(lambda obj: isinstance(obj, str)))
sdp.add(Event("event"), "entry1", "foo")
sdp.add(Event("event"), "entry1", "bar")
sdp.add(Event("event"), "entry1", "baz", use_ref=True)
sdp.add(Event("event"), "entry2", "xyz")
sdp.add(evt_digest, "entry1", "foo")
sdp.add(evt_digest, "entry1", "bar")
sdp.add(evt_digest, "entry1", "baz", use_ref=True)
sdp.add(evt_digest, "entry2", "xyz")
result = sdp.list("entry1")
@@ -592,10 +593,10 @@ def test_i_can_list_when_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.serializer.register(PickleSerializer(lambda obj: isinstance(obj, ObjWithKey)))
sdp.add(Event("event"), "entry1", {"1": "foo"})
sdp.add(Event("event"), "entry1", {"2": "bar"})
sdp.add(Event("event"), "entry1", ObjWithKey("3", "value"), use_ref=True)
sdp.add(Event("event"), "entry2", {"4": "xxx"})
sdp.add(evt_digest, "entry1", {"1": "foo"})
sdp.add(evt_digest, "entry1", {"2": "bar"})
sdp.add(evt_digest, "entry1", ObjWithKey("3", "value"), use_ref=True)
sdp.add(evt_digest, "entry2", {"4": "xxx"})
result = sdp.list("entry1")
@@ -604,8 +605,8 @@ def test_i_can_list_when_key():
def test_i_can_list_when_one_element():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", "foo")
sdp.add(Event("event"), "entry2", "baz")
sdp.add(evt_digest, "entry1", "foo")
sdp.add(evt_digest, "entry2", "baz")
result = sdp.list("entry1")
@@ -614,8 +615,8 @@ def test_i_can_list_when_one_element():
def test_i_can_filter_on_key_for_dict():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", {"1": "foo"})
sdp.add(Event("event"), "entry1", {"2": "bar"})
sdp.add(evt_digest, "entry1", {"1": "foo"})
sdp.add(evt_digest, "entry1", {"2": "bar"})
result = sdp.list("entry1", lambda k, o: k == "1")
@@ -624,8 +625,8 @@ def test_i_can_filter_on_key_for_dict():
def test_i_can_filter_on_key_for_objects():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", ObjWithKey("a1", "b1"))
sdp.add(Event("event"), "entry1", ObjWithKey("a2", "b2"))
sdp.add(evt_digest, "entry1", ObjWithKey("a1", "b1"))
sdp.add(evt_digest, "entry1", ObjWithKey("a2", "b2"))
result = sdp.list("entry1", lambda k, o: k == "a1")
@@ -634,8 +635,8 @@ def test_i_can_filter_on_key_for_objects():
def test_i_can_filter_on_attribute_for_dict():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", {"1": {"a": "a1", "b": "b1"}})
sdp.add(Event("event"), "entry1", {"2": {"a": "a2", "b": "b2"}})
sdp.add(evt_digest, "entry1", {"1": {"a": "a1", "b": "b1"}})
sdp.add(evt_digest, "entry1", {"2": {"a": "a2", "b": "b2"}})
result = sdp.list("entry1", lambda k, o: o["a"] == "a2")
@@ -644,8 +645,8 @@ def test_i_can_filter_on_attribute_for_dict():
def test_i_can_filter_on_attribute_for_object():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", ObjWithKey("a1", "b1"))
sdp.add(Event("event"), "entry1", ObjWithKey("a2", "b2"))
sdp.add(evt_digest, "entry1", ObjWithKey("a1", "b1"))
sdp.add(evt_digest, "entry1", ObjWithKey("a2", "b2"))
result = sdp.list("entry1", lambda k, o: o.b == "b2")
@@ -654,8 +655,8 @@ def test_i_can_filter_on_attribute_for_object():
def test_i_can_filter_a_list():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", "foo")
sdp.add(Event("event"), "entry1", "bar")
sdp.add(evt_digest, "entry1", "foo")
sdp.add(evt_digest, "entry1", "bar")
result = sdp.list("entry1", lambda o: o == "bar")
@@ -664,8 +665,8 @@ def test_i_can_filter_a_list():
def test_i_can_filter_a_list_of_object():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", ObjNoKey("a1", "b1"))
sdp.add(Event("event"), "entry1", ObjNoKey("a2", "b2"))
sdp.add(evt_digest, "entry1", ObjNoKey("a1", "b1"))
sdp.add(evt_digest, "entry1", ObjNoKey("a2", "b2"))
result = sdp.list("entry1", lambda o: o.b == "b1")
@@ -674,10 +675,10 @@ def test_i_can_filter_a_list_of_object():
def test_i_can_remove_all_elements():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", "foo")
sdp.add(Event("event"), "entry1", "bar")
sdp.add(evt_digest, "entry1", "foo")
sdp.add(evt_digest, "entry1", "bar")
state_digest = sdp.remove(Event("event"), "entry1")
state_digest = sdp.remove(evt_digest, "entry1")
result = sdp.list("entry1")
assert read_text_file(path.join(sdp.root, SheerkaDataProvider.HeadFile)) == state_digest
@@ -686,10 +687,10 @@ def test_i_can_remove_all_elements():
def test_i_can_remove_a_element():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", "foo")
sdp.add(Event("event"), "entry1", "bar")
sdp.add(evt_digest, "entry1", "foo")
sdp.add(evt_digest, "entry1", "bar")
sdp.remove(Event("event"), "entry1", lambda o: o == "foo")
sdp.remove(evt_digest, "entry1", lambda o: o == "foo")
result = sdp.list("entry1")
assert list(result) == ["bar"]
@@ -697,10 +698,10 @@ def test_i_can_remove_a_element():
def test_i_can_remove_dict_by_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", {"1": ObjNoKey("a1", "b1")})
sdp.add(Event("event"), "entry1", {"2": ObjNoKey("a2", "b2")})
sdp.add(evt_digest, "entry1", {"1": ObjNoKey("a1", "b1")})
sdp.add(evt_digest, "entry1", {"2": ObjNoKey("a2", "b2")})
sdp.remove(Event("event"), "entry1", lambda k, o: k == "2")
sdp.remove(evt_digest, "entry1", lambda k, o: k == "2")
result = sdp.list("entry1")
assert list(result) == [ObjNoKey("a1", "b1")]
@@ -708,9 +709,9 @@ def test_i_can_remove_dict_by_key():
def test_i_can_remove_when_only_one_element():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", "foo")
sdp.add(evt_digest, "entry1", "foo")
sdp.remove(Event("event"), "entry1", lambda o: o == "foo")
sdp.remove(evt_digest, "entry1", lambda o: o == "foo")
result = sdp.list("entry1")
assert list(result) == []
@@ -719,7 +720,7 @@ def test_i_can_remove_when_only_one_element():
def test_i_cannot_remove_if_entry_does_not_exist():
sdp = SheerkaDataProvider(".sheerka")
with pytest.raises(IndexError) as e:
sdp.remove(Event("event"), "entry")
sdp.remove(evt_digest, "entry")
assert str(e) == "entry"
@@ -727,17 +728,17 @@ def test_i_cannot_modify_an_entry_without_a_key():
sdp = SheerkaDataProvider(".sheerka")
with pytest.raises(SheerkaDataProviderError) as error:
sdp.modify(Event("event"), "entry", None, "baz")
sdp.modify(evt_digest, "entry", None, "baz")
assert error.value.args[0] == "Key is mandatory."
def test_i_can_modify_dict_with_a_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", {"key1": "foo"})
sdp.add(Event("event"), "entry", {"key2": "bar"})
sdp.add(evt_digest, "entry", {"key1": "foo"})
sdp.add(evt_digest, "entry", {"key2": "bar"})
entry, key = sdp.modify(Event("event"), "entry", "key1", "baz")
entry, key = sdp.modify(evt_digest, "entry", "key1", "baz")
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {"key1": "baz", "key2": "bar"}}
@@ -747,10 +748,10 @@ def test_i_can_modify_dict_with_a_key():
def test_i_can_modify_an_object_with_a_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", ObjWithKey("key1", "foo"))
sdp.add(Event("event"), "entry", ObjWithKey("key2", "bar"))
sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo"))
sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"))
entry, key = sdp.modify(Event("event"), "entry", "key1", ObjWithKey("key1", "baz"))
entry, key = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key1", "baz"))
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {"key1": ObjWithKey("key1", "baz"), "key2": ObjWithKey("key2", "bar")}}
@@ -760,10 +761,10 @@ def test_i_can_modify_an_object_with_a_key():
def test_i_can_modify_an_object_while_changing_the_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", ObjWithKey("key1", "foo"))
sdp.add(Event("event"), "entry", ObjWithKey("key2", "bar"))
sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo"))
sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"))
entry, key = sdp.modify(Event("event"), "entry", "key1", ObjWithKey("key3", "baz"))
entry, key = sdp.modify(evt_digest, "entry", "key1", ObjWithKey("key3", "baz"))
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {"key2": ObjWithKey("key2", "bar"), "key3": ObjWithKey("key3", "baz")}}
@@ -773,10 +774,10 @@ def test_i_can_modify_an_object_while_changing_the_key():
def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", ObjWithKey("key1", "foo"))
sdp.add(Event("event"), "entry", ObjWithKey("key2", "bar"))
sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo"))
sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"))
entry, key = sdp.modify(Event("event"), "entry", "key2", ObjWithKey("key1", "bar"))
entry, key = sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key1", "bar"))
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {"key1": [ObjWithKey("key1", "foo"), ObjWithKey("key1", "bar")]}}
@@ -793,14 +794,14 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_list(
sdp = SheerkaDataProvider(".sheerka")
sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
sdp.add(Event("event"), "entry", ObjDumpJson("key1", "value11"))
sdp.add(Event("event"), "entry", ObjDumpJson("key1", "value12"))
sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value21"))
sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value22"))
sdp.add(evt_digest, "entry", ObjDumpJson("key1", "value11"))
sdp.add(evt_digest, "entry", ObjDumpJson("key1", "value12"))
sdp.add(evt_digest, "entry", ObjDumpJson("key2", "value21"))
sdp.add(evt_digest, "entry", ObjDumpJson("key2", "value22"))
new_value = ObjDumpJson("key1", "value13")
setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest())
entry, key = sdp.modify(Event("event"), "entry", "key2", new_value)
entry, key = sdp.modify(evt_digest, "entry", "key2", new_value)
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {
@@ -820,12 +821,12 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_nothi
sdp = SheerkaDataProvider(".sheerka")
sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value21"))
sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value22"))
sdp.add(evt_digest, "entry", ObjDumpJson("key2", "value21"))
sdp.add(evt_digest, "entry", ObjDumpJson("key2", "value22"))
new_value = ObjDumpJson("key1", "value13")
setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest())
entry, key = sdp.modify(Event("event"), "entry", "key2", new_value)
entry, key = sdp.modify(evt_digest, "entry", "key2", new_value)
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {
@@ -845,13 +846,13 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_one_i
sdp = SheerkaDataProvider(".sheerka")
sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
sdp.add(Event("event"), "entry", ObjDumpJson("key1", "value11"))
sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value21"))
sdp.add(Event("event"), "entry", ObjDumpJson("key2", "value22"))
sdp.add(evt_digest, "entry", ObjDumpJson("key1", "value11"))
sdp.add(evt_digest, "entry", ObjDumpJson("key2", "value21"))
sdp.add(evt_digest, "entry", ObjDumpJson("key2", "value22"))
new_value = ObjDumpJson("key1", "value13")
setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key2", "value21").get_digest())
entry, key = sdp.modify(Event("event"), "entry", "key2", new_value)
entry, key = sdp.modify(evt_digest, "entry", "key2", new_value)
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {
@@ -865,10 +866,10 @@ def test_i_can_modify_an_object_while_changing_the_key_to_an_existing_when_one_i
def test_i_can_modify_a_ref():
sdp = SheerkaDataProvider(".sheerka")
sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithKey)))
sdp.add(Event("event"), "entry", ObjWithKey("key1", "foo"))
entry, key = sdp.add(Event("event"), "entry", ObjWithKey("key2", "bar"), use_ref=True)
sdp.add(evt_digest, "entry", ObjWithKey("key1", "foo"))
entry, key = sdp.add(evt_digest, "entry", ObjWithKey("key2", "bar"), use_ref=True)
sdp.modify(Event("event"), "entry", "key2", ObjWithKey("key2", "baz"))
sdp.modify(evt_digest, "entry", "key2", ObjWithKey("key2", "baz"))
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {
"key1": ObjWithKey("key1", "foo"),
@@ -881,30 +882,30 @@ def test_i_cannot_modify_an_entry_that_does_not_exist():
sdp = SheerkaDataProvider(".sheerka")
with pytest.raises(IndexError) as e:
sdp.modify(Event("event"), "entry", "key", "foo")
sdp.modify(evt_digest, "entry", "key", "foo")
assert str(e.value) == "entry"
def test_i_cannot_modify_a_key_that_does_not_exist():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", {"1": "foo"})
sdp.add(evt_digest, "entry1", {"1": "foo"})
with pytest.raises(IndexError) as e:
sdp.modify(Event("event"), "entry1", "2", "bar")
sdp.modify(evt_digest, "entry1", "2", "bar")
assert str(e) == "entry1.2"
def test_i_cannot_modify_a_list_when_origin_is_unknown():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", ObjWithKey("key", "value1"))
sdp.add(Event("event"), "entry", ObjWithKey("key", "value2")) # same they
sdp.add(evt_digest, "entry", ObjWithKey("key", "value1"))
sdp.add(evt_digest, "entry", ObjWithKey("key", "value2")) # same they
state = sdp.load_state(sdp.get_snapshot())
with pytest.raises(SheerkaDataProviderError) as error:
sdp.modify(Event("event"), "entry", "key", ObjWithKey("key", "value2"))
sdp.modify(evt_digest, "entry", "key", ObjWithKey("key", "value2"))
assert error.value.obj == ObjWithKey("key", "value2")
assert error.value.args[0] == "Multiple entries under 'entry.key'"
@@ -913,13 +914,13 @@ def test_i_cannot_modify_a_list_when_origin_is_unknown():
def test_i_can_modify_a_list_when_the_origin_is_known():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry", ObjDumpJson("key", "value1"))
sdp.add(Event("event"), "entry", ObjDumpJson("key", "value2")) # same they
sdp.add(evt_digest, "entry", ObjDumpJson("key", "value1"))
sdp.add(evt_digest, "entry", ObjDumpJson("key", "value2")) # same they
new_value = ObjDumpJson("key", "value3")
setattr(new_value, Serializer.ORIGIN, ObjDumpJson("key", "value1").get_digest())
sdp.modify(Event("event"), "entry", "key", new_value)
sdp.modify(evt_digest, "entry", "key", new_value)
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {"key": [ObjDumpJson("key", "value3"), ObjDumpJson("key", "value2")]}}
@@ -934,13 +935,13 @@ def test_i_can_modify_a_list_when_the_origin_is_known_2():
sdp = SheerkaDataProvider(".sheerka")
sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
sdp.add(Event("event"), "entry", ObjDumpJson("key", "value1"), use_ref=True)
sdp.add(Event("event"), "entry", ObjDumpJson("key", "value2"), use_ref=True) # same they
sdp.add(evt_digest, "entry", ObjDumpJson("key", "value1"), use_ref=True)
sdp.add(evt_digest, "entry", ObjDumpJson("key", "value2"), use_ref=True) # same they
objs = sdp.get("entry", "key") # origin is automatically set to the loaded objects
objs[0].value = "value3"
sdp.modify(Event("event"), "entry", "key", objs[0])
sdp.modify(evt_digest, "entry", "key", objs[0])
state = sdp.load_state(sdp.get_snapshot())
assert state.data == {"entry": {"key": [
@@ -955,8 +956,8 @@ def test_i_can_modify_a_list_when_the_origin_is_known_2():
def test_i_can_get_the_entire_entry():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", "foo")
sdp.add(Event("event"), "entry1", "bar")
sdp.add(evt_digest, "entry1", "foo")
sdp.add(evt_digest, "entry1", "bar")
result = sdp.get("entry1")
result_safe = sdp.get_safe("entry1")
@@ -967,7 +968,7 @@ def test_i_can_get_the_entire_entry():
def test_i_can_get_an_entry_with_on_object():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", "foo")
sdp.add(evt_digest, "entry1", "foo")
result = sdp.get("entry1")
result_safe = sdp.get_safe("entry1")
@@ -978,8 +979,8 @@ def test_i_can_get_an_entry_with_on_object():
def test_i_can_get_an_entry_by_key():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", {"1": "foo"})
sdp.add(Event("event"), "entry1", {"2": "bar"})
sdp.add(evt_digest, "entry1", {"1": "foo"})
sdp.add(evt_digest, "entry1", {"2": "bar"})
result = sdp.get("entry1", "2")
result_safe = sdp.get_safe("entry1", "2")
@@ -993,7 +994,7 @@ def test_i_can_get_object_save_by_reference():
obj = ObjDumpJson("my_key", "value1")
sdp.serializer.register(ObjectSerializer(BaseSerializer.get_full_qualified_name(obj)))
entry, key = sdp.add(Event("event"), "entry", obj, use_ref=True)
entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
loaded = sdp.get(entry, key)
assert loaded == obj
@@ -1003,8 +1004,8 @@ def test_i_can_get_objects_from_list_when_saved_by_reference():
sdp = SheerkaDataProvider(".sheerka")
sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjDumpJson)))
sdp.add(Event("event"), "entry", ObjDumpJson("key", "value1"), use_ref=True)
sdp.add(Event("event"), "entry", ObjDumpJson("key", "value2"), use_ref=True) # same they
sdp.add(evt_digest, "entry", ObjDumpJson("key", "value1"), use_ref=True)
sdp.add(evt_digest, "entry", ObjDumpJson("key", "value2"), use_ref=True) # same they
objs = sdp.get("entry", "key")
@@ -1023,7 +1024,7 @@ def test_i_cannot_get_an_entry_that_does_not_exist():
def test_i_cannot_get_a_key_that_does_not_exist():
sdp = SheerkaDataProvider(".sheerka")
sdp.add(Event("event"), "entry1", {"1": "foo"})
sdp.add(evt_digest, "entry1", {"1": "foo"})
assert sdp.get_safe("entry1", "2") is None
with pytest.raises(IndexError) as e:
@@ -1090,7 +1091,7 @@ def test_i_can_test_than_an_entry_exits():
sdp = SheerkaDataProvider(".sheerka")
assert not sdp.exists("entry")
sdp.add(Event("event"), "entry", "value")
sdp.add(evt_digest, "entry", "value")
assert sdp.exists("entry")
@@ -1099,7 +1100,7 @@ def test_i_can_save_and_load_object_ref_with_history():
obj = ObjDumpJson("my_key", "value1")
sdp.serializer.register(ObjectSerializer(BaseSerializer.get_full_qualified_name(obj)))
entry, key = sdp.add(Event("event"), "entry", obj, use_ref=True)
entry, key = sdp.add(evt_digest, "entry", obj, use_ref=True)
loaded = sdp.get(entry, key)
history = getattr(loaded, Serializer.HISTORY)
@@ -1118,7 +1119,7 @@ def test_i_can_save_and_load_object_ref_with_history():
previous_modification_time = history[Serializer.MODIFICATION_DATE]
previous_parents = history[Serializer.PARENTS]
sdp.modify(Event("event"), "entry", key, loaded)
sdp.modify(evt_digest, "entry", key, loaded)
loaded = sdp.get(entry, key)
history = getattr(loaded, Serializer.HISTORY)
@@ -1129,7 +1130,7 @@ def test_i_can_save_and_load_object_ref_with_history():
previous_digest = loaded.get_digest()
loaded.value = "value2"
sdp.modify(Event("event"), "entry", key, loaded)
sdp.modify(evt_digest, "entry", key, loaded)
loaded2 = sdp.get(entry, key)
history2 = getattr(loaded2, Serializer.HISTORY)