Refactored sheerka execution flow + Enhanced log management

sdp/sheerkaDataProvider.py

@@ -5,10 +5,8 @@ import zlib
 
 from sdp.sheerkaDataProviderIO import SheerkaDataProviderIO
 from sdp.sheerkaSerializer import Serializer, SerializerContext
-import logging
+from core.sheerka_logger import get_logger
 
-log = logging.getLogger(__name__)
-init_log = logging.getLogger(__name__ + ".init")
 
 def json_default_converter(o):
     """
@@ -276,7 +274,9 @@ class SheerkaDataProvider:
     REF_PREFIX = "##REF##:"
 
     def __init__(self, root=None):
-        init_log.debug("Initializing sdp.")
+        self.log = get_logger(__name__)
+        self.init_log = get_logger("init." + __name__)
+        self.init_log.debug("Initializing sdp.")
 
         self.io = SheerkaDataProviderIO.get(root)
         self.first_time = self.io.first_time
@@ -323,7 +323,6 @@ class SheerkaDataProvider:
 
         return None
 
-
     @staticmethod
     def get_stream_digest(stream):
         sha256_hash = hashlib.sha256()
@@ -353,14 +352,14 @@ class SheerkaDataProvider:
         snapshot = self.get_snapshot()
         state = self.load_state(snapshot)
 
-        log.debug(f"Adding obj '{obj}' in entry '{entry}' (allow_multiple={allow_multiple}, use_ref={use_ref})")
+        self.log.debug(f"Adding obj '{obj}' in entry '{entry}' (allow_multiple={allow_multiple}, use_ref={use_ref})")
 
         if not isinstance(obj, ObjToUpdate):
             obj = ObjToUpdate(obj)
 
         # check uniqueness, cannot add the same key twice if allow_multiple == False
         key = obj.get_key()
-        log.debug(f"key found : '{key}'") if key else log.debug("No key found")
+        self.log.debug(f"key found : '{key}'") if key else self.log.debug("No key found")
         if not allow_multiple:
             if isinstance(obj.obj, dict):
                 for k in obj.obj:
@@ -505,7 +504,10 @@ class SheerkaDataProvider:
             filter_to_use = (lambda k, o: True) if filter is None else filter
             for key, element in elements.items():
                 if filter_to_use(key, element):
-                    yield self.load_ref_if_needed(element)[0]
+                    if isinstance(element, list):
+                        yield [self.load_ref_if_needed(e)[0] for e in element]
+                    else:
+                        yield self.load_ref_if_needed(element)[0]
         else:
             # manage when no key is defined for the elements
             if not isinstance(elements, list) and not isinstance(elements, set):
@@ -643,7 +645,7 @@ class SheerkaDataProvider:
 
     def save_state(self, state: State):
         digest = state.get_digest()
-        log.debug(f"Saving new state. digest={digest}")
+        self.log.debug(f"Saving new state. digest={digest}")
         target_path = self.io.get_obj_path(SheerkaDataProvider.StateFolder, digest)
         if self.io.exists(target_path):
             return digest
@@ -660,18 +662,18 @@ class SheerkaDataProvider:
             return self.serializer.deserialize(f, None)
 
     def save_obj(self, obj):
-        log.debug(f"Saving '{obj}' as reference...")
+        self.log.debug(f"Saving '{obj}' as reference...")
         stream = self.serializer.serialize(obj, SerializerContext(user_name="kodjo"))
         digest = obj.get_digest() if hasattr(obj, "get_digest") else self.get_stream_digest(stream)
 
         target_path = self.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, digest)
         if self.io.exists(target_path):
-            log.debug(f"...already saved. digest is {digest}")
+            self.log.debug(f"...already saved. digest is {digest}")
             return digest
 
         self.io.write_binary(target_path, stream.read())
 
-        log.debug(f"...digest={digest}.")
+        self.log.debug(f"...digest={digest}.")
         return digest
 
     def load_obj(self, digest, add_origin=True):

sdp/sheerkaDataProviderIO.py

@@ -2,13 +2,15 @@ import io
 from os import path
 import os
 from fs.memoryfs import MemoryFS
-import logging
+from core.sheerka_logger import get_logger
 
 
 class SheerkaDataProviderIO:
 
     def __init__(self, root):
         self.root = root
+        self.log = get_logger(__name__)
+        self.init_log = get_logger("init." + __name__)
 
     def exists(self, file_path):
         pass
@@ -48,16 +50,15 @@ class SheerkaDataProviderIO:
 class SheerkaDataProviderFileIO(SheerkaDataProviderIO):
 
     def __init__(self, root):
-        self.log = logging.getLogger(self.__class__.__name__ + ".init")
         root = path.abspath(path.join(path.expanduser("~"), ".sheerka")) \
             if root is None \
             else path.abspath(root)
         super().__init__(root)
 
-        self.log.debug("root is set to '" + self.root + "'")
+        self.init_log.debug("root is set to '" + self.root + "'")
 
         if not path.exists(self.root):
-            self.log.debug("root folder not found. Creating it.")
+            self.init_log.debug("root folder not found. Creating it.")
             os.makedirs(self.root)
             self.first_time = True
         else:
@@ -96,12 +97,12 @@ class SheerkaDataProviderFileIO(SheerkaDataProviderIO):
 
 
 class SheerkaDataProviderMemoryIO(SheerkaDataProviderIO):
-    log = logging.getLogger("MemoryIO")
 
     def __init__(self):
         super().__init__("")
+
         self.mem_fs = MemoryFS()
-        self.log.debug("Initializing memory file.")
+        self.init_log.debug("Initializing memory file.")
         self.first_time = True
 
     def open(self, file_path, mode):
@@ -133,12 +134,11 @@ class SheerkaDataProviderMemoryIO(SheerkaDataProviderIO):
 
 
 class SheerkaDataProviderDictionaryIO(SheerkaDataProviderIO):
-    log = logging.getLogger("DictionaryIO")
 
     def __init__(self):
         super().__init__("")
         self.cache = {}
-        self.log.debug("Initializing dictionary file.")
+        self.init_log.debug("Initializing dictionary file.")
         self.first_time = True
 
     def exists(self, file_path):

sdp/sheerkaSerializer.py

@@ -4,17 +4,13 @@ import datetime
 import struct
 import io
 from dataclasses import dataclass
-import logging
+from core.sheerka_logger import get_logger
 from enum import Enum
 
 import core.utils
 
 from core.concept import Concept
 
-log = logging.getLogger(__name__)
-init_log = logging.getLogger(__name__ + ".init")
-
-
 def json_default_converter(o):
     """
     Default formatter for json
@@ -45,7 +41,9 @@ class Serializer:
     HISTORY = "##history##"
 
     def __init__(self):
-        init_log.debug("Initializing serializers")
+        self.log = get_logger(__name__)
+        self.init_log = get_logger("init." + __name__)
+        self.init_log.debug("Initializing serializers")
         self._cache = []
 
         # add builtin serializers
@@ -60,7 +58,7 @@ class Serializer:
         :param serializer:
         :return:
         """
-        init_log.debug(f"Adding serializer {serializer}")
+        self.init_log.debug(f"Adding serializer {serializer}")
         self._cache.append(serializer)
 
     def serialize(self, obj, context):