Fixed #8

Fixed #12
Fixed #13
Fixed #14

Makefile

@@ -0,0 +1,23 @@
+.PHONY: test
+
+test:
+	pytest
+
+coverage:
+	coverage run --source=src -m pytest
+	coverage html
+
+clean:
+	rm -rf build
+	rm -rf htmlcov
+	rm -rf .coverage
+	rm -rf docs/build
+	rm -rf docs/source/_build
+	rm -rf prof
+	rm -rf tests/prof
+	rm -rf tests/build
+	rm -rf Untitled*.ipynb
+	rm -rf .ipynb_checkpoints
+	find . -name '.pytest_cache' -exec rm -rf {} +
+	find . -name '__pycache__' -exec rm -rf {} +
+	find . -name 'debug.txt' -exec rm -rf {} +

README.md

@@ -8,4 +8,24 @@ My personnal AI
 ```shell
 cd src
 uvicorn server:app --reload
-```
+```
+
+## To start the client
+
+```shell
+python $DEV_HOME/src/client.py --username <username> --password <password>
+```
+
+## to test
+
+```shell
+pytest
+```
+
+## To run the coverage
+```shell
+coverage run --source=src -m pytest
+
+coverage report
+coverage html
+```

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.pytest.ini_options]
+pythonpath = "src tests"

requirements.txt

@@ -1,18 +1,26 @@
 anyio==3.6.2
+arrow==1.2.3
 attrs==22.2.0
 bcrypt==4.0.1
 certifi==2022.12.7
 cffi==1.15.1
 charset-normalizer==3.0.1
 click==8.1.3
+coverage==7.1.0
 cryptography==39.0.0
 ecdsa==0.18.0
 exceptiongroup==1.1.0
 fastapi==0.89.1
 h11==0.14.0
+httpcore==0.16.3
 httptools==0.5.0
+httpx==0.23.3
 idna==3.4
 iniconfig==2.0.0
+Jinja2==3.1.2
+jinja2-time==0.2.0
+make==0.1.6.post2
+MarkupSafe==2.1.2
 oauthlib==3.2.2
 packaging==23.0
 passlib==1.7.4
@@ -22,12 +30,14 @@ pyasn1==0.4.8
 pycparser==2.21
 pydantic==1.10.4
 pytest==7.2.0
+python-dateutil==2.8.2
 python-dotenv==0.21.0
 python-jose==3.3.0
 python-multipart==0.0.5
 PyYAML==6.0
 requests==2.28.2
 requests-oauthlib==1.3.1
+rfc3986==1.5.0
 rsa==4.9
 six==1.16.0
 sniffio==1.3.0

src/caching/BaseCache.py

@@ -0,0 +1,442 @@
+from threading import RLock
+
+from common.global_symbols import NotFound
+from common.utils import sheerka_deepcopy
+
+MAX_INITIALIZED_KEY = 100
+
+
+class BaseCache:
+    """
+    An in memory FIFO cache object
+    When the max_size is reach the first element that was put is removed
+    When you put the same key twice, the previous element is overridden
+    """
+
+    def __init__(self, max_size: int = None, default=NotFound, extend_exists=None, alt_sdp_get=None, sdp=None):
+        self._cache = {}
+        self._max_size = max_size
+        self._default = default  # default value to return when key is not found. It can be a callable of key
+        self._extend_exists = extend_exists  # search in remote
+        self._alt_sdp_get = alt_sdp_get  # How to get the value when called by alt_sdp
+        self._sdp = sdp  # current instance of SheerkaDataProvider
+        self._lock = RLock()
+        self._current_size = 0
+        self._initialized_keys = set()  # to keep the list of the keys already requested (using get())
+        self._is_cleared = False  # indicate that clear() was called
+
+        self.to_add = set()
+        self.to_remove = set()
+
+        # Explanation on _initialized_keys
+        # everytime you try to get an item, its key is added to _initialized_keys
+        # If the item is found, the entru is i
+
+    def __len__(self):
+        """
+        Return the number of items in the cache
+        :return:
+        """
+        with self._lock:
+            return self._current_size
+
+    def __contains__(self, key: str):
+        with self._lock:
+            return key in self._cache
+
+    def __iter__(self):
+        with self._lock:
+            keys = self._cache.copy()
+        yield from keys
+
+    def __next__(self):
+        return next(iter(self._cache))
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(size={self._current_size}, #keys={len(self._cache)})"
+
+    def configure(self, max_size: int = None, default=NotFound, extend_exists=None, alt_sdp_get=None, sdp=None):
+        if max_size is not None:
+            self._max_size = max_size
+
+        if default is not NotFound:
+            self._default = default
+
+        if extend_exists is not None:
+            self._extend_exists = extend_exists
+
+        if alt_sdp_get is not None:
+            self._alt_sdp_get = alt_sdp_get
+
+        if sdp is not None:
+            self._sdp = sdp
+
+        return self
+
+    def auto_configure(self, cache_name: str):
+        """
+        Convenient way to configure the cache
+        :param cache_name:
+        :return:
+        """
+        self._default = lambda sdp, key: sdp.get(cache_name, key)
+        self._extend_exists = lambda sdp, key: sdp.exists(cache_name, key)
+        self._alt_sdp_get = lambda sdp, key: sdp.alt_get(cache_name, key)  # by default, same than get
+
+        return self
+
+    def disable_default(self):
+        self._default = (lambda sdp, key: NotFound) if self._sdp else (lambda key: NotFound)
+
+    def put(self, key: str, value: object, alt_sdp=None):
+        """
+        Add a new entry in cache
+        :param key:
+        :param value:
+        :param alt_sdp:
+        :return:
+        """
+        with self._lock:
+            if self._put(key, value, alt_sdp):
+                self._current_size += 1
+
+    def get(self, key: str, alt_sdp=None):
+        """
+        Retrieve an entry from the cache
+        If the entry does not exist, will use the 'default' value or delegate
+        :param key:
+        :param alt_sdp: if not found in cache._sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            return self._get(key, alt_sdp)
+
+    def alt_get(self, key: str):
+        """
+        Alternate way to get an entry, from concept cache
+        This is mainly used for IncCache, in order to get the value without increasing it
+        It used for another cache, it must return the value from key WITHOUT modifying the state of the cache
+        :param key:
+        :return:
+        """
+        with self._lock:
+            return self._alt_get(key)
+
+    def get_all(self):
+        """
+        Retrieve all items already in cache
+        This method does not fetch in the remoter repository
+        :return:
+        """
+        with self._lock:
+            return self._cache.values()
+
+    def inner_get(self, key: str):
+        return self._cache[key]
+
+    def update(self, old_key: str, old_value: object, new_key: str, new_value: object, alt_sdp=None):
+        """
+        Update an entry in the cache
+        :param old_key: key of the previous version of the entry
+        :param old_value: previous version of the entry
+        :param new_key: key of the entry
+        :param new_value: new value
+        :param alt_sdp: new value
+        :return:
+        """
+        with self._lock:
+            self._update(old_key, old_value, new_key, new_value, alt_sdp)
+
+    def delete(self, key: str, value: object = None, alt_sdp=None):
+        with self._lock:
+            try:
+                self._sync(key)
+                self._delete(key, value, alt_sdp)
+                return True
+            except KeyError:
+                return False
+
+    def populate(self, populate_function: callable, get_key_function: callable, reset_events=False):
+        """
+        Initialise the cache with a bunch of data
+        :param populate_function: iterable that produces item
+        :param get_key_function: how to compute the key of an item
+        :param reset_events: if TRUE, events are not updated
+        :return:
+        """
+        with self._lock:
+            if reset_events:
+                to_add_copy = self.to_add.copy()
+                to_remove_copy = self.to_remove.copy()
+
+            for item in (populate_function(self._sdp) if self._sdp else populate_function()):
+                self.put(get_key_function(item), item)
+
+            if reset_events:
+                self.to_add = to_add_copy
+                self.to_remove = to_remove_copy
+
+    def force_value(self, key: str, value: object):
+        """
+        Force a value into a key without raising any event
+        """
+        with self._lock:
+            self._cache[key] = value
+
+    def remove_initialized_key(self, key: str):
+        """
+        When a value is requested by alt_sdp, we should not keep track of the request
+        As the outcome is not known
+        """
+        with self._lock:
+            self._initialized_keys.remove(key)
+
+    def has(self, key: str):
+        """
+        Return True if the key is in the cache
+        Never use extend_exist
+        :param key:
+        :return:
+        """
+        with self._lock:
+            return key in self._cache
+
+    def exists(self, key: str):
+        """
+        Return True if the key is in the cache
+        Can use extend_exist
+        :param key:
+        :return:
+        """
+        with self._lock:
+            if key in self._cache:
+                return True
+
+            if self._extend_exists:
+                return self._extend_exists(self._sdp, key) if self._sdp else self._extend_exists(key)
+            else:
+                return False
+
+    def evict(self, nb_items: int):
+        """
+        Remove nb_items from the cache, using the replacement policy
+        :return:
+        """
+        with self._lock:
+            nb_items = self._current_size if self._current_size < nb_items else nb_items
+            to_remove = []
+            iter_cache = iter(self._cache)
+            try:
+                while nb_items > 0:
+                    key = next(iter_cache)
+                    if key in self.to_add or key in self.to_remove:
+                        continue  # cannot remove an item that is not yet committed
+                    else:
+                        to_remove.append(key)
+                        nb_items -= 1
+            except StopIteration:
+                pass
+
+            for key in to_remove:
+                del (self._cache[key])
+                try:
+                    self._initialized_keys.remove(key)
+                except KeyError:
+                    pass
+
+            self._current_size -= len(to_remove)
+
+        return len(to_remove)
+
+    def evict_by_key(self, predicate: callable):
+        """
+        Remove entries that matches the predicate
+        :param predicate:
+        :return:
+        """
+        to_delete = []
+        with self._lock:
+            for key in self._cache:
+                if predicate(key):
+                    to_delete.append(key)
+
+            for key in to_delete:
+                del (self._cache[key])
+                try:
+                    self._initialized_keys.remove(key)
+                except KeyError:
+                    pass
+
+            self._current_size -= len(to_delete)
+
+        return len(to_delete)
+
+    def clear(self, set_is_cleared: bool = True):
+        with self._lock:
+            # Seems that remote sdp is not correctly updated
+            self._cache.clear()
+            self._current_size = 0
+            self._initialized_keys.clear()
+            self.to_add.clear()
+            self.to_remove.clear()
+            if set_is_cleared:
+                self._is_cleared = True
+
+    def dump(self):
+        with self._lock:
+            return {
+                "current_size": self._current_size,
+                "cache": self._cache.copy()
+            }
+
+    def copy(self):
+        with self._lock:
+            return self._cache.copy()
+
+    def init_from_dump(self, dump: dict):
+        with self._lock:
+            self._current_size = dump["current_size"]
+            self._cache = dump["cache"].copy()
+            return self
+
+    def reset_events(self):
+        with self._lock:
+            self.to_add.clear()
+            self.to_remove.clear()
+            self._is_cleared = False
+
+    def reset_initialized_keys(self):
+        """
+        Use when an ontology is put back. Reset all the previous requests as alt_sdp is a new one
+        """
+        with self._lock:
+            self._initialized_keys.clear()
+
+    def is_cleared(self):
+        with self._lock:
+            return self._is_cleared
+
+    def clone(self):
+        return type(self)(self._max_size, self._default, self._extend_exists, self._alt_sdp_get, self._sdp)
+
+    def test_only_reset(self):
+        """
+        Clears the cache, but does not set is_cleared to True
+        It's a convenient way to clear the cache without altering alt_sdp behaviour
+        """
+        self.clear(set_is_cleared=False)
+
+    def _sync(self, *keys):
+        # KSI 2020-12-29. DO not try to use alt_sdp here
+        # Sync must only sync with the current sdp
+        for key in keys:
+            if key not in self._initialized_keys and callable(self._default):
+                # to keep sync with the remote repo is needed
+                # first check self._initialized_keys to prevent infinite loop
+                self.get(key)
+
+    def _add_to_add(self, key: str):
+        """
+        Adds the key to the list of recently added keys
+        :param key:
+        :type key:
+        :return:
+        :rtype:
+        """
+        self.to_add.add(key)
+        try:
+            self.to_remove.remove(key)
+        except KeyError:
+            pass
+
+    def _add_to_remove(self, key: str):
+        """
+        Adds the key to the list of recently removed keys
+        :param key:
+        :type key:
+        :return:
+        :rtype:
+        """
+        self.to_remove.add(key)
+        try:
+            self.to_add.remove(key)
+        except KeyError:
+            pass
+
+    def _get(self, key: str, alt_sdp=None):
+        try:
+            value = self._cache[key]
+        except KeyError:
+            if len(self._initialized_keys) == MAX_INITIALIZED_KEY:
+                self._initialized_keys.clear()
+            if callable(self._default):
+                if key in self._initialized_keys:
+                    # it means that we have already asked the repository
+                    return NotFound
+
+                simple_copy = True
+
+                # first, tries to use the default value
+                value = self._default(self._sdp, key) if self._sdp else self._default(key)
+                if value is NotFound and alt_sdp and not self._is_cleared:
+                    # try in the alternate (Sheerka) Data Provider
+                    value = self._alt_sdp_get(alt_sdp, key)
+                    simple_copy = False  # in the case, make sure to make a deep copy
+
+                if value is not NotFound:
+                    value = value if simple_copy else sheerka_deepcopy(value)
+                    self._cache[key] = value
+
+                    # update _current_size
+                    if isinstance(value, (list, set)):
+                        self._current_size += len(value)
+                    else:
+                        self._current_size += 1
+
+                    if self._max_size and self._current_size > self._max_size:
+                        self.evict(self._current_size - self._max_size)
+
+                # else 'value' remains NotFound
+            else:
+                value = self._default
+            self._initialized_keys.add(key)
+
+        return value
+
+    def _alt_get(self, key: str):
+        return self._get(key)  # by default, point to _get
+
+    def _put(self, key: str, value: object, alt_sdp):
+        """
+        To be defined in subclass
+        :param key:
+        :type key:
+        :param value:
+        :type value:
+        :param alt_sdp:
+        :type alt_sdp:
+        :return:
+        :rtype:
+        """
+        pass
+
+    def _update(self, old_key: str, old_value: object, new_key: str, new_value: object, alt_sdp):
+        """
+        To be defined in subclass
+        :param old_key:
+        :type old_key:
+        :param old_value:
+        :type old_value:
+        :param new_key:
+        :type new_key:
+        :param new_value:
+        :type new_value:
+        :param alt_sdp:
+        :type alt_sdp:
+        :return:
+        :rtype:
+        """
+        pass
+
+    def _delete(self, key: str, value: object, alt_sdp):
+        raise NotImplementedError("_delete BaseCache")

src/caching/Cache.py

@@ -0,0 +1,40 @@
+from caching.BaseCache import BaseCache
+from common.global_symbols import Removed
+
+
+class Cache(BaseCache):
+    """
+    An in memory FIFO cache object
+    When the max_size is reach the first element that was put is removed
+    When you put the same key twice, the previous element is overridden
+    """
+
+    def _put(self, key: str, value: object, alt_sdp):
+        res = key not in self._cache
+        self._cache[key] = value
+        self._add_to_add(key)
+        return res
+
+    def _update(self, old_key: str, old_value: object, new_key: str, new_value: object, alt_sdp):
+        self._cache[new_key] = new_value
+        self._add_to_add(new_key)
+
+        if new_key != old_key:
+            self._sync(old_key)
+            if not self._is_cleared and alt_sdp and self._extend_exists and self._extend_exists(alt_sdp, old_key):
+                self._cache[old_key] = Removed
+                self._add_to_add(old_key)
+                self._current_size += 1
+            else:
+                del (self._cache[old_key])
+                self._add_to_remove(old_key)
+
+    def _delete(self, key: str, value: object, alt_sdp):
+        if not self._is_cleared and alt_sdp and self._extend_exists and self._extend_exists(alt_sdp, key):
+            self._cache[key] = Removed
+            self._add_to_add(key)
+            # do not decrease self._current_size as 'Removed' takes on slot
+        else:
+            del (self._cache[key])
+            self._add_to_remove(key)
+            self._current_size -= 1

src/caching/CacheManager.py

@@ -0,0 +1,342 @@
+from dataclasses import dataclass, field
+from threading import RLock
+from typing import Callable
+
+from caching.BaseCache import BaseCache
+from common.global_symbols import NotFound
+from core.concept import Concept, ConceptMetadata
+
+
+@dataclass
+class MultipleEntryError(Exception):
+    """
+    Exception raised when trying to alter an entry with multiple element
+    without giving the origin of the element
+    """
+
+    key: str
+
+
+@dataclass
+class ConceptNotFound(Exception):
+    """
+    Thrown when you try to remove a concept that is not found
+    """
+    concept: object
+
+
+@dataclass
+class CacheDefinition:
+    cache: BaseCache
+    use_ref: bool
+    get_key: Callable[[ConceptMetadata], str] | None = field(repr=False)
+    persist: bool = True
+
+
+class CacheManager:
+    """
+    Single class to manage all the caches
+    """
+
+    def __init__(self, sdp=None):
+        """
+        Manager for all the caches.
+        Make the link between the cache and the SheerkaDataProvider
+        :param sdp:
+        :type sdp:
+        """
+        self.sdp = sdp
+        self.caches: dict[str, CacheDefinition] = {}  # dict {cache_name: CacheDefinition}
+        self.concept_caches = []
+        self.is_dirty = False  # to indicate that the value of a cache has changed
+
+        self._lock = RLock()
+
+    def register_concept_cache(self, name, cache, get_key, use_ref):
+        """
+        Special caches to manage concept definition
+        They store concepts metadata, using specific index
+        For example, you may declare an index to store metadata by id, by key or by whatever you need.
+
+        It a convenient way to manage indexes for concepts definitions
+        Underneath, there are simple `Cache` objects.
+        :param name:
+        :param cache:
+        :param get_key:
+        :param use_ref:
+        :return:
+        """
+        with self._lock:
+            if self.sdp:
+                cache.configure(sdp=self.sdp)
+            self.caches[name] = CacheDefinition(cache, use_ref, get_key)
+            self.concept_caches.append(name)
+
+    def register_cache(self, name, cache, persist=True, use_ref=False):
+        """
+        Define which type of cache along with how to compute the key
+        :param name:
+        :param cache:
+        :param persist:
+        :param use_ref:
+        :return:
+        """
+        with self._lock:
+            if self.sdp:
+                cache.configure(sdp=self.sdp)
+
+            self.caches[name] = CacheDefinition(cache, use_ref, None, persist)
+
+    def add_concept(self, metadata: ConceptMetadata, alt_sdp=None):
+        """
+        We need multiple indexes to retrieve a concept
+        So the new concept is dispatched into multiple caches
+        :param metadata:
+        :param alt_sdp: if not found in self.sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            for name in self.concept_caches:
+                cache_def = self.caches[name]
+                key = cache_def.get_key(metadata)
+                if key is None:
+                    raise KeyError("")
+                cache_def.cache.put(key, metadata, alt_sdp)
+
+            self.is_dirty = True
+
+    def update_concept(self, old: ConceptMetadata, new: ConceptMetadata, alt_sdp=None):
+        """
+        Update a concept.
+        :param old:  old version of the concept
+        :param new: new version of the concept
+        :param alt_sdp: if not found in self.sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            for cache_name in self.concept_caches:
+                cache_def = self.caches[cache_name]
+
+                old_key = cache_def.get_key(old)
+                new_key = cache_def.get_key(new)
+                cache_def.cache.update(old_key, old, new_key, new, alt_sdp=alt_sdp)
+
+            self.is_dirty = True
+
+    def remove_concept(self, concept: ConceptMetadata, alt_sdp=None):
+        """
+        Remove a concept from all caches
+        :param concept:
+        :param alt_sdp: if not found in self.sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            # the first concept cache must the one where all concept are unique
+            # eg it has to be the concept by id
+            ref_cache_def = self.caches[self.concept_caches[0]]
+            concept_id = ref_cache_def.get_key(concept)
+            ref_concept = ref_cache_def.cache.get(concept_id)
+
+            if ref_concept is NotFound and alt_sdp:
+                ref_concept = alt_sdp.get(self.concept_caches[0], concept_id)
+
+            if ref_concept is NotFound:
+                raise ConceptNotFound(concept)
+
+            for cache_name in self.concept_caches:
+                cache_def = self.caches[cache_name]
+                key = cache_def.get_key(ref_concept)
+                cache_def.cache.delete(key, ref_concept, alt_sdp=alt_sdp)
+
+            self.is_dirty = True
+
+    def get(self, cache_name, key, alt_sdp=None):
+        """
+        From concept cache, get an entry
+        :param cache_name:
+        :param key:
+        :param alt_sdp: if not found in self.sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            return self.caches[cache_name].cache.get(key, alt_sdp)
+
+    def alt_get(self, cache_name, key):
+        """
+        Alternate way to get an entry, from concept cache
+        This is mainly used for IncCache, in order to get the value without increasing it
+        :param cache_name:
+        :param key:
+        :return:
+        """
+        with self._lock:
+            return self.caches[cache_name].cache.alt_get(key)
+
+    def put(self, cache_name, key, value, alt_sdp=None):
+        """
+        Add to a cache
+        :param cache_name:
+        :param key:
+        :param value:
+        :param alt_sdp: if not found in self.sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            self.caches[cache_name].cache.put(key, value, alt_sdp)
+            self.is_dirty = True
+
+    def delete(self, cache_name, key, value=None, alt_sdp=None):
+        """
+        Delete an entry from the cache
+        :param cache_name:
+        :param key:
+        :param value:
+        :param alt_sdp: if not found in self.sdp, look in other repositories
+        :return:
+        """
+        with self._lock:
+            if self.caches[cache_name].cache.delete(key, value, alt_sdp):
+                self.is_dirty = True
+
+    def get_inner_cache(self, cache_name) -> BaseCache:
+        """
+        Return the BaseCache object
+        :param cache_name:
+        :return:
+        """
+        with self._lock:
+            return self.caches[cache_name].cache
+
+    def copy(self, cache_name) -> dict:
+        """
+        get a copy the content of the whole cache as a dictionary
+        :param self:
+        :param cache_name:
+        :return:
+        """
+        return self.caches[cache_name].cache.copy()
+
+    def populate(self, cache_name, populate_function, get_key_function, reset_events=False):
+        """
+        Populate a specific cache with a bunch of items
+        :param cache_name:
+        :param populate_function: how to get the items
+        :param get_key_function: how to get the key, out of an item
+        :param reset_events: reset to_add and to_remove events after populate
+        :return:
+        """
+        with self._lock:
+            self.caches[cache_name].cache.populate(populate_function, get_key_function, reset_events)
+
+    def force_value(self, cache_name, key, value):
+        """
+        Update the content of the cache, but does not raise any event
+        """
+        with self._lock:
+            self.caches[cache_name].cache.force_value(key, value)
+
+    def remove_initialized_key(self, cache_name, key):
+        """
+
+        """
+        with self._lock:
+            self.caches[cache_name].cache.remove_initialized_key(key)
+
+    def has(self, cache_name, key):
+        """
+        True if the value is in cache only. Never try to look in a remote repository
+        :param cache_name:
+        :param key:
+        :return:
+        """
+        with self._lock:
+            return self.caches[cache_name].cache.has(key)
+
+    def exists(self, cache_name, key):
+        """
+        True if the value is in cache.
+        If not found, may search in a remote repository
+        :param cache_name:
+        :param key:
+        :return:
+        """
+
+        with self._lock:
+            return self.caches[cache_name].cache.exists(key)
+
+    def commit(self, context):
+        """
+        Persist all the caches into a physical persistence storage
+        :param context:
+        :return:
+        """
+
+        def update_full_serialisation(items, value):
+            # Take care, infinite recursion is not handled !!
+            if isinstance(items, (list, set, tuple)):
+                for item in items:
+                    update_full_serialisation(item, value)
+            elif isinstance(items, dict):
+                for values in items.values():
+                    update_full_serialisation(values, value)
+            elif isinstance(items, Concept):
+                items.get_metadata().full_serialization = value
+
+        with self._lock:
+            with self.sdp.get_transaction(context.event.get_digest()) as transaction:
+                for cache_name, cache_def in self.caches.items():
+                    if not cache_def.persist:
+                        continue
+
+                    if cache_def.cache.is_cleared():
+                        transaction.clear(cache_name)
+
+                    for key in cache_def.cache.to_remove:
+                        transaction.remove(cache_name, key)
+
+                    for key in cache_def.cache.to_add:
+                        if key == "*self*":
+                            transaction.add(cache_name, None, cache_def.cache.dump()["cache"])
+                        else:
+                            to_save = cache_def.cache.inner_get(key)
+                            update_full_serialisation(to_save, True)
+                            transaction.add(cache_name, key, to_save, cache_def.use_ref)
+                            update_full_serialisation(to_save, False)
+
+                    cache_def.cache.reset_events()
+            self.is_dirty = False
+
+    def clear(self, cache_name=None, set_is_cleared=True):
+        with self._lock:
+            if cache_name:
+                self.caches[cache_name].cache.clear(set_is_cleared)
+            else:
+                for cache_def in self.caches.values():
+                    cache_def.cache.clear(set_is_cleared)
+
+    def dump(self):
+        """
+        For test purpose, dumps the whole content of the cache manager
+        :return:
+        """
+        with self._lock:
+            res = {}
+            for cache_name, cache_def in self.caches.items():
+                res[cache_name] = cache_def.cache.dump()
+
+            return res
+
+    def init_from_dump(self, dump):
+        with self._lock:
+            for cache_name, content in dump.items():
+                if cache_name in self.caches:
+                    self.caches[cache_name].cache.init_from_dump(content)
+
+        return self
+
+    def reset(self):
+        """For unit test speed enhancement"""
+        self.clear()
+        self.caches.clear()
+        self.concept_caches.clear()
+        self.is_dirty = False

src/caching/DictionaryCache.py

@@ -0,0 +1,88 @@
+from caching.BaseCache import MAX_INITIALIZED_KEY
+from caching.Cache import BaseCache
+from common.global_symbols import NotFound
+
+
+class DictionaryCache(BaseCache):
+    """
+    Kind of all or nothing dictionary database
+    You can get the values key by by
+    But when you want to put, you must put the whole database
+    For this reason, alt_sdp is not supported. The top ontology layer contains the whole database
+    """
+
+    def auto_configure(self, cache_name):
+        """
+        Convenient way to configure the cache
+        :param cache_name:
+        :return:
+        """
+        self._default = lambda sdp, key: sdp.get(cache_name)  # retrieve the whole entry
+        self._extend_exists = None  # not used
+        self._alt_sdp_get = None  # not used
+
+        return self
+
+    def _get(self, key, alt_sdp=None):
+        """
+        Management of the default is different
+        :param key:
+        :return:
+        """
+        try:
+            return self._cache[key]
+        except KeyError:
+            if key in self._initialized_keys:
+                return NotFound
+
+            if len(self._initialized_keys) == MAX_INITIALIZED_KEY:
+                self._initialized_keys.clear()
+
+            self._initialized_keys.add(key)
+
+            if callable(self._default):
+                default_values = self._default(self._sdp, key) if self._sdp else self._default(key)
+            else:
+                default_values = self._default
+
+            if isinstance(default_values, dict):
+                self._cache.update(default_values)  # update the whole cache dictionary to resync with remote sdp
+
+            self._count_items()
+            return self._cache[key] if key in self._cache else NotFound
+
+    def _put(self, key, value, alt_sdp):
+        """
+        Adds a whole dictionary
+        :param key: True to append, False to reset
+        :param value: dictionary
+        :param alt_sdp: NOT SUPPORTED as the values from alt_sdp must be retrieved and computed BEFORE the put
+        :return:
+        """
+        if not isinstance(key, bool):
+            raise KeyError
+
+        if not isinstance(value, dict):
+            raise ValueError
+
+        if key:  # update the current cache
+            if self._cache is None:
+                self._cache = value.copy()
+            else:
+                self._cache.update(value)
+        else:  # reset the current cache
+            self._cache = value
+
+        self._count_items()
+
+        # special meaning for to_add
+        self._add_to_add("*self*")
+        return False
+
+    def _delete(self, key, value, alt_sdp):
+        raise NotImplementedError("_delete DictionaryCache")
+
+    def _count_items(self):
+        self._current_size = 0
+        for v in self._cache.values():
+            self._current_size += len(v) if hasattr(v, "__len__") and not isinstance(v, str) else 1

src/caching/FastCache.py

@@ -0,0 +1,71 @@
+from common.global_symbols import NotFound
+
+
+class FastCache:
+    """
+    Simplest LRU cache
+    """
+
+    def __init__(self, max_size=256, default=None):
+        self.max_size = max_size
+        self.cache = {}
+        self.lru = []
+        self.default = default
+        self.calls = {}
+
+    def __contains__(self, item):
+        return self.has(item)
+
+    def __iter__(self):
+        yield from self.cache
+
+    def __next__(self):
+        return next(iter(self.cache))
+
+    def __len__(self):
+        return len(self.cache)
+
+    def put(self, key, value):
+        if len(self.cache) == self.max_size:
+            del self.cache[self.lru.pop(0)]
+
+        if key in self.cache:
+            self.lru.remove(key)
+
+        self.cache[key] = value
+        self.lru.append(key)
+        self.calls[key] = 0
+
+    def has(self, key):
+        return key in self.cache
+
+    def get(self, key):
+        try:
+            res = self.cache[key]
+            self.calls[key] += 1
+            return res
+        except KeyError:
+            if self.default:
+                value = self.default(key)
+                self.put(key, value)
+                return value
+
+            return NotFound
+
+    def evict_by_key(self, predicate):
+        to_remove = []
+
+        for k, v in self.cache.items():
+            if predicate(k):
+                to_remove.append(k)
+
+        for k in to_remove:
+            self.lru.remove(k)
+            del self.cache[k]
+
+    def copy(self):
+        return self.cache.copy()
+
+    def clear(self):
+        self.cache.clear()
+        self.lru.clear()

src/caching/IncCache.py

@@ -0,0 +1,24 @@
+from caching.Cache import Cache
+from common.global_symbols import NotFound, Removed
+
+
+class IncCache(Cache):
+    """
+    Increment the value of the key every time it's accessed
+    """
+
+    def _get(self, key, alt_sdp=None):
+        value = super()._get(key, alt_sdp=alt_sdp)
+        if value in (NotFound, Removed):
+            value = 0
+        value += 1
+        self._put(key, value, alt_sdp)
+        return value
+
+    def _put(self, key, value, alt_sdp):
+        self._cache[key] = value
+        self._add_to_add(key)
+        return True
+
+    def _alt_get(self, key):
+        return super()._get(key)  # point to parent, not to self

src/caching/ListCache.py

@@ -0,0 +1,66 @@
+from caching.Cache import BaseCache
+from common.global_symbols import NotFound, Removed
+from common.utils import sheerka_deepcopy
+
+
+
+class ListCache(BaseCache):
+    """
+    An in memory FIFO cache object
+    When the max_size is reach the first element that was put is removed
+    Items of this cache are list
+    """
+
+    def _put(self, key, value, alt_sdp):
+        if key in self._cache:
+            self._cache[key].append(value)
+        else:
+            self._sync(key)
+
+            if key not in self._cache and alt_sdp and not self._is_cleared:
+                previous = self._alt_sdp_get(alt_sdp, key)
+                if previous not in (NotFound, Removed):
+                    self._cache[key] = sheerka_deepcopy(previous)
+
+            if key in self._cache:
+                self._cache[key].append(value)
+            else:
+                self._cache[key] = [value]
+
+        self._add_to_add(key)
+        return True
+
+    def _update(self, old_key, old_value, new_key, new_value, alt_sdp):
+        self._sync(old_key, new_key)
+
+        if old_key not in self._cache and alt_sdp and not self._is_cleared:
+            # no value found in local cache or remote repository
+            # Use the values from alt_sdp
+            previous = self._alt_sdp_get(alt_sdp, old_key)
+            if previous in (NotFound, Removed):
+                raise KeyError(old_key)
+
+            self._cache[old_key] = sheerka_deepcopy(previous)
+            self._current_size += len(previous)
+
+        if old_key != new_key:
+            self._cache[old_key].remove(old_value)
+            if len(self._cache[old_key]) == 0:
+                if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, old_key):
+                    self._cache[old_key] = Removed
+                    self._add_to_add(old_key)
+                    self._current_size += 1
+                else:
+                    del (self._cache[old_key])
+                    self._add_to_remove(old_key)
+            else:
+                self._add_to_add(old_key)
+
+            self._put(new_key, new_value, alt_sdp)
+            self._add_to_add(new_key)
+        else:
+            for i in range(len(self._cache[new_key])):
+                if self._cache[new_key][i] == old_value:
+                    self._cache[new_key][i] = new_value  # avoid add and remove in dict
+                    break  # only the first one is affected
+            self._add_to_add(new_key)

src/caching/ListIfNeededCache.py

@@ -0,0 +1,134 @@
+from caching.Cache import BaseCache
+from common.global_symbols import NotFound, Removed
+from common.utils import sheerka_deepcopy
+
+
+class ListIfNeededCache(BaseCache):
+    """
+    An in memory FIFO cache object
+    When the max_size is reach the first element that was put is removed
+    When you put the same key twice, you now have a list of two elements
+    """
+
+    def _put(self, key, value, alt_sdp):
+        if key in self._cache:
+            if isinstance(self._cache[key], list):
+                self._cache[key].append(value)
+            else:
+                self._cache[key] = value if self._cache[key] is Removed else [self._cache[key], value]
+        else:
+            self._sync(key)
+
+            if key not in self._cache and alt_sdp and not self._is_cleared:
+                previous = self._alt_sdp_get(alt_sdp, key)
+                if previous not in (NotFound, Removed):
+                    self._cache[key] = sheerka_deepcopy(previous)
+
+            if key in self._cache:
+                if isinstance(self._cache[key], list):
+                    self._cache[key].append(value)
+                else:
+                    self._cache[key] = value if self._cache[key] is Removed else [self._cache[key], value]
+            else:
+                self._cache[key] = value
+        self._add_to_add(key)
+        return True
+
+    def _update(self, old_key, old_value, new_key, new_value, alt_sdp):
+
+        self._sync(old_key, new_key)
+
+        if old_key not in self._cache and alt_sdp and not self._is_cleared:
+            # no value found in local cache or remote repository
+            # Use the values from alt_sdp
+            previous = self._alt_sdp_get(alt_sdp, old_key)
+            if previous in (NotFound, Removed):
+                raise KeyError(old_key)
+
+            self._cache[old_key] = sheerka_deepcopy(previous)
+            self._current_size += len(previous) if isinstance(previous, list) else 1
+
+        if old_key != new_key:
+            if isinstance(self._cache[old_key], list):
+                self._cache[old_key].remove(old_value)
+                if len(self._cache[old_key]) == 1:
+                    self._cache[old_key] = self._cache[old_key][0]
+                self._add_to_add(old_key)
+            else:
+                if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, old_key):
+                    self._cache[old_key] = Removed
+                    self._add_to_add(old_key)
+                    self._current_size += 1
+                else:
+                    del (self._cache[old_key])
+                    self._add_to_remove(old_key)
+
+            self._put(new_key, new_value, alt_sdp)
+            self._add_to_add(new_key)
+        else:
+            if isinstance(self._cache[new_key], list):
+                for i in range(len(self._cache[new_key])):
+                    if self._cache[new_key][i] == old_value:
+                        self._cache[new_key][i] = new_value  # avoid add and remove in dict
+                        break
+            else:
+                self._cache[new_key] = new_value
+            self._add_to_add(new_key)
+
+    def _delete(self, key, value, alt_sdp):
+        if value is None:
+            # Remove the whole key
+            if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key):
+                if key in self._cache:
+                    previous = self._cache[key]
+                    if isinstance(previous, list):
+                        self._current_size -= len(previous) + 1
+                else:
+                    self._current_size += 1
+
+                self._cache[key] = Removed
+                self._add_to_add(key)
+            else:
+                previous = self._cache[key]
+                self._current_size -= len(previous) if isinstance(previous, list) else 1
+                del self._cache[key]
+                self._add_to_remove(key)
+
+        else:
+            # Remove a single value
+            try:
+                previous = self._cache[key]
+                if isinstance(previous, list):
+                    previous.remove(value)
+                    self._cache[key] = previous[0] if len(previous) == 1 else previous
+                    self._current_size -= 1
+                    self.to_add.add(key)
+                else:
+                    if previous == value:
+                        # I am about to delete the entry
+                        if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key):
+                            self._cache[key] = Removed
+                            self.to_add.add(key)
+                            # self._current_size -= 1 # Do not decrease size, as it's replaced by 'Removed'
+                        else:
+                            del self._cache[key]
+                            self._current_size -= 1
+                            self.to_remove.add(key)
+            except KeyError as ex:
+                previous = self._alt_sdp_get(alt_sdp, key) if not self._is_cleared and alt_sdp else NotFound
+                if previous in (NotFound, Removed):
+                    raise ex
+
+                if isinstance(previous, list):
+                    previous = sheerka_deepcopy(previous)
+                    previous.remove(value)  # raise an exception if value in not in the list
+                    self._cache[key] = previous[0] if len(previous) == 1 else previous
+                    self._current_size -= 1
+                    self.to_add.add(key)
+                else:
+                    if previous == value:
+                        self._cache[key] = Removed
+                        self.to_add.add(key)
+                        self._current_size -= 1
+
+        return True

src/caching/SetCache.py

@@ -0,0 +1,122 @@
+from caching.Cache import BaseCache
+from common.global_symbols import NotFound, Removed
+from common.utils import sheerka_deepcopy
+
+
+class SetCache(BaseCache):
+    """
+    An in memory FIFO cache object
+    When the max_size is reach the first element that was put is removed
+
+    You can use the same key multiple times, but the elements under this key will be unique
+    When there are multiple elements, a python set is used
+
+    >> self.put('key', 'value1')
+    >> assert {'value1'} == self.get('key')
+    >> self.put('key', 'value2')
+    >> assert {'value1', 'value2'} == self.get('key')
+    """
+
+    def _put(self, key, value, alt_sdp):
+        if key in self._cache:
+            if self._cache[key] is Removed:
+                self._cache[key] = {value}
+            elif value in self._cache[key]:
+                return False
+            else:
+                self._cache[key].add(value)
+        else:
+            self._sync(key)
+
+            if key not in self._cache and alt_sdp and not self._is_cleared:
+                previous = self._alt_sdp_get(alt_sdp, key)
+                if previous not in (NotFound, Removed):
+                    self._cache[key] = sheerka_deepcopy(previous)
+
+            if key in self._cache:
+                if self._cache[key] == Removed:
+                    self._cache[key] = {value}
+                else:
+                    self._cache[key].add(value)
+            else:
+                self._cache[key] = {value}
+
+        self._add_to_add(key)
+        return True
+
+    def _update(self, old_key, old_value, new_key, new_value, alt_sdp):
+        self._sync(old_key, new_key)
+
+        if old_key not in self._cache and alt_sdp and not self._is_cleared:
+            # no value found in local cache or remote repository
+            # Use the values from alt_sdp
+            previous = self._alt_sdp_get(alt_sdp, old_key)
+            if previous in (NotFound, Removed):
+                raise KeyError(old_key)
+
+            self._cache[old_key] = sheerka_deepcopy(previous)
+            self._current_size += len(previous)
+
+        if old_key != new_key:
+            if isinstance(self._cache[old_key], set):
+                self._cache[old_key].remove(old_value)
+                if len(self._cache[old_key]) == 0:
+                    if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, old_key):
+                        self._cache[old_key] = Removed
+                        self._add_to_add(old_key)
+                        self._current_size += 1
+                    else:
+                        del (self._cache[old_key])
+                        self._add_to_remove(old_key)
+                else:
+                    self._add_to_add(old_key)
+
+            self._put(new_key, new_value, alt_sdp)
+            self._add_to_add(new_key)
+        else:
+            self._cache[new_key].remove(old_value)
+            self._put(new_key, new_value, alt_sdp)
+            self._add_to_add(new_key)
+
+    def _delete(self, key, value, alt_sdp):
+        if value is None:
+            if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key):
+                self._current_size += 1 - len(self._cache[key]) if key in self._cache else 1
+                self._cache[key] = Removed
+                self._add_to_add(key)
+            else:
+                self._current_size -= len(self._cache[key])
+                del self._cache[key]
+                self._add_to_remove(key)
+
+        else:
+            try:
+                self._cache[key].remove(value)
+                if len(self._cache[key]) == 0:
+                    if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key):
+                        self._cache[key] = Removed
+                        self._add_to_add(key)
+                        # self._current_size -= 1 # Do not decrease size, as it's replaced by 'Removed'
+                    else:
+                        del self._cache[key]
+                        self._add_to_remove(key)
+                        self._current_size -= 1
+                else:
+                    self._add_to_add(key)
+                    self._current_size -= 1
+            except KeyError as ex:
+                previous = self._alt_sdp_get(alt_sdp, key) if not self._is_cleared and alt_sdp else NotFound
+                if previous in (NotFound, Removed):
+                    raise ex
+
+                previous = sheerka_deepcopy(previous)
+                previous.remove(value)  # will raise a KeyError if value is not in the set
+                if len(previous) == 0:
+                    self._cache[key] = Removed
+                    self._current_size += 1
+                else:
+                    self._cache[key] = previous
+                    self._current_size += len(previous)
+                self._add_to_add(key)
+
+        return True

src/client.py

@@ -30,6 +30,7 @@ class SheerkaClient:
         self.url = f"{self.hostname}:{self.port}" if self.port else f"{self.hostname}"
         self.history_file = path.abspath(path.join(path.expanduser("~"), ".sheerka", "history.txt"))
         self.token = None
+        self.user = None
 
     def init_folder(self):
         root_path = path.dirname(self.history_file)
@@ -56,7 +57,9 @@ class SheerkaClient:
             form_data = {"username": username, "password": password}
             res = requests.post(token_url, data=form_data)
             if res:
-                self.token = res.json()["access_token"]
+                as_json = res.json()
+                self.token = as_json["access_token"]
+                self.user = {"first_name": as_json["first_name"], "last_name": as_json["last_name"]}
                 return TestResponse(True, f"Connected as {username}")
             else:
                 self.token = None
@@ -65,6 +68,14 @@ class SheerkaClient:
             self.token = None
             return TestResponse(False, str(ex))
 
+    def help(self):
+        self.print_info("Basic commands:")
+        self.print_info("  clear")
+        self.print_info("  connect")
+        if not self.token:
+            self.print_error("You are not connected.")
+            self.print_info("You should use the command `connect('username', 'password'`)")
+
     def run(self):
         while True:
             try:
@@ -80,6 +91,10 @@ class SheerkaClient:
                     prompt_toolkit.shortcuts.clear()
                     continue
 
+                if _in == "help":
+                    self.help()
+                    continue
+
                 # allow reconnection
                 m = connect_regex.match(_in)
                 if m:
@@ -92,9 +107,9 @@ class SheerkaClient:
                 # Call Sheerka
                 if self.token:
                     headers = {"Authorization": f"Bearer {self.token}"}
-                    response = requests.post(f"{self.url}/echo/{_in}", headers=headers)
+                    response = requests.post(f"{self.url}/command/{_in}", headers=headers)
                 else:
-                    response = requests.post(f"{self.url}/echo/{_in}")
+                    response = requests.post(f"{self.url}/command/{_in}")
 
                 # read the response from the server
                 if response:

src/common/global_symbols.py

@@ -0,0 +1,66 @@
+EVENT_CONCEPT_PRECEDENCE_MODIFIED = "evt_cp_m"
+EVENT_RULE_PRECEDENCE_MODIFIED = "evt_rp_m"
+EVENT_CONTEXT_DISPOSED = "evt_ctx_d"
+EVENT_USER_INPUT_EVALUATED = "evt_ui_e"
+EVENT_CONCEPT_CREATED = "evt_c_c"
+EVENT_CONCEPT_MODIFIED = "evt_c_m"
+EVENT_CONCEPT_DELETED = "evt_c_d"
+EVENT_CONCEPT_ID_DELETED = "evt_c_id_d"
+EVENT_RULE_CREATED = "evt_r_c"
+EVENT_RULE_DELETED = "evt_r_d"
+EVENT_RULE_ID_DELETED = "evt_r_id_d"
+EVENT_ONTOLOGY_CREATED = "evt_o_c"
+EVENT_ONTOLOGY_DELETED = "evt_o_d"
+
+VARIABLE_PREFIX = "__var__"
+
+
+class CustomType:
+    """
+    Base class for custom types used in Sheerka
+    A custom type is a type that has only one instance across the application and have a semantic meaning
+    For example the type 'None' is a singleton which have a semantic meaning.
+    We need to define others in Sheerka
+    """
+
+    def __init__(self, value):
+        self.value = value
+
+    def __repr__(self):
+        return self.value
+
+    def __eq__(self, other):
+        return isinstance(other, CustomType) and self.value == other.value
+
+    def __hash__(self):
+        return hash(self.value)
+
+
+class NotInitType(CustomType):
+    def __init__(self):
+        super(NotInitType, self).__init__("**NotInit**")
+
+
+class NotFoundType(CustomType):
+    """
+    Using when an entry in not found in Cache or in sdp
+    """
+
+    def __init__(self):
+        super(NotFoundType, self).__init__("**NotFound**")
+
+
+class RemovedType(CustomType):
+    def __init__(self):
+        super(RemovedType, self).__init__("**Removed**")
+
+
+class NoFirstTokenType(CustomType):
+    def __init__(self):
+        super(NoFirstTokenType, self).__init__("**NoFirstToken**")
+
+
+NotInit = NotInitType()
+NotFound = NotFoundType()
+Removed = RemovedType()
+NoFirstToken = NoFirstTokenType()

src/common/utils.py

@@ -0,0 +1,293 @@
+import importlib
+import pkgutil
+from copy import deepcopy
+
+from common.global_symbols import CustomType
+
+
+def get_class(qname):
+    """
+    Loads a class from its full qualified name
+    :param qname:
+    :return:
+    """
+    parts = qname.split('.')
+    module = ".".join(parts[:-1])
+    m = __import__(module)
+    for comp in parts[1:]:
+        m = getattr(m, comp)
+    return m
+
+
+def get_module(qname):
+    """
+    Loads a module from its full qualified name
+    :param qname:
+    :return:
+    """
+    parts = qname.split('.')
+    m = __import__(qname)
+    for comp in parts[1:]:
+        m = getattr(m, comp)
+    return m
+
+
+def get_full_qualified_name(obj):
+    """
+    Returns the full qualified name of a class (including its module name )
+    :param obj:
+    :return:
+    """
+    if obj.__class__ == type:
+        module = obj.__module__
+        if module is None or module == str.__class__.__module__:
+            return obj.__name__  # Avoid reporting __builtin__
+        else:
+            return module + '.' + obj.__name__
+    else:
+        module = obj.__class__.__module__
+        if module is None or module == str.__class__.__module__:
+            return obj.__class__.__name__  # Avoid reporting __builtin__
+        else:
+            return module + '.' + obj.__class__.__name__
+
+
+def get_logger_name(name):
+    """
+    Wrapper to fancy how the name of the loggers are returned
+    :param name:
+    :type name:
+    :return:
+    :rtype:
+    """
+    if name is None:
+        return None
+    return name.split(".")[-1]
+
+
+def import_module_and_sub_module(module_name):
+    """
+    Import the module, and one sub level
+    :param module_name:
+    :return:
+    """
+    mod = get_module(module_name)
+    for (module_loader, name, ispkg) in pkgutil.iter_modules(mod.__path__, module_name + "."):
+        importlib.import_module(name)
+
+
+def get_sub_classes(package_name, base_class):
+    def _get_class(name):
+        modname, _, clsname = name.rpartition('.')
+        mod = importlib.import_module(modname)
+        cls = getattr(mod, clsname)
+        return cls
+
+    base_class = _get_class(base_class) if isinstance(base_class, str) else base_class
+    all_class = set(base_class.__subclasses__()).union(
+        [s for c in base_class.__subclasses__() for s in get_sub_classes(package_name, c)])
+
+    # limit to the classes of the package
+    return [c for c in all_class if c.__module__.startswith(package_name)]
+
+
+def sheerka_deepcopy(obj):
+    """
+    Internal implementation of deepcopy that will eventually handle Concept's circular references
+    :param obj:
+    :return:
+    """
+    from core.concept import Concept
+    already_seen = {}
+
+    def copy_concept(c: Concept):
+        id_c = id(c)
+        if id_c in already_seen:
+            ref = already_seen[id_c]
+            if ref == '_##_REF_##_':
+                raise Exception("Circular Ref not managed yet!")
+            else:
+                return ref
+
+        already_seen[id_c] = '_##_REF_##_'
+
+        cls = type(c)
+        instance = cls()
+        # update the metadata
+        for prop_name, prop_value in vars(c.get_metadata()).items():
+            if prop_name != "props":
+                setattr(instance.get_metadata(), prop_name, prop_value)
+            else:
+                setattr(instance.get_metadata(), prop_name, sheerka_deepcopy(prop_value))
+
+        # update the values
+        for prop_name, prop_value in c.values().items():
+            setattr(instance, prop_name, prop_value)
+
+        already_seen[id_c] = instance
+        return instance
+
+    if isinstance(obj, CustomType):
+        return obj
+    elif isinstance(obj, dict):
+        res = {sheerka_deepcopy(k): sheerka_deepcopy(v) for k, v in obj.items()}
+        return res
+    elif isinstance(obj, list):
+        return [sheerka_deepcopy(item) for item in obj]
+    elif isinstance(obj, set):
+        return {sheerka_deepcopy(item) for item in obj}
+    elif isinstance(obj, tuple):
+        return tuple((sheerka_deepcopy(item) for item in obj))
+    elif isinstance(obj, Concept):
+        return copy_concept(obj)
+    else:
+        return deepcopy(obj)
+
+
+def str_concept(t, drop_name=None, prefix="c:"):
+    """
+    The key,id identifiers of a concept are stored in a tuple
+    we want to return the key and the id, separated by a pipe
+    None value must be replaced by an empty string
+
+    >>> assert str_concept(("key", "id")) == "c:key#id:"
+    >>> assert str_concept((None, "id")) == "c:#id:"
+    >>> assert str_concept(("key", None)) == "c:key:"
+    >>> assert str_concept((None, None)) == ""
+    >>> assert str_concept(Concept(name="foo", id="bar")) == "c:foo#bar:"
+    >>> assert str_concept(Concept(name="foo", id="bar"), drop_name=True) == "c:#bar:"
+    >>> assert str_concept(("key", "id"), prefix='r:') == "r:key|id:"
+    :param t:
+    :param drop_name: True if we only want the id (and not the key)
+    :param prefix:
+    :return:
+    """
+    if isinstance(t, tuple):
+        name, id_ = t[0], t[1]
+    else:
+        name, id_ = t.name, t.id
+
+    if name is None and id_ is None:
+        return ""
+
+    result = prefix if (name is None or drop_name) else prefix + name
+    if id_:
+        result += "#" + id_
+    return result + ":"
+
+
+def unstr_concept(concept_repr, prefix='c:'):
+    """
+    if concept_repr is like :c:key:id:
+    return the key and the id
+    >>> assert unstr_concept("c:key:") == ("key", None)
+    >>> assert unstr_concept("c:key#id:") == ("key", "id")
+    >>> assert unstr_concept("c:#id:") == ("None", "id")
+    >>> assert unstr_concept("c:key#:") == ("key", "None")
+    >>> assert unstr_concept("r:key#id:", prefix='r:') == ("key", "id")
+    >>> # Otherwise, return (None,None)
+
+    :param concept_repr:
+    :return:
+    """
+    if not (concept_repr and
+            isinstance(concept_repr, str) and
+            concept_repr.startswith(prefix) and
+            concept_repr.endswith(":")):
+        return None, None
+
+    i = 2
+    length = len(concept_repr)
+    key = ""
+    while i < length:
+        c = concept_repr[i]
+        i += 1
+        if c in (":", "#"):
+            break
+        key += c
+    else:
+        return None, None
+
+    if c == ":":
+        return key if key != "" and i == length else None, None
+
+    c_id = ""
+    while i < length:
+        c = concept_repr[i]
+        i += 1
+        if c == ":":
+            break
+        c_id += c
+    else:
+        return None, None
+
+    if i != length:
+        return None, None
+
+    return key if key != "" else None, c_id if c_id != "" else None
+
+
+def compute_hash(obj):
+    """
+    Helper to get the hash from collection
+    :param obj:
+    :type obj:
+    :return:
+    :rtype:
+    """
+    try:
+        if isinstance(obj, (list, tuple)):
+            return hash(tuple([compute_hash(o) for o in obj]))
+
+        if isinstance(obj, set):
+            return hash(tuple([compute_hash(o) for o in sorted(list(obj))]))
+
+        if isinstance(obj, dict):
+            return hash(repr(obj))
+
+        return hash(obj)
+    except:
+        return 0
+
+
+def decode_enum(enum_repr: str):
+    """
+    Tries to transform ClassName.Name into an enum
+    :param enum_repr:
+    :return:
+    """
+    if not (enum_repr and isinstance(enum_repr, str)):
+        return None
+
+    try:
+        idx = enum_repr.rindex(".")
+        if idx == len(enum_repr):
+            return None
+
+        cls_name = enum_repr[:idx]
+        cls = get_class(cls_name)
+        name = enum_repr[idx + 1:]
+        return cls[name]
+
+    except ValueError:
+        return None
+
+    except TypeError:
+        return None
+
+
+def to_dict(items, get_attr):
+    """
+    Create a dictionary from a list when duplicates keys are merged into lists
+    :param items:
+    :type items:
+    :param get_attr:
+    :type get_attr:
+    :return:
+    :rtype:
+    """
+    res = {}
+    for item in items:
+        res.setdefault(get_attr(item), []).append(item)
+
+    return res

src/core/BuiltinConcepts.py

@@ -0,0 +1,7 @@
+class BuiltinConcepts:
+    SHEERKA = "__SHEERKA"
+
+    NEW_CONCEPT = "__NEW_CONCEPT"
+    UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT"
+    USER_INPUT = "__USER_INPUT"
+    PARSER_INPUT = "__PARSER_INPUT"

src/core/ErrorContext.py

@@ -0,0 +1,35 @@
+from common.utils import compute_hash
+from core.ExecutionContext import ExecutionContext
+
+
+class SheerkaException(Exception):
+    pass
+
+
+class ErrorContext:
+    """
+    This class represents the result of a data flow processing
+    """
+
+    def __init__(self, who: str, context: ExecutionContext, value: object = None):
+        self.who = who
+        self.context = context
+        self.value = value
+        self.parents = None
+
+    def __repr__(self):
+        return f"Error(who={self.who}, context_id={self.context.long_id}, value={self.value})"
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, ErrorContext):
+            return False
+
+        return self.who == other.who and \
+            self.context.id == other.context.id and \
+            self.value == other.value
+
+    def __hash__(self):
+        return hash((self.who, self.context.id, compute_hash(self.value)))

src/core/Event.py

@@ -0,0 +1,69 @@
+import hashlib
+from datetime import datetime
+
+
+class Event(object):
+    """
+    Class that represents something that modifies the state of the system
+    """
+
+    def __init__(self, message="", user_id="", date=None, parents=None):
+        self.user_id: str = user_id  # id of the user that triggers the modification
+        self.date: datetime | None = date or datetime.now()  # when
+        self.message: str = message  # user input or whatever that modifies the system
+        self.parents: list[str] = parents  # digest(s) of the parent(s) of this event
+        self._digest: str | None = None  # digest of the event
+
+    def __str__(self):
+        return f"{self.date.strftime('%d/%m/%Y %H:%M:%S')} {self.message}"
+
+    def __repr__(self):
+        return f"{self.get_digest()[:12]} {self.message}"
+
+    def get_digest(self):
+        """
+        Returns the digest of the event
+        :return: sha256 of the event
+        """
+
+        if self._digest:
+            return self._digest
+
+        if self.user_id == "":
+            # only possible during the unit test
+            # We use this little trick to speed up the unit test
+            self._digest = self.message[6:] if self.message.startswith("TEST::") else "xxx"
+            return self._digest
+
+        if not isinstance(self.message, str):
+            raise NotImplementedError(f"message={self.message}")
+
+        to_hash = f"Event:{self.user_id}{self.date}{self.message}{self.parents}".encode("utf-8")
+        self._digest = hashlib.sha256(to_hash).hexdigest()
+        return self._digest
+
+    def to_dict(self):
+        return self.__dict__
+
+    def from_dict(self, as_dict):
+        self.user_id = as_dict["user_id"]
+        self.date = datetime.fromisoformat(as_dict["date"])
+        self.message = as_dict["message"]
+        self.parents = as_dict["parents"]
+        self._digest = as_dict["_digest"]  # freeze the digest
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if isinstance(other, Event):
+            return (self.user_id == other.user_id and
+                    self.date == other.date and
+                    self.message == other.message and
+                    self.parents == other.parents)
+
+        return False
+
+    def __hash__(self):
+        return hash(self.get_digest())
+

src/core/ExecutionContext.py

@@ -1,2 +1,181 @@
+from __future__ import annotations
+
+import time
+
+from core.Event import Event
+
+
+class ExecutionContextActions:
+    TESTING = "Testing"
+    INIT_SHEERKA = "Init Sheerka"
+    EVALUATE_USER_INPUT = "Evaluate user input"
+    EVALUATING_STEP = "Evaluating step"
+    EVALUATING_ITERATION = "Evaluating iteration"
+    BEFORE_PARSING = "Before parsing"
+    PARSING = "Parsing"
+    AFTER_PARSING = "After parsing"
+    BEFORE_EVALUATION = "Before evaluation"
+    EVALUATION = "Evaluation"
+    AFTER_EVALUATION = "After Evaluation"
+
+
+class ContextHint:
+    REDUCE_CONCEPTS = "Reduce Concepts"  # to tell the process to only keep the meaningful results
+
+
+ids = {}  # keep track of the next execution context id, for a given event id
+
+
+def get_next_id(event_digest):
+    """
+    For a given event, give the next id
+    :param event_digest:
+    :type event_digest:
+    :return:
+    :rtype:
+    """
+    if event_digest in ids:
+        ids[event_digest] += 1
+    else:
+        ids[event_digest] = 0
+    return ids[event_digest]
+
+
 class ExecutionContext:
-    pass
+    """
+    To keep track of the execution of a request
+    Note that the protected hints are working correctly only if the hint is added BEFORE the creation of the child
+    """
+
+    def __init__(self,
+                 who: str,
+                 event: Event,
+                 sheerka,
+                 action: ExecutionContextActions,
+                 action_context: object,
+                 desc: str = None,
+                 logger=None,
+                 global_hints=None,
+                 protected_hints=None,
+                 parent: ExecutionContext = None):
+        self._id = get_next_id(event.get_digest())
+        self._parent = parent
+        self._children = []
+        self._start = 0  # when the execution starts (to measure elapsed time)
+        self._stop = 0  # when the execution stops (to measure elapses time)
+        self._logger = logger
+
+        self.who = who  # who is asking
+        self.event = event  # what was the (original) trigger
+        self.sheerka = sheerka  # sheerka
+        self.action = action
+        self.action_context = action_context
+        self.desc = desc  # human description of what is going on
+
+        self.private_hints = set()
+        self.protected_hints = set() if protected_hints is None else protected_hints.copy()
+        self.global_hints = set() if global_hints is None else global_hints
+
+        self.inputs = {}  # what were the parameters of the execution context
+        self.values = {}  # what was produced by the execution context
+
+    def __repr__(self):
+        msg = f"ExecutionContext(who={self.who}, id={self._id}, action={self.action}, context={self.action_context}"
+        if self.desc:
+            msg += f", desc='{self.desc}'"
+        msg += ")"
+        return msg
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, ExecutionContext):
+            return False
+
+        return self.long_id == other.long_id
+
+    def __hash__(self):
+        return hash(self.long_id)
+
+    @property
+    def long_id(self):
+        return f"{self.event.get_digest()}:{self._id}"
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def elapsed(self):
+        if self._start == 0:
+            return 0
+
+        return (self._stop if self._stop > 0 else time.time_ns()) - self._start
+
+    @property
+    def elapsed_str(self):
+        nano_sec = self.elapsed
+        dt = nano_sec / 1e6
+        return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
+
+    def add_inputs(self, **kwargs):
+        """
+        When entering stacking an ExecutionContext, list of variable that are worth to trace
+        :param kwargs:
+        :type kwargs:
+        :return:
+        :rtype:
+        """
+        self.inputs.update(kwargs)
+        return self
+
+    def add_values(self, **kwargs):
+        """
+        When popping from an ExecutionContext, list of variable that are worth to trace
+        :param kwargs:
+        :type kwargs:
+        :return:
+        :rtype:
+        """
+        self.values.update(kwargs)
+        return self
+
+    def push(self,
+             who: str,
+             action: ExecutionContextActions,
+             action_context: object,
+             desc: str = None,
+             logger=None):
+        child = ExecutionContext(
+            who,
+            self.event,
+            self.sheerka,
+            action,
+            action_context,
+            desc,
+            logger or self._logger,
+            self.global_hints,
+            self.protected_hints,
+            self
+        )
+        self._children.append(child)
+        return child
+
+    def get_children(self, level=-1):
+        """
+        recursively look for children
+        :return:
+        :rtype:
+        """
+        for child in self._children:
+            yield child
+            if level != 1:
+                yield from child.get_children(level - 1)
+
+    def __enter__(self):
+        self._start = time.time_ns()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._stop = time.time_ns()

src/core/ReturnValue.py

@@ -0,0 +1,32 @@
+from typing import Any
+
+from common.utils import compute_hash
+
+
+class ReturnValue:
+    """
+    This class represents the result of a data flow processing
+    """
+
+    def __init__(self, who: str = None, status: bool = None, value: Any = None, parents: list = None):
+        self.who = who
+        self.status = status
+        self.value = value
+        self.parents = parents
+
+    def __repr__(self):
+        return f"ReturnValue(who={self.who}, status={self.status}, value={self.value})"
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, ReturnValue):
+            return False
+
+        return self.who == other.who and \
+            self.status == other.status and \
+            self.value == other.value
+
+    def __hash__(self):
+        return hash((self.who, self.status, compute_hash(self.value)))

src/core/Sheerka.py

@@ -1,2 +1,324 @@
+import inspect
+import logging
+import sys
+from dataclasses import dataclass
+from operator import attrgetter
+from os import path
+from typing import Callable
+
+from caching.Cache import Cache
+from caching.IncCache import IncCache
+from common.utils import get_logger_name, get_sub_classes, import_module_and_sub_module
+from core.BuiltinConcepts import BuiltinConcepts
+from core.ErrorContext import ErrorContext
+from core.Event import Event
+from core.ExecutionContext import ContextHint, ExecutionContext, ExecutionContextActions
+from core.ReturnValue import ReturnValue
+from core.concept import Concept, ConceptMetadata
+from ontologies.SheerkaOntologyManager import SheerkaOntologyManager
+from server.authentication import User
+
+EXECUTE_STEPS = [
+    ExecutionContextActions.BEFORE_PARSING,
+    ExecutionContextActions.PARSING,
+    ExecutionContextActions.AFTER_PARSING,
+    ExecutionContextActions.BEFORE_EVALUATION,
+    ExecutionContextActions.EVALUATION,
+    ExecutionContextActions.AFTER_EVALUATION
+]
+
+
+@dataclass
+class SheerkaConfig:
+    """
+    After each execution, persist the whole executions flow as a file
+    This file will be used by the debugger
+    """
+    save_execution_context: bool = True
+
+
+@dataclass
+class SheerkaMethod:
+    """
+    Wrapper to sheerka method, to indicate if it's safe to call
+    """
+    name: str
+    service: str
+    method: Callable
+    has_side_effect: bool
+
+    def __repr__(self):
+        return self.name
+
+    def __hash__(self):
+        return hash((self.name, self.service))
+
+
 class Sheerka:
-    pass
+    OBJECTS_IDS_ENTRY = "Objects_Ids"
+    CHICKEN_AND_EGG_CONCEPTS_ENTRY = "Chicken_And_Egg_Concepts"
+
+    def __init__(self):
+        """
+        Engine of the so called Sheerka
+        """
+        self.name = "Sheerka"
+        self.om: SheerkaOntologyManager = None
+        self.config = SheerkaConfig()
+        self.during_initialisation = False
+        self.log = logging.getLogger(get_logger_name(__name__))
+        self.init_log = logging.getLogger(get_logger_name("init." + __name__))
+
+        self.services = {}  # sheerka plugins
+        self.evaluators = {}  # cache for evaluators
+
+        self.sheerka_methods = {}
+        self.methods_with_context = set()  # only the names, the method is defined in sheerka_methods
+        self.global_context_hints = set()
+
+    def bind_service_method(self, service_name, bound_method, can_modify_state, as_name=None, visible=True):
+        """
+        Bind service method to sheerka instance for ease of use ?
+        :param service_name:
+        :param bound_method:
+        :param can_modify_state: Can update the state of Sheerka => can produce side_effect
+        :param as_name: give another name to the method
+        :param visible: make the method visible to Sheerka
+        :return:
+        """
+        if as_name is None:
+            as_name = bound_method.__name__
+
+        if visible:
+            signature = inspect.signature(bound_method)
+            if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context":
+                self.methods_with_context.add(as_name)
+            self.sheerka_methods[as_name] = SheerkaMethod(as_name, service_name, bound_method, can_modify_state)
+
+        setattr(self, bound_method.__name__, bound_method)
+
+    def initialize(self, root_folder: str = None, **kwargs):
+        """
+        Starting Sheerka
+        Loads the current configuration
+        Notes that when it's the first time, it also creates the needed working folders
+        :param root_folder: root configuration folder
+        :return: ReturnValue(Success or Error)
+        """
+
+        if root_folder is None:
+            root_folder = path.abspath(path.join(path.expanduser("~"), ".sheerka"))
+
+        self.initialize_logging(False, root_folder)
+
+        self.config.save_execution_context = kwargs.get("save_execution_context", self.config.save_execution_context)
+
+        try:
+            self.init_log.info("Starting Sheerka")
+            self.during_initialisation = True
+            # from sheerkapickle.sheerka_handlers import initialize_pickle_handlers
+            # initialize_pickle_handlers()
+
+            self.om = SheerkaOntologyManager(self, root_folder)
+            # self.builtin_cache, self.builtin_cache_by_class_name = self.get_builtins_classes_as_dict()
+
+            self.initialize_caching()
+            self.initialize_evaluators()
+            self.initialize_services()
+            # self.initialize_builtin_evaluators()
+            # self.om.init_subscriptions()
+
+            event = Event("Initializing Sheerka.", user_id=self.name)
+            self.om.save_event(event)
+            with ExecutionContext(self.name,
+                                  event,
+                                  self,
+                                  ExecutionContextActions.INIT_SHEERKA,
+                                  None,
+                                  desc="Initializing Sheerka.") as exec_context:
+                if self.om.current_sdp().first_time:
+                    self.first_time_initialisation(exec_context)
+
+                self.initialize_services_deferred(exec_context, self.om.current_sdp().first_time)
+
+                res = ReturnValue(self.name, True, self.get_startup_config())
+
+                exec_context.add_values(return_values=res)
+
+                if self.om.is_dirty():
+                    self.om.commit(exec_context)
+
+                if self.config.save_execution_context:
+                    self.om.save_execution_context(exec_context, is_admin=True)
+
+                # append the other ontologies if needed
+                self.om.freeze()
+                self.initialize_ontologies(exec_context)
+
+                # self.init_log.debug(f"Sheerka successfully initialized")
+
+        except IOError as e:
+            res = ReturnValue(self.name, False, ErrorContext(self.name, exec_context, e))
+
+        finally:
+            self.during_initialisation = False
+
+        return res
+
+    @staticmethod
+    def initialize_logging(is_debug, root_folder):
+        if is_debug:
+            # log_format = "%(asctime)s %(name)s"
+            log_format = "[%(levelname)s] [%(name)s]"
+            log_format += " %(message)s"
+            log_level = logging.DEBUG
+        else:
+            log_format = "%(message)s"
+            log_level = logging.INFO
+
+        logging.basicConfig(format=log_format, level=log_level, handlers=[logging.StreamHandler(sys.stdout)])
+        logging.addLevelName(logging.ERROR, f"\033[1;41m%s\033[1;0m{logging.getLevelName(logging.ERROR)}")
+
+    def initialize_ontologies(self, context):
+        ontologies = self.om.current_sdp().load_ontologies()
+        if not ontologies:
+            return
+        for ontology_name in list(reversed(ontologies))[1:]:
+            self.om.push_ontology(ontology_name, False)
+            # self.initialize_services_deferred(context, False)
+
+    def first_time_initialisation(self, context):
+        pass
+        # self.record_var(context, self.name, "save_execution_context", self.save_execution_context)
+
+    def initialize_caching(self):
+
+        cache = IncCache().auto_configure(self.OBJECTS_IDS_ENTRY)
+        self.om.register_cache(self.OBJECTS_IDS_ENTRY, cache)
+
+        cache = Cache().auto_configure(self.CHICKEN_AND_EGG_CONCEPTS_ENTRY)
+        self.om.register_cache(self.CHICKEN_AND_EGG_CONCEPTS_ENTRY, cache, persist=False)
+
+    def initialize_services(self):
+        """
+        Introspect to find services and bind them
+        :return:
+        """
+        self.init_log.info("Initializing services")
+
+        import_module_and_sub_module('core.services')
+        base_class = "core.services.BaseService.BaseService"
+        services = [service(self) for service in get_sub_classes("core.services", base_class)]
+        services.sort(key=attrgetter("order"))
+        for service in services:
+            if hasattr(service, "initialize"):
+                service.initialize()
+            self.services[service.NAME] = service
+
+        self.init_log.info(f"{len(services)} service(s) found.")
+
+    def initialize_services_deferred(self, context, is_first_time):
+        """
+        Initialize part of services that may take some time or that need the execution context
+        :return:
+        """
+        self.init_log.debug(f"Initializing services (deferred, {is_first_time=})")
+        for service in self.services.values():
+            if hasattr(service, "initialize_deferred"):
+                service.initialize_deferred(context, is_first_time)
+
+    def initialize_evaluators(self):
+        self.init_log.info("Initializing evaluators")
+
+        base_class1 = "evaluators.base_evaluator.OneReturnValueEvaluator"
+        base_class2 = "evaluators.base_evaluator.AllReturnValuesEvaluator"
+        import_module_and_sub_module('evaluators')
+
+        evaluators = [evaluator() for evaluator in get_sub_classes("evaluators", base_class1)] + \
+                     [evaluator() for evaluator in get_sub_classes("evaluators", base_class2)]
+
+        self.evaluators = {e.NAME: e for e in evaluators}
+        self.init_log.info(f"{len(evaluators)} evaluator(s) found.")
+
+    def bind_services_methods(self):
+        # init methods
+        # self.bind_service_method(self.name, self.test, False)
+        # self.bind_service_method(self.name, self.test_using_context, False)
+        # self.bind_service_method(self.name, self.test_dict, False)
+        # self.bind_service_method(self.name, self.test_error, False)
+        # self.bind_service_method(self.name, self.is_sheerka, False)
+        # self.bind_service_method(self.name, self.objvalue, False)
+        pass
+
+    def get_startup_config(self):
+        """
+        Return a dictionary with current configuration, used for initialization
+        :return:
+        :rtype:
+        """
+        return {
+            "config": self.config.__dict__
+        }
+
+    def publish(self, context, topic, data=None):
+        """
+        To be removed as it must be part of the EventManager service
+        :param context:
+        :type context:
+        :param topic:
+        :type topic:
+        :param data:
+        :type data:
+        :return:
+        :rtype:
+        """
+        pass
+
+    def evaluate_user_input(self, command: str, user: User):
+        self.log.info("Processing '%s' from '%s'", command, user.email)
+
+        event = Event(command, user_id=user.email)
+        self.om.save_event(event)
+        with ExecutionContext(user.email,
+                              event,
+                              self,
+                              ExecutionContextActions.EVALUATE_USER_INPUT,
+                              command,
+                              desc=f"Evaluating '{command}'",
+                              global_hints=self.global_context_hints.copy()) as exec_context:
+            user_input = ReturnValue(self.name, True, self.newn(BuiltinConcepts.USER_INPUT, command=command))
+            exec_context.private_hints.add(ContextHint.REDUCE_CONCEPTS)
+
+            # KSI : 2023-04-30
+            # Il me manque le execute et toute la classe SheerkaProcessUserInput
+            exec_context.add_inputs(user_input=user_input)
+            ret = self.execute(exec_context, [user_input], EXECUTE_STEPS)
+            exec_context.add_values(return_values=ret)
+
+            if self.om.is_dirty():
+                self.om.commit(exec_context)
+
+        return ret
+
+    def isinstance(self, a, b):
+        """
+        Returns true if 'a' is a concept of type 'b'
+        Note that this function can be moved into ConceptManager
+        I keep it here for quick access
+        :param a:
+        :type a:
+        :param b:
+        :type b:
+        :return:
+        :rtype:
+        """
+        if not isinstance(a, Concept):
+            return False
+
+        if isinstance(b, (Concept, ConceptMetadata)):
+            return a.id == b.id
+
+        if b.startswith("c:#"):
+            return a.id == b[3:-1]
+
+        return a.key == b

src/core/concept.py

@@ -0,0 +1,201 @@
+from dataclasses import dataclass
+
+from common.global_symbols import NotFound, NotInit
+
+
+class ConceptDefaultProps:
+    """
+    Lists metadata that can contains some code
+    """
+    WHERE = "#where#"
+    PRE = "#pre#"
+    POST = "#post#"
+    BODY = "#body#"
+    RET = "#ret#"
+
+
+DefaultProps = [v for k, v in ConceptDefaultProps.__dict__.items() if not k.startswith("_")]
+
+
+class DefinitionType:
+    DEFAULT = "Default"
+    BNF = "Bnf"
+
+
+@dataclass
+class ConceptMetadata:
+    """
+    Static information of the Concept
+
+    """
+    id: str  # unique identifier for a concept. The id will never be modified (but the key can)
+    name: str
+    key: str
+    is_builtin: bool
+    is_unique: bool
+    body: str  # main method, can also be the value of the concept
+    where: str  # condition to recognize variables in name
+    pre: str  # list of preconditions before calling the main function
+    post: str  # list of post conditions after calling the main function
+    ret: str  # variable to return when a concept is recognized
+    definition: str  # regex used to define the concept
+    definition_type: DefinitionType  # definition can be done with something else than regex
+    desc: str  # possible description for the concept
+    autouse: bool  # indicates if eval must be automatically called on the concept once validated
+    bound_body: str  # which property must be considered have default value for the concept
+    props: dict  # hashmap of properties, values
+    variables: tuple  # list of concept variables(tuple), with their default values
+    parameters: tuple  # list of variables that are part of the name of the concept
+    digest: str = None
+    all_attrs: tuple = None
+
+
+@dataclass
+class ConceptRuntimeInfo:
+    """
+    Dynamic information of the Concept
+    They are related to the instance of the concept
+    """
+    is_evaluated: bool = False  # True is the concept is evaluated by sheerka.eval_concept()
+    need_validation: bool = False  # True if the properties of the concept need to be validated
+    recognized_by: str = None  # RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME, RECOGNIZED_BY_KEY (from Sheerka.py)
+
+    def copy(self):
+        return ConceptRuntimeInfo(self.is_evaluated,
+                                  self.need_validation,
+                                  self.recognized_by)
+
+
+class Concept:
+    """
+    Default concept object
+    A concept is the base object of our universe
+    Everything is a concept
+    """
+
+    def __init__(self, metadata: ConceptMetadata):
+
+        self._metadata: ConceptMetadata = metadata
+        self._compiled = {}  # cached ast for the where, pre, post and body parts and variables
+        self._compiled_context_hints = {}  # context hints to use when evaluating compiled
+        self._bnf = None  # compiled bnf expression
+        self._runtime_info = ConceptRuntimeInfo()  # runtime settings for the concept
+        self._all_attrs = None
+
+    def __repr__(self):
+        text = f"({self._metadata.id}){self._metadata.name}"
+        if self._metadata.pre:
+            text += f", #pre={self._metadata.pre}"
+
+        for attr in [attr for attr in self.all_attrs() if not attr.startswith("#")]:
+            text += f", {attr}={self.get_value(attr)}"
+
+        return text
+
+    def __eq__(self, other):
+        # I don't want this test to be part of the recursion
+        # So let's just get rif ogf it
+        if not isinstance(other, Concept):
+            return False
+
+        # I chose to use an iterative algorithm in order to be able to spot circular reference
+        # without inner functions.
+        # I also think that it's a better approach for a function that can be massively called
+
+        stack = [self, other]
+        id_self = id(self)
+
+        while stack:
+            right = stack.pop()
+            left = stack.pop()
+
+            if id(left) == id(right):
+                return True
+
+            # 1. in order for two concepts to be equal, they must have the same definition
+            # 2. They must have the same properties and variables
+            if left.get_definition_digest() != right.get_definition_digest():
+                return False
+
+            if left.all_attrs() != right.all_attrs():
+                return False
+
+            for attr in left.all_attrs():
+                value = left.get_value(attr)
+                other_value = right.get_value(attr)
+
+                if isinstance(value, Concept) and isinstance(other_value, Concept):
+                    if id(value) == id_self or id(other_value) == id_self:
+                        # infinite recursion detected
+                        pass
+                    else:
+                        stack.extend([value, other_value])
+                else:
+                    if value != other_value:
+                        return False
+
+        return True
+
+    def __hash__(self):
+        return self._metadata.digest
+
+    @property
+    def id(self):
+        return self._metadata.id
+
+    @property
+    def name(self):
+        return self._metadata.name
+
+    @property
+    def key(self):
+        return self._metadata.key
+
+    @property
+    def body(self):
+        return self.get_value(ConceptDefaultProps.BODY)
+
+    @property
+    def str_id(self):
+        return f"c:#{self.id}:" if self.id else f"c:{self.name}:"
+
+    def get_definition_digest(self):
+        return self._metadata.digest
+
+    def all_attrs(self):
+        if self._all_attrs is None:
+            return self._metadata.all_attrs
+
+        return self._all_attrs
+
+    def get_metadata(self) -> ConceptMetadata:
+        return self._metadata
+
+    def set_value(self, name: str, value: object):
+        """
+        Set the resolved value of a metadata or a variable (not the metadata itself)
+        :param name:
+        :param value:
+        :return:
+        """
+        setattr(self, name, value)
+        if name == self._metadata.bound_body:
+            setattr(self, ConceptDefaultProps.BODY, value)
+        elif self._metadata.bound_body and name == ConceptDefaultProps.BODY:
+            setattr(self, self._metadata.bound_body, value)
+
+        return self
+
+    def get_value(self, name: str):
+        """
+        Gets the resolved value of a metadata
+        :param name:
+        :return:
+        """
+        try:
+            return getattr(self, name)
+        except AttributeError:
+            return NotInit if name in self.all_attrs() else NotFound
+
+    def get_runtime_info(self):
+        return self._runtime_info

src/core/global_symbols.py

@@ -1,31 +0,0 @@
-class CustomType:
-    """
-    Base class for custom types used in Sheerka
-    A custom type is a type that has only one instance across the application and have a semantic meaning
-    For example the type 'None' is a singleton which have a semantic meaning.
-    We need to define others in Sheerka
-    """
-
-    def __init__(self, value):
-        self.value = value
-
-    def __repr__(self):
-        return self.value
-
-    def __eq__(self, other):
-        return isinstance(other, CustomType) and self.value == other.value
-
-    def __hash__(self):
-        return hash(self.value)
-
-
-class NotFoundType(CustomType):
-    """
-    Using when an entry in not found in Cache or in sdp
-    """
-
-    def __init__(self):
-        super(NotFoundType, self).__init__("**NotFound**")
-
-
-NotFound = NotFoundType()

src/core/services/BaseService.py

@@ -0,0 +1,49 @@
+from common.global_symbols import NotFound
+from common.utils import sheerka_deepcopy
+from core.Sheerka import Sheerka
+
+
+class BaseService:
+    """
+    Base class for services
+    """
+
+    def __init__(self, sheerka: Sheerka, order=999):
+        self.sheerka = sheerka
+        self.order = order  # initialisation order. The lowest is initialized first
+
+    def initialize(self):
+        """
+        Adds cache or bind methods
+        :return:
+        """
+        pass
+
+    def state_properties(self):
+        pass
+
+    def push_state(self, context):
+        """
+        Use variable Manager to store the state of the service
+        """
+        args = self.state_properties()
+        if args:
+            for prop_name in args:
+                self.sheerka.record_var(context, self.NAME, prop_name, sheerka_deepcopy(getattr(self, prop_name)))
+
+    def pop_state(self):
+        """
+        Use Variable Manager to restore the state of a service
+        :return:
+        """
+        args = self.state_properties()
+        if args:
+            for prop_name in args:
+                if (value := self.sheerka.load_var(self.NAME, prop_name)) is not NotFound:
+                    setattr(self, prop_name, value)
+
+    def store_var(self, context, var_name):
+        """
+        Store/record the value of an attribute
+        """
+        self.sheerka.record_var(context, self.NAME, var_name, getattr(self, var_name))

src/core/services/SheerkaConceptManager.py

@@ -0,0 +1,343 @@
+import hashlib
+import logging
+from dataclasses import dataclass
+
+from caching.Cache import Cache
+from caching.FastCache import FastCache
+from caching.ListIfNeededCache import ListIfNeededCache
+from common.global_symbols import NotFound, NotInit, VARIABLE_PREFIX
+from common.utils import get_logger_name
+from core.BuiltinConcepts import BuiltinConcepts
+from core.ErrorContext import ErrorContext, SheerkaException
+from core.ExecutionContext import ExecutionContext
+from core.ReturnValue import ReturnValue
+from core.concept import Concept, ConceptMetadata, DefaultProps, DefinitionType
+from core.services.BaseService import BaseService
+from parsers.tokenizer import TokenKind, Tokenizer, strip_tokens
+
+PROPERTIES_FOR_DIGEST = ("name", "key",
+                         "definition", "definition_type",
+                         "is_builtin", "is_unique",
+                         "where", "pre", "post", "body", "ret",
+                         "desc", "bound_body", "autouse", "props", "variables", "parameters")
+
+
+@dataclass
+class ConceptAlreadyDefined(SheerkaException):
+    concept: ConceptMetadata
+    already_defined_id: str
+
+
+@dataclass
+class InvalidBnf(SheerkaException):
+    bnf: str
+
+
+@dataclass
+class FirstItemError(SheerkaException):
+    pass
+
+
+class ConceptManager(BaseService):
+    """
+    The service is used for the administration of concepts
+    You can define new concept, modify or delete them
+
+    There are also function to help retrieve them easily (like first token cache)
+    Already instantiated concepts are managed by the Memory service
+    """
+
+    NAME = "ConceptManager"
+
+    USER_CONCEPTS_IDS = "User_Concepts_IDs"  # incremented everytime a new concept is created
+    CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID"  # to store all the concepts
+    CONCEPTS_BY_KEY_ENTRY = "ConceptManager:Concepts_By_Key"
+    CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name"
+    CONCEPTS_BY_HASH_ENTRY = "ConceptManager:Concepts_By_Hash"  # sto
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka, order=11)
+
+        self.log = logging.getLogger(get_logger_name(__name__))
+        self.init_log = logging.getLogger(get_logger_name("init." + __name__))
+        self.bnf_expr_cache = FastCache()
+
+    def initialize(self):
+        self.init_log.debug(f"Initializing ConceptManager, order={self.order}")
+        self.sheerka.bind_service_method(self.NAME, self.define_new_concept, True)
+        self.sheerka.bind_service_method(self.NAME, self.newn, True)
+        self.sheerka.bind_service_method(self.NAME, self.newi, True)
+
+        register_concept_cache = self.sheerka.om.register_concept_cache
+
+        # Cache of concept metadata, organized by id
+        cache = Cache().auto_configure(self.CONCEPTS_BY_ID_ENTRY)
+        register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True)
+
+        cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_KEY_ENTRY)
+        register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True)
+
+        cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_NAME_ENTRY)
+        register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True)
+
+        cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_HASH_ENTRY)
+        register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.digest, True)
+
+    def initialize_deferred(self, context, is_first_time):
+        if is_first_time:
+            self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000)
+
+            _ = self._create_builtin_concept
+            _(1, BuiltinConcepts.SHEERKA, desc="Sheerka")
+            _(2, BuiltinConcepts.NEW_CONCEPT, desc="On new concept creation", variables=("metadata",))
+            _(3, BuiltinConcepts.UNKNOWN_CONCEPT, desc="Unknown concept", variables=("requested_name", "requested_id"))
+            _(4, BuiltinConcepts.USER_INPUT, desc="Any external input", variables=("command",))
+            _(5, BuiltinConcepts.PARSER_INPUT, desc="tokenized input", variables=("pi",))
+
+            self.init_log.debug('%s builtin concepts created',
+                                len(self.sheerka.om.current_cache_manager().concept_caches))
+
+    def define_new_concept(self, context: ExecutionContext,
+                           name: str,
+                           is_builtin: bool = False,  # is the concept defined Sheerka
+                           is_unique: bool = False,  # is the concept a singleton
+                           body: str = "",  # return value of the concept
+                           where: str = "",  # condition to recognize variables in name
+                           pre: str = "",  # list of preconditions before calling the main function
+                           post: str = "",  # list of post conditions after calling the main function
+                           ret: str = "",  # variable to return when a concept is recognized
+                           definition: str = "",  # regex used to define the concept
+                           definition_type: DefinitionType = DefinitionType.DEFAULT,
+                           autouse: bool = False,  # indicate if the concept must be automatically evaluated
+                           bound_body: str = None,  #
+                           desc: str = "",  # possible description for the concept
+                           props: dict = None,  # hashmap of default properties
+                           variables: list = None,  # list of concept variables(tuple), with their default values
+                           parameters: list = None  # list of variables that are part of the name of the concept
+                           ) -> ReturnValue:
+        """
+        Adds the definition of a new concept
+        :return:
+        :rtype:
+        """
+        concept_key = self.create_concept_key(name, definition, variables)
+        concept_id = "waiting for id"
+
+        metadata = ConceptMetadata(
+            concept_id,
+            name,
+            concept_key,
+            is_builtin,
+            is_unique,
+            body,
+            where,
+            pre,
+            post,
+            ret,
+            definition,
+            definition_type,
+            desc,
+            autouse,
+            bound_body,
+            props or {},
+            variables or (),
+            parameters or (),
+        )
+
+        digest = self.compute_metadata_digest(metadata)
+        if self.sheerka.om.exists_in_current(self.CONCEPTS_BY_HASH_ENTRY, digest):
+            already_defined = self.sheerka.om.get(self.CONCEPTS_BY_HASH_ENTRY, digest)
+            error = ErrorContext(self.NAME, context, ConceptAlreadyDefined(metadata, already_defined.id))
+            return ReturnValue(self.NAME, False, error)
+
+        metadata.digest = digest
+        metadata.all_attrs = self.compute_all_attrs(variables)
+
+        # bnf_expr = None
+        # if definition_type == DefinitionType.BNF:
+        #     try:
+        #         bnf_expr = self.compute_concept_bnf(definition)
+        #     except InvalidBnf as ex:
+        #         error = ErrorContext(self.NAME, context, ex)
+        #         return ReturnValue(self.NAME, False, error)
+
+        # try:
+        #     first_item_res = self.recompute_first_items(context, None, [metadata])
+        # except FirstItemError as ex:
+        #     return ReturnValue(self.NAME, False, ex)
+
+        # at this point everything is fine. let's get the id and save everything
+        om = self.sheerka.om
+        metadata.id = str(self.sheerka.om.get(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS))
+        om.add_concept(metadata)
+        # self.update_first_items_caches(context, first_item_res)
+        # if bnf_expr:
+        #     self.bnf_expr_cache.put(metadata.id, bnf_expr)
+        #     # update references
+        #     for ref in self.compute_references(bnf_expr):
+        #         om.put(self.CONCEPTS_REFERENCES_ENTRY, ref, metadata.id)
+
+        return ReturnValue(self.NAME, True, self.newn(BuiltinConcepts.NEW_CONCEPT, metadata=metadata))
+
+    def newn(self, concept_name: str, **kwargs):
+        """
+        new_by_name
+        Creates and returns an instance of a new concept by its name
+        :param concept_name:
+        :type concept_name:
+        :param kwargs:
+        :type kwargs:
+        :return:
+        :rtype:
+        """
+        metadata = self.get_by_name(concept_name)
+        if metadata is NotFound:
+            return self._inner_new(self.get_by_name(BuiltinConcepts.UNKNOWN_CONCEPT), requested_name=concept_name)
+
+        if isinstance(metadata, list):
+            return [self._inner_new(m, **kwargs) for m in metadata]
+
+        return self._inner_new(metadata, **kwargs)
+
+    def newi(self, concept_id: str, **kwargs):
+        """
+        new_by_id
+        Creates and returns an instance of a new concept by its id
+        :param concept_id:
+        :type concept_id:
+        :param kwargs:
+        :type kwargs:
+        :return:
+        :rtype:
+        """
+        metadata = self.get_by_id(concept_id)
+        if metadata is NotFound:
+            return self._inner_new(self.get_by_name(BuiltinConcepts.UNKNOWN_CONCEPT), requested_id=concept_id)
+        return self._inner_new(metadata, **kwargs)
+
+    def get_by_name(self, key: str):
+        """
+        Returns a concept metadata, using its name
+        :param key:
+        :type key:
+        :return:
+        :rtype:
+        """
+        return self.sheerka.om.get(self.CONCEPTS_BY_NAME_ENTRY, key)
+
+    def get_by_id(self, concept_id: str):
+        """
+        Returns a concept metadata, using its name
+        :param concept_id:
+        :type concept_id:
+        :return:
+        :rtype:
+        """
+        return self.sheerka.om.get(self.CONCEPTS_BY_ID_ENTRY, concept_id)
+
+    def get_by_key(self, key: str):
+        """
+        Returns a concept metadata, using its name
+        :param key:
+        :type key:
+        :return:
+        :rtype:
+        """
+        return self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, key)
+
+    @staticmethod
+    def compute_metadata_digest(metadata: ConceptMetadata):
+        """
+        Compute once for all the digest of the definition of a concept
+        :param metadata: 
+        :type metadata: 
+        :return: 
+        :rtype: 
+        """
+        as_dict = {p: getattr(metadata, p) for p in PROPERTIES_FOR_DIGEST}
+        return hashlib.sha256(f"{as_dict}".encode("utf-8")).hexdigest()
+
+    @staticmethod
+    def compute_all_attrs(variables: tuple | None):
+        """
+        Compute the list of available attributes for a concept
+        :param variables:
+        :return:
+        :rtype:
+        """
+        all_attrs = DefaultProps.copy()
+        if variables:
+            all_attrs += [k for k, v in variables]
+
+        return tuple(all_attrs)
+
+    @staticmethod
+    def compute_concept_bnf(definition):
+        pass
+
+    @staticmethod
+    def create_concept_key(name: str, definition: str | None, variables: tuple | None):
+        """
+        Creates the key from the definition
+        :param name:
+        :type name:
+        :param definition:
+        :type definition:
+        :param variables:
+        :type variables:
+        :return:
+        :rtype:
+        """
+
+        definition_to_use = definition or name
+        tokens = list(Tokenizer(definition_to_use, yield_eof=False))
+
+        if variables is None or len(strip_tokens(tokens, True)) == 1:
+            variables_to_use = []
+        else:
+            variables_to_use = [k for k, v in variables]
+
+        parts = []
+        for token in tokens:
+            if token.type == TokenKind.WHITESPACE:
+                continue
+            if token.value in variables_to_use:
+                parts.append(VARIABLE_PREFIX + str(variables_to_use.index(token.value)))
+            else:
+                parts.append(token.value)
+
+        return " ".join(parts)
+
+    def _create_builtin_concept(self, concept_id: int, name: str, desc: str, variables: tuple = ()):
+        variables_to_use = tuple((k, NotInit) for k in variables)
+        concept_key = self.create_concept_key(name, None, variables_to_use)
+        metadata = ConceptMetadata(
+            str(concept_id),
+            name,
+            concept_key,
+            True,
+            False,
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            DefinitionType.DEFAULT,
+            desc,
+            False,
+            variables[0] if variables else "",
+            {},
+            variables_to_use,
+            variables,
+        )
+        metadata.digest = self.compute_metadata_digest(metadata)
+        metadata.all_attrs = self.compute_all_attrs(variables_to_use)
+        self.sheerka.om.add_concept(metadata)
+
+    @staticmethod
+    def _inner_new(_metadata_def: ConceptMetadata, **kwargs):
+        concept = Concept(_metadata_def)
+        for k, v in kwargs.items():
+            concept.set_value(k, v)
+        return concept

src/core/services/SheerkaEngine.py

@@ -0,0 +1,223 @@
+from dataclasses import dataclass
+
+from common.utils import to_dict
+from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+from core.ReturnValue import ReturnValue
+from core.services.BaseService import BaseService
+from evaluators.base_evaluator import AllReturnValuesEvaluator, BaseEvaluator, OneReturnValueEvaluator
+
+
+@dataclass
+class EvaluationPlan:
+    sorted_priorities: list[int]  # list of available priorities
+    evaluators: dict[int, list[BaseEvaluator]]
+
+
+class SheerkaEngine(BaseService):
+    """
+    This service is used to process user input
+    It is responsible to parse and evaluate the information
+    It also holds the rule engine
+    """
+    NAME = "Engine"
+
+    def __init__(self, sheerka):
+        super().__init__(sheerka, order=15)
+        self.execution_plan = None  # { ExecutionContextActions : { priority : [evaluators] }}
+        self.no_evaluation_plan = EvaluationPlan([], {})
+
+    def initialize(self):
+        self.execution_plan = self.compute_execution_plan(self.sheerka.evaluators.values())
+        self.sheerka.bind_service_method(self.NAME, self.execute, True)
+
+    def call_evaluators(self,
+                        context: ExecutionContext,
+                        return_values: list[ReturnValue],
+                        step: ExecutionContextActions):
+        """
+        Calls all evaluators defined for a given step
+        :param context:
+        :type context:
+        :param return_values:
+        :type return_values:
+        :param step:
+        :type step:
+        :return:
+        :rtype:
+        """
+
+        plan = self.get_evaluation_plan(context, step)
+        iteration = 0
+        while True:
+            with context.push(self.NAME,
+                              ExecutionContextActions.EVALUATING_ITERATION,
+                              {"step": step, "iteration": iteration},
+                              desc=f"iteration #{iteration}") as iteration_context:
+                simple_digest = return_values.copy()
+                iteration_context.add_inputs(return_values=simple_digest)
+
+                for priority in plan.sorted_priorities:
+
+                    return_values_copy = return_values.copy()
+                    new_return_values = {}
+                    return_values_to_delete = set()
+                    for evaluator in plan.evaluators[priority]:
+
+                        sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})"
+                        with iteration_context.push(self.NAME,
+                                                    step,
+                                                    {"step": step,
+                                                     "iteration": iteration,
+                                                     "evaluator": evaluator.name},
+                                                    desc=sub_context_desc) as evaluator_context:
+                            evaluator_context.add_inputs(return_values=return_values_copy)
+
+                            # process evaluators that work on one simple return value at the time
+                            if isinstance(evaluator, OneReturnValueEvaluator):
+                                self.call_one_return_value_evaluator(evaluator_context,
+                                                                     evaluator,
+                                                                     return_values_copy,
+                                                                     new_return_values,
+                                                                     return_values_to_delete)
+
+                            # process evaluators that work on all return values
+                            else:
+                                self.call_all_return_values_evaluator(evaluator_context,
+                                                                      evaluator,
+                                                                      return_values_copy,
+                                                                      new_return_values,
+                                                                      return_values_to_delete)
+
+                    # Recreate the new return_value
+                    # Try to keep the order of what replaces what
+                    return_values = []
+                    for item in return_values_copy:
+                        if item not in return_values_to_delete:
+                            return_values.append(item)
+                        if item in new_return_values:
+                            return_values.extend(new_return_values[item])
+
+                iteration_context.add_values(return_values=return_values.copy())
+
+            iteration += 1
+            if simple_digest == return_values:
+                # I can use a variable like 'has_changed', but I think that this comparison is explicit
+                # It explains that I stay in the loop if something was modified
+                break
+
+        return return_values
+
+    def execute(self,
+                context: ExecutionContext,
+                return_values: list[ReturnValue],
+                steps: list[ExecutionContextActions]):
+        """
+        Runs the processing engine on the return_values
+        :param context:
+        :type context:
+        :param return_values:
+        :type return_values:
+        :param steps:
+        :type steps:
+        :return:
+        :rtype:
+        """
+        for step in steps:
+            copy = return_values.copy()
+            with context.push(self.NAME, ExecutionContextActions.EVALUATING_STEP, {"step": step}) as sub_context:
+                sub_context.add_inputs(return_values=copy)
+
+                return_values = self.call_evaluators(sub_context, return_values, step)
+
+                sub_context.add_values(return_values=return_values)
+                sub_context.add_values(has_changed=(copy != return_values))
+
+        return return_values
+
+    def get_evaluation_plan(self, context: ExecutionContext, step: ExecutionContextActions) -> EvaluationPlan:
+        if step not in self.execution_plan:
+            return self.no_evaluation_plan
+
+        evaluators = self.execution_plan[step]
+        return EvaluationPlan(sorted(evaluators.keys(), reverse=True), evaluators)
+
+    @staticmethod
+    def call_one_return_value_evaluator(context: ExecutionContext,
+                                        evaluator: OneReturnValueEvaluator,
+                                        return_values: list[ReturnValue],
+                                        new_return_values: dict[ReturnValue, list[ReturnValue]],
+                                        return_values_to_delete: set[ReturnValue]):
+        """
+
+        :param context:
+        :type context:
+        :param evaluator:
+        :type evaluator:
+        :param return_values:
+        :type return_values:
+        :param new_return_values:
+        :type new_return_values:
+        :param return_values_to_delete:
+        :type return_values_to_delete:
+        :return:
+        :rtype:
+        """
+        context_trace = []
+        for item in return_values:
+            debug = {"item": item}
+            context_trace.append(debug)
+
+            m = evaluator.matches(context, item)
+            debug["match"] = m.status
+
+            if m.status:
+                result = evaluator.eval(context, m.obj, item)
+                return_values_to_delete.update(result.eaten)
+                new_return_values.setdefault(item, []).extend(result.new)
+                debug["new"] = result.new
+                debug["eaten"] = result.eaten
+
+        context.add_values(evaluation=context_trace)
+
+    @staticmethod
+    def call_all_return_values_evaluator(context: ExecutionContext,
+                                         evaluator: AllReturnValuesEvaluator,
+                                         return_values: list[ReturnValue],
+                                         new_return_values: dict[ReturnValue, list[ReturnValue]],
+                                         return_values_to_delete: set[ReturnValue]):
+        """
+
+        :param context:
+        :type context:
+        :param evaluator:
+        :type evaluator:
+        :param return_values:
+        :type return_values:
+        :param new_return_values:
+        :type new_return_values:
+        :param return_values_to_delete:
+        :type return_values_to_delete:
+        :return:
+        :rtype:
+        """
+        debug = {}
+        m = evaluator.matches(context, return_values)
+        debug["match"] = m.status
+
+        if m.status:
+            result = evaluator.eval(context, m.obj, return_values)
+            return_values_to_delete.update(result.eaten)
+            new_return_values.setdefault(result.new[0].parents[0], []).extend(result.new)
+            debug["new"] = result.new
+            debug["eaten"] = result.eaten
+
+        context.add_values(evaluation=debug)
+
+    @staticmethod
+    def compute_execution_plan(evaluators):
+        evaluators = [e for e in evaluators if e.enabled]
+        by_step = to_dict(evaluators, lambda e: e.step)
+        for k, v in by_step.items():
+            by_step[k] = to_dict(v, lambda e: e.priority)
+
+        return by_step

src/core/utils.py

@@ -1,32 +0,0 @@
-def get_class(qname):
-    """
-    Loads a class from its full qualified name
-    :param qname:
-    :return:
-    """
-    parts = qname.split('.')
-    module = ".".join(parts[:-1])
-    m = __import__(module)
-    for comp in parts[1:]:
-        m = getattr(m, comp)
-    return m
-
-
-def get_full_qualified_name(obj):
-    """
-    Returns the full qualified name of a class (including its module name )
-    :param obj:
-    :return:
-    """
-    if obj.__class__ == type:
-        module = obj.__module__
-        if module is None or module == str.__class__.__module__:
-            return obj.__name__  # Avoid reporting __builtin__
-        else:
-            return module + '.' + obj.__name__
-    else:
-        module = obj.__class__.__module__
-        if module is None or module == str.__class__.__module__:
-            return obj.__class__.__name__  # Avoid reporting __builtin__
-        else:
-            return module + '.' + obj.__class__.__name__

src/evaluators/CreateParserInput.py

@@ -0,0 +1,30 @@
+from core.BuiltinConcepts import BuiltinConcepts
+from core.ErrorContext import ErrorContext
+from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+from core.ReturnValue import ReturnValue
+from evaluators.base_evaluator import EvaluatorEvalResult, EvaluatorMatchResult, OneReturnValueEvaluator
+from parsers.ParserInput import ParserInput
+
+
+class CreateParserInput(OneReturnValueEvaluator):
+    NAME = "CreateParserInput"
+
+    def __init__(self):
+        super().__init__(self.NAME, ExecutionContextActions.BEFORE_EVALUATION, 50)
+
+    def matches(self, context: ExecutionContext, return_value: ReturnValue) -> EvaluatorMatchResult:
+        if return_value.status and \
+                context.sheerka.isinstance(return_value.value, BuiltinConcepts.USER_INPUT):
+            return EvaluatorMatchResult(True)
+        return EvaluatorMatchResult(False)
+
+    def eval(self, context, evaluator_context, return_value):
+        parser_input = ParserInput(return_value.value.get_value("command"))
+        if parser_input.init():
+            parser_input_concept = context.sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=parser_input)
+            new_ret_val = ReturnValue(self.NAME, True, parser_input_concept, parents=[return_value])
+            return EvaluatorEvalResult([new_ret_val], [return_value])
+        else:
+            error = ErrorContext(self.NAME, context, parser_input)
+            new_ret_val = ReturnValue(self.NAME, False, error, parents=[return_value])
+            return EvaluatorEvalResult([new_ret_val], [return_value])

src/evaluators/base_evaluator.py

@@ -0,0 +1,73 @@
+from dataclasses import dataclass
+
+from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+from core.ReturnValue import ReturnValue
+
+
+@dataclass
+class EvaluatorMatchResult:
+    status: bool
+    obj: object = None
+
+
+@dataclass
+class EvaluatorEvalResult:
+    new: list[ReturnValue] = None
+    eaten: list[ReturnValue] = None
+
+
+class BaseEvaluator:
+    """
+    Base class to evaluate ReturnValues
+    """
+
+    def __init__(self, name, step: ExecutionContextActions, priority: int, enabled=True):
+        self.name = name
+        self.step = step
+        self.priority = priority
+        self.enabled = enabled
+
+    def __repr__(self):
+        return f"{self.name} ({self.priority})"
+
+    def __eq__(self, other):
+        if not isinstance(other, BaseEvaluator):
+            return False
+
+        return self.name == other.name and \
+            self.priority == other.priority and \
+            self.step == other.step and \
+            self.enabled == other.enabled
+
+    def __hash__(self):
+        return hash((self.name, self.priority, self.step, self.enabled))
+
+
+class OneReturnValueEvaluator(BaseEvaluator):
+    """
+    Evaluate one specific return value
+    """
+
+    def matches(self, context: ExecutionContext,
+                return_value: ReturnValue) -> EvaluatorMatchResult:
+        pass
+
+    def eval(self, context: ExecutionContext,
+             evaluation_context: object,
+             return_value: ReturnValue) -> EvaluatorEvalResult:
+        pass
+
+
+class AllReturnValuesEvaluator(BaseEvaluator):
+    """
+    Evaluates the groups of ReturnValues
+    """
+
+    def matches(self, context: ExecutionContext,
+                return_values: list[ReturnValue]) -> EvaluatorMatchResult:
+        pass
+
+    def eval(self, context: ExecutionContext,
+             evaluation_context: object,
+             return_values: list[ReturnValue]) -> EvaluatorEvalResult:
+        pass

src/ontologies/Exceptions.py

@@ -0,0 +1,37 @@
+class OntologyManagerFrozen(Exception):
+    """
+    Raised when you try to add a cache manager while the ontology manager is frozen
+    """
+    pass
+
+
+class OntologyManagerNotFrozen(Exception):
+    """
+    Raised when you try to push or pop a cache manager while the ontology manager is not frozen
+    """
+    pass
+
+
+class OntologyManagerCannotPopLatest(Exception):
+    """
+    Raised when you try pop the latest cache manager
+    """
+    pass
+
+
+class OntologyAlreadyExists(Exception):
+    """
+    When the ontology exists AND is not the top layer
+    """
+
+    def __init__(self, name):
+        self.name = name
+
+
+class OntologyNotFound(Exception):
+    """
+    During revert(), when the requested ontology does not exist
+    """
+
+    def __init__(self, ontology):
+        self.ontology = ontology

src/ontologies/SheerkaOntologyManager.py

@@ -0,0 +1,534 @@
+from caching.Cache import Cache
+from caching.CacheManager import CacheManager
+from caching.DictionaryCache import DictionaryCache
+from caching.SetCache import SetCache
+from common.global_symbols import EVENT_CONCEPT_ID_DELETED, \
+    EVENT_RULE_ID_DELETED, NotFound, \
+    Removed
+from ontologies.Exceptions import OntologyAlreadyExists, OntologyManagerCannotPopLatest, OntologyManagerFrozen, \
+    OntologyManagerNotFrozen, OntologyNotFound
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+
+
+class AlternateSdp:
+    """
+    Other Cache managers that are linked together
+    """
+
+    def __init__(self, ontologies):
+        self.names = [o.name for o in ontologies]
+        self.cache_managers = [o.cache_manager for o in ontologies]
+
+    def get(self, cache_name: str, key: str):
+        last = len(self.cache_managers) - 1
+        for i, cache_manager in enumerate(self.cache_managers):
+            value = cache_manager.get(cache_name, key)
+            if value is not NotFound:
+                return value
+
+            if i != last:
+                # forget than the key was requested
+                cache_manager.remove_initialized_key(cache_name, key)
+
+        return NotFound
+
+    def alt_get(self, cache_name: str, key: str):
+        last = len(self.cache_managers) - 1
+        for i, cache_manager in enumerate(self.cache_managers):
+            value = cache_manager.alt_get(cache_name, key)
+            if value is not NotFound:
+                return value
+
+            if i != last:
+                # forget than the key was requested
+                cache_manager.remove_initialized_key(cache_name, key)
+
+        return NotFound
+
+    def exists(self, cache_name: str, key: str):
+        for cache_manager in self.cache_managers:
+            if cache_manager.exists(cache_name, key):
+                return True
+        return False
+
+
+class Ontology:
+    """
+    an Ontology is
+     * a CacheManager (which is linked to a sdp)
+     * and a way to get to the next Ontologies (as AlternateSdp)
+    """
+
+    def __init__(self, name, depth, cache_manager: CacheManager, alt_sdp: AlternateSdp | None):
+        self.name = name
+        self.depth = depth
+        self.cache_manager = cache_manager
+        self.alt_sdp = alt_sdp
+        self.concepts_attributes = None
+
+    def __repr__(self):
+        return f"Ontology('{self.name}')"
+
+    def __eq__(self, other):
+        if not isinstance(other, Ontology):
+            return False
+
+        return self.name == other.name and self.depth == other.depth
+
+    def __hash__(self):
+        return hash((self.name, self.depth))
+
+
+class SheerkaOntologyManager:
+    ROOT_ONTOLOGY_NAME = "__default__"
+    SELF_CACHE_MANAGER = "__ontology_manager__"  # cache to store SheerkaOntologyManager info
+    CONCEPTS_BY_ONTOLOGY_ENTRY = "ConceptsByOntologyEntry"
+    RULES_BY_ONTOLOGY_ENTRY = "RulesByOntologyEntry"
+    ONTOLOGY_BY_CONCEPT_ENTRY = "OntologyByConceptEntry"
+    ONTOLOGY_BY_RULE_ENTRY = "OntologyByRuleEntry"
+
+    def __init__(self, sheerka, root_folder):
+        self.sheerka = sheerka
+        self.root_folder = root_folder
+        self.frozen = False
+
+        # We create the first cache manager
+        # For the time being, this there is no AltSdp yet !
+        ref_cache_manager = CacheManager(sdp=SheerkaDataProvider(root_folder, self.sheerka))
+        self.ontologies = [Ontology(self.ROOT_ONTOLOGY_NAME, 0, ref_cache_manager, None)]
+
+        # Ontology manager also needs to keep track of its own stuff
+        # So we create a separate sdp
+        internal_sdp = SheerkaDataProvider(root_folder, self.sheerka, self.SELF_CACHE_MANAGER)
+        self.internal_cache_manager = CacheManager(sdp=internal_sdp)
+
+        # add cache to track all the concepts defined under a given ontology
+        # key : name of the ontology
+        # value : sets of all concepts id
+        cache = SetCache(max_size=None).auto_configure(self.CONCEPTS_BY_ONTOLOGY_ENTRY)
+        self.internal_cache_manager.register_cache(self.CONCEPTS_BY_ONTOLOGY_ENTRY, cache)
+
+        # add cache to track all the rules defined under a given ontology
+        # key : name of the ontology
+        # value : sets of all rules id
+        cache = SetCache(max_size=None).auto_configure(self.RULES_BY_ONTOLOGY_ENTRY)
+        self.internal_cache_manager.register_cache(self.RULES_BY_ONTOLOGY_ENTRY, cache)
+
+        # add cache to track where the concept is created (under which ontology)
+        # key : concept id
+        # value : name of the ontology
+        cache = Cache(max_size=None).auto_configure(self.ONTOLOGY_BY_CONCEPT_ENTRY)
+        self.internal_cache_manager.register_cache(self.ONTOLOGY_BY_CONCEPT_ENTRY, cache)
+
+        # add cache to track where the rule is created (under which ontology)
+        # key : rule id
+        # value : name of the ontology
+        cache = Cache(max_size=None).auto_configure(self.ONTOLOGY_BY_RULE_ENTRY)
+        self.internal_cache_manager.register_cache(self.ONTOLOGY_BY_RULE_ENTRY, cache)
+
+    @property
+    def ontologies_names(self):
+        return [o.name for o in self.ontologies]
+
+    def freeze(self):
+        """
+        Once frozen, you can no longer register cache
+        It's to ensure consistency between ontologies
+
+        The same caches must exist for all ontologies
+        if an information is not found in a cache manager, we can request to the parent ontology.
+        This is possible only if the same cache exists in the parent ontology.
+
+        To ensure that.
+        1. You `register_cache()` all the caches
+        2. You call `freeze()`
+
+        You won't be able to `add` or `push` new ontologies if the `freeze()` is not called
+        :return:
+        :rtype:
+        """
+        self.frozen = True
+        return self
+
+    def push_ontology(self, name) -> Ontology:
+        """
+        Add an ontology layer
+        :param name: name of the layer
+        :param cache_only:
+        """
+        if not self.frozen:
+            raise OntologyManagerNotFrozen()
+
+        # pseudo clone cache manager
+        cache_manager = CacheManager(sdp=self.get_sdp(name))
+        for cache_name, cache_def in self.current_cache_manager().caches.items():
+            clone = cache_def.cache.clone()
+            if cache_name in self.current_cache_manager().concept_caches:
+                cache_manager.register_concept_cache(cache_name, clone, cache_def.get_key, cache_def.use_ref)
+            else:
+                cache_manager.register_cache(cache_name, clone, cache_def.persist, cache_def.use_ref)
+
+            # Dictionary cache special treatment
+            if isinstance(clone, DictionaryCache):
+                clone.put(False, cache_def.cache.copy())  # only a shadow copy for now
+                clone.reset_events()
+
+        alt_sdp = AlternateSdp(self.ontologies)
+        new_ontology = Ontology(name, len(self.ontologies), cache_manager, alt_sdp)
+        self.ontologies.insert(0, new_ontology)
+        return new_ontology
+
+    def pop_ontology(self, context):
+        """
+        Remove the top ontology layer
+        """
+        if not self.frozen:
+            raise OntologyManagerNotFrozen()
+
+        if len(self.ontologies) == 1:
+            raise OntologyManagerCannotPopLatest()
+
+        # remove concepts and rules tracking for the ontology to pop
+        ontology_name = self.current_ontology().name
+        concepts = self.internal_cache_manager.get(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name)
+        if concepts is not NotFound:
+            for concept in concepts:
+                self.sheerka.publish(context, EVENT_CONCEPT_ID_DELETED, concept)
+                self.internal_cache_manager.delete(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept)
+            self.internal_cache_manager.delete(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name)
+
+        rules = self.internal_cache_manager.get(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name)
+        if rules is not NotFound:
+            for rule in rules:
+                self.sheerka.publish(context, EVENT_RULE_ID_DELETED, rule)
+                self.internal_cache_manager.delete(self.ONTOLOGY_BY_RULE_ENTRY, rule)
+            self.internal_cache_manager.delete(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name)
+
+        return self.ontologies.pop(0)
+
+    def add_ontology(self, ontology: Ontology):
+        """
+        Put back a previously created ontology
+        :param ontology: how to get the items
+        """
+        if not self.frozen:
+            raise OntologyManagerNotFrozen()
+
+        ontology.alt_sdp = AlternateSdp(self.ontologies)
+        self.ontologies.insert(0, ontology)
+        for cache_def in ontology.cache_manager.caches.values():
+            cache_def.cache.reset_initialized_keys()
+
+        return self
+
+    def revert_ontology(self, context, ontology) -> Ontology:
+        """
+        Pop every ontology until the requested one is found.
+        The requested one is also popped
+        :param context:
+        :type context:
+        :param ontology:
+        :type ontology:
+        :return:
+        :rtype:
+        """
+        if ontology not in self.ontologies:
+            raise OntologyNotFound(ontology)
+
+        while self.current_ontology() != ontology:
+            self.pop_ontology(context)
+        self.pop_ontology(context)
+
+        return self.current_ontology()
+
+    def get_ontology(self, name=None):
+        """
+        Return the first ontology with the corresponding name
+        When no name is given, return the top ontology
+        """
+        if name is None:
+            return self.ontologies[0]
+
+        for ontology in self.ontologies:
+            if ontology.name == name:
+                return ontology
+
+        raise KeyError(name)
+
+    def save_ontologies_names(self):
+        self.current_sdp().save_ontologies(self.ontologies_names)
+
+    def already_on_top(self, name):
+        """
+        Returns True if the ontology 'name' is already on the top
+        Raises a OntologyAlreadyExists exception if the ontology exists, but not at the top
+        """
+        if self.ontologies[0].name == name:
+            return True
+
+        if name in self.ontologies_names:
+            raise OntologyAlreadyExists(name)
+
+        return False
+
+    def record_sheerka_state(self):
+        """
+        The current ontology can keep extra information
+        """
+        pass
+        # # TODO persist these information ?
+        # self.current_ontology().concepts_attributes = copy_concepts_attrs()
+
+    def reset_sheerka_state(self):
+        pass
+        # if self.current_ontology().concepts_attributes is not None:
+        #     load_concepts_attrs(self.current_ontology().concepts_attributes)
+
+    def current_cache_manager(self) -> CacheManager:
+        return self.ontologies[0].cache_manager
+
+    def current_sdp(self) -> SheerkaDataProvider:
+        return self.ontologies[0].cache_manager.sdp
+
+    def current_ontology(self) -> Ontology:
+        return self.ontologies[0]
+
+    def register_concept_cache(self, name, cache, get_key, use_ref):
+        """
+        Define which type of cache along with how to compute the key
+        :param name:
+        :param cache:
+        :param get_key:
+        :param use_ref:
+        :return:
+        """
+        if self.frozen:
+            raise OntologyManagerFrozen
+
+        return self.current_cache_manager().register_concept_cache(name, cache, get_key, use_ref)
+
+    def register_cache(self, name, cache, persist=True, use_ref=False):
+        """
+        Define which type of cache along with how to compute the key
+        :param name:
+        :param cache:
+        :param persist:
+        :param use_ref:
+        :return:
+        """
+        if self.frozen:
+            raise OntologyManagerFrozen
+
+        return self.current_cache_manager().register_cache(name, cache, persist, use_ref)
+
+    def add_concept(self, concept):
+        """
+        We need multiple indexes to retrieve a concept
+        So the new concept is dispatched into multiple caches
+        :param concept:
+        :return:
+        """
+        self.current_cache_manager().add_concept(concept, self.ontologies[0].alt_sdp)
+
+        # update internal states
+        self._on_concept_created(concept)
+
+    def update_concept(self, old, new):
+        """
+        Update a concept.
+        :param old:  old version of the concept
+        :param new: new version of the concept
+        :return:
+        """
+        self.current_cache_manager().update_concept(old, new, self.ontologies[0].alt_sdp)
+
+        # update internal states
+        self._on_concept_deleted(old)
+        self._on_concept_created(new)
+
+    def remove_concept(self, concept):
+        """
+        Remove a concept from all caches
+        :param concept:
+        :return:
+        """
+        self.current_cache_manager().remove_concept(concept, self.ontologies[0].alt_sdp)
+
+        # update internal states
+        self._on_concept_deleted(concept)
+
+    def get(self, cache_name, key):
+        """
+        Browses the ontologies, looking for the data 'key' in entry 'cache_name'
+        If a value is found in a low level cache, updates the top level one
+        :param cache_name:
+        :param key:
+        :return:
+        """
+        value = self.current_cache_manager().get(cache_name, key, self.ontologies[0].alt_sdp)
+        return NotFound if value is Removed else value
+
+    def exists(self, cache_name, key):
+        """
+        Browses the ontologies to check if the data 'key' is defined in entry 'cache_name'
+        :param cache_name:
+        :param key:
+        :return:
+        """
+        for ontology in self.ontologies:
+            if ontology.cache_manager.exists(cache_name, key):
+                return True
+
+        return False
+
+    def exists_in_current(self, cache_name, key):
+        """
+        Checks if the keys exists in the current ontology only
+        :param cache_name:
+        :type cache_name:
+        :param key:
+        :type key:
+        :return:
+        :rtype:
+        """
+        return self.current_cache_manager().exists(cache_name, key)
+
+    def list(self, entry, cache_only=False):
+        """
+        list all entries
+        """
+        return list(self.get_all(entry, cache_only).values())
+
+    def get_all(self, entry, cache_only=False):
+        """
+        Return all <key, value> from all ontologies
+        First look in sdp, then override with the cache, for all ontologies
+        :param entry: cache name / sdp entry
+        :param cache_only: Do no fetch data from remote sdp
+        """
+        res = {}
+        for ontology in reversed(self.ontologies):
+
+            if not cache_only:
+                # get values from sdp
+                values = ontology.cache_manager.sdp.get(entry)
+                if values is Removed:
+                    res.clear()
+
+                elif values is not NotFound:
+                    for k, v in values.items():
+                        if v is Removed:
+                            del res[k]
+                        else:
+                            res[k] = v
+
+            # override with the values from cache
+            try:
+                cache = ontology.cache_manager.get_inner_cache(entry)
+
+                if cache.is_cleared():
+                    res.clear()
+
+                for k in cache:
+                    v = cache.alt_get(k)  # Do not use get(), because of IncCache()
+                    if v is Removed:
+                        del res[k]
+                    else:
+                        res[k] = v
+
+            except KeyError:
+                pass
+
+        return res
+
+    def put(self, cache_name, key, value):
+        """
+        Add to a cache
+        :param cache_name:
+        :param key:
+        :param value:
+        :return:
+        """
+        return self.current_cache_manager().put(cache_name, key, value, self.ontologies[0].alt_sdp)
+
+    def delete(self, cache_name, key, value=None):
+        """
+        Delete an entry
+        :param cache_name:
+        :param key:
+        :param value:
+        :return:
+        """
+        return self.current_cache_manager().delete(cache_name, key, value, self.ontologies[0].alt_sdp)
+
+    def populate(self, cache_name, populate_function, get_key_function, reset_events=False, all_ontologies=False):
+        """
+        Populate a specific cache with a bunch of items
+        :param cache_name:
+        :param populate_function: how to get the items
+        :param get_key_function: how to get the key, out of an item
+        :param reset_events: reset the to_add and to_remove events after the populate
+        :param all_ontologies: populate all ontology layers
+        :return:
+        """
+        self.current_cache_manager().populate(cache_name, populate_function, get_key_function, reset_events)
+        if all_ontologies:
+            for ontology in self.ontologies[1:]:
+                ontology.cache_manager.populate(cache_name, populate_function, get_key_function, reset_events)
+
+    def copy(self, cache_name):
+        """
+        get a copy the content of the top ontology layer
+        :param self:
+        :param cache_name:
+        :return:
+        """
+        return self.current_cache_manager().caches[cache_name].cache.copy()
+
+    def commit(self, context):
+        """
+        Persist all the caches into a physical persistence storage
+        :param context:
+        :return:
+        """
+        self.internal_cache_manager.commit(context)
+        return self.current_cache_manager().commit(context)
+
+    def clear(self, cache_name=None):
+        return self.current_cache_manager().clear(cache_name)
+
+    def get_sdp(self, name=None):
+        """
+        Return new instance of SheerkaDataProvider
+        """
+        if name:
+            return SheerkaDataProvider(self.root_folder, self.sheerka, name)
+        else:
+            return self.current_sdp()
+
+    def save_event(self, event):
+        return self.current_sdp().save_event(event)
+
+    def save_execution_context(self, execution_context, is_admin):
+        return self.current_sdp().save_execution_context(execution_context, is_admin)
+
+    def is_dirty(self):
+        return self.current_cache_manager().is_dirty
+
+    def _on_concept_created(self, concept):
+        self.internal_cache_manager.put(self.CONCEPTS_BY_ONTOLOGY_ENTRY, self.current_ontology().name, concept.id)
+        self.internal_cache_manager.put(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id, self.current_ontology().name)
+
+    def _on_concept_deleted(self, concept):
+        ontology_name = self.internal_cache_manager.get(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id)
+        self.internal_cache_manager.delete(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name, concept.id)
+        self.internal_cache_manager.delete(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id)
+
+    def _on_rule_created(self, rule):
+        self.internal_cache_manager.put(self.RULES_BY_ONTOLOGY_ENTRY, self.current_ontology().name, rule.id)
+        self.internal_cache_manager.put(self.ONTOLOGY_BY_RULE_ENTRY, rule.id, self.current_ontology().name)
+
+    def _on_rule_deleted(self, rule):
+        ontology_name = self.internal_cache_manager.get(self.ONTOLOGY_BY_RULE_ENTRY, rule.id)
+        self.internal_cache_manager.delete(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name, rule.id)
+        self.internal_cache_manager.delete(self.ONTOLOGY_BY_RULE_ENTRY, rule.id)

src/parsers/ParserInput.py

@@ -0,0 +1,19 @@
+from parsers.tokenizer import Tokenizer
+
+
+class ParserInput:
+    def __init__(self, text, yield_oef=True):
+        self.original_text = text
+        self.yield_oef = yield_oef
+        self.tokens = None
+        self.exception = None
+
+    def init(self) -> bool:
+        try:
+            # the eof if forced, but will not be yield if not set to.
+            self.tokens = list(Tokenizer(self.original_text, yield_eof=True))
+            return True
+        except Exception as ex:
+            self.tokens = None
+            self.exception = ex
+            return False

src/parsers/tokenizer.py

@@ -0,0 +1,569 @@
+from dataclasses import dataclass, field
+from enum import Enum
+
+from common.global_symbols import VARIABLE_PREFIX
+
+
+class TokenKind(Enum):
+    EOF = "eof"
+    WHITESPACE = "whitespace"
+    NEWLINE = "newline"
+    IDENTIFIER = "identifier"
+    CONCEPT = "concept"
+    RULE = "rule"
+    EXPR = "expression"
+    STRING = "string"
+    NUMBER = "number"
+    LPAR = "lpar"
+    RPAR = "rpar"
+    LBRACKET = "lbracket"
+    RBRACKET = "rbracket"
+    LBRACE = "lbrace"
+    RBRACE = "rbrace"
+    PLUS = "plus"
+    MINUS = "minus"
+    STAR = "star"
+    SLASH = "slash"
+    PERCENT = "percent"
+    COMMA = "comma"  # ,
+    SEMICOLON = "semicolon"  # ;
+    COLON = "colon"  # :
+    DOT = "dot"  # .
+    QMARK = "qmark"
+    VBAR = "vbar"
+    AMPER = "amper"
+    EQUALS = "="
+    AT = "at"
+    BACK_QUOTE = "bquote"  # `
+    BACK_SLASH = "bslash"  # \
+    CARAT = "carat"  # ^
+    DOLLAR = "dollar"  # $
+    EURO = "dollar"  # €
+    STERLING = "steling"  # £
+    EMARK = "emark"  # !
+    GREATER = "greater"  # >
+    LESS = "less"  # <
+    HASH = "HASH"  # #
+    TILDE = "tilde"  # ~
+    UNDERSCORE = "underscore"  # _
+    DEGREE = "degree"  # °
+    QUOTE = "'"  # '
+    WORD = "word"
+    EQUALSEQUALS = "=="
+    STARSTAR = "**"
+    SLASHSLASH = "//"
+    VAR_DEF = "concept variable"  # __var__
+    REGEX = "regex"  # r'xxx' or r\"xxx\" or r|xxx| or r/xxx/ but not r:xxx: which means rules
+
+
+@dataclass()
+class Token:
+    type: TokenKind
+    value: object
+    index: int
+    line: int
+    column: int
+
+    _strip_quote: str = field(default=None, repr=False, compare=False, hash=None)
+    _str_value: str = field(default=None, repr=False, compare=False, hash=None)
+    _repr_value: str = field(default=None, repr=False, compare=False, hash=None)
+
+    def __repr__(self):
+        return f"Token({self.repr_value})"
+
+    @property
+    def strip_quote(self):
+        if self._strip_quote:
+            return self._strip_quote
+
+        if self.type in (TokenKind.STRING, TokenKind.REGEX):
+            self._strip_quote = self.value[1:-1]
+        else:
+            self._strip_quote = self.value
+        return self._strip_quote
+
+    @property
+    def str_value(self):
+        if self._str_value:
+            return self._str_value
+
+        self._str_value = self.to_str(False)
+        return self._str_value
+
+    @property
+    def repr_value(self):
+        if self._repr_value:
+            return self._repr_value
+
+        if self.type == TokenKind.EOF:
+            self._repr_value = "<EOF>"
+        elif self.type == TokenKind.WHITESPACE:
+            self._repr_value = "<!ws>" if self.value == "" else "<tab>" if self.value[0] == "\t" else "<ws>"
+        elif self.type == TokenKind.NEWLINE:
+            self._repr_value = "<nl>"
+        elif self.type == TokenKind.CONCEPT:
+            self._repr_value = str_concept(self.value)
+        elif self.type == TokenKind.RULE:
+            self._repr_value = str_concept(self.value, prefix="r:")
+        else:
+            self._repr_value = self.str_value
+        return self._repr_value
+
+    @staticmethod
+    def is_whitespace(token):
+        return token and token.type == TokenKind.WHITESPACE
+
+    def to_str(self, strip_quote):
+        if strip_quote and self.type == TokenKind.STRING:
+            return self.value[1:-1]
+        elif self.type == TokenKind.CONCEPT:
+            return str_concept(self.value)
+        elif self.type == TokenKind.RULE:
+            return str_concept(self.value, prefix="r:")
+        elif self.type == TokenKind.REGEX:
+            return "r" + self.value
+        else:
+            return str(self.value)
+
+    def clone(self):
+        return Token(self.type, self.value, self.index, self.line, self.column)
+
+
+@dataclass()
+class LexerError(Exception):
+    message: str
+    text: str
+    index: int
+    line: int
+    column: int
+
+
+class Keywords(Enum):
+    DEF = "def"
+    CONCEPT = "concept"
+    RULE = "rule"
+    FROM = "from"
+    BNF = "bnf"
+    AS = "as"
+    WHERE = "where"
+    PRE = "pre"
+    POST = "post"
+    ISA = "isa"
+    RET = "ret"
+    WHEN = "when"
+    PRINT = "print"
+    THEN = "then"
+    AUTO_EVAL = "auto_eval"
+    DEF_VAR = "def_var"
+
+
+class Tokenizer:
+    """
+    Class that can iterate on the tokens
+    """
+
+    def __init__(self, text, yield_eof=True, parse_word=False):
+        self.text = text
+        self.text_len = len(text)
+        self.column = 1
+        self.line = 1
+        self.i = 0
+        self.yield_eof = yield_eof
+        self.parse_word = parse_word
+
+    def __iter__(self):
+
+        while self.i < self.text_len:
+            c = self.text[self.i]
+            if c == "+":
+                if self.i + 1 < self.text_len and self.text[self.i + 1].isdigit():
+                    number = self.eat_number(self.i)
+                    yield Token(TokenKind.NUMBER, number, self.i, self.line, self.column)
+                    self.i += len(number)
+                    self.column += len(number)
+                else:
+                    yield Token(TokenKind.PLUS, "+", self.i, self.line, self.column)
+                    self.i += 1
+                    self.column += 1
+            elif c == "-":
+                if self.i + 1 < self.text_len and self.text[self.i + 1].isdigit():
+                    number = self.eat_number(self.i)
+                    yield Token(TokenKind.NUMBER, number, self.i, self.line, self.column)
+                    self.i += len(number)
+                    self.column += len(number)
+                else:
+                    yield Token(TokenKind.MINUS, "-", self.i, self.line, self.column)
+                    self.i += 1
+                    self.column += 1
+            elif c == "_":
+                if self.i + 7 < self.text_len and \
+                        self.text[self.i: self.i + 7] == VARIABLE_PREFIX and \
+                        self.text[self.i + 7].isdigit():
+                    number = self.eat_number(self.i + 7)
+                    yield Token(TokenKind.VAR_DEF, VARIABLE_PREFIX + number, self.i, self.line, self.column)
+                    self.i += 7 + len(number)
+                    self.column += 7 + len(number)
+                elif self.i + 1 < self.text_len and (self.text[self.i + 1].isalpha() or self.text[self.i + 1] == "_"):
+                    identifier = self.eat_identifier(self.i)
+                    yield Token(TokenKind.IDENTIFIER, identifier, self.i, self.line, self.column)
+                    self.i += len(identifier)
+                    self.column += len(identifier)
+                else:
+                    yield Token(TokenKind.UNDERSCORE, "_", self.i, self.line, self.column)
+                    self.i += 1
+                    self.column += 1
+            elif c == "/":
+                if self.i + 1 < self.text_len and self.text[self.i + 1] == "/":
+                    yield Token(TokenKind.SLASHSLASH, "//", self.i, self.line, self.column)
+                    self.i += 2
+                    self.column += 2
+                else:
+                    yield Token(TokenKind.SLASH, "/", self.i, self.line, self.column)
+                    self.i += 1
+                    self.column += 1
+            elif c == "*":
+                if self.i + 1 < self.text_len and self.text[self.i + 1] == "*":
+                    yield Token(TokenKind.STARSTAR, "**", self.i, self.line, self.column)
+                    self.i += 2
+                    self.column += 2
+                else:
+                    yield Token(TokenKind.STAR, "*", self.i, self.line, self.column)
+                    self.i += 1
+                    self.column += 1
+            elif c == "{":
+                yield Token(TokenKind.LBRACE, "{", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "}":
+                yield Token(TokenKind.RBRACE, "}", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "(":
+                yield Token(TokenKind.LPAR, "(", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == ")":
+                yield Token(TokenKind.RPAR, ")", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "[":
+                yield Token(TokenKind.LBRACKET, "[", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "]":
+                yield Token(TokenKind.RBRACKET, "]", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "=":
+                if self.i + 1 < self.text_len and self.text[self.i + 1] == "=":
+                    yield Token(TokenKind.EQUALSEQUALS, "==", self.i, self.line, self.column)
+                    self.i += 2
+                    self.column += 2
+                else:
+                    yield Token(TokenKind.EQUALS, "=", self.i, self.line, self.column)
+                    self.i += 1
+                    self.column += 1
+            elif c == " " or c == "\t":
+                whitespace = self.eat_whitespace(self.i)
+                yield Token(TokenKind.WHITESPACE, whitespace, self.i, self.line, self.column)
+                self.i += len(whitespace)
+                self.column += len(whitespace)
+            elif c == ",":
+                yield Token(TokenKind.COMMA, ",", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == ".":
+                yield Token(TokenKind.DOT, ".", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == ";":
+                yield Token(TokenKind.SEMICOLON, ";", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == ":":
+                yield Token(TokenKind.COLON, ":", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "?":
+                yield Token(TokenKind.QMARK, "?", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "|":
+                yield Token(TokenKind.VBAR, "|", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "&":
+                yield Token(TokenKind.AMPER, "&", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "<":
+                yield Token(TokenKind.LESS, "<", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == ">":
+                yield Token(TokenKind.GREATER, ">", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "!":
+                yield Token(TokenKind.EMARK, "!", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "`":
+                yield Token(TokenKind.BACK_QUOTE, "`", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "\\":
+                yield Token(TokenKind.BACK_SLASH, "\\", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "^":
+                yield Token(TokenKind.CARAT, "^", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "$":
+                yield Token(TokenKind.DOLLAR, "$", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "€":
+                yield Token(TokenKind.EURO, "€", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "£":
+                yield Token(TokenKind.STERLING, "£", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "#":
+                yield Token(TokenKind.HASH, "#", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "°":
+                yield Token(TokenKind.DEGREE, "°", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "~":
+                yield Token(TokenKind.TILDE, "~", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "%":
+                yield Token(TokenKind.PERCENT, "%", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "\n" or c == "\r":
+                newline = self.eat_newline(self.i)
+                yield Token(TokenKind.NEWLINE, newline, self.i, self.line, self.column)
+                self.i += len(newline)
+                self.column = 1
+                self.line += 1
+            elif c == "c" and self.i + 1 < self.text_len and self.text[self.i + 1] == ":":
+                name, id, length = self.eat_concept(self.i + 2, self.line, self.column + 2)
+                yield Token(TokenKind.CONCEPT, (name, id), self.i, self.line, self.column)
+                self.i += length + 2
+                self.column += length + 2
+            elif c == "r" and self.i + 1 < self.text_len and self.text[self.i + 1] == ":":
+                name, id, length = self.eat_concept(self.i + 2, self.line, self.column + 2)
+                yield Token(TokenKind.RULE, (name, id), self.i, self.line, self.column)
+                self.i += length + 2
+                self.column += length + 2
+            elif c == "r" and self.i + 1 < self.text_len and self.text[self.i + 1] in "'\"|/":
+                string, newlines, column_index = self.eat_string(self.i + 1, self.line, self.column)
+                yield Token(TokenKind.REGEX, string, self.i, self.line, self.column)  # quotes are kept
+                self.i += len(string) + 1
+                self.column = column_index  # 1 if newlines > 0 else self.column + len(string)
+                self.line += newlines
+            elif self.parse_word and (c.isalpha() or c.isdigit()):
+                word = self.eat_word(self.i)
+                yield Token(TokenKind.WORD, word, self.i, self.line, self.column)
+                self.i += len(word)
+                self.column += len(word)
+            elif c.isalpha():
+                identifier = self.eat_identifier(self.i)
+                yield Token(TokenKind.IDENTIFIER, identifier, self.i, self.line, self.column)
+                self.i += len(identifier)
+                self.column += len(identifier)
+            elif c.isdigit():
+                number = self.eat_number(self.i)
+                yield Token(TokenKind.NUMBER, number, self.i, self.line, self.column)
+                self.i += len(number)
+                self.column += len(number)
+            elif c == "'" and self.i > 0 and self.text[self.i - 1] != " ":
+                yield Token(TokenKind.QUOTE, "'", self.i, self.line, self.column)
+                self.i += 1
+                self.column += 1
+            elif c == "'" or c == '"':
+                string, newlines, column_index = self.eat_string(self.i, self.line, self.column)
+                yield Token(TokenKind.STRING, string, self.i, self.line, self.column)  # quotes are kept
+                self.i += len(string)
+                self.column = column_index  # 1 if newlines > 0 else self.column + len(string)
+                self.line += newlines
+            else:
+                raise LexerError(f"Unknown token '{c}'", self.text, self.i, self.line, self.column)
+
+        if self.yield_eof:
+            yield Token(TokenKind.EOF, "", self.i, self.line, self.column)
+
+    def eat_concept(self, start, line, column):
+        key, id, buffer = None, None, ""
+        i = start
+        processing_key = True
+
+        while i < self.text_len:
+
+            c = self.text[i]
+            if c == "\n":
+                raise LexerError(f"New line in concept name", self.text[start:i], i, line, column + i - start)
+
+            if c == ":":
+                if processing_key:
+                    key = buffer if buffer else None
+                else:
+                    id = buffer if buffer else None
+                i += 1  # eat the colon
+                break
+
+            if c == "#":
+                key = buffer if buffer else None
+                buffer = ""
+                processing_key = False
+                i += 1
+                continue
+
+            buffer += c
+            i += 1
+        else:
+            raise LexerError(f"Missing ending colon", self.text[start:i], i, line, column + i - start)
+
+        if (key, id) == (None, None):
+            raise LexerError(f"Concept identifiers not found", "", start, line, column)
+
+        return key, id, i - start
+
+    def eat_whitespace(self, start):
+        result = self.text[start]
+        i = start + 1
+        while i < self.text_len:
+            c = self.text[i]
+            if c == " " or c == "\t":
+                result += c
+                i += 1
+            else:
+                break
+
+        return result
+
+    def eat_newline(self, start):
+        if start + 1 == self.text_len:
+            return self.text[start]
+
+        current = self.text[start]
+        next = self.text[start + 1]
+        if current == "\n" and next == "\r" or current == "\r" and next == "\n":
+            return current + next
+
+        return current
+
+    def eat_identifier(self, start):
+        result = self.text[start]
+        i = start + 1
+        while i < self.text_len:
+            c = self.text[i]
+            if c.isalpha() or c == "_" or c == "-" or c.isdigit():
+                result += c
+                i += 1
+            else:
+                break
+
+        return result
+
+    def eat_number(self, start):
+        result = self.text[start]
+        i = start + 1
+        while i < self.text_len:
+            c = self.text[i]
+            if c.isdigit() or c == ".":
+                result += c
+                i += 1
+            else:
+                break
+
+        return result
+
+    def eat_string(self, start_index, start_line, start_column):
+        quote = self.text[start_index]
+        result = self.text[start_index]
+        lines_count = 0
+        column_index = start_column + 1
+
+        i = start_index + 1
+        escape = False
+        # newline = None
+        while i < self.text_len:
+            c = self.text[i]
+            result += c
+            i += 1
+            column_index += 1
+
+            if c == "\n":
+                lines_count += 1
+                column_index = 1
+
+            if c == "\\":
+                escape = True
+            elif c == quote and not escape:
+                break
+            else:
+                escape = False
+
+        # # add trailing new line if needed
+        # if newline:
+        #     lines_count += 1
+        #     column_index = 1
+
+        if result[-1] != quote or len(result) == 1:
+            raise LexerError("Missing Trailing quote", result, i, start_line + lines_count,
+                             1 if lines_count > 0 else start_column + len(result))
+
+        return result, lines_count, column_index
+
+    def eat_word(self, start):
+        """
+        Word is an alphanum (no space)
+        :param start:
+        :return:
+        """
+        result = self.text[start]
+        i = start + 1
+        while i < self.text_len:
+            c = self.text[i]
+            if c.isalpha() or c.isdigit():
+                result += c
+                i += 1
+            else:
+                break
+
+        return result
+
+
+def strip_tokens(tokens, strip_eof=False):
+    """
+    Remove the starting and trailing spaces and newline
+    """
+    if tokens is None:
+        return None
+
+    start = 0
+    length = len(tokens)
+    while start < length and tokens[start].type in (TokenKind.WHITESPACE, TokenKind.NEWLINE):
+        start += 1
+
+    if start == length:
+        return []
+
+    end_tokens = (TokenKind.WHITESPACE, TokenKind.NEWLINE, TokenKind.EOF) \
+        if strip_eof \
+        else (TokenKind.WHITESPACE, TokenKind.NEWLINE)
+
+    end = length - 1
+    while end > 0 and tokens[end].type in end_tokens:
+        end -= 1
+
+    return tokens[start: end + 1]

src/sdp/readme.md

@@ -0,0 +1,24 @@
+# How to serialize ?
+
+## General rule
+- 1 byte : type of object code
+- int : version of the encoder
+- data : can be the json representation of the object
+
+### Current supported types
+- J : Json object
+- P : pickle
+- E : events
+- S : state
+- X : executionContext
+- M : context metadata
+
+Not verified yet !
+
+- C : concept (with history management)
+- D : concept definitions (no history management)
+- R : executionContext ('R' stands for Result or ReturnValue, no history management)
+- O : ServiceObj (from pickle)
+- M : MemoryObject (using SheerkaPickle)
+- X : Rule (from sheerkaPickle, 'X' stands for nothing, I am running out of meaningful letters)
+- T : CustomType

src/sdp/sheerkaDataProvider.py

@@ -8,9 +8,10 @@ from os import path
 from threading import RLock
 from typing import Callable
 
+from common.utils import get_logger_name
+from core.Event import Event
 from core.ExecutionContext import ExecutionContext
-from core.Sheerka import Sheerka
-from core.global_symbols import NotFound
+from common.global_symbols import NotFound
 from sdp.sheerkaDataProviderIO import SheerkaDataProviderIO
 from sdp.sheerkaSerializer import Serializer, SerializerContext
 
@@ -27,72 +28,6 @@ def json_default_converter(o):
         return o.isoformat()
 
 
-class Event(object):
-    """
-    Class that represents something that modifies the state of the system
-    """
-
-    def __init__(self, message="", user_id="", date=None, parents=None):
-        self.user_id: str = user_id  # id of the user that triggers the modification
-        self.date: datetime | None = date or datetime.now()  # when
-        self.message: str = message  # user input or whatever that modifies the system
-        self.parents: list[str] = parents  # digest(s) of the parent(s) of this event
-        self._digest: str | None = None  # digest of the event
-
-    def __str__(self):
-        return f"{self.date.strftime('%d/%m/%Y %H:%M:%S')} {self.message}"
-
-    def __repr__(self):
-        return f"{self.get_digest()[:12]} {self.message}"
-
-    def get_digest(self):
-        """
-        Returns the digest of the event
-        :return: sha256 of the event
-        """
-
-        if self._digest:
-            return self._digest
-
-        if self.user_id == "":
-            # only possible during the unit test
-            # We use this little trick to speed up the unit test
-            self._digest = self.message[6:] if self.message.startswith("TEST::") else "xxx"
-            return self._digest
-
-        if not isinstance(self.message, str):
-            raise NotImplementedError(f"message={self.message}")
-
-        to_hash = f"Event:{self.user_id}{self.date}{self.message}{self.parents}".encode("utf-8")
-        self._digest = hashlib.sha256(to_hash).hexdigest()
-        return self._digest
-
-    def to_dict(self):
-        return self.__dict__
-
-    def from_dict(self, as_dict):
-        self.user_id = as_dict["user_id"]
-        self.date = datetime.fromisoformat(as_dict["date"])
-        self.message = as_dict["message"]
-        self.parents = as_dict["parents"]
-        self._digest = as_dict["_digest"]  # freeze the digest
-
-    def __eq__(self, other):
-        if id(self) == id(other):
-            return True
-
-        if isinstance(other, Event):
-            return (self.user_id == other.user_id and
-                    self.date == other.date and
-                    self.message == other.message and
-                    self.parents == other.parents)
-
-        return False
-
-    def __hash__(self):
-        return hash(self.get_digest())
-
-
 class State:
     """
     Class that represents the state of the system (dictionary of all known entries)
@@ -118,7 +53,7 @@ class SheerkaDataProviderTransaction:
     Note that Transaction within Transaction is not supported
     """
 
-    def __init__(self, sdp, event):
+    def __init__(self, sdp, event: Event):
         self.sdp: SheerkaDataProvider = sdp
         self.event: Event = event
         self.state: State | None = None
@@ -181,6 +116,7 @@ class SheerkaDataProviderTransaction:
                 pass
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        # TODO: Manage exception in order to rollback
         self.state.parents = [] if self.snapshot is None else [self.snapshot]
         self.state.events = [self.event_digest]
         self.state.date = datetime.now()
@@ -207,10 +143,11 @@ class SheerkaDataProvider:
     REF_PREFIX = "##REF##:"
 
     def __init__(self, root=None, sheerka=None, name="__default__"):
-        self.log = logging.getLogger(__name__)
-        self.init_log = logging.getLogger("init." + __name__)
-        self.init_log.debug("Initializing sdp.")
+        self.log = logging.getLogger(get_logger_name(__name__))
+        self.init_log = logging.getLogger(get_logger_name("init." + __name__))
+        self.init_log.debug(f"Initializing sdp '{name}'")
 
+        from core.Sheerka import Sheerka
         self.sheerka: Sheerka = sheerka
         self.io = SheerkaDataProviderIO.get(root)
         self.first_time = self.io.first_time

src/sdp/sheerkaDataProviderIO.py

@@ -3,13 +3,15 @@ import logging
 import os
 from os import path
 
+from common.utils import get_logger_name
+
 
 class SheerkaDataProviderIO:
 
     def __init__(self, root):
         self.root = root
-        self.log = logging.getLogger(__name__)
-        self.init_log = logging.getLogger("init." + __name__)
+        self.log = logging.getLogger(get_logger_name(__name__))
+        self.init_log = logging.getLogger(get_logger_name("init." + __name__))
 
     def exists(self, file_path):
         pass

src/sdp/sheerkaSerializer.py

@@ -7,9 +7,12 @@ import struct
 from dataclasses import dataclass
 from enum import Enum
 
-from core.utils import get_class, get_full_qualified_name
+from common.global_symbols import CustomType, NotFound, NotInit, Removed
+from common.utils import get_class, get_full_qualified_name, get_logger_name
 
 
+# https://stackoverflow.com/questions/15721363/preserve-python-tuples-with-json
+
 def json_default_converter(o):
     """
     Default formatter for json
@@ -24,7 +27,7 @@ def json_default_converter(o):
     if isinstance(o, Enum):
         return o.name
 
-    raise Exception(f"Cannot serialize object '{o}', class='{o.__class__.__name__}'")
+    # raise Exception(f"Cannot serialize object '{o}', class='{o.__class__.__name__}'")
     # In debug mode, just
     #
     # with open("json_encoding_error.txt", "a") as f:
@@ -47,21 +50,23 @@ class Serializer:
     HISTORY = "##history##"
 
     def __init__(self):
-        self.log = logging.getLogger(__name__)
-        self.init_log = logging.getLogger("init." + __name__)
+        self.log = logging.getLogger(get_logger_name(__name__))
+        self.init_log = logging.getLogger(get_logger_name("init." + __name__))
         self.init_log.debug("Initializing serializers")
         self._cache = []
 
         # add builtin serializers
         self.register(EventSerializer())
         self.register(StateSerializer())
+        self.register(ExecutionContextSerializer())
+        self.register(ConceptMetadataSerializer())
+        self.register(CustomTypeSerializer())
         # self.register(ConceptSerializer())
         # self.register(DictionarySerializer())
-        # self.register(ExecutionContextSerializer())
         # self.register(MemoryObjectSerializer())  # before ServiceObjSerializer
         # self.register(ServiceObjSerializer())
         # self.register(RuleSerializer())
-        # self.register(CustomTypeSerializer())
+        #
 
     def register(self, serializer):
         """
@@ -204,12 +209,35 @@ class JsonSerializer(BaseSerializer):
         return obj
 
 
+class CustomTypeSerializer(BaseSerializer):
+    def __init__(self):
+        BaseSerializer.__init__(self, "T", 1)
+
+    def matches(self, obj):
+        return isinstance(obj, CustomType)
+
+    def dump(self, stream, obj, context):
+        stream.write(obj.value.encode("utf-8"))
+        stream.seek(0)
+        return stream
+
+    def load(self, stream, context):
+        value = stream.read().decode("utf-8")
+        if value == NotInit.value:
+            return NotInit
+        elif value == NotFound.value:
+            return NotFound
+        elif value == Removed.value:
+            return Removed
+        raise NotImplemented(f"CustomTypeSerializer.load({value})")
+
+
 class EventSerializer(BaseSerializer):
     def __init__(self):
         BaseSerializer.__init__(self, "E", 1)
 
     def matches(self, obj):
-        return get_full_qualified_name(obj) == "sdp.sheerkaDataProvider.Event"
+        return get_full_qualified_name(obj) == "core.Event.Event"
 
     def dump(self, stream, obj, context):
         stream.write(json.dumps(obj.to_dict(), default=json_default_converter).encode("utf-8"))
@@ -231,6 +259,21 @@ class StateSerializer(PickleSerializer):
             "S",
             1)
 
+
+class ExecutionContextSerializer(JsonSerializer):
+    CLASS_NAME = "core.ExecutionContext.ExecutionContext"
+
+    def __init__(self):
+        super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "X", 1)
+
+
+class ConceptMetadataSerializer(PickleSerializer):
+    CLASS_NAME = "core.concept.ConceptMetadata"
+
+    def __init__(self):
+        super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "M", 1)
+
+#
 #
 #
 # class JsonSerializer(BaseSerializer):
@@ -359,24 +402,3 @@ class StateSerializer(PickleSerializer):
 #         super().__init__(lambda obj: isinstance(obj, Rule), "X", 1)
 #
 #
-# class CustomTypeSerializer(BaseSerializer):
-#     def __init__(self):
-#         BaseSerializer.__init__(self, "T", 1)
-#
-#     def matches(self, obj):
-#         return isinstance(obj, CustomType)
-#
-#     def dump(self, stream, obj, context):
-#         stream.write(obj.value.encode("utf-8"))
-#         stream.seek(0)
-#         return stream
-#
-#     def load(self, stream, context):
-#         value = stream.read().decode("utf-8")
-#         if value == NotInit.value:
-#             return NotInit
-#         elif value == NotFound.value:
-#             return NotFound
-#         elif value == Removed.value:
-#             return Removed
-#         raise NotImplemented(f"CustomTypeSerializer.load({value})")

src/server/authentication.py

@@ -19,7 +19,8 @@ oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
 fake_users_db = {
     "kodjo": {
         "username": "kodjo",
-        "full_name": "Kodjo Sossouvi",
+        "firstname": "Kodjo",
+        "lastname": "Sossouvi",
         "email": "kodjo.sossouvi@gmail.com",
         "hashed_password": "$2b$12$fb9jW7QUZ9KIEAAtVmWMEOGtehKy9FafUr7Zfrsb3ZMhsBbzZs7SC",  # password is kodjo
         "disabled": False,
@@ -42,7 +43,8 @@ class User(BaseModel):
     """
     username: str
     email: str | None = None
-    full_name: str | None = None
+    firstname: str | None = None
+    lastname: str | None = None
     disabled: bool | None = None
 
 

src/server/main.py

@@ -4,12 +4,32 @@ import uvicorn
 from fastapi import Depends, FastAPI, HTTPException
 from fastapi.security import OAuth2PasswordRequestForm
 from starlette import status
+from starlette.middleware.cors import CORSMiddleware
 
 from constants import CLIENT_OPERATION_QUIT, EXIT_COMMANDS, SHEERKA_PORT
+from core.Sheerka import Sheerka
 from server.authentication import ACCESS_TOKEN_EXPIRE_MINUTES, User, authenticate_user, create_access_token, \
     fake_users_db, get_current_active_user
 
 app = FastAPI()
+origins = [
+    "http://localhost:56426",
+    "http://localhost:5173",
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+#  NEED TO FIND A WAY TO INIT SHEERKA within the __name__ == "__main__" section
+#  As of now, if we do that, sheerka is not properly initialized using the command
+#  'uvicorn server.main:app' from the shell
+sheerka = Sheerka()
+sheerka.initialize()
 
 
 @app.get("/")
@@ -34,7 +54,10 @@ async def login(form_data: OAuth2PasswordRequestForm = Depends()):
 
     access_token_expires = timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
     access_token = create_access_token(data={"sub": user.username}, expires_delta=access_token_expires)
-    return {"access_token": access_token, "token_type": "bearer"}
+    return {"first_name": user.firstname,
+            "last_name": user.lastname,
+            "access_token": access_token,
+            "token_type": "bearer"}
 
 
 @app.post("/echo/{message}", status_code=status.HTTP_200_OK, response_model=dict)
@@ -62,5 +85,21 @@ async def echo(message: str, current_user: User = Depends(get_current_active_use
     }
 
 
+@app.post("/command/{message}", status_code=status.HTTP_200_OK, response_model=dict)
+async def command(message: str, current_user: User = Depends(get_current_active_user)) -> dict:
+    if message in EXIT_COMMANDS:
+        return {
+            "status": True,
+            "response": "Take care.",
+            "command": CLIENT_OPERATION_QUIT
+        }
+    res = sheerka.evaluate_user_input(message, current_user)
+    return {
+        "status": res[0].status,
+        "response": res[0].value.body,
+        "command": None
+    }
+
+
 if __name__ == "__main__":
     uvicorn.run("server.main:app", port=SHEERKA_PORT, log_level="info")

src/sheerkapickle/__init__.py

@@ -0,0 +1,7 @@
+from .sheerkaplicker import encode
+from .sheerkaunpickler import decode
+
+__all__ = ('encode', 'decode')
+
+# register built-in handlers
+__import__('sheerkapickle.handlers', level=0)

src/sheerkapickle/handlers.py

@@ -0,0 +1,197 @@
+import re
+import threading
+import uuid
+
+from sheerkapickle import utils
+
+
+class ToReduce:
+    def __init__(self, predicate, get_value):
+        self.predicate = predicate
+        self.get_value = get_value
+
+
+class SheerkaRegistry(object):
+
+    def __init__(self):
+        self._handlers = {}
+        self._base_handlers = {}
+
+    def get(self, cls_or_name: str, default=None):
+        """
+        :param cls_or_name: the type or its fully qualified name
+        :param default: default value, if no matching handler is not found
+
+        Looks up a handler by type reference or its fully
+        qualified name. If a direct match
+        is not found, the search is performed over all
+        handlers registered with base=True.
+        """
+        handler = self._handlers.get(cls_or_name)
+        # attempt to find a base class
+        if handler is None and utils.is_type(cls_or_name):
+            for cls, base_handler in self._base_handlers.items():
+                if issubclass(cls_or_name, cls):
+                    return base_handler
+        return default if handler is None else handler
+
+    def register(self, cls, handler: 'BaseHandler' = None, base: bool = False):
+        """Register the custom handler for a class
+
+        :param cls: The custom object class to handle
+        :param handler: The custom handler class (if
+            None, a decorator wrapper is returned)
+        :param base: Indicates whether the handler should
+            be registered for all subclasses
+
+        This function can be also used as a decorator
+        by omitting the `handler` argument::
+
+            @jsonpickle.handlers.register(Foo, base=True)
+            class FooHandler(jsonpickle.handlers.BaseHandler):
+                pass
+
+        """
+        if handler is None:
+            def _register(handler_cls):
+                self.register(cls, handler=handler_cls, base=base)
+                return handler_cls
+
+            return _register
+        if not utils.is_type(cls):
+            raise TypeError('{!r} is not a class/type'.format(cls))
+        # store both the name and the actual type for the ugly cases like
+        # _sre.SRE_Pattern that cannot be loaded back directly
+        self._handlers[utils.importable_name(cls)] = \
+            self._handlers[cls] = handler
+        if base:
+            # only store the actual type for subclass checking
+            self._base_handlers[cls] = handler
+
+    def unregister(self, cls):
+        self._handlers.pop(cls, None)
+        self._handlers.pop(utils.importable_name(cls), None)
+        self._base_handlers.pop(cls, None)
+
+
+registry = SheerkaRegistry()
+register = registry.register
+unregister = registry.unregister
+get = registry.get
+
+
+class BaseHandler(object):
+
+    def __init__(self, sheerka, context):
+        """
+        Initialize a new handler to handle a registered type.
+
+        :Parameters:
+          - `context`: reference to pickler/unpickler
+
+        """
+        self.sheerka = sheerka
+        self.context = context
+
+    def __call__(self, sheerka, context):
+        """This permits registering either Handler instances or classes
+
+        :Parameters:
+          - `context`: reference to pickler/unpickler
+        """
+        self.sheerka = sheerka
+        self.context = context
+        return self
+
+    def flatten(self, obj: object, data: dict):
+        """
+        Flatten `obj` into a json-friendly form and write result to `data`.
+
+        :param object obj: The object to be serialized.
+        :param dict data: A partially filled dictionary which will contain the
+            json-friendly representation of `obj` once this method has
+            finished.
+        """
+        raise NotImplementedError('You must implement flatten() in %s' %
+                                  self.__class__)
+
+    def new(self, data: dict):
+        raise NotImplementedError('You must implement new() in %s' %
+                                  self.__class__)
+
+    def restore(self, data: dict, instance: object):
+        """
+        Restore an object of the registered type from the json-friendly
+        representation `obj` and return it.
+        """
+        raise NotImplementedError('You must implement restore() in %s' %
+                                  self.__class__)
+
+    @classmethod
+    def handles(self, cls: str):
+        """
+        Register this handler for the given class. Suitable as a decorator,
+        e.g.::
+
+            @MyCustomHandler.handles
+            class MyCustomClass:
+                def __reduce__(self):
+                    ...
+        """
+        registry.register(cls, self)
+        return cls
+
+
+class RegexHandler(BaseHandler):
+    """Flatten _sre.SRE_Pattern (compiled regex) objects"""
+
+    def flatten(self, obj, data):
+        data['pattern'] = obj.pattern
+        return data
+
+    def new(self, data):
+        return re.compile(data['pattern'])
+
+    def restore(self, data, instance):
+        return instance
+
+
+RegexHandler.handles(type(re.compile('')))
+
+
+class UUIDHandler(BaseHandler):
+    """Serialize uuid.UUID objects"""
+
+    def flatten(self, obj, data):
+        data['hex'] = obj.hex
+        return data
+
+    def new(self, data):
+        return uuid.UUID(data['hex'])
+
+    def restore(self, data, instance):
+        return instance
+
+
+UUIDHandler.handles(uuid.UUID)
+
+
+class LockHandler(BaseHandler):
+    """Serialize threading.Lock objects"""
+
+    def flatten(self, obj, data):
+        data['locked'] = obj.locked()
+        return data
+
+    def new(self, data):
+        lock = threading.Lock()
+        if data.get('locked', False):
+            lock.acquire()
+        return lock
+
+    def restore(self, data, instance):
+        return instance
+
+
+_lock = threading.Lock()
+LockHandler.handles(_lock.__class__)

src/sheerkapickle/sheerka_handlers.py

@@ -0,0 +1,240 @@
+from common.global_symbols import NotInit
+from core.concept import Concept
+from sheerkapickle.handlers import BaseHandler, registry
+
+default_concept = Concept()
+default_concept_values = default_concept.values()
+CONCEPT_ID = "concept/id"
+RULE_ID = "rule/id"
+
+
+class ConceptHandler(BaseHandler):
+
+    def flatten(self, obj: Concept, data):
+        pickler = self.context
+        sheerka = self.sheerka
+
+        if obj.get_metadata().full_serialization:
+            ref = default_concept
+            ref_values = default_concept_values
+        else:
+            ref = sheerka.get_by_id(obj.id, allow_dynamic=True)
+            ref_values = ref.values()
+            data[CONCEPT_ID] = obj.id
+
+        # transform metadata
+        for name in CONCEPT_PROPERTIES_TO_SERIALIZE:
+            value = getattr(obj.get_metadata(), name)
+            ref_value = getattr(ref.get_metadata(), name)
+            if value != ref_value:
+                value_to_use = [list(t) for t in value] if name == "variables" else value
+                data["meta." + name] = pickler.flatten(value_to_use)
+
+        # # transform values
+        for name, value in obj.values().items():
+            if name not in ref_values or value != ref_values[name]:
+                if "values" not in data:
+                    data["values"] = []
+                data["values"].append((pickler.flatten(name), pickler.flatten(value)))
+
+        return data
+
+    def new(self, data):
+        sheerka = self.sheerka
+        if CONCEPT_ID in data:
+            return sheerka.new((None, data[CONCEPT_ID]), allow_dynamic=True)
+        else:
+            return Concept()
+
+    def restore(self, data, instance):
+        pickler = self.context
+
+        for key, value in data.items():
+            if key.startswith("_sheerka/") or key == CONCEPT_ID:
+                continue
+
+            resolved_value = pickler.restore(data[key])
+
+            if key.startswith("meta."):
+                # get metadata
+                resolved_prop = key[5:]
+                if resolved_prop == "variables":
+                    for prop_name, prop_value in resolved_value:
+                        instance.def_var(prop_name, prop_value)
+                else:
+                    setattr(instance.get_metadata(), resolved_prop, resolved_value)
+            elif key == "values":
+                # get properties
+                for prop_name, prop_value in resolved_value:
+                    instance.set_value(prop_name, NotInit if prop_value is None else prop_value)
+            else:
+                raise Exception("Sanity check as it's not possible yet")
+
+        # instance.freeze_definition_hash()
+        return instance
+
+
+class UserInputHandler(ConceptHandler):
+
+    def flatten(self, obj: UserInputConcept, data):
+        data[CONCEPT_ID] = (obj.key, obj.id)
+        data["user_name"] = obj.user_name
+        data["text"] = core.utils.get_text_from_tokens(obj.text) if isinstance(obj.text, list) else \
+            obj.text.as_text() if isinstance(obj.text, ParserInput) else \
+                obj.text
+        return data
+
+    def new(self, data):
+        return UserInputConcept.__new__(UserInputConcept)
+
+    def restore(self, data, instance):
+        instance.__init__(data["text"], data["user_name"])
+        instance.get_metadata().key = data[CONCEPT_ID][0]
+        instance.get_metadata().id = data[CONCEPT_ID][1]
+        instance.freeze_definition_hash()
+        return instance
+
+
+class ReturnValueHandler(BaseHandler):
+
+    def flatten(self, obj: ReturnValueConcept, data):
+        pickler = self.context
+
+        data[CONCEPT_ID] = (obj.key, obj.id)
+        data["who"] = f"c:{obj.who.id}:" if isinstance(obj.who, Concept) else \
+            obj.who.name if isinstance(obj.who, (BaseParser, BaseEvaluator)) else \
+                obj.who
+        data["status"] = obj.status
+        data["value"] = pickler.flatten(obj.value)
+        if obj.parents:
+            data["parents"] = pickler.flatten(obj.parents)
+        return data
+
+    def new(self, data):
+        return ReturnValueConcept.__new__(ReturnValueConcept)
+
+    def restore(self, data, instance):
+        pickler = self.context
+
+        instance.__init__(data["who"], data["status"], pickler.restore(data["value"]))
+        if "parents" in data:
+            instance.parents = pickler.restore(data["parents"])
+
+        instance.get_metadata().key = data[CONCEPT_ID][0]
+        instance.get_metadata().id = data[CONCEPT_ID][1]
+        instance.freeze_definition_hash()
+        return instance
+
+
+class SheerkaHandler(ConceptHandler):
+
+    # def flatten(self, obj: BuiltinConcepts, data):
+    #     return ConceptHandler().flatten()data
+
+    def new(self, data):
+        return self.sheerka
+
+    # def restore(self, data, instance):
+    #     return instance
+
+
+class ExecutionContextHandler(BaseHandler):
+
+    def flatten(self, obj, data):
+        pickler = self.context
+
+        for prop in CONTEXT_PROPERTIES_TO_SERIALIZE:
+            if prop in ("who", "action", "action_context"):
+                value = str(getattr(obj, prop))
+            else:
+                value = getattr(obj, prop)
+
+            if value is not None:
+                data[prop] = pickler.flatten(value)
+
+        return data
+
+    def new(self, data):
+        return ExecutionContext(data["who"], None, self.sheerka, BuiltinConcepts.NOP, None)
+
+    def restore(self, data, instance):
+        pickler = self.context
+
+        for prop in CONTEXT_PROPERTIES_TO_SERIALIZE:
+            if prop not in data or prop == "who":
+                continue
+            setattr(instance, prop, pickler.restore(data[prop]))
+
+        return instance
+
+
+class RuleContextHandler(BaseHandler):
+
+    def flatten(self, obj, data):
+        data[RULE_ID] = obj.id
+        data["name"] = obj.metadata.name
+        data["predicate"] = obj.metadata.predicate
+        data["action_type"] = obj.metadata.action_type
+        data["action"] = obj.metadata.action
+        return data
+
+    def new(self, data):
+        return Rule(data["action_type"], data["name"], data["predicate"], data["action"])
+
+    def restore(self, data, instance):
+        instance.metadata.id = data[RULE_ID]
+        return instance
+
+
+class PythonNodeHandler(BaseHandler):
+
+    def flatten(self, obj, data):
+        pickler = self.context
+
+        data["source"] = obj.source
+        data["ast_str"] = obj.ast_str
+        data["objects"] = pickler.flatten(obj.objects)
+        return data
+
+    def new(self, data):
+        return PythonNode.__new__(PythonNode)
+
+    def restore(self, data, instance):
+        pickler = self.context
+
+        instance.__init__(data["source"], objects=pickler.restore(data["objects"]))
+        instance.ast_str = data["ast_str"]
+        return instance
+
+
+class DefRuleNodeHandler(BaseHandler):
+    def flatten(self, obj, data):
+        pickler = self.context
+
+        data["tokens"] = pickler.flatten(obj.tokens)
+        data["name"] = pickler.flatten(obj.name)
+        return data
+
+    def new(self, data):
+        return DefRuleNode.__new__(DefRuleNode)
+
+    def restore(self, data, instance):
+        pickler = self.context
+
+        instance.__init__(data["source"], objects=pickler.restore(data["objects"]))
+        instance.tokens = pickler.restore(data["tokens"])
+        instance.name = pickler.restore(data["name"])
+        return instance
+
+
+def initialize_pickle_handlers():
+    registry.register(Concept, ConceptHandler, True)
+    registry.register(UserInputConcept, UserInputHandler, True)
+    registry.register(ReturnValueConcept, ReturnValueHandler, True)
+    registry.register(Sheerka, SheerkaHandler, True)
+    registry.register(ExecutionContext, ExecutionContextHandler, True)
+    registry.register(Rule, RuleContextHandler, True)
+    registry.register(PythonNode, PythonNodeHandler, True)
+    registry.register(DefRuleNode, DefRuleNodeHandler, True)
+    registry.register(DefExecRuleNode, DefRuleNodeHandler, True)  # TODO: fix inheritance that does not work
+    registry.register(DefFormatRuleNode, DefRuleNodeHandler, True)  # TODO: fix inheritance that does not work

src/sheerkapickle/sheerkaplicker.py

@@ -0,0 +1,163 @@
+import json
+from logging import Logger
+
+from common.utils import get_full_qualified_name, str_concept
+from core.concept import Concept
+from sheerkapickle import utils, tags, handlers
+
+
+def encode(sheerka, obj):
+    pickler = SheerkaPickler(sheerka)
+    flatten = pickler.flatten(obj)
+    return json.dumps(flatten)
+
+
+class ToReduce:
+    def __init__(self, predicate, get_value):
+        self.predicate = predicate
+        self.get_value = get_value
+
+
+class SheerkaPickler:
+    """
+    Json sheerkapickle
+    Inspired by jsonpickle (https://github.com/jsonpickle/jsonpickle)
+    which failed to work in my environment
+    """
+
+    def __init__(self, sheerka):
+        self.ids = {}
+        self.objs = []
+        self.id_count = -1
+        self.sheerka = sheerka
+        self.to_reduce = []
+
+        self.to_reduce.append(ToReduce(lambda o: isinstance(o, Logger), lambda o: None))
+        # from parsers.BaseParser import BaseParser
+        # from evaluators.BaseEvaluator import BaseEvaluator
+        from ontologies.SheerkaOntologyManager import Ontology
+        # from core.sheerka.Sheerka import SheerkaMethod
+        # self.to_reduce.append(ToReduce(lambda o: isinstance(o, (BaseParser, BaseEvaluator)), lambda o: o.name))
+        # self.to_reduce.append(ToReduce(lambda o: isinstance(o, ParserInput), lambda o: o.as_text()))
+        self.to_reduce.append(ToReduce(lambda o: isinstance(o, Ontology), lambda o: f"__ONTOLOGY:{o.name}__"))
+        # self.to_reduce.append(ToReduce(lambda o: isinstance(o, SheerkaMethod), lambda o: o.name))
+
+    def flatten(self, obj):
+        if utils.is_to_discard(obj):
+            return str(obj)
+
+        if utils.is_primitive(obj):
+            return obj
+
+        if utils.is_custom_type(obj):
+            return self._flatten_custom_type(obj)
+
+        if utils.is_type(obj):
+            return str(obj)
+
+        if utils.is_tuple(obj):
+            return {tags.TUPLE: [self.flatten(v) for v in obj]}
+
+        if utils.is_set(obj):
+            return {tags.SET: [self.flatten(v) for v in obj]}
+
+        if utils.is_list(obj):
+            return [self.flatten(v) for v in obj]
+
+        if utils.is_dictionary(obj):
+            return self._flatten_dict(obj)
+
+        if utils.is_enum(obj):
+            return self._flatten_enum(obj)
+
+        if utils.is_object(obj):
+            return self._flatten_obj_instance(obj)
+
+        raise Exception(f"Cannot flatten '{obj}'")
+
+    def _flatten_dict(self, obj):
+        data = {}
+        for k, v in obj.items():
+            if k is None:
+                k_str = "null"
+            elif utils.is_enum(k):
+                k_str = get_full_qualified_name(k) + "." + k.name
+            elif isinstance(k, Concept):
+                k_str = str_concept(k)
+            else:
+                k_str = k
+
+            data[k_str] = self.flatten(v)
+
+        return data
+
+    def _flatten_enum(self, obj):
+        # check if the object was already seen
+        exists, _id = self.exist(obj)
+        if exists:
+            return {tags.ID: _id}
+        else:
+            self.id_count = self.id_count + 1
+            self.ids[id(obj)] = self.id_count
+            self.objs.append(obj)
+
+        data = {}
+        class_name = get_full_qualified_name(obj)
+        data[tags.ENUM] = class_name + "." + obj.name
+        return data
+
+    def _flatten_obj_instance(self, obj):
+        for reduce in self.to_reduce:
+            if reduce.predicate(obj):
+                return reduce.get_value(obj)
+
+        # check if the object was already seen
+        exists, _id = self.exist(obj)
+        if exists:
+            return {tags.ID: _id}
+        else:
+            self.id_count = self.id_count + 1
+            self.ids[id(obj)] = self.id_count
+            self.objs.append(obj)
+
+        # flatten
+        data = {}
+        cls = obj.__class__ if hasattr(obj, '__class__') else type(obj)
+        class_name = utils.importable_name(cls)
+        data[tags.OBJECT] = class_name
+
+        handler = handlers.get(class_name)
+        if handler is not None:
+            return handler(self.sheerka, self).flatten(obj, data)
+
+        if hasattr(obj, "__dict__"):
+            for k, v in obj.__dict__.items():
+                data[k] = self.flatten(v)
+
+        return data
+
+    def _flatten_custom_type(self, obj):
+        # check if the object was already seen
+        exists, _id = self.exist(obj)
+        if exists:
+            return {tags.ID: _id}
+        else:
+            self.id_count = self.id_count + 1
+            self.ids[id(obj)] = self.id_count
+            self.objs.append(obj)
+
+        return {tags.CUSTOM: obj.value}
+
+    def exist(self, obj):
+        try:
+            v = self.ids[id(obj)]
+            return True, v
+        except KeyError:
+            return False, None
+
+    # def exist(self, obj):
+    #     for k, v in self.ids.items():
+    #         if k == id(obj):
+    #             return True, v
+    #
+    #     return False, None

src/sheerkapickle/sheerkaunpickler.py

@@ -0,0 +1,129 @@
+import json
+
+from common.global_symbols import NoFirstToken, NotFound, NotInit, Removed
+from common.utils import decode_enum, get_class, unstr_concept
+from sheerkapickle import tags, utils, handlers
+
+
+def decode(sheerka, obj):
+    return SheerkaUnpickler(sheerka).restore(json.loads(obj))
+
+
+class SheerkaUnpickler:
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+        self.objs = []
+
+    def restore(self, obj):
+        if has_tag(obj, tags.ID):
+            return self._restore_id(obj)
+
+        if has_tag(obj, tags.TUPLE):
+            return self._restore_tuple(obj)
+
+        if has_tag(obj, tags.CUSTOM):
+            return self._restore_custom(obj)
+
+        if has_tag(obj, tags.SET):
+            return self._restore_set(obj)
+
+        if has_tag(obj, tags.ENUM):
+            return self._restore_enum(obj)
+
+        if has_tag(obj, tags.OBJECT):
+            return self._restore_obj(obj)
+
+        if utils.is_list(obj):
+            return self._restore_list(obj)
+
+        if utils.is_dictionary(obj):
+            return self._restore_dict(obj)
+
+        return obj
+
+    def _restore_list(self, obj):
+        return [self.restore(v) for v in obj]
+
+    def _restore_tuple(self, obj):
+        return tuple([self.restore(v) for v in obj[tags.TUPLE]])
+
+    def _restore_custom(self, obj):
+        if obj[tags.CUSTOM] == NotInit.value:
+            instance = NotInit
+        elif obj[tags.CUSTOM] == NotFound.value:
+            instance = NotFound
+        elif obj[tags.CUSTOM] == Removed.value:
+            instance = Removed
+        elif obj[tags.CUSTOM] == NoFirstToken.value:
+            instance = NoFirstToken
+        else:
+            raise KeyError(f"unknown  {obj[tags.CUSTOM]}")
+
+        self.objs.append(instance)
+        return instance
+
+    def _restore_set(self, obj):
+        return set([self.restore(v) for v in obj[tags.SET]])
+
+    def _restore_enum(self, obj):
+        instance = decode_enum(obj[tags.ENUM])
+        self.objs.append(instance)
+        return instance
+
+    def _restore_dict(self, obj):
+        data = {}
+        for k, v in obj.items():
+            resolved_key = self._resolve_key(k)
+            data[resolved_key] = self.restore(v)
+        return data
+
+    def _restore_id(self, obj):
+        try:
+            return self.objs[obj[tags.ID]]
+        except IndexError:
+            pass
+
+    def _restore_obj(self, obj):
+        handler = handlers.get(obj[tags.OBJECT])
+
+        if handler:
+            handler = handler(self.sheerka, self)
+            instance = handler.new(obj)
+            self.objs.append(instance)
+            instance = handler.restore(obj, instance)
+        else:
+            # KSI 202011: Hack because Property is removed
+            # To suppress asap
+            if obj[tags.OBJECT] == "core.concept.Property":
+                return self.restore(obj["value"])
+
+            cls = get_class(obj[tags.OBJECT])
+            instance = cls.__new__(cls)
+            self.objs.append(instance)
+
+            for k, v in obj.items():
+                if k == tags.OBJECT:
+                    continue
+                value = self.restore(v)
+                setattr(instance, k, value)
+
+        return instance
+
+    def _resolve_key(self, key):
+
+        if key == "null":
+            return None
+
+        concept_key, concept_id = unstr_concept(key)
+        if concept_key is not None:
+            return self.sheerka.new((concept_key, concept_id)) if concept_id else self.sheerka.new(concept_key)
+
+        as_enum = decode_enum(key)
+        if as_enum is not None:
+            return as_enum
+
+        return key
+
+
+def has_tag(obj, tag):
+    return type(obj) is dict and tag in obj

src/sheerkapickle/tags.py

@@ -0,0 +1,6 @@
+ID = "_sheerka/id"
+TUPLE = "_sheerka/tuple"
+SET = "_sheerka/set"
+OBJECT = "_sheerka/obj"
+ENUM = "_sheerka/enum"
+CUSTOM = "_sheerka/custom"

src/sheerkapickle/utils.py

@@ -0,0 +1,101 @@
+import base64
+import types
+from enum import Enum
+
+from common.global_symbols import CustomType
+
+class_types = (type,)
+PRIMITIVES = (str, bool, type(None), int, float)
+
+
+def is_type(obj):
+    """Returns True is obj is a reference to a type.
+    """
+    # use "isinstance" and not "is" to allow for metaclasses
+    return isinstance(obj, class_types)
+
+
+def is_enum(obj):
+    return isinstance(obj, Enum)
+
+
+def is_custom_type(obj):
+    return isinstance(obj, CustomType)
+
+
+def is_object(obj):
+    """Returns True is obj is a reference to an object instance."""
+
+    return (isinstance(obj, object) and
+            not isinstance(obj, (type,
+                                 types.FunctionType,
+                                 types.BuiltinFunctionType,
+                                 types.GeneratorType)))
+
+
+def is_to_discard(obj):
+    return isinstance(obj, (types.GeneratorType, types.CodeType))
+
+
+def is_primitive(obj):
+    return isinstance(obj, PRIMITIVES)
+
+
+def is_dictionary(obj):
+    return isinstance(obj, dict)
+
+
+def is_list(obj):
+    return isinstance(obj, list)
+
+
+def is_set(obj):
+    return isinstance(obj, set)
+
+
+def is_bytes(obj):
+    return isinstance(obj, bytes)
+
+
+def is_tuple(obj):
+    return isinstance(obj, tuple)
+
+
+def is_class(obj):
+    return isinstance(obj, type)
+
+
+def b64encode(data):
+    """
+    Encode binary data to ascii text in base64. Data must be bytes.
+    """
+    return base64.b64encode(data).decode('ascii')
+
+
+def translate_module_name(module):
+    """Rename builtin modules to a consistent module name.
+
+    Prefer the more modern naming.
+
+    This is used so that references to Python's `builtins` module can
+    be loaded in both Python 2 and 3.  We remap to the "__builtin__"
+    name and unmap it when importing.
+
+    Map the Python2 `exceptions` module to `builtins` because
+    `builtins` is a superset and contains everything that is
+    available in `exceptions`, which makes the translation simpler.
+
+    See untranslate_module_name() for the reverse operation.
+    """
+    lookup = dict(__builtin__='builtins', exceptions='builtins')
+    return lookup.get(module, module)
+
+
+def importable_name(cls):
+    """
+    Fully qualified name (prefixed by builtin when needed)
+    """
+    # Use the fully-qualified name if available (Python >= 3.3)
+    name = getattr(cls, '__qualname__', cls.__name__)
+    module = translate_module_name(cls.__module__)
+    return '{}.{}'.format(module, name)

tests/base.py

@@ -0,0 +1,35 @@
+import os
+import shutil
+from os import path
+
+import pytest
+
+from core.Sheerka import Sheerka
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+
+
+class BaseTest:
+    @pytest.fixture()
+    def sdp(self) -> SheerkaDataProvider:
+        return SheerkaDataProvider("mem://", name="test")
+
+
+class UsingFileBasedSheerka(BaseTest):
+    TESTS_ROOT_DIRECTORY = path.abspath("../build/tests")
+    SHEERKA_ROOT_DIR = os.path.join(TESTS_ROOT_DIRECTORY, ".sheerka")
+
+    @pytest.fixture(scope="class")
+    def sheerka_fb(self):
+        """
+        the default fixture to get Sheerka is overriden
+        :return:
+        :rtype:
+        """
+        # first, make sure to create a fresh environment
+        if path.exists(self.SHEERKA_ROOT_DIR):
+            shutil.rmtree(self.SHEERKA_ROOT_DIR)
+
+        # create the new Sheerka instance
+        sheerka = Sheerka()
+        sheerka.initialize(root_folder=self.SHEERKA_ROOT_DIR)
+        return sheerka

tests/caching/__init__.py

@@ -0,0 +1,18 @@
+class FakeSdp:
+    def __init__(self, /, get_value=None, extend_exists=None, get_alt_value=None, populate=None):
+        self.get_value = get_value
+        self.extend_exists = extend_exists
+        self.populate_function = populate
+        self.get_alt_value = get_alt_value
+
+    def get(self, cache_name, key):
+        return self.get_value(cache_name, key)
+
+    def exists(self, cache_name, key):
+        return self.extend_exists(cache_name, key)
+
+    def alt_get(self, cache_name, key):
+        return self.get_alt_value(cache_name, key)
+
+    def populate(self):
+        return self.populate_function() if callable(self.populate_function) else self.populate_function

tests/caching/test_CacheManager.py

@@ -0,0 +1,288 @@
+from dataclasses import dataclass
+
+import pytest
+
+from base import BaseTest
+from caching.Cache import Cache
+from caching.CacheManager import CacheManager, ConceptNotFound
+from caching.ListIfNeededCache import ListIfNeededCache
+from common.global_symbols import NotFound
+from helpers import get_metadata
+from tests.caching import FakeSdp
+
+
+@dataclass
+class Obj:
+    key: str
+    value: str
+
+
+class TestCacheManager(BaseTest):
+
+    def test_i_can_push_into_sdp(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache(), persist=True)
+        cache_manager.put("test", "key", "value")
+
+        cache_manager.commit(context)
+        assert sdp.exists("test", "key")
+
+    def test_sdp_given_to_the_cache_manager_can_be_used_by_the_cache(self, context, sdp):
+        """
+        When the sdp is given to the cache_manager, the 'Cache' object can use the lambda with two parameters
+        :param context:
+        :type context:
+        :return:
+        :rtype:
+        """
+        cache_manager = CacheManager(sdp)
+        cache = Cache(default=lambda _sdp, k: _sdp.get("test", k))
+        cache_manager.register_cache("test", cache)
+
+        with cache_manager.sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        assert cache_manager.get("test", "key") == "value"
+
+    def test_cache_can_use_auto_configure_when_sdp_is_given_to_the_cache_manager(self, context, sdp):
+        """
+        When sdp is given to the cache manager,
+        Cache can be configured using auto_configure as the sdp will be provided by the cache_manager
+        :param context:
+        :type context:
+        :return:
+        :rtype:
+        """
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        cache_manager = CacheManager(sdp)
+        cache = Cache().auto_configure("test")
+        cache_manager.register_cache("test", cache)
+
+        assert cache_manager.get("test", "key") == "value"
+
+    def test_i_can_get_value_when_the_sdp_is_given_to_the_cache(self, context, sdp):
+        """
+        If the sdp is not given to the cache manager, the cache will need to explicitly provide the sdp
+        during its configuration
+        :param context:
+        :type context:
+        :return:
+        :rtype:
+        """
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        cache_manager = CacheManager()
+        cache = Cache(default=lambda k: sdp.get("test", k))
+        cache_manager.register_cache("test", cache)
+
+        assert cache_manager.get("test", "key") == "value"
+
+    def test_i_can_get_value_from_alt_sdp(self, sdp):
+        """
+        When nothing is found in cache and in sdp, we use alternate sdp
+        :param sdp:
+        :type sdp:
+        :return:
+        :rtype:
+        """
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !")
+        assert cache_manager.get("test", "key", alt_sdp=alt_sdp) == "value found !"
+
+    def test_i_can_commit_simple_cache(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+        cache = cache_manager.caches["test"].cache
+
+        cache_manager.put("test", "key", "value")
+
+        cache_manager.commit(context)
+
+        cache.clear()
+        assert cache_manager.sdp.get("test", "key") == "value"
+        assert cache.get("key") == "value"
+
+        cache.update("key", "value", "key", "another_value")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") == "another_value"
+
+        cache.update("key", "another_value", "key2", "another_value")
+        cache_manager.commit(context)
+        assert cache_manager.sdp.get("test", "key") is NotFound
+        assert cache_manager.sdp.get("test", "key2") == "another_value"
+
+        # sanity check
+        # sdp 'test' has value, but sdp '__default__' does not
+        assert cache_manager.sdp.name == "test"
+        assert cache_manager.sdp.state.data == {'test': {'key2': 'another_value'}}
+
+    def test_i_can_use_concept_cache(self):
+        cache_manager = CacheManager()
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("name", ListIfNeededCache(), lambda c: c.name, True)
+
+        # caches are correctly created
+        assert cache_manager.concept_caches == ["id", "name"]
+        assert "id" in cache_manager.caches
+        assert "name" in cache_manager.caches
+
+        # caches are correctly updated on insertion
+        meta1 = get_metadata(id="1", name="foo")
+        meta2 = get_metadata(id="2", name="bar")
+        meta3 = get_metadata(id="3", name="foo")
+
+        for metadata in meta1, meta2, meta3:
+            cache_manager.add_concept(metadata)
+
+        assert cache_manager.get_inner_cache("id").copy() == {"1": meta1, "2": meta2, "3": meta3}
+        assert cache_manager.get_inner_cache("name").copy() == {"foo": [meta1, meta3], "bar": meta2}
+
+        # caches are correctly updated on modification
+        meta3prime = get_metadata(id="3", name="bar")
+        cache_manager.update_concept(meta3, meta3prime)
+
+        assert cache_manager.get_inner_cache("id").copy() == {"1": meta1, "2": meta2, "3": meta3prime}
+        assert cache_manager.get_inner_cache("name").copy() == {"foo": meta1, "bar": [meta2, meta3prime]}
+
+        # caches are correctly updated on removal
+        cache_manager.remove_concept(meta3prime)
+        assert cache_manager.get_inner_cache("id").copy() == {"1": meta1, "2": meta2}
+        assert cache_manager.get_inner_cache("name").copy() == {"foo": meta1, "bar": meta2}
+
+    def test_i_cannot_remove_a_concept_that_does_not_exists(self):
+        cache_manager = CacheManager()
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+        cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True)
+
+        meta1 = get_metadata(id="1", name="foo")
+
+        with pytest.raises(ConceptNotFound) as ex:
+            cache_manager.remove_concept(meta1)
+
+        assert ex.value.concept == meta1
+
+    def test_nothing_is_sent_to_sdp_if_persist_is_false(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache(), persist=False)
+
+        cache_manager.put("test", "key", "value")
+        cache_manager.commit(context)
+
+        value = sdp.get("test", "key")
+        assert value is NotFound
+
+    def test_i_can_delete_from_cache_manager(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache(), persist=True)
+        cache_manager.put("test", "key", "value")
+        cache_manager.commit(context)
+
+        # sanity check
+        assert cache_manager.get("test", "key") == "value"
+        assert sdp.get("test", "key") == "value"
+
+        # I remove but I don't commit
+        cache_manager.delete("test", "key")
+        assert cache_manager.get("test", "key") is NotFound
+        assert sdp.get("test", "key") == "value"
+
+        # commit
+        cache_manager.commit(context)
+        assert cache_manager.get("test", "key") is NotFound
+        assert sdp.get("test", "key") is NotFound
+
+    def test_i_can_get_the_inner_cache(self):
+        cache_manager = CacheManager()
+        cache = Cache()
+        cache_manager.register_cache("test", cache)
+
+        inner_cache = cache_manager.get_inner_cache("test")
+        assert id(inner_cache) == id(cache)
+
+    def test_i_can_get_a_copy_of_a_cache(self):
+        cache_manager = CacheManager()
+        cache = Cache()
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+        cache_manager.register_cache("test", cache)
+
+        copy = cache_manager.copy("test")
+
+        assert isinstance(copy, dict)
+        assert copy == {"key1": "value1", "key2": "value2"}
+
+    def test_i_can_populate(self):
+        cache_manager = CacheManager()
+        cache = Cache()
+        cache_manager.register_cache("test", cache)
+
+        obj1, obj2 = Obj("key1", "value1"), Obj("key2", "value2")
+        cache_manager.populate("test", lambda: [obj1, obj2], lambda o: o.key)
+
+        assert cache_manager.copy("test") == {"key1": obj1, "key2": obj2}
+
+    def test_has(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+
+        with cache_manager.sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key_from_sdp", "value")
+
+        cache_manager.put("test", "key_in_cache", "value")
+
+        assert cache_manager.has("test", "key_in_cache")
+        assert not cache_manager.has("test", "key_from_sdp")
+
+    def test_exist(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+
+        with cache_manager.sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key_from_sdp", "value")
+
+        cache_manager.put("test", "key_in_cache", "value")
+
+        assert cache_manager.exists("test", "key_in_cache")
+        assert cache_manager.exists("test", "key_from_sdp")
+
+    def test_i_can_clear_a_single_cache(self, context, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test", Cache().auto_configure("test"))
+        cache_manager.put("test", "key1", "value1")
+        cache_manager.put("test", "key2", "value2")
+
+        cache_manager.commit(context)
+
+        cache_manager.clear("test")
+        assert cache_manager.copy("test") == {}
+        assert sdp.exists("test", "key1")
+        assert sdp.exists("test", "key2")
+
+        cache_manager.commit(context)
+        assert cache_manager.copy("test") == {}
+        assert not sdp.exists("test", "key1")
+        assert not sdp.exists("test", "key2")
+
+    def test_i_can_clear_all_caches(self, sdp):
+        cache_manager = CacheManager(sdp)
+        cache_manager.register_cache("test1", Cache())
+        cache_manager.register_cache("test2", Cache())
+        cache_manager.put("test1", "key", "value")
+        cache_manager.put("test2", "key", "value")
+
+        cache_manager.clear()
+        assert cache_manager.copy("test1") == {}
+        assert cache_manager.copy("test2") == {}
+
+    def test_i_cannot_add_null_keys_into_concept_cache(self):
+        cache_manager = CacheManager()
+        cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True)
+
+        with pytest.raises(KeyError):
+            meta1 = get_metadata(name="foo")
+            cache_manager.add_concept(meta1)

tests/caching/test_DictionaryCache.py

@@ -0,0 +1,165 @@
+import pytest
+
+from base import BaseTest
+from caching.DictionaryCache import DictionaryCache
+from common.global_symbols import NotFound
+from tests.caching import FakeSdp
+
+
+class TestDictionaryCache(BaseTest):
+    @pytest.mark.parametrize('key', [None, "str_value", 0, 1.0])
+    def test_key_must_be_true_or_false(self, key):
+        cache = DictionaryCache()
+
+        # key must be true False
+        with pytest.raises(KeyError):
+            cache.put("key", key)
+
+    def test_value_must_be_a_dictionary(self):
+        cache = DictionaryCache()
+
+        with pytest.raises(ValueError):
+            cache.put(True, "value")
+
+        with pytest.raises(ValueError):
+            cache.put(False, "value")
+
+    def test_i_can_put_and_retrieve_value_from_dictionary_cache(self):
+        cache = DictionaryCache()
+
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(False, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) == id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+        # I can append values
+        cache.put(True, {"key": "another_value", "key3": "value3"})
+        assert len(cache) == 4
+        assert cache.get("key") == "another_value"
+        assert cache.get("key2") == ["value21", "value22"]
+        assert cache.get("key3") == "value3"
+
+        # I can reset
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(False, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) == id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+        assert cache.get("key3") is NotFound
+
+        assert cache.copy() == {'key': 'value', 'key2': ['value21', 'value22']}
+
+    def test_i_can_get_a_value_that_does_not_exist_without_compromising_the_cache(self):
+        cache = DictionaryCache()
+        cache.put(False, {"key": "value"})
+
+        assert cache.get("key2") is NotFound
+        assert cache.copy() == {"key": "value"}
+
+    def test_i_can_append_to_a_dictionary_cache_even_if_it_is_new(self):
+        cache = DictionaryCache()
+
+        entry = {"key": "value", "key2": ["value21", "value22"]}
+        cache.put(True, entry)
+        assert len(cache) == 3
+        assert id(cache._cache) != id(entry)
+        assert cache.get("key") == "value"
+        assert cache.get("key2") == ["value21", "value22"]
+
+    def test_exists_in_dictionary_cache(self):
+        cache = DictionaryCache()
+        assert not cache.exists("key")
+
+        cache.put(True, {"key": "value"})
+        assert cache.exists("key")
+
+    def test_default_for_dictionary_cache(self):
+        cache = DictionaryCache(default={"key": "value", "key2": "value2"})
+
+        # cache is fully set when the value is found
+        assert cache.get("key") == "value"
+        assert cache.copy() == {"key": "value", "key2": "value2"}
+
+        # cache is fully set when the value is not found
+        cache.test_only_reset()
+        assert cache.get("key3") is NotFound
+        assert cache.copy() == {"key": "value", "key2": "value2"}
+
+        # cache is not corrupted when value is found
+        cache.put(True, {"key3": "value3", "key4": "value4"})
+        assert cache.get("key3") == "value3"
+        assert cache.copy() == {"key": "value", "key2": "value2", "key3": "value3", "key4": "value4"}
+
+        # cache is not corrupted when value is not found
+        cache._cache["key"] = "another value"  # operation that is normally not possible
+        assert cache.get("key5") is NotFound
+        assert cache.copy() == {"key": "value", "key2": "value2", "key3": "value3", "key4": "value4"}
+
+    def test_default_callable_for_dictionary_cache(self):
+        cache = DictionaryCache(default=lambda k: {"key": "value", "key2": "value2"})
+
+        assert cache.get("key") == "value"
+        assert "key2" in cache
+        assert len(cache) == 2
+
+        cache.clear()
+        assert cache.get("key3") is NotFound
+        assert len(cache) == 2
+        assert "key" in cache
+        assert "key2" in cache
+
+    def test_default_callable_with_internal_sdp_for_dictionary_cache(self):
+        cache = DictionaryCache(default=lambda sdp, key: sdp.get("cache_name", key),
+                                sdp=FakeSdp(lambda entry, k: {"key": "value", "key2": "value2"}))
+
+        assert cache.get("key") == "value"
+        assert "key2" in cache
+        assert len(cache) == 2
+
+        cache.clear()
+        assert cache.get("key3") is NotFound
+        assert len(cache) == 2
+        assert "key" in cache
+        assert "key2" in cache
+
+    def test_dictionary_cache_cannot_be_null(self):
+        cache = DictionaryCache(default=lambda k: NotFound)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=NotFound)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=lambda k: None)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+        cache = DictionaryCache(default=None)
+        assert cache.get("key") is NotFound
+        assert cache._cache == {}
+
+    def test_auto_configure_retrieves_the_whole_remote_repository(self, sdp, context):
+        cache = DictionaryCache(sdp=sdp).auto_configure("test")
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key1", "value1")
+            transaction.add("test", "key2", "value2")
+
+        # when call for a value that is not in the cache, Dictionary cache is configured to retrieve the repo
+        cache.get("value")
+
+        assert cache.copy() == {'key1': 'value1', 'key2': 'value2'}
+
+    def test_we_do_no_go_twice_in_repo_when_not_found(self, sdp, context):
+        cache = DictionaryCache(sdp=sdp).auto_configure("test")
+
+        assert cache.get("key") is NotFound
+
+        # now add a value in remote repo
+        with sdp.get_transaction(context.event) as transaction:
+            transaction.add("test", "key", "value")
+
+        assert cache.get("key") is NotFound  # the key was previously requested

tests/caching/test_FastCache.py

@@ -0,0 +1,111 @@
+from caching.FastCache import FastCache
+from common.global_symbols import NotFound
+
+
+def test_i_can_put_an_retrieve_values():
+    cache = FastCache()
+    cache.put("key", "value")
+
+    assert cache.get("key") == "value"
+    assert cache.cache == {"key": "value"}
+    assert cache.lru == ["key"]
+
+
+def test_i_can_put_and_retrieve_multiple_items():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    assert cache.cache == {"key1": "value1", "key2": "value2", "key3": "value3"}
+    assert cache.lru == ["key1", "key2", "key3"]
+
+
+def test_i_the_least_used_is_remove_first():
+    cache = FastCache(3)
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    cache.put("key4", "value4")
+    assert cache.cache == {"key4": "value4", "key2": "value2", "key3": "value3"}
+    assert cache.lru == ["key2", "key3", "key4"]
+
+    cache.put("key5", "value5")
+    assert cache.cache == {"key4": "value4", "key5": "value5", "key3": "value3"}
+    assert cache.lru == ["key3", "key4", "key5"]
+
+
+def test_i_can_put_the_same_key_several_times():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key1", "value3")
+
+    assert cache.cache == {"key1": "value3", "key2": "value2"}
+    assert cache.lru == ["key2", "key1"]
+
+
+def test_none_is_returned_when_not_found():
+    cache = FastCache()
+    assert cache.get("foo") is NotFound
+
+
+def test_i_can_evict_by_key():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("to_keep1", "to_keep_value1")
+    cache.put("key2", "value2")
+    cache.put("to_keep2", "to_keep_value2")
+    cache.put("key3", "value3")
+    cache.put("to_keep3", "to_keep_value3")
+
+    cache.evict_by_key(lambda k: k.startswith("key"))
+    assert cache.cache == {"to_keep1": "to_keep_value1",
+                           "to_keep2": "to_keep_value2",
+                           "to_keep3": "to_keep_value3"}
+
+    assert cache.lru == ["to_keep1", "to_keep2", "to_keep3"]
+
+
+def test_i_can_get_default_value():
+    cache = FastCache(max_size=3, default=lambda key: key + 1)
+
+    assert cache.get(1) == 2
+    assert cache.get(2) == 3
+    assert cache.get(3) == 4
+    assert cache.get(4) == 5
+
+    assert cache.cache == {2: 3, 3: 4, 4: 5}  # only 3 values
+
+
+def test_i_can_iter_on_entries():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    res = []
+    for k in cache:
+        assert k in cache
+        res.append(k)
+
+    assert res == ["key1", "key2", "key3"]
+
+
+def test_i_can_count():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    assert len(cache) == 3
+
+
+def test_i_can_copy():
+    cache = FastCache()
+    cache.put("key1", "value1")
+    cache.put("key2", "value2")
+    cache.put("key3", "value3")
+
+    assert cache.copy() == {"key1": "value1", "key2": "value2", "key3": "value3"}

tests/caching/test_IncCache.py

@@ -0,0 +1,87 @@
+from base import BaseTest
+from caching.IncCache import IncCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class FakeIncSdp:
+    """
+    FakeIncSdp has two levels of "ontology"
+    if a value of
+    """
+    def __init__(self, init_value1, init_value2):
+        self.level1 = IncCache()
+        self.level1.put("key", init_value1)
+
+        self.level2 = IncCache()
+        self.level2.put("key", init_value2)
+
+    def alt_get(self, cache_name, key):
+        for cache in [self.level1, self.level2]:
+            value = cache.alt_get(key)
+            if value is not NotFound:
+                return value
+
+        return NotFound
+
+
+class TestIncCache(BaseTest):
+    def test_i_can_put_and_retrieve_values_from_inc_cache(self):
+        cache = IncCache()
+
+        assert cache.get("key") == 1
+        assert cache.get("key") == 2
+        assert cache.get("key") == 3
+        assert cache.get("key2") == 1
+        assert cache.get("key2") == 2
+
+        cache.put("key", 100)
+        assert cache.get("key") == 101
+
+        assert cache.copy() == {'key': 101, 'key2': 2}
+
+    def test_i_can_alt_get(self):
+        cache = IncCache()
+
+        assert cache.get("key") == 1
+        assert cache.get("key") == 2
+        assert cache.alt_get("key") == 2
+        assert cache.alt_get("key") == 2
+        assert cache.get("key") == 3
+
+    def test_current_cache_takes_precedence_over_alt_sdp(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        assert cache.get("key") == 1
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: 10)
+        assert cache.get("key", alt_sdp=alt_sdp) == 2
+
+    def test_remote_repository_takes_precedence_over_alt_sdp(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: 5)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: 10)
+        assert cache.get("key", alt_sdp=alt_sdp) == 6
+        assert cache.get("key", alt_sdp=alt_sdp) == 7  # then we use the value from the cache
+
+    def test_i_can_take_value_from_alt_sdp(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeIncSdp(10, NotFound)
+        assert cache.get("key", alt_sdp=alt_sdp) == 11
+        assert cache.get("key", alt_sdp=alt_sdp) == 12  # then we use the value from the cache
+
+    def test_i_can_get_when_alt_sdp_and_cache_is_cleared(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        alt_sdp = FakeIncSdp(10, NotFound)
+        assert cache.get("key", alt_sdp=alt_sdp) == 1
+        assert cache.get("key", alt_sdp=alt_sdp) == 2  # then we use the value from the cache
+
+    def test_i_can_manage_when_the_value_from_alt_sdp_is_removed(self):
+        cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeIncSdp(Removed, 10)
+        assert cache.get("key", alt_sdp=alt_sdp) == 1
+        assert cache.get("key", alt_sdp=alt_sdp) == 2  # then we use the value from the cache

tests/caching/test_ListCache.py

@@ -0,0 +1,281 @@
+import pytest
+
+from base import BaseTest
+from caching.ListCache import ListCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class TestListCache(BaseTest):
+
+    def test_i_can_put_and_retrieve_value_from_list_cache(self):
+        cache = ListCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == ["value"]
+        assert len(cache) == 1
+
+        cache.put("key", "value2")  # we can append to this list
+        assert cache.get("key") == ["value", "value2"]
+        assert len(cache) == 2
+
+        cache.put("key2", "value")
+        assert cache.get("key2") == ["value"]
+        assert len(cache) == 3
+
+        # duplicates are allowed
+        cache.put("key", "value")
+        assert cache.get("key") == ["value", "value2", "value"]
+        assert len(cache) == 4
+
+        assert cache.copy() == {'key': ['value', 'value2', 'value'], 'key2': ['value']}
+
+    def test_i_can_put_in_list_cache_when_alt_sdp_returns_values(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]))
+        assert cache.get("key") == ["value1", "value2"]
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == ["value1"]
+
+    def test_i_can_put_in_list_cache_when_alt_sdp_returns_values_and_cache_is_cleared(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]))
+        assert cache.get("key") == ["value2"]
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == ["value1"]
+
+    def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_list_cache(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value1")
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_list_cache(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: ["value1"])).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_i_can_get_when_alt_sdp(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.get("key", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]))
+        assert cache.get("key") == ["value1"]
+
+    def test_i_can_update_from_list_cache(self):
+        cache = ListCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "another value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value2", "value"]  # only the first one is affected
+
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key2") == ["value2"]
+
+        cache.update("key2", "value2", "key3", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key3") == ["value2"]
+        assert cache.get("key2") is NotFound
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self):
+        cache = ListCache(default=lambda sdp, key: sdp.get("cache_name", key),
+                          extend_exists=lambda sdp, key: sdp.exists("cache_name", key),
+                          sdp=FakeSdp(get_value=lambda cache_name, key: NotFound))
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == ["new_value"]
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self):
+        # There is nothing in cache or remote repository.
+        # We must ust the value from alt_sdp
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        previous_value = ["old_1", "old_2", "value"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: previous_value)
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == ["old_1", "old_2", "new_value"]
+        assert previous_value == ["old_1", "old_2", "value"]
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self):
+        # keys are different
+        # make sure that current cache take precedence over alt_sdp
+        # In this test, the values from alt_sdp are never used
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: ["xxx1"] if key == "key1" else NotFound)
+
+        # one values in 'key1'
+        cache.put("key1", "old_1")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2"]
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self):
+        # keys are different
+        # make sure that current repo take precedence over alt_sdp
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1"] if key == "key1" else NotFound)
+        cache = ListCache(sdp=remote).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: ["xxx1"] if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2"] if key == "key1" else NotFound)
+        cache = ListCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2"]
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self):
+        # keys are different
+        # No value found in cache or remote repository,
+        # Will use values from alt_sdp
+        # The old key is the same, so it has to be marked as Removed
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one values in 'key1'
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: ["old_1"] if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.test_only_reset()
+        old_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2"]
+        assert cache.get("key2") == ["new_value"]
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == ["old_1", "old_2"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It takes precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: ["xxx2"] if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.put("key2", "old_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It takes precedence
+        # If no value is found, we must use the value from alt_sdp
+        remote_repo = FakeSdp(get_value=lambda cache_name, key: ["old_value"] if key == "key2" else NotFound)
+        cache = ListCache(sdp=remote_repo).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: ["xxx2"] if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It takes precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key1", "source_value")
+        previous_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ["old_1", "old_2", 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+        assert previous_values == ["old_1", "old_2"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])
+        cache.clear()
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp)
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp)
+
+    def test_default_is_called_before_updating_list_cache(self):
+        cache = ListCache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"])
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == ["new_value", "other old value"]
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else NotFound)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == ["other old value"]
+        assert cache.get("new_key") == ["new_value"]
+
+        cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else ["other new"])
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == ["other old value"]
+        assert cache.get("new_key") == ["other new", "new_value"]

tests/caching/test_ListIfNeededCache.py

@@ -0,0 +1,648 @@
+import pytest
+
+from base import BaseTest
+from caching.ListIfNeededCache import ListIfNeededCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class TestListIfNeededCache(BaseTest):
+    def test_i_can_put_and_retrieve_value_from_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == "value"
+
+        # second time with the same key creates a list
+        cache.put("key", "value2")
+        assert cache.get("key") == ["value", "value2"]
+        assert len(cache) == 2
+
+        # third time, we now have a list
+        cache.put("key", "value3")
+        assert cache.get("key") == ["value", "value2", "value3"]
+        assert len(cache) == 3
+
+        # other keys are not affected
+        cache.put("key2", "value")
+        assert cache.get("key2") == "value"
+        assert len(cache) == 4
+
+        # duplicates are allowed
+        cache.put("key", "value")
+        assert cache.get("key") == ["value", "value2", "value3", "value"]
+        assert len(cache) == 5
+
+    def test_i_can_put_in_list_if_need_cache_when_alt_sdp_returns_values(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "value1"))
+        assert cache.get("key") == ["value1", "value2"]
+
+        cache.put("key2", "value3", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2"]))
+        assert cache.get("key2") == ["value1", "value2", "value3"]
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == "value1"
+
+    def test_i_can_put_in_list_if_need__cache_when_alt_sdp_returns_values_and_cache_is_cleared(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "value1"))
+        assert cache.get("key") == "value2"
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == "value1"
+
+    def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_list_if_needed_cache(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value1")
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_list_if_needed_cache(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: "value1")).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == ["value1", "value2"]
+
+    def test_i_can_update_from_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value")
+
+        # only the first 'value' is affected
+        cache.update("key", "value", "key", "another value")
+        assert len(cache._cache) == 1
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value2", "value"]
+
+        # change the key
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key2") == "value2"
+
+        # rename the newly created key
+        cache.update("key2", "value2", "key3", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == ["another value", "value"]
+        assert cache.get("key3") == "value2"
+        assert cache.get("key2") is NotFound
+
+        # from list to single item and vice versa
+        cache.update("key", "value", "key3", "value")
+        assert len(cache._cache) == 2
+        assert len(cache) == 3
+        assert cache.get("key") == "another value"  # 'key' is no longer a list
+        assert cache.get("key3") == ["value2", "value"]  # 'key3' is now a list
+        assert cache.get("key2") is NotFound
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == "new_value"
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one value in alt_sdp
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: "old_value")
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == "new_value"
+
+        # multiple values in alt_sdp
+        cache.test_only_reset()
+        previous_value = ["old_1", "old_2", "value"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: previous_value)
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == ["old_1", "old_2", "new_value"]
+        assert previous_value == ["old_1", "old_2", "value"]
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self):
+        # keys are different
+        # make sure that current cache take precedence over alt_sdp
+        # In this test, the values from alt_sdp are never used
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: "xxx1" if key == "key1" else NotFound)
+
+        # one values in 'key1'
+        cache.put("key1", "old_1")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == "old_2"
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # List of values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+        cache.put("key1", "old_3")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2", "old_3"]
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self):
+        # keys are different
+        # make sure that current repo take precedence over alt_sdp
+        remote = FakeSdp(get_value=lambda cache_name, key: "old_1" if key == "key1" else NotFound)
+        cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: "xxx1" if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2"] if key == "key1" else NotFound)
+        cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == "old_2"
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # List of values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2", "old_3"] if key == "key1" else NotFound)
+        cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2", "old_3"]
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self):
+        # keys are different
+        # No value found in cache or remote repository,
+        # Will use values from alt_sdp
+        # The old key is the same, so it has to be marked as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one values in 'key1'
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: "old_1" if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.test_only_reset()
+        old_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == "old_2"
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == ["old_1", "old_2"]  # not modified
+
+        # List of values in 'key1'
+        cache.test_only_reset()
+        old_values = ["old_1", "old_2", "old_3"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == ["old_2", "old_3"]
+        assert cache.get("key2") == "new_value"
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == ["old_1", "old_2", "old_3"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.put("key2", "old_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        remote_repo = FakeSdp(get_value=lambda cache_name, key: "old_value" if key == "key2" else NotFound)
+        cache = ListIfNeededCache(sdp=remote_repo).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        # one value in 'key2'
+        cache.put("key1", "source_value")
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "old_value" if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ['old_value', 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+        # Multiple values in 'key2'
+        cache.test_only_reset()
+        cache.put("key1", "source_value")
+        previous_values = ["old_1", "old_2"]
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == ["old_1", "old_2", 'new_value']
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+        assert previous_values == ["old_1", "old_2"]  # not modified
+
+    def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1")
+        cache.clear()
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp)
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp)
+
+    def test_default_is_called_before_updating_list_if_needed_cache(self):
+        cache = ListIfNeededCache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = ListIfNeededCache(default=lambda k: "old_value")
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == "new_value"
+
+        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"])
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == ["new_value", "other old value"]
+
+        cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else NotFound)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == "other old value"
+        assert cache.get("new_key") == "new_value"
+
+    def test_i_can_delete_key_and_values(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+        cache.put("key", "value11")
+        cache.put("key2", "value2")
+        cache.put("key2", "value22")
+        cache.put("key2", "value222")
+        cache.put("key3", "value3")
+        cache.put("key3", "value33")
+        cache.put("key4", "value4")
+        cache.reset_events()
+
+        assert len(cache) == 8
+
+        # I can remove a whole key
+        cache.delete("key")
+        assert cache.get("key") is NotFound
+        assert len(cache) == 6
+        assert cache.to_remove == {"key"}
+        assert cache.to_add == set()
+
+        # I can remove an element while a list is remaining
+        cache.reset_events()
+        cache.delete("key2", "value22")
+        assert cache.get("key2") == ["value2", "value222"]
+        assert len(cache) == 5
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while a single element is remaining
+        cache.reset_events()
+        cache.delete("key3", "value33")
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 4
+        assert cache.to_add == {"key3"}
+        assert cache.to_remove == set()
+
+        # I can remove an element while nothing remains
+        cache.reset_events()
+        cache.delete("key4", "value4")
+        assert cache.get("key4") is NotFound
+        assert len(cache) == 3
+        assert cache.to_remove == {"key4"}
+        assert cache.to_add == set()
+
+        # I do not remove when the value is not the same
+        cache.reset_events()
+        cache.delete("key3", "value33")  # value33 was already remove
+        assert cache.get("key3") == "value3"
+        assert len(cache) == 3
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_and_then_put_back(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)  # remove all values
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_one_value(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+        cache.put("key", "value2")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": "value2"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.put("key", "value3")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ['value2', 'value3']}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure(
+            "cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: "value")).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository_and_then_put_back(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure(
+            "cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value=None, alt_sdp=alt_sdp)  # remove all values
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_remaining_one_value(self):
+        # There is a value in alt_cache_manager,
+        # But this, there are remaining values in current cache after deletion
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure(
+            "cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": "value2"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1, value2"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1",
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_and_then_put_back(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1",
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_one_value_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, one value remains in the cache
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": "value2"}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_multiple_values_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, one value remains in the cache
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2", "value3"],
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": ["value2", "value3"]}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_an_already_removed_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # But the alternate sdp returns Removed, which means that previous value was deleted
+        # It's like there is nothing to delete
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_deleting_an_entry_that_does_not_exist_is_not_an_error(self):
+        cache = ListIfNeededCache()
+        cache.put("key", "value1")
+
+        cache.reset_events()
+        cache.delete("key3")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key3", "value")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value2")
+        assert len(cache) == 1
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value",
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.clear()
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository(self):
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: Removed)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository_from_alt_sdp(self):
+        # The key is removed in the sub layers
+        # We can put it back
+        cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.put("key", "value", alt_sdp=alt_sdp)
+
+        assert cache.copy() == {"key": "value"}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}

tests/caching/test_SetCache.py

@@ -0,0 +1,540 @@
+import pytest
+
+from base import BaseTest
+from caching.SetCache import SetCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class TestSetCache(BaseTest):
+
+    def test_i_can_put_and_retrieve_values_from_set_cache(self):
+        cache = SetCache()
+
+        cache.put("key", "value")
+        assert cache.get("key") == {"value"}
+        assert len(cache) == 1
+
+        # we can add to this set
+        cache.put("key", "value2")
+        assert cache.get("key") == {"value", "value2"}
+        assert len(cache) == 2
+
+        # other keys are not affected
+        cache.put("key2", "value")
+        assert cache.get("key2") == {"value"}
+        assert len(cache) == 3
+
+        # duplicates are removed
+        cache.put("key", "value")
+        assert cache.get("key") == {"value", "value2"}
+        assert len(cache) == 3
+
+        assert cache.copy() == {'key': {'value', 'value2'}, 'key2': {'value'}}
+
+    def test_i_can_put_in_set_cache_when_alt_sdp_returns_values(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: {"value1"}))
+        assert cache.get("key") == {"value1", "value2"}
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == {"value1"}
+
+    def test_i_can_put_in_set_cache_when_alt_sdp_returns_values_and_cache_is_cleared(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: {"value1"}))
+        assert cache.get("key") == {"value2"}
+
+        cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed))
+        assert cache.get("key3") == {"value1"}
+
+    def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_set_cache(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value1")
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == {"value1", "value2"}
+
+    def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_set_cache(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: {"value1"})).auto_configure("cache_name")
+
+        cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx"))
+        assert cache.get("key") == {"value1", "value2"}
+
+    def test_i_can_update_from_set_cache(self):
+        cache = SetCache()
+
+        cache.put("key", "value")
+        cache.put("key", "value2")
+        cache.update("key", "value", "key", "another value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 2
+        assert cache.get("key") == {"another value", "value2"}
+
+        cache.update("key", "value2", "key2", "value2")
+        assert len(cache._cache) == 2
+        assert len(cache) == 2
+        assert cache.get("key") == {"another value"}
+        assert cache.get("key2") == {"value2"}
+
+        cache.update("key", "another value", "key3", "another value")
+        assert len(cache._cache) == 2
+        assert len(cache) == 2
+        assert cache.get("key") is NotFound
+        assert cache.get("key2") == {"value2"}
+        assert cache.get("key3") == {"another value"}
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == {"new_value"}
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self):
+        # There is nothing in cache or remote repository.
+        # We must ust the value from alt_sdp
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        previous_value = {"old_1", "old_2", "value"}
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True,
+                          get_alt_value=lambda cache_name, key: previous_value)
+        cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key") == {"old_1", "old_2", "new_value"}
+        assert previous_value == {"old_1", "old_2", "value"}
+
+    def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self):
+        # keys are different
+        # make sure that current cache take precedence over alt_sdp
+        # In this test, the values from alt_sdp are never used
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: {"xxx1"} if key == "key1" else NotFound)
+
+        # one values in 'key1'
+        cache.put("key1", "old_1")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.clear()
+        cache.put("key1", "old_1")
+        cache.put("key1", "old_2")
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == {"old_2"}
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self):
+        # keys are different
+        # make sure that current repo take precedence over alt_sdp
+        remote = FakeSdp(get_value=lambda cache_name, key: {"old_1"} if key == "key1" else NotFound)
+        cache = SetCache(sdp=remote).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: {"xxx1"} if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        remote = FakeSdp(get_value=lambda cache_name, key: {"old_1", "old_2"} if key == "key1" else NotFound)
+        cache = SetCache(sdp=remote).auto_configure("cache_name")
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == {"old_2"}
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+    def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self):
+        # keys are different
+        # No value found in cache or remote repository,
+        # Will use values from alt_sdp
+        # The old key is the same, so it has to be marked as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        # one values in 'key1'
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: {"old_1"} if key == "key1" else NotFound)
+
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == Removed
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+
+        # Multiple values in 'key1'
+        cache.test_only_reset()
+        old_values = {"old_1", "old_2"}
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1",
+                          get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound)
+        cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == {"old_2"}
+        assert cache.get("key2") == {"new_value"}
+        assert cache.to_add == {"key2", "key1"}
+        assert cache.to_remove == set()
+        assert old_values == {"old_1", "old_2"}  # not modified
+
+    def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: {"xxx2"} if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.put("key2", "old_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == {'old_value', 'new_value'}
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        remote_repo = FakeSdp(get_value=lambda cache_name, key: {"old_value"} if key == "key2" else NotFound)
+        cache = SetCache(sdp=remote_repo).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound)
+        cache.put("key1", "source_value")
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == {'old_value', 'new_value'}
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+
+    def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self):
+        # If a value exists in destination key, either in local cache or remote repository
+        # It take precedence
+        # If no value is found, we must use the value from alt_sdp
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        cache.put("key1", "source_value")
+        previous_values = {"old_1", "old_2"}
+        alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2",
+                          get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound)
+
+        cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp)
+        assert cache.get("key1") == NotFound
+        assert cache.get("key2") == {"old_1", "old_2", 'new_value'}
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key1"}
+        assert previous_values == {"old_1", "old_2"}  # not modified
+
+    def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])
+        cache.clear()
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp)
+
+        with pytest.raises(KeyError):
+            cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp)
+
+    def test_default_is_called_before_updating_set_cache(self):
+        cache = SetCache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"})
+        cache.update("old_key", "old_value", "old_key", "new_value")
+        assert cache.get("old_key") == {"new_value", "other old value"}
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else NotFound)
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == {"other old value"}
+        assert cache.get("new_key") == {"new_value"}
+
+        cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else {"other new"})
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("old_key") == {"other old value"}
+        assert cache.get("new_key") == {"other new", "new_value"}
+
+    def test_i_can_delete_values_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+        cache.reset_events()
+
+        cache.delete("key", "fake_value")
+        assert cache.get("key") == {"value1", "value2"}
+        assert len(cache) == 2
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", "value1")
+        assert cache.get("key") == {"value2"}
+        assert cache.to_add == {"key"}
+        assert len(cache) == 1
+
+        cache.delete("key", "value2")
+        assert cache.get("key") is NotFound
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_key_from_set_cache(self):
+        cache = SetCache()
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+
+        cache.delete("key")
+        assert cache.get("key") is NotFound
+        assert cache.to_remove == {"key"}
+        assert len(cache) == 0
+
+    def test_i_can_delete_a_key_that_does_not_exists(self):
+        cache = SetCache()
+        cache.delete("key")
+
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_and_then_put_back(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True)
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But there is a value in the current cache after deletion
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        cache.put("key", "value1")
+        cache.put("key", "value2")
+
+        cache.delete("key", value="value1", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": {"value2"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value"})).auto_configure("cache_name")
+
+        cache.delete("key", value="value", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_and_then_put_back(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_remaining_values(self):
+        # There is a value in alt_cache_manager,
+        # But there is a value in the current cache after deletion
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name")
+
+        cache.delete("key", value="value1", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": {"value2"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_key_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1, value2"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_and_then_put_back(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_one_value_remaining(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1", "value2"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {"key": {"value2"}}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_key_that_does_not_exist_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=(lambda cache_name, key: {"value1", "value2"} if key == "key" else NotFound),
+                          extend_exists=lambda cache_name, key: key == "key")
+
+        cache.delete("key2", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_a_value_that_does_not_exist_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # After value deletion, the key is empty
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1", "value2"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.delete("key", value="value4", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_an_already_removed_value_from_alt_sdp(self):
+        # alt_cache_manager is used because no value in cache or in remote repository
+        # But the alternate sdp returns Removed, which means that previous value was deleted
+        # It's like there is nothing to delete
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.delete("key", value="value1", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value"},
+                          extend_exists=lambda cache_name, key: True)
+
+        cache.clear()
+        cache.delete("key", value=None, alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key", value="value", alt_sdp=alt_sdp)
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository(self):
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: Removed)).auto_configure("cache_name")
+        cache.put("key", "value")
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}
+
+    def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository_from_alt_sdp(self):
+        # The key is removed in the sub layers
+        # We can put it back
+        cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name")
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed,
+                          extend_exists=lambda cache_name, key: False)
+
+        cache.put("key", "value", alt_sdp=alt_sdp)
+
+        assert cache.copy() == {"key": {"value"}}
+        assert cache.to_remove == set()
+        assert cache.to_add == {"key"}

tests/caching/test_cache.py

@@ -0,0 +1,512 @@
+import pytest
+
+from caching.BaseCache import BaseCache, MAX_INITIALIZED_KEY
+from caching.Cache import Cache
+from caching.DictionaryCache import DictionaryCache
+from caching.IncCache import IncCache
+from caching.ListCache import ListCache
+from caching.ListIfNeededCache import ListIfNeededCache
+from caching.SetCache import SetCache
+from common.global_symbols import NotFound, Removed
+from tests.caching import FakeSdp
+
+
+class TestCache:
+    def test_i_can_configure(self):
+        cache = Cache()
+        cache.configure(max_size=256,
+                        default="default_delegate",
+                        extend_exists="extend_exists_delegate",
+                        alt_sdp_get="alt_sdp_delegate",
+                        sdp=FakeSdp())
+
+        # Caution, in this test, I initialize default, extend_exists and alt_get_delegate with string
+        # to simplify the test, but in real usage, they are lambda
+        # default = lambda sdp, key: sdp.get(cache_name, key) or lambda key: func(key)
+        # extend_exists = lambda sdp, key: sdp.exists(cache_name, key) or lambda key: func(key)
+        # alt_sdp_get = lambda sdp, key: sdp.alt_get(cache_name, key)
+
+        assert cache._max_size == 256
+        assert cache._default == "default_delegate"
+        assert cache._extend_exists == "extend_exists_delegate"
+        assert cache._alt_sdp_get == "alt_sdp_delegate"
+        assert cache._sdp is not None
+
+    def test_i_can_auto_configure(self):
+        sdp = FakeSdp(get_value=lambda cache_name, key: key + 1 if cache_name == "cache_name" else NotFound,
+                      extend_exists=lambda cache_name, key: True if cache_name == "cache_name" else False,
+                      get_alt_value=lambda cache_name, key: key + 2 if cache_name == "cache_name" else NotFound)
+
+        cache = Cache(sdp=sdp).auto_configure("cache_name")
+        assert cache._default(cache._sdp, 10) == 11
+        assert cache._extend_exists(cache._sdp, 10) is True
+        assert cache._alt_sdp_get(cache._sdp, 10) == 12
+
+        cache = Cache(sdp=sdp).auto_configure("another_cache")
+        assert cache._default(cache._sdp, 10) == NotFound
+        assert cache._extend_exists(cache._sdp, 10) is False
+        assert cache._alt_sdp_get(cache._sdp, 10) == NotFound
+
+    def test_i_can_get_an_retrieve_value_from_cache(self):
+        cache = Cache()
+        cache.put("key", "value")
+        assert cache.get("key") == "value"
+        assert len(cache) == 1
+
+        cache.put("key", "another value")  # another value in the cache replace the previous one
+        assert cache.get("key") == "another value"
+        assert len(cache) == 1
+
+        cache.put("key2", "value2")  # another key
+        assert cache.get("key2") == "value2"
+        assert len(cache) == 2
+        assert cache.copy() == {"key": "another value", "key2": "value2"}
+
+    def test_i_do_not_evict_when_put(self):
+        """
+        It's because we evict on get()
+        :return:
+        :rtype:
+        """
+        maxsize = 5
+        cache = Cache(max_size=5)
+
+        for key in range(maxsize + 2):
+            cache.put(str(key), key)
+
+        assert len(cache) == maxsize + 2
+        assert cache.copy() == {
+            "0": 0,
+            "1": 1,
+            "2": 2,
+            "3": 3,
+            "4": 4,
+            "5": 5,
+            "6": 6,
+        }
+
+    def test_i_can_evict_when_get(self):
+        maxsize = 5
+        cache = Cache(max_size=5, default=lambda k: int(k))
+
+        for key in range(maxsize + 2):
+            cache.get(str(key))
+
+        assert len(cache) == maxsize
+        assert cache.copy() == {
+            "2": 2,
+            "3": 3,
+            "4": 4,
+            "5": 5,
+            "6": 6,
+        }
+
+    def test_i_do_not_evict_when_items_are_not_committed(self):
+        maxsize = 5
+        cache = Cache(max_size=5, default=lambda k: k)
+
+        for key in range(maxsize + 2):
+            cache.put(str(key), key)
+
+        assert len(cache) == maxsize + 2
+
+        cache.get("-1")
+        assert len(cache) == maxsize + 2
+        assert cache.copy() == {
+            "0": 0,
+            "1": 1,
+            "2": 2,
+            "3": 3,
+            "4": 4,
+            "5": 5,
+            "6": 6,
+        }
+
+    def test_i_can_get_a_value_from_alt_sdp(self):
+        cache = Cache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !")
+        assert cache.get("key", alt_sdp=alt_sdp) == "value found !"
+
+        # The value is now in cache
+        assert cache.copy() == {'key': 'value found !'}
+
+    def test_i_cannot_get_a_value_from_alt_sdp_when_cache_is_cleared(self):
+        cache = Cache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
+        cache.clear()
+
+        alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !")
+        assert cache.get("key", alt_sdp=alt_sdp) is NotFound
+        assert cache.copy() == {}
+
+    def test_i_can_get_default_value_from_simple_cache(self):
+        cache = Cache()
+        assert cache.get("key") is NotFound
+
+        cache = Cache(default=10)
+        assert cache.get("key") == 10
+        assert "key" not in cache  # default value are not put in cache
+
+        cache = Cache(default=lambda key: key + "_not_found")
+        assert cache.get("key") == "key_not_found"
+        assert "key" in cache  # default callable are put in cache
+
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda entry, key: key + "_not_found"))
+        assert cache.get("key") == "key_not_found"
+        assert "key" in cache  # default callable are put in cache
+
+    def test_i_do_not_ask_the_remote_repository_twice(self):
+        nb_request = []
+
+        cache = Cache(default=lambda key: nb_request.append("requested"))
+        assert cache.get("key") is None
+        assert cache.get("key") is None
+        assert len(nb_request) == 1
+
+    def test_i_can_update_from_simple_cache(self):
+        cache = Cache()
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value")
+
+        assert len(cache._cache) == 1
+        assert len(cache) == 1
+        assert cache.get("key") == "new_value"
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.reset_events()
+        cache.update("key", "new_value", "another_key", "another_value")
+        assert len(cache._cache) == 1
+        assert len(cache) == 1
+        assert cache.get("key") is NotFound
+        assert cache.get("another_key") == "another_value"
+        assert cache.to_add == {"another_key"}
+        assert cache.to_remove == {"key"}
+
+        with pytest.raises(KeyError):
+            cache.update("wrong key", "value", "key", "value")
+
+    def test_i_can_update_when_alt_sdp_same_keys(self):
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key),
+                      extend_exists=lambda sdp, key: sdp.exists("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda cache_name, key: NotFound))
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key") == "new_value"
+
+    def test_i_can_update_when_alt_sdp_different_keys(self):
+        cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key),
+                      extend_exists=lambda sdp, key: sdp.exists("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda cache_name, key: NotFound))
+
+        cache.put("key", "value")
+        cache.update("key", "value", "key2", "value2", FakeSdp(extend_exists=lambda cache_name, key: True))
+
+        assert cache.get("key2") == "value2"
+        assert cache.get("key") == Removed
+        assert cache.to_add == {"key", "key2"}
+        assert cache.to_remove == set()
+
+    @pytest.mark.parametrize("cache", [
+        Cache(), ListCache(), ListIfNeededCache(), SetCache(), IncCache()
+    ])
+    def test_i_can_manage_cache_events(self, cache: BaseCache):
+        cache.put("key", "value")
+
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "value", "key", "another value")
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "another value", "key2", "value2")
+        assert cache.to_add == {"key2"}
+        assert cache.to_remove == {"key"}
+
+        cache.update("key2", "value2", "key", "value")
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == {"key2"}
+
+    @pytest.mark.parametrize("cache", [
+        ListCache(), SetCache(), ListIfNeededCache()
+    ])
+    def test_i_can_manage_list_and_set_cache_events(self, cache):
+        cache.put("key", "value")
+        cache.put("key", "value2")
+
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "value", "key", "another value")
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "value2", "key2", "value2")
+        assert cache.to_add == {"key", "key2"}
+        assert cache.to_remove == set()
+
+        cache.update("key", "another value", "key3", "another value")
+        assert cache.to_add == {"key2", "key3"}
+        assert cache.to_remove == {"key"}
+
+    @pytest.mark.parametrize("cache", [
+        Cache(), ListCache(), SetCache(), ListIfNeededCache(), IncCache()
+    ])
+    def test_exists(self, cache):
+        assert not cache.exists("key")
+        cache.put("key", "value")
+        assert cache.exists("key")
+
+    def test_exists_extend(self):
+        cache = Cache(extend_exists=lambda k: True if k == "special_key" else False)
+        assert not cache.exists("key")
+        assert cache.exists("special_key")
+
+    def test_i_can_extend_exists_when_internal_sdp(self):
+        cache = Cache(extend_exists=lambda sdp, k: True if k == "special_key" else False, sdp=FakeSdp)
+        assert not cache.exists("key")
+        assert cache.exists("special_key")
+
+    @pytest.mark.parametrize("cache, default, new_value, expected", [
+        (ListCache(), lambda k: NotFound, "value", ["value"]),
+        (ListCache(), lambda k: ["value"], "value", ["value", "value"]),
+        (ListIfNeededCache(), lambda k: NotFound, "value", "value"),
+        (ListIfNeededCache(), lambda k: "value", "value1", ["value", "value1"]),
+        (ListIfNeededCache(), lambda k: ["value1", "value2"], "value1", ["value1", "value2", "value1"]),
+        (SetCache(), lambda k: NotFound, "value", {"value"}),
+        (SetCache(), lambda k: {"value"}, "value", {"value"}),
+        (SetCache(), lambda k: {"value1"}, "value2", {"value1", "value2"}),
+    ])
+    def test_default_is_called_before_put_to_keep_in_sync(self, cache, default, new_value, expected):
+        cache.configure(default=default)
+        cache.put("key", new_value)
+
+        assert cache.get("key") == expected
+
+    def test_default_is_called_before_updating_simple_cache(self):
+        cache = Cache(default=lambda k: NotFound)
+        with pytest.raises(KeyError):
+            cache.update("old_key", "old_value", "new_key", "new_value")
+
+        cache = Cache(default=lambda k: "old_value")
+        cache.update("old_key", "old_value", "new_key", "new_value")
+        assert cache.get("new_key") == "new_value"
+
+    def test_i_can_delete_an_entry_from_cache(self):
+        cache = Cache()
+        cache.put("key", "value")
+
+        assert cache.get("key") == "value"
+        cache.delete("key")
+        assert cache.get("key") is NotFound
+        assert cache.to_remove == {"key"}
+
+    def test_i_can_delete_when_entry_is_only_in_db(self):
+        cache = Cache(default=lambda k: "value" if k == 'key' else NotFound)
+
+        cache.delete("another_key")
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+        cache.delete("key")
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == {"key"}
+
+    def test_i_can_delete_an_entry_from_cache_when_alt_sdp_and_value_in_cache(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = Cache(extend_exists=lambda sdp, k: sdp.exists("cache_name", k))
+        cache.put("key", "value")
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_an_entry_from_cache_when_alt_sdp_when_in_remote_repository(self):
+        # There is a value in alt_cache_manager,
+        # No remaining value in current cache after deletion
+        # The key must be flagged as Removed
+        cache = Cache(default=lambda k: "value", extend_exists=lambda sdp, k: sdp.exists("cache_name", k))
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_i_can_delete_an_entry_from_cache_when_alt_sdp_and_no_value_in_cache_or_remote_repository(self):
+        # alt_cache_manager is used when no value found
+        cache = Cache(default=lambda sdp, k: sdp.get("cache_name", k),
+                      extend_exists=lambda sdp, k: sdp.exists("cache_name", k),
+                      sdp=FakeSdp(get_value=lambda entry, k: NotFound))
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True))
+        assert cache.copy() == {"key": Removed}
+        assert cache.to_add == {"key"}
+        assert cache.to_remove == set()
+
+    def test_no_error_when_deleting_a_key_that_does_not_exists_when_alt_sdp(self):
+        # alt_cache_manager is used when no value found
+        cache = Cache(default=lambda sdp, k: sdp.get("cache_name", k),
+                      extend_exists=lambda sdp, k: sdp.exists("cache_name", k),
+                      sdp=FakeSdp(get_value=lambda entry, k: NotFound))
+
+        cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: False))
+        assert cache.copy() == {}
+        assert cache.to_add == set()
+        assert cache.to_remove == set()
+
+    def test_initialized_key_is_removed_when_the_entry_is_found(self):
+        caches = [Cache(), ListCache(), ListIfNeededCache(), SetCache()]
+
+        for cache in caches:
+            cache.put("key", "value")
+            cache.get("key")
+
+            assert len(cache._initialized_keys) == 0
+
+        cache = IncCache()
+        cache.put("key", 10)
+        cache.get("key")
+        assert len(cache._initialized_keys) == 0
+
+    def test_initialized_keys_are_reset_when_max_length_is_reached(self):
+        cache = Cache()
+        for i in range(MAX_INITIALIZED_KEY):
+            cache.get(str(i))
+
+        assert len(cache._initialized_keys) == MAX_INITIALIZED_KEY
+
+        cache.get(str(MAX_INITIALIZED_KEY + 1))
+        assert len(cache._initialized_keys) == 1
+
+    def test_i_can_populate(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache()
+
+        cache.populate(lambda: items, lambda item: item[0])
+
+        assert len(cache) == 3
+        assert cache.get("1") == ("1", "1")
+        assert cache.get("2") == ("2", "2")
+        assert cache.get("3") == ("3", "3")
+
+        assert cache.to_add == {"1", "2", "3"}
+        assert cache.to_remove == set()
+
+    def test_i_can_populate_using_internal_sdp(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache(sdp=FakeSdp(populate=items))
+
+        cache.populate(lambda sdp: sdp.populate(), lambda item: item[0])
+
+        assert len(cache) == 3
+        assert cache.get("1") == ("1", "1")
+        assert cache.get("2") == ("2", "2")
+        assert cache.get("3") == ("3", "3")
+
+        assert cache.to_add == {"1", "2", "3"}
+        assert cache.to_remove == set()
+
+    def test_i_can_reset_the_event_after_populate(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache()
+        cache.to_add = {"some_value"}
+        cache.to_remove = {"some_other_value"}
+
+        cache.populate(lambda: items, lambda item: item[0], reset_events=True)
+
+        assert len(cache) == 3
+        assert cache.copy() == {"1": ("1", "1"),
+                                "2": ("2", "2"),
+                                "3": ("3", "3")}
+        assert cache.to_add == {"some_value"}
+        assert cache.to_remove == {"some_other_value"}
+
+    def test_i_can_get_all(self):
+        items = [("1", "1"), ("2", "2"), ("3", "3")]
+        cache = Cache()
+
+        cache.populate(lambda: items, lambda item: item[0])
+
+        res = cache.get_all()
+        assert len(res) == 3
+        assert list(res) == [('1', '1'), ('2', '2'), ('3', '3')]
+
+    def test_i_can_clone_cache(self):
+        cache = Cache(max_size=256,
+                      default=lambda sdp, key: sdp.get("cache_name", key),
+                      extend_exists=False,
+                      alt_sdp_get=lambda sdp, key: sdp.alt_get("cache_name", key),
+                      sdp=FakeSdp(get_value=lambda entry, key: key + "_not_found"))
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+
+        clone = cache.clone()
+        assert type(cache) == type(clone)
+        assert clone._max_size == cache._max_size
+        assert clone._default == cache._default
+        assert clone._extend_exists == cache._extend_exists
+        assert clone._alt_sdp_get == cache._alt_sdp_get
+        assert clone._sdp == cache._sdp
+        assert clone._cache == {}  # value are not copied
+        assert clone._initialized_keys == set()
+        assert clone._current_size == 0
+        assert clone.to_add == set()
+        assert clone.to_remove == set()
+
+        clone.configure(sdp=FakeSdp(lambda entry, key: key + " found !"))
+
+        assert cache.get("key3") == "key3_not_found"
+        assert clone.get("key3") == "key3 found !"
+
+    @pytest.mark.parametrize("cache", [
+        Cache(),
+        DictionaryCache(),
+        IncCache(),
+        ListCache(),
+        ListIfNeededCache()
+    ])
+    def test_i_can_clone_all_caches(self, cache):
+        clone = cache.clone()
+        assert type(clone) == type(cache)
+
+    def test_sanity_check_on_list_if_needed_cache(self):
+        cache = ListIfNeededCache()
+        clone = cache.clone()
+
+        clone.put("key", "value1")
+        clone.put("key", "value2")
+
+        assert clone.get("key") == ["value1", "value2"]
+
+    def test_i_can_clear_when_alt_sdp(self):
+        cache = Cache().auto_configure("cache_name")
+
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+
+        cache.clear()
+
+        assert cache.copy() == {}
+        assert cache._is_cleared
+
+    def test_i_can_iter_on_the_content(self):
+        cache = Cache()
+        cache.put("key1", "value1")
+        cache.put("key2", "value2")
+        cache.put("key3", "value3")
+
+        res = []
+        for k in cache:
+            assert k in cache
+            res.append(k)
+
+        assert res == ["key1", "key2", "key3"]

tests/common/test_utils.py

@@ -0,0 +1,122 @@
+from dataclasses import dataclass
+
+import pytest
+
+from common.utils import decode_enum, get_class, to_dict, str_concept, unstr_concept
+from helpers import get_concept
+from parsers.tokenizer import Keywords, Token, TokenKind
+
+
+@dataclass
+class Obj:
+    prop1: str
+    prop2: str
+
+    def __hash__(self):
+        return hash((self.prop1, self.prop1))
+
+    def __eq__(self, other):
+        if not isinstance(other, Obj):
+            return False
+
+        return self.prop1 == other.prop1 and self.prop2 == other.prop2
+
+
+@dataclass
+class Obj2:
+    prop1: object
+    prop2: object
+
+
+def get_tokens(lst):
+    res = []
+    for e in lst:
+        if e == " ":
+            res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0))
+        elif e == "\n":
+            res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0))
+        elif e == "<EOF>":
+            res.append(Token(TokenKind.EOF, "\n", 0, 0, 0))
+        else:
+            res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0))
+
+    return res
+
+
+def test_i_can_get_class():
+    # example of classes that should be in the result
+    create_parser_input = get_class("evaluators.CreateParserInput.CreateParserInput")
+
+    assert isinstance(create_parser_input, type)
+
+
+@pytest.mark.parametrize("text, expected_key, expected_id", [
+    (None, None, None),
+    (10, None, None),
+    ("", None, None),
+    ("xxx", None, None),
+    ("c:", None, None),
+    ("c:key", None, None),
+    ("c:key:", "key", None),
+    ("c:key#id", None, None),
+    ("c:key#id:", "key", "id"),
+    ("c:#id:", None, "id"),
+    ("c:key#:", "key", None),
+    ("c:key#id:x", None, None),
+    ("c:one: plus c:two:", None, None),
+    ("c:one#id: plus c:two:", None, None),
+])
+def test_i_can_unstr_concept(text, expected_key, expected_id):
+    k, i = unstr_concept(text)
+    assert k == expected_key
+    assert i == expected_id
+
+
+@pytest.mark.parametrize("text, expected_key, expected_id", [
+    ("r:key:", "key", None),
+    ("r:key#id:", "key", "id"),
+])
+def test_i_can_unstr_concept_rules(text, expected_key, expected_id):
+    k, i = unstr_concept(text, prefix="r:")
+    assert k == expected_key
+    assert i == expected_id
+
+
+def test_i_can_str_concept():
+    assert str_concept(("key", "id")) == "c:key#id:"
+    assert str_concept((None, "id")) == "c:#id:"
+    assert str_concept(("key", None)) == "c:key:"
+    assert str_concept((None, None)) == ""
+    assert str_concept(("key", "id"), drop_name=True) == "c:#id:"
+
+    concept = get_concept("foo")
+    assert str_concept(concept) == "c:foo:"
+
+    concept.get_metadata().id = "1001"
+    assert str_concept(concept) == "c:foo#1001:"
+    assert str_concept(concept, drop_name=True) == "c:#1001:"
+
+    assert str_concept(("key", "id"), prefix='r:') == "r:key#id:"
+
+
+@pytest.mark.parametrize("text, expected", [
+    (None, None),
+    (10, None),
+    ("", None),
+    ("xxx", None),
+    ("xxx.", None),
+    ("xxx.yyy", None),
+    ("parsers.tokenizer.Keywords.CONCEPT", Keywords.CONCEPT),
+])
+def test_i_can_decode_enum(text, expected):
+    actual = decode_enum(text)
+    assert actual == expected
+
+
+@pytest.mark.parametrize("items, expected", [
+    ([], {}),
+    ([Obj("a", "1"), Obj("a", "2"), Obj("b", "3")], {"a": [Obj("a", "1"), Obj("a", "2")],
+                                                     "b": [Obj("b", "3")]}),
+])
+def test_i_can_to_dict(items, expected):
+    assert to_dict(items, lambda obj: obj.prop1) == expected

tests/conftest.py

@@ -0,0 +1,84 @@
+import pytest
+
+from helpers import GetNextId
+
+
+@pytest.fixture(scope="session")
+def sheerka():
+    from core.Sheerka import Sheerka
+
+    sheerka = Sheerka()
+    sheerka.initialize("mem://")
+    return sheerka
+
+
+@pytest.fixture(scope="module", autouse=True)
+def on_new_module(sheerka, request):
+    """
+    For each new module, make sure to create a new ontology
+    Remove it at the end of the module
+    :param sheerka:
+    :type sheerka:
+    :param request:
+    :type request:
+    :return:
+    :rtype:
+    """
+    from core.Event import Event
+    from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+    module_name = request.module.__name__.split(".")[-1]
+    context = ExecutionContext("test",
+                               Event(message=f"Executing module {module_name}"),
+                               sheerka,
+                               ExecutionContextActions.TESTING,
+                               None)
+
+    ontology = sheerka.om.push_ontology(module_name)
+    yield
+    sheerka.om.revert_ontology(context, ontology)
+
+
+@pytest.fixture(scope="function")
+def context(sheerka):
+    from core.Event import Event
+    from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+
+    return ExecutionContext("test",
+                            Event(message=""),
+                            sheerka,
+                            ExecutionContextActions.TESTING,
+                            None)
+
+
+@pytest.fixture()
+def next_id():
+    return GetNextId()
+
+
+class TestUsingFileBasedSheerka:
+    @pytest.fixture(scope="class")
+    def sheerka(self):
+        sheerka = Sheerka()
+        sheerka.initialize()
+        return sheerka
+
+
+class NewOntology:
+    """
+    For some test who may need to declare the same concepts across the tests
+    """
+    from core.ExecutionContext import ExecutionContext
+
+    def __init__(self, context: ExecutionContext, name="current_test"):
+        self.sheerka = context.sheerka
+        self.context = context
+        self.name = name
+        self.ontology = None
+
+    def __enter__(self):
+        self.ontology = self.sheerka.om.push_ontology(self.name)
+        return self.ontology
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.sheerka.om.revert_ontology(self.context, self.ontology)
+        return False

tests/core/test_concept.py

@@ -0,0 +1,103 @@
+from common.global_symbols import NotFound, NotInit
+from core.concept import ConceptDefaultProps
+from helpers import GetNextId, get_concept
+
+
+def test_i_can_retrieve_concept_properties():
+    foo = get_concept("a plus b", "a + b", variables=("a", "b"), id="1001")
+
+    assert foo.name == "a plus b"
+    assert foo.id == "1001"
+    assert foo.str_id == "c:#1001:"
+    assert foo.all_attrs() == ('#where#', '#pre#', '#post#', '#body#', '#ret#', 'a', 'b')
+    assert foo.get_definition_digest() == "3a2cfcda8ffd0d99a7f8c7d2f1ffc4a99fc96162f3be7b9875f30751d3691af6"
+
+    # sanity check to make sure that 'get_concept' works as expected
+    assert foo.get_metadata().variables == (("a", NotInit), ("b", NotInit))
+
+
+def test_i_can_set_and_get_value():
+    foo = get_concept("foo", variables=["a"])
+    foo.set_value("a", "some value")
+    assert foo.get_value("a") == "some value"
+
+
+def test_i_can_set_and_get_value_from_bound_attr():
+    foo = get_concept("foo", variables=["a"], bound_body="a")
+
+    foo.set_value("a", "some value")
+    assert foo.get_value(ConceptDefaultProps.BODY) == "some value"
+
+    foo.set_value(ConceptDefaultProps.BODY, "another value")
+    assert foo.get_value("a") == "another value"
+
+
+def test_i_can_test_concept_equality():
+    foo1 = get_concept("foo", "a + b", variables=["a", "b"], id=5)
+    foo2 = get_concept("foo", "a + b", variables=["a", "b"], id=6)
+    foo1.set_value("a", 10).set_value("b", 20)
+    foo2.set_value("a", 10).set_value("b", 20)
+
+    assert foo1 == foo2
+
+
+def test_i_can_detect_when_concepts_are_not_equal():
+    foo1 = get_concept("foo", "a + b", variables=["a", "b"], id=5)
+    foo2 = get_concept("foo", "a + b", variables=["a", "b"], id=6)
+    foo1.set_value("a", 10).set_value("b", 20)
+    foo2.set_value("a", 10).set_value("b", 25)
+
+    assert foo1 != foo2
+
+
+def test_i_can_test_concept_equality_in_case_of_infinite_recursion():
+    foo1 = get_concept("foo", "a + b", variables=["a"], id=5)
+    foo2 = get_concept("foo", "a + b", variables=["a"], id=6)
+
+    # foo1 and foo2 are equals
+    assert foo1 == foo2
+
+    foo1.set_value("a", foo1)
+    foo2.set_value("a", foo2)
+    assert foo1 == foo2
+
+    foo1.set_value("a", foo2)
+    foo2.set_value("a", foo1)
+    assert foo1 == foo2
+
+
+def test_i_can_test_concept_equality_in_case_of_infinite_recursion_with_more_than_two_concepts():
+    foo1 = get_concept("foo", "a + b", variables=["a"], id=5)
+    foo2 = get_concept("foo", "a + b", variables=["a"], id=6)
+    foo3 = get_concept("foo", "a + b", variables=["a"], id=7)
+
+    foo1.set_value("a", foo2)
+    foo2.set_value("a", foo3)
+    foo3.set_value("a", foo1)
+    assert foo1 == foo2
+
+    foo1.set_value("a", foo2)
+    foo2.set_value("a", foo3)
+    foo3.set_value("a", foo3)
+    assert foo1 == foo2
+
+
+def test_i_cannot_get_an_attribute_which_is_not_defined():
+    next_id = GetNextId()
+    foo = get_concept("add a b", definition="add", variables=["a", "b"], sequence=next_id)
+
+    assert foo.get_value("a") is NotInit
+    assert foo.get_value("b") is NotInit
+    assert foo.get_value("c") is NotFound
+
+
+def test_i_can_repr_a_concept():
+    next_id = GetNextId()
+    foo = get_concept("foo", sequence=next_id)
+    assert repr(foo) == "(1001)foo"
+
+    bar = get_concept("bar", pre="is an int", sequence=next_id)
+    assert repr(bar) == "(1002)bar, #pre=is an int"
+
+    baz = get_concept("baz", definition="add a b", variables=["a", "b"], sequence=next_id)
+    assert repr(baz) == "(1003)baz, a=**NotInit**, b=**NotInit**"

tests/core/test_execution_context.py

@@ -0,0 +1,151 @@
+from core.Event import Event
+from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+
+
+def test_i_can_create_execution_context(sheerka):
+    event = Event("myEvent", "fake_userid")
+    context1 = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value1", "my desc")
+
+    assert context1.who == "who"
+    assert context1.event == event
+    assert context1.sheerka == sheerka
+    assert context1.action == ExecutionContextActions.TESTING
+    assert context1.action_context == "value1"
+    assert context1.desc == "my desc"
+    assert context1.id == 0
+    assert context1.long_id == f"{event.get_digest()}:{context1.id}"
+
+
+def test_i_can_push(sheerka):
+    event = Event("test")
+    context = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value")
+    with context.push("pusher", ExecutionContextActions.PARSING, "action_context", "my desc") as sub_context:
+        assert sub_context.who == "pusher"
+        assert sub_context.event == event
+        assert sub_context.sheerka == sheerka
+        assert sub_context.action == ExecutionContextActions.PARSING
+        assert sub_context.action_context == "action_context"
+        assert sub_context.desc == "my desc"
+        assert sub_context.id == context.id + 1
+
+
+def test_i_can_increment_ids(sheerka):
+    # The id of an execution context is linked to the event
+    # If the event is the same, the id is incremented
+
+    event = Event("TEST::myEvent", "fake_userid")
+    context1 = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value")
+    context2 = context1.push("who1", ExecutionContextActions.TESTING, "value1")
+    context3 = context2.push("who2", ExecutionContextActions.TESTING, "value2")
+    context4 = context1.push("who1", ExecutionContextActions.TESTING, "value3")
+    context5 = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value4")
+
+    assert context1.id == 0
+    assert context2.id == 1
+    assert context3.id == 2
+    assert context4.id == 3
+    assert context5.id == 4
+
+    event2 = Event("TEST::myEvent2", "fake_userid")
+    context6 = ExecutionContext("who", event2, sheerka, ExecutionContextActions.TESTING, "value")
+    assert context6.id == 0
+
+
+def test_i_can_manage_global_hints(context):
+    context2 = context.push("pusher", ExecutionContextActions.TESTING, None)
+    context3 = context2.push("pusher", ExecutionContextActions.TESTING, None)
+    context4 = context3.push("pusher", ExecutionContextActions.TESTING, None)
+    context5 = context.push("pusher", ExecutionContextActions.TESTING, None)
+
+    context.global_hints.add("new_hint")
+    assert context.global_hints == {"new_hint"}
+    assert context2.global_hints == {"new_hint"}
+    assert context3.global_hints == {"new_hint"}
+    assert context4.global_hints == {"new_hint"}
+    assert context5.global_hints == {"new_hint"}
+
+    context4.global_hints.add("another_hint")
+    assert context.global_hints == {"new_hint", "another_hint"}
+    assert context2.global_hints == {"new_hint", "another_hint"}
+    assert context3.global_hints == {"new_hint", "another_hint"}
+    assert context4.global_hints == {"new_hint", "another_hint"}
+    assert context5.global_hints == {"new_hint", "another_hint"}
+
+
+def test_i_can_manage_protected_hint(context):
+    # Note that protected hint only works if the hint is added BEFORE the creation of the child
+    context.protected_hints.add("new_hint")
+    context2 = context.push("pusher", ExecutionContextActions.TESTING, None)
+    context3 = context2.push("pusher", ExecutionContextActions.TESTING, None)
+    context3.protected_hints.add("another_hint")
+    context4 = context3.push("pusher", ExecutionContextActions.TESTING, None)
+    context5 = context.push("pusher", ExecutionContextActions.TESTING, None)
+
+    assert context.protected_hints == {"new_hint"}
+    assert context2.protected_hints == {"new_hint"}
+    assert context3.protected_hints == {"new_hint", "another_hint"}
+    assert context4.protected_hints == {"new_hint", "another_hint"}
+    assert context5.protected_hints == {"new_hint"}
+
+
+def test_i_can_manage_private_hints(context):
+    context.private_hints.add("new_hint")
+    context2 = context.push("pusher", ExecutionContextActions.TESTING, None)
+    context3 = context2.push("pusher", ExecutionContextActions.TESTING, None)
+    context3.private_hints.add("another_hint")
+    context4 = context3.push("pusher", ExecutionContextActions.TESTING, None)
+    context5 = context.push("pusher", ExecutionContextActions.TESTING, None)
+
+    assert context.private_hints == {"new_hint"}
+    assert context2.private_hints == set()
+    assert context3.private_hints == {"another_hint"}
+    assert context4.private_hints == set()
+    assert context5.private_hints == set()
+
+
+def test_i_can_keep_track_of_children(context):
+    context2 = context.push("pusher", ExecutionContextActions.TESTING, None)
+    context3 = context.push("pusher", ExecutionContextActions.TESTING, None)
+    context4 = context2.push("pusher2", ExecutionContextActions.TESTING, None)
+
+    assert len(context._children) == 2
+    assert len(context2._children) == 1
+    assert len(context3._children) == 0
+    assert len(context4._children) == 0
+
+
+def test_i_can_get_children(context):
+    context1 = context.push("child 1", ExecutionContextActions.TESTING, None)
+    context2 = context.push("child 2", ExecutionContextActions.TESTING, None)
+    context3 = context.push("child 3", ExecutionContextActions.TESTING, None)
+    context21 = context2.push("child 21", ExecutionContextActions.TESTING, None)
+    context22 = context2.push("child 22", ExecutionContextActions.TESTING, None)
+    context211 = context21.push("child 211", ExecutionContextActions.TESTING, None)
+    context31 = context3.push("child 31", ExecutionContextActions.TESTING, None)
+
+    assert list(context1.get_children()) == []
+
+    assert list(context.get_children()) == [
+        context1,
+        context2,
+        context21,
+        context211,
+        context22,
+        context3,
+        context31
+    ]
+
+    assert list(context.get_children(level=1)) == [
+        context1,
+        context2,
+        context3
+    ]
+
+    assert list(context.get_children(level=2)) == [
+        context1,
+        context2,
+        context21,
+        context22,
+        context3,
+        context31,
+    ]

tests/core/test_sheerka.py

@@ -0,0 +1,36 @@
+from os import path
+
+from base import UsingFileBasedSheerka
+from helpers import get_concept, get_concepts, get_file_content
+
+
+class TestSheerka(UsingFileBasedSheerka):
+
+    def test_i_can_initialize_sheerka(self, sheerka_fb):
+        sheerka = sheerka_fb
+        assert path.exists(self.SHEERKA_ROOT_DIR)
+
+        last_event_path = path.join(self.SHEERKA_ROOT_DIR, "LAST_EVENT")
+        assert path.exists(last_event_path)
+
+        last_event_digest = get_file_content(last_event_path)
+        last_event_folder = path.join(self.SHEERKA_ROOT_DIR, "events", last_event_digest[:24], last_event_digest)
+        assert path.exists(last_event_folder)
+        assert path.exists(last_event_folder + "_admin_context")
+
+        assert len(sheerka.services) > 0
+        assert len(sheerka.evaluators) > 0
+
+        # add test to validate that we can access bind methods
+
+    def test_i_can_use_isinstance(self, sheerka, context):
+        foo, bar = get_concepts(context, "foo", "bar", use_sheerka=True)
+        assert sheerka.isinstance(foo, foo.key)
+        assert sheerka.isinstance(foo, foo.str_id)
+        assert sheerka.isinstance(foo, foo)
+        assert sheerka.isinstance(foo, foo.get_metadata())
+
+        assert not sheerka.isinstance(foo, bar.key)
+        assert not sheerka.isinstance(foo, bar.str_id)
+        assert not sheerka.isinstance(foo, bar)
+        assert not sheerka.isinstance(foo, bar.get_metadata())

tests/evaluators/test_CreateParserInput.py

@@ -0,0 +1,23 @@
+import pytest
+
+from base import BaseTest
+from core.BuiltinConcepts import BuiltinConcepts
+from evaluators.CreateParserInput import CreateParserInput
+from helpers import _rv, _rvf
+
+
+class TestCreateParserInput(BaseTest):
+
+    @pytest.fixture()
+    def evaluator(self, sheerka):
+        return sheerka.evaluators[CreateParserInput.NAME]
+
+    def test_i_can_match(self, sheerka, context, evaluator):
+        ret_val = _rv(sheerka.newn(BuiltinConcepts.USER_INPUT, command="hello sheerka"))
+        assert evaluator.matches(context, ret_val).status is True
+
+        ret_val = _rv(sheerka.newn(BuiltinConcepts.UNKNOWN_CONCEPT))  # it responds to USER_INPUT only
+        assert evaluator.matches(context, ret_val).status is False
+
+        ret_val = _rvf(sheerka.newn(BuiltinConcepts.USER_INPUT, command="hello sheerka"))  # status should be true
+        assert evaluator.matches(context, ret_val).status is False

tests/helpers.py

@@ -0,0 +1,377 @@
+from common.global_symbols import NotInit
+from core.ExecutionContext import ExecutionContext
+from core.ReturnValue import ReturnValue
+from core.concept import Concept, ConceptMetadata, DefinitionType
+from core.services.SheerkaConceptManager import ConceptManager
+
+
+class GetNextId:
+    def __init__(self):
+        self.seq = 1000
+
+    def next(self):
+        self.seq += 1
+        return self.seq
+
+
+def get_concept(name=None, body=None,
+                id=None,
+                key=None,
+                where=None,
+                pre=None,
+                post=None,
+                ret=None,
+                definition=None,
+                definition_type=None,
+                desc=None,
+                props=None,
+                variables=None,
+                parameters=None,
+                bound_body=None,
+                is_builtin=False,
+                is_unique=False,
+                autouse=False,
+                sequence=None) -> Concept:
+    """
+    Create a Concept objet
+    Caution : 'id' and 'key' are not initialized
+
+    :param name:
+    :type name:
+    :param body:
+    :type body:
+    :param id:
+    :type id:
+    :param key:
+    :type key:
+    :param where:
+    :type where:
+    :param pre:
+    :type pre:
+    :param post:
+    :type post:
+    :param ret:
+    :type ret:
+    :param definition:
+    :type definition:
+    :param definition_type:
+    :type definition_type:
+    :param desc:
+    :type desc:
+    :param props:
+    :type props:
+    :param variables:
+    :type variables:
+    :param parameters:
+    :type parameters:
+    :param bound_body:
+    :type bound_body:
+    :param is_builtin:
+    :type is_builtin:
+    :param is_unique:
+    :type is_unique:
+    :param autouse:
+    :type autouse:
+    :param sequence:
+    :type sequence:
+    :return:
+    :rtype:
+    """
+    metadata = get_metadata(
+        name, body,
+        id,
+        key,
+        where,
+        pre,
+        post,
+        ret,
+        definition,
+        definition_type,
+        desc,
+        props,
+        variables,
+        parameters,
+        bound_body,
+        is_builtin,
+        is_unique,
+        autouse
+    )
+    if sequence:
+        metadata.auto_init(sequence)
+    else:
+        metadata.digest = ConceptManager.compute_metadata_digest(metadata)
+        metadata.all_attrs = ConceptManager.compute_all_attrs(metadata.variables)
+    return Concept(metadata)
+
+
+def get_metadata(name=None, body=None,
+                 id=None,
+                 key=None,
+                 where=None,
+                 pre=None,
+                 post=None,
+                 ret=None,
+                 definition=None,
+                 definition_type=DefinitionType.DEFAULT,
+                 desc=None,
+                 props=None,
+                 variables=None,
+                 parameters=None,
+                 bound_body=None,
+                 is_builtin=False,
+                 is_unique=False,
+                 autouse=False,
+                 digest=None,
+                 all_attrs=None):
+    new_variables = []
+    if variables:
+        for v in variables:
+            if isinstance(v, tuple):
+                new_variables.append(v)
+            else:
+                new_variables.append((v, NotInit))
+
+    return ConceptMetadata(
+        id,
+        name,
+        key,
+        is_builtin,
+        is_unique,
+        body,
+        where,
+        pre,
+        post,
+        ret,
+        definition,
+        definition_type,
+        desc,
+        autouse,
+        bound_body,
+        props or {},
+        tuple(new_variables),
+        parameters or [],
+        digest,
+        all_attrs,
+    )
+
+
+def metadata_auto_init(self: ConceptMetadata, sequence) -> ConceptMetadata:
+    """
+    Helper function for the unit tests.
+    This method will be added to the `ConceptMetadata` to ease the writing of the unit tests
+    It properly initializes the ConceptMetadata
+    :param self:
+    :type self:
+    :param sequence:
+    :type sequence:
+    :return:
+    :rtype:
+    """
+    if not self.id:
+        self.id = str(sequence.next())
+    if not self.key:
+        self.key = ConceptManager.create_concept_key(self.name, self.definition, self.variables)
+    if not self.is_unique:
+        self.is_unique = False
+    if not self.is_builtin:
+        self.is_builtin = False
+    if not self.definition_type:
+        self.definition_type = DefinitionType.DEFAULT
+    if not self.all_attrs:
+        self.all_attrs = ConceptManager.compute_all_attrs(self.variables)
+    if not self.digest:
+        self.digest = ConceptManager.compute_metadata_digest(self)
+
+    # Note that I do not automatically update the digest as I don't want to make unnecessary computations
+
+    return self
+
+
+def metadata_clone(self: ConceptMetadata, name=None, body=None,
+                   key=None,
+                   where=None,
+                   pre=None,
+                   post=None,
+                   ret=None,
+                   definition=None,
+                   definition_type=None,
+                   desc=None,
+                   props=None,
+                   variables=None,
+                   parameters=None,
+                   bound_body=None,
+                   is_builtin=None,
+                   is_unique=None,
+                   autouse=None,
+                   digest=None,
+                   all_attrs=None) -> ConceptMetadata:
+    """
+    Helper function for the unit tests.
+    This method will be added to the `ConceptMetadata` to ease the writing of the unit tests
+    It clones a ConceptMetadata, but can override some attributes if requested
+    :param self:
+    :type self:
+    :param name:
+    :type name:
+    :param body:
+    :type body:
+    :param key:
+    :type key:
+    :param where:
+    :type where:
+    :param pre:
+    :type pre:
+    :param post:
+    :type post:
+    :param ret:
+    :type ret:
+    :param definition:
+    :type definition:
+    :param definition_type:
+    :type definition_type:
+    :param desc:
+    :type desc:
+    :param props:
+    :type props:
+    :param variables:
+    :type variables:
+    :param parameters:
+    :type parameters:
+    :param bound_body:
+    :type bound_body:
+    :param is_builtin:
+    :type is_builtin:
+    :param is_unique:
+    :type is_unique:
+    :param autouse:
+    :type autouse:
+    :param digest:
+    :type digest:
+    :param all_attrs:
+    :type all_attrs:
+    :return:
+    :rtype:
+    """
+    return ConceptMetadata(
+        id=self.id,
+        name=self.name if name is None else name,
+        body=self.body if body is None else body,
+        key=self.key if key is None else key,
+        where=self.where if where is None else where,
+        pre=self.pre if pre is None else pre,
+        post=self.post if post is None else post,
+        ret=self.ret if ret is None else ret,
+        definition=self.definition if definition is None else definition,
+        definition_type=self.definition_type if definition_type is None else definition_type,
+        desc=self.desc if desc is None else desc,
+        props=self.props if props is None else props,
+        variables=self.variables if variables is None else variables,
+        parameters=self.parameters if parameters is None else parameters,
+        bound_body=self.bound_body if bound_body is None else bound_body,
+        is_builtin=self.is_builtin if is_builtin is None else is_builtin,
+        is_unique=self.is_unique if is_unique is None else is_unique,
+        autouse=self.autouse if autouse is None else autouse,
+        digest=self.digest if digest is None else digest,
+        all_attrs=self.all_attrs if all_attrs is None else all_attrs,
+    )
+
+
+# Helpers functions for unit tests
+setattr(ConceptMetadata, 'auto_init', metadata_auto_init)
+setattr(ConceptMetadata, 'clone', metadata_clone)
+
+
+def get_metadatas(*args, **kwargs):
+    as_metadatas = [arg if isinstance(arg, ConceptMetadata) else get_metadata(arg) for arg in args]
+    next_id = kwargs.get("next_id", None)
+    if next_id:
+        for metadata in as_metadatas:
+            metadata_auto_init(metadata, next_id)
+
+    return as_metadatas
+
+
+def get_concepts(context: ExecutionContext, *concepts, **kwargs) -> list[Concept]:
+    """
+    Simple and quick way to get initialize concepts for a test
+    :param sheerka:
+    :type sheerka:
+    :param context:
+    :type context:
+    :param concepts:
+    :type concepts:
+    :param kwargs:
+    :type kwargs:
+    :return:
+    :rtype:
+    """
+    res = []
+    use_sheerka = kwargs.pop("use_sheerka", False)
+    sequence = kwargs.pop("sequence", None)
+    for c in concepts:
+        if use_sheerka:
+            c = define_new_concept(context, c)
+        elif isinstance(c, str):
+            c = get_concept(c)
+
+        if sequence:
+            c.get_metadata().auto_init(sequence)
+
+        res.append(c)
+
+    return res
+
+
+def define_new_concept(context: ExecutionContext, c: str | Concept) -> Concept:
+    sheerka = context.sheerka
+    if isinstance(c, str):
+        retval = sheerka.define_new_concept(context, c)
+    else:
+        metadata = c.get_metadata()
+        retval = sheerka.define_new_concept(context,
+                                            metadata.name,
+                                            metadata.is_builtin,
+                                            metadata.is_unique,
+                                            metadata.body,
+                                            metadata.where,
+                                            metadata.pre,
+                                            metadata.post,
+                                            metadata.ret,
+                                            metadata.definition,
+                                            metadata.definition_type,
+                                            metadata.autouse,
+                                            metadata.bound_body,
+                                            metadata.desc,
+                                            metadata.props,
+                                            metadata.variables,
+                                            metadata.parameters)
+
+    assert retval.status
+    concept = sheerka.newn(retval.value.metadata.name)
+    return concept
+
+
+def get_file_content(file_name):
+    with open(file_name) as f:
+        return f.read()
+
+
+def _rv(value, who="Test"):
+    return ReturnValue(who=who, status=True, value=value)
+
+
+def _rvc(concept_name, who="Test"):
+    next_id = GetNextId()
+    concept = get_concept(concept_name, sequence=next_id)
+    return ReturnValue(who=who, status=True, value=concept)
+
+
+def _rvf(value, who="Test"):
+    """
+    Return Value False
+    :param value:
+    :type value:
+    :return:
+    :rtype:
+    """
+    return ReturnValue(who=who, status=False, value=value)

tests/parsers/test_parser_input.py

@@ -0,0 +1,14 @@
+from parsers.ParserInput import ParserInput
+from parsers.tokenizer import LexerError
+
+
+def test_i_can_parser_input():
+    parser_input = ParserInput("def concept a")
+    assert parser_input.init() is True
+    assert parser_input.exception is None
+
+
+def test_i_can_detect_errors():
+    parser_input = ParserInput('def concept "a')
+    assert parser_input.init() is False
+    assert isinstance(parser_input.exception, LexerError)

tests/parsers/test_tokenizer.py

@@ -0,0 +1,211 @@
+import pytest
+
+from parsers.tokenizer import LexerError, Token, TokenKind, Tokenizer
+
+
+def test_i_can_tokenize():
+    source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&<>c:name:"
+    source += "$£€!_identifier°~_^\\`==#__var__10r/regex\nregex/r:xxx#1:**//%that's"
+    tokens = list(Tokenizer(source))
+    assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
+    assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
+    assert tokens[2] == Token(TokenKind.MINUS, "-", 2, 1, 3)
+    assert tokens[3] == Token(TokenKind.SLASH, "/", 3, 1, 4)
+    assert tokens[4] == Token(TokenKind.LBRACE, "{", 4, 1, 5)
+    assert tokens[5] == Token(TokenKind.RBRACE, "}", 5, 1, 6)
+    assert tokens[6] == Token(TokenKind.LBRACKET, "[", 6, 1, 7)
+    assert tokens[7] == Token(TokenKind.RBRACKET, "]", 7, 1, 8)
+    assert tokens[8] == Token(TokenKind.LPAR, "(", 8, 1, 9)
+    assert tokens[9] == Token(TokenKind.RPAR, ")", 9, 1, 10)
+    assert tokens[10] == Token(TokenKind.WHITESPACE, "    ", 10, 1, 11)
+    assert tokens[11] == Token(TokenKind.COMMA, ",", 14, 1, 15)
+    assert tokens[12] == Token(TokenKind.SEMICOLON, ";", 15, 1, 16)
+    assert tokens[13] == Token(TokenKind.COLON, ":", 16, 1, 17)
+    assert tokens[14] == Token(TokenKind.DOT, ".", 17, 1, 18)
+    assert tokens[15] == Token(TokenKind.QMARK, "?", 18, 1, 19)
+    assert tokens[16] == Token(TokenKind.NEWLINE, "\n", 19, 1, 20)
+    assert tokens[17] == Token(TokenKind.NEWLINE, "\n\r", 20, 2, 1)
+    assert tokens[18] == Token(TokenKind.NEWLINE, "\r", 22, 3, 1)
+    assert tokens[19] == Token(TokenKind.NEWLINE, "\r\n", 23, 4, 1)
+    assert tokens[20] == Token(TokenKind.IDENTIFIER, "identifier_0", 25, 5, 1)
+    assert tokens[21] == Token(TokenKind.WHITESPACE, "\t  \t", 37, 5, 13)
+    assert tokens[22] == Token(TokenKind.NUMBER, "10.15", 41, 5, 17)
+    assert tokens[23] == Token(TokenKind.WHITESPACE, " ", 46, 5, 22)
+    assert tokens[24] == Token(TokenKind.NUMBER, "10", 47, 5, 23)
+    assert tokens[25] == Token(TokenKind.WHITESPACE, " ", 49, 5, 25)
+    assert tokens[26] == Token(TokenKind.STRING, "'string\n'", 50, 5, 26)
+    assert tokens[27] == Token(TokenKind.WHITESPACE, " ", 59, 6, 2)
+    assert tokens[28] == Token(TokenKind.STRING, '"another string"', 60, 6, 3)
+    assert tokens[29] == Token(TokenKind.EQUALS, '=', 76, 6, 19)
+    assert tokens[30] == Token(TokenKind.VBAR, '|', 77, 6, 20)
+    assert tokens[31] == Token(TokenKind.AMPER, '&', 78, 6, 21)
+    assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 22)
+    assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 23)
+    assert tokens[34] == Token(TokenKind.CONCEPT, ('name', None), 81, 6, 24)
+    assert tokens[35] == Token(TokenKind.DOLLAR, '$', 88, 6, 31)
+    assert tokens[36] == Token(TokenKind.STERLING, '£', 89, 6, 32)
+    assert tokens[37] == Token(TokenKind.EURO, '€', 90, 6, 33)
+    assert tokens[38] == Token(TokenKind.EMARK, '!', 91, 6, 34)
+    assert tokens[39] == Token(TokenKind.IDENTIFIER, '_identifier', 92, 6, 35)
+    assert tokens[40] == Token(TokenKind.DEGREE, '°', 103, 6, 46)
+    assert tokens[41] == Token(TokenKind.TILDE, '~', 104, 6, 47)
+    assert tokens[42] == Token(TokenKind.UNDERSCORE, '_', 105, 6, 48)
+    assert tokens[43] == Token(TokenKind.CARAT, '^', 106, 6, 49)
+    assert tokens[44] == Token(TokenKind.BACK_SLASH, '\\', 107, 6, 50)
+    assert tokens[45] == Token(TokenKind.BACK_QUOTE, '`', 108, 6, 51)
+    assert tokens[46] == Token(TokenKind.EQUALSEQUALS, '==', 109, 6, 52)
+    assert tokens[47] == Token(TokenKind.HASH, '#', 111, 6, 54)
+    assert tokens[48] == Token(TokenKind.VAR_DEF, '__var__10', 112, 6, 55)
+    assert tokens[49] == Token(TokenKind.REGEX, '/regex\nregex/', 121, 6, 64)
+    assert tokens[50] == Token(TokenKind.RULE, ("xxx", "1"), 135, 7, 7)
+    assert tokens[51] == Token(TokenKind.STARSTAR, "**", 143, 7, 15)
+    assert tokens[52] == Token(TokenKind.SLASHSLASH, "//", 145, 7, 17)
+    assert tokens[53] == Token(TokenKind.PERCENT, "%", 147, 7, 19)
+    assert tokens[54] == Token(TokenKind.IDENTIFIER, "that", 148, 7, 20)
+    assert tokens[55] == Token(TokenKind.QUOTE, "'", 152, 7, 24)
+    assert tokens[56] == Token(TokenKind.IDENTIFIER, "s", 153, 7, 25)
+
+    assert tokens[57] == Token(TokenKind.EOF, '', 154, 7, 26)
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("_ident", True),
+    ("__ident", True),
+    ("___ident", True),
+    ("ident", True),
+    ("ident123", True),
+    ("ident_123", True),
+    ("ident-like-this", True),
+    ("àèùéû", True),
+    ("011254", False),
+    ("0abcd", False),
+    ("-abcd", False)
+])
+def test_i_can_tokenize_identifiers(text, expected):
+    tokens = list(Tokenizer(text))
+    comparison = tokens[0].type == TokenKind.IDENTIFIER
+    assert comparison == expected
+
+
+@pytest.mark.parametrize("text", [
+    "123abc",
+    "123",
+    "abc",
+    "abc123"
+])
+def test_i_can_parse_word(text):
+    tokens = list(Tokenizer(text, parse_word=True))
+    assert tokens[0].type == TokenKind.WORD
+    assert tokens[0].value == text
+    assert tokens[1].index == len(text)
+
+
+@pytest.mark.parametrize("text", [
+    "__var__0",
+    "__var__1",
+    "__var__10",
+    "__var__999",
+])
+def test_i_can_parse_var_def(text):
+    tokens = list(Tokenizer(text))
+    assert len(tokens) == 2
+    assert tokens[0].type == TokenKind.VAR_DEF
+    assert tokens[0].value == text
+
+
+@pytest.mark.parametrize("text, message, error_text, index, line, column", [
+    ("'string", "Missing Trailing quote", "'string", 7, 1, 8),
+    ('"string', "Missing Trailing quote", '"string', 7, 1, 8),
+    ('"a" + "string', "Missing Trailing quote", '"string', 13, 1, 14),
+    ('"a"\n\n"string', "Missing Trailing quote", '"string', 12, 3, 8),
+    ('"', "Missing Trailing quote", '"', 1, 1, 2),
+    ("c::", "Concept identifiers not found", "", 2, 1, 3),
+    ("c:foo\nbar:", "New line in concept name", "foo", 5, 1, 6),
+    ("c:foo", "Missing ending colon", "foo", 5, 1, 6)
+])
+def test_i_can_detect_tokenizer_errors(text, message, error_text, index, line, column):
+    with pytest.raises(LexerError) as e:
+        list(Tokenizer(text))
+    assert e.value.message == message
+    assert e.value.text == error_text
+    assert e.value.index == index
+    assert e.value.line == line
+    assert e.value.column == column
+
+
+@pytest.mark.parametrize("text, expected_text, expected_newlines, expected_column", [
+    ("'foo'", "'foo'", 0, 6),
+    ('"foo"', '"foo"', 0, 6),
+    ("'foo\nbar'", "'foo\nbar'", 1, 5),
+    ("'foo\rbar'", "'foo\rbar'", 0, 10),
+    ("'foo\n\rbar'", "'foo\n\rbar'", 1, 6),
+    ("'foo\r\nbar'", "'foo\r\nbar'", 1, 5),
+    ("'foo\n\nbar'", "'foo\n\nbar'", 2, 5),
+    ("'foo\r\n\n\rbar'", "'foo\r\n\n\rbar'", 2, 6),
+    ("'\nfoo\nbar\n'", "'\nfoo\nbar\n'", 3, 2),
+    ("'\n\rfoo\r\n'", "'\n\rfoo\r\n'", 2, 2),
+    (r"'foo\'bar'", r"'foo\'bar'", 0, 11),
+    (r'"foo\"bar"', r'"foo\"bar"', 0, 11),
+    ('"foo"bar"', '"foo"', 0, 6),
+    ("'foo'bar'", "'foo'", 0, 6),
+])
+def test_i_can_parse_strings(text, expected_text, expected_newlines, expected_column):
+    lexer = Tokenizer(text)
+    text_found, nb_of_newlines, column_index = lexer.eat_string(0, 1, 1)
+
+    assert text_found == expected_text
+    assert nb_of_newlines == expected_newlines
+    assert column_index == expected_column
+
+
+@pytest.mark.parametrize("text", [
+    "1", "3.1415", "0.5", "01", "-5", "-5.10"
+])
+def test_i_can_parse_numbers(text):
+    tokens = list(Tokenizer(text))
+    assert tokens[0].type == TokenKind.NUMBER
+    assert tokens[0].value == text
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("c:key:", ("key", None)),
+    ("c:key|id:", ("key", "id")),
+    ("c:key|:", ("key", None)),
+    ("c:|id:", (None, "id")),
+    ("c:125:", ("125", None)),
+])
+def test_i_can_parse_concept_token(text, expected):
+    tokens = list(Tokenizer(text))
+
+    assert tokens[0].type == TokenKind.CONCEPT
+    assert tokens[0].value == expected
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("r:key:", ("key", None)),
+    ("r:key#id:", ("key", "id")),
+    ("r:key#:", ("key", None)),
+    ("r:#id:", (None, "id")),
+    ("r:125:", ("125", None)),
+])
+def test_i_can_parse_concept_token(text, expected):
+    tokens = list(Tokenizer(text))
+
+    assert tokens[0].type == TokenKind.RULE
+    assert tokens[0].value == expected
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("r|regex|", "|regex|"),
+    ("r/regex/", "/regex/"),
+    ("r'regex'", "'regex'"),
+    ('r"regex"', '"regex"'),
+])
+def test_i_can_parse_regex_token(text, expected):
+    tokens = list(Tokenizer(text))
+
+    assert tokens[0].type == TokenKind.REGEX
+    assert tokens[0].value == expected
+    assert tokens[0].str_value == "r" + expected
+    assert tokens[0].repr_value == "r" + expected
+    assert tokens[0].strip_quote == expected[1:-1]

tests/sdp/test_sheerkaDataProvider.py

@@ -6,11 +6,11 @@ from os import path
 
 import pytest
 
-from core.global_symbols import NotFound
+from common.global_symbols import NotFound
 from sdp.sheerkaDataProvider import Event, SheerkaDataProvider
 from sdp.sheerkaSerializer import JsonSerializer, PickleSerializer
 
-tests_root = path.abspath("../../build/tests")
+tests_root = path.abspath("../build/tests")
 evt_digest = "3a571cb6034ef6fc8d7fe91948d0d29728eed74de02bac7968b0e9facca2c2d7"
 
 
@@ -71,7 +71,7 @@ class ObjWithDigestWithKey:
 
 
 @pytest.fixture(autouse=True)
-def init_test():
+def init_current_dir():
     """
     I test both SheerkaDataProviderFileIO and SheerkaDataProviderDictionaryIO
     So it's important to reset the folders between two tests
@@ -407,11 +407,11 @@ def test_i_can_add_an_object_and_save_it_as_a_reference(root):
 
     state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
     assert state.data == {
-        "entry": {'key1': '##REF##:fbc2b1c60ed753b49217cae851e342371ee39ebabc9778105f450812e615a513',
-                  'key2': ['##REF##:fbc2b1c60ed753b49217cae851e342371ee39ebabc9778105f450812e615a513',
-                           '##REF##:448420dbc57d61401d10a98759fccdabbe50e2e825b6da3bd018c190926bcda4'],
-                  'key3': {'##REF##:448420dbc57d61401d10a98759fccdabbe50e2e825b6da3bd018c190926bcda4',
-                           '##REF##:fbc2b1c60ed753b49217cae851e342371ee39ebabc9778105f450812e615a513'}}
+        "entry": {'key1': '##REF##:4d20621e3c45e8977504016caa2539c0d518850d3a8f92eb20f3e9e5192c41cf',
+                  'key2': ['##REF##:4d20621e3c45e8977504016caa2539c0d518850d3a8f92eb20f3e9e5192c41cf',
+                           '##REF##:c142f0a14ae4b52afa7cdcbb88dc16563468ca3fa99584323083968099cbaf6b'],
+                  'key3': {'##REF##:4d20621e3c45e8977504016caa2539c0d518850d3a8f92eb20f3e9e5192c41cf',
+                           '##REF##:c142f0a14ae4b52afa7cdcbb88dc16563468ca3fa99584323083968099cbaf6b'}}
     }
 
 

tests/sdp/test_sheerkaSerializer.py

@@ -26,7 +26,7 @@ class ObjNoKey:
 
 
 def test_i_can_json_serialize():
-    json_serializer = JsonSerializer(lambda obj: True)
+    json_serializer = JsonSerializer(lambda o: True)
     obj = ObjNoKey("a", "b")
     stream = io.BytesIO()
 

tests/server/test_server.py

@@ -0,0 +1,19 @@
+from starlette.testclient import TestClient
+
+from server.main import app
+
+client = TestClient(app)
+
+
+def test_i_can_authenticate():
+    data = {
+        "username": "kodjo",
+        "password": "kodjo"
+    }
+    response = client.post("/token", data=data)
+    assert response.status_code == 200
+    as_json = response.json()
+    assert 'access_token' in as_json
+    assert 'first_name' in as_json
+    assert 'last_name' in as_json
+    assert 'token_type' in as_json

tests/services/test_ConceptManager.py

@@ -0,0 +1,186 @@
+import pytest
+
+from base import BaseTest
+from common.global_symbols import NotFound, NotInit
+from conftest import NewOntology
+from core.BuiltinConcepts import BuiltinConcepts
+from core.ErrorContext import ErrorContext
+from core.concept import ConceptMetadata
+from core.services.SheerkaConceptManager import ConceptAlreadyDefined, ConceptManager
+from helpers import get_metadata
+
+
+class TestConceptManager(BaseTest):
+
+    @pytest.fixture()
+    def service(self, sheerka):
+        return sheerka.services[ConceptManager.NAME]
+
+    def test_i_can_compute_concept_digest(self, service):
+        """
+        Two concepts with the same definition share the same digest
+        :return:
+        :rtype:
+        """
+        metadata = get_metadata("foo", "body")
+        digest = service.compute_metadata_digest(metadata)
+        assert digest == "21a1c2f420da62f4dc60f600c95b19dd9527b19dd28fd38e17f5c0e28963d176"
+
+        another_metadata = get_metadata("foo", "body")
+        other_digest = service.compute_metadata_digest(another_metadata)
+
+        assert digest == other_digest
+
+    def test_id_is_not_part_of_the_digest(self, service):
+        metadata1 = get_metadata("foo", "body", id=1)
+        metadata2 = get_metadata("foo", "body", id=2)
+
+        assert service.compute_metadata_digest(metadata1) == service.compute_metadata_digest(metadata2)
+
+    def test_i_can_compute_concept_attributes_based_on_the_metadata(self, service):
+        compute_all_attrs = service.compute_all_attrs
+
+        m1 = get_metadata("foo")
+        assert compute_all_attrs(m1.variables) == ('#where#', '#pre#', '#post#', '#body#', '#ret#')
+
+        m2 = get_metadata("bar", variables=[("var1", None), ("var2", None)])
+        assert compute_all_attrs(m2.variables) == ('#where#', '#pre#', '#post#', '#body#', '#ret#', 'var1', 'var2')
+
+    @pytest.mark.parametrize("definition, variables, expected", [
+        ("foo", [], "foo"),
+        ("foo(bar)", [], "foo ( bar )"),
+        ("foo a", ["a"], "foo __var__0"),
+        ("a foo b", ["a", "b"], "__var__0 foo __var__1"),
+        ("a foo b", ["b", "a"], "__var__1 foo __var__0"),
+        ("foo", ["foo"], "foo"),
+        ("foo a", ["foo"], "__var__0 a"),
+        ("foo       a   b", ["a"], "foo __var__0 b"),
+        ("'foo'", [], "'foo'"),
+        ("my name is a", ["a"], "my name is __var__0"),
+        ("a b c d", ["b", "c"], "a __var__0 __var__1 d"),
+        ("a 'b c' d", ["b", "c"], "a 'b c' d"),
+        ("a | b", ["a", "b"], "__var__0 | __var__1"),
+        ("a b a c", ["a", "b"], "__var__0 __var__1 __var__0 c"),
+        ("a b a c", ["b", "a"], "__var__1 __var__0 __var__1 c"),
+        ("def concept a", ["a"], "def concept __var__0"),
+    ])
+    def test_i_can_create_concept_key(self, service, definition, variables, expected):
+        expanded_variables = tuple((v, NotInit) for v in variables)
+        key = service.create_concept_key(definition, None, expanded_variables)
+
+        assert key == expected
+
+    def test_the_key_is_created_from_the_definition_if_it_is_set(self, service):
+        assert service.create_concept_key("from name", "from definition", None) == "from definition"
+        assert service.create_concept_key("from name", None, None) == "from name"
+
+    def test_i_can_define_a_new_concept(self, context, service):
+        with NewOntology(context, "test_i_can_define_a_new_concept"):
+            res = service.define_new_concept(context, "name", body="body")
+
+            assert res.status is True
+
+            metadata = res.value.metadata
+            assert isinstance(metadata, ConceptMetadata)
+            assert metadata.id == "1001"
+            assert metadata.name == "name"
+            assert metadata.key == "name"
+            assert metadata.body == "body"
+            assert metadata.digest == "eb0620bd4a317af8a403c0ae1e185a528f9b58f8b0878d990e62278f89cf10d5"
+            assert metadata.all_attrs == ('#where#', '#pre#', '#post#', '#body#', '#ret#')
+
+            # is sorted in db
+            om = context.sheerka.om
+            assert om.get(ConceptManager.CONCEPTS_BY_ID_ENTRY, metadata.id) == metadata
+            assert om.get(ConceptManager.CONCEPTS_BY_NAME_ENTRY, metadata.name) == metadata
+            assert om.get(ConceptManager.CONCEPTS_BY_KEY_ENTRY, metadata.key) == metadata
+            assert om.get(ConceptManager.CONCEPTS_BY_HASH_ENTRY, metadata.digest) == metadata
+
+    def test_i_cannot_create_the_same_concept_twice(self, context, service):
+        with NewOntology(context, "test_i_cannot_create_the_same_concept_twice"):
+            res = service.define_new_concept(context, "name", body="body")
+            assert res.status
+
+            res = service.define_new_concept(context, "name", body="body")
+            assert not res.status
+            assert isinstance(res.value, ErrorContext)
+            assert isinstance(res.value.value, ConceptAlreadyDefined)
+
+    def test_i_can_add_the_same_concept_on_different_ontologies(self, context, service):
+        with NewOntology(context, "test_i_can_add_the_same_concept_on_different_ontologies"):
+            res = service.define_new_concept(context, "name", body="body")
+            assert res.status
+
+            sheerka = context.sheerka
+            om = sheerka.om
+            om.push_ontology("my_new_ontology")
+            res = service.define_new_concept(context, "name", body="body")
+            assert res.status is True
+
+    def test_i_can_get_a_newly_created_concept(self, context, service):
+        with NewOntology(context, "test_i_can_get_a_newly_created_concept"):
+            res = service.define_new_concept(context, "name", body="body")
+            assert res.status
+            metadata = res.value.metadata
+
+            assert service.get_by_id(metadata.id).id == metadata.id
+            assert service.get_by_name(metadata.name).name == metadata.name
+            assert service.get_by_key(metadata.key).key == metadata.key
+
+    def test_i_can_instantiate_a_new_concept_by_its_name(self, context, service):
+        with NewOntology(context, "test_i_can_instantiate_a_new_concept_by_its_name"):
+            res = service.define_new_concept(context, "foo", variables=[("var1", None), ("var2", None)])
+            assert res.status
+
+            foo = service.newn("foo", var1="value1", var2="value2")
+
+            assert foo.id == "1001"
+            assert foo.key == "foo"
+            assert foo.name == "foo"
+            assert foo.str_id == "c:#1001:"
+            assert foo.var1 == "value1"
+            assert foo.var2 == "value2"
+
+    def test_i_can_instantiate_a_new_concept_by_its_id(self, context, service):
+        with NewOntology(context, "test_i_can_instantiate_a_new_concept_by_its_id"):
+            res = service.define_new_concept(context, "foo", variables=[("var1", None), ("var2", None)])
+            assert res.status
+
+            foo = service.newi("1001", var1="value1", var2="value2")
+
+            assert foo.id == "1001"
+            assert foo.key == "foo"
+            assert foo.name == "foo"
+            assert foo.str_id == "c:#1001:"
+            assert foo.var1 == "value1"
+            assert foo.var2 == "value2"
+
+    def test_i_cannot_instantiate_a_concept_which_does_not_exist(self, context, service):
+        foo = service.newn("foo", var1="value1", var2="value2")
+        assert foo.key == BuiltinConcepts.UNKNOWN_CONCEPT
+        assert foo.requested_name == "foo"
+
+        foo = service.newi("1001", var1="value1", var2="value2")
+        assert foo.key == BuiltinConcepts.UNKNOWN_CONCEPT
+        assert foo.requested_id == "1001"
+
+    def test_i_can_instantiate_by_name_when_multiple_results(self, context, service):
+        with NewOntology(context, "test_i_can_instantiate_by_name_when_multiple_results"):
+            service.define_new_concept(context, "foo", body="body1")
+            service.define_new_concept(context, "foo", body="body2")
+
+            concepts = service.newn("foo")
+
+            assert len(concepts) == 2
+            assert concepts[0].id == "1001"
+            assert concepts[0].get_metadata().body == "body1"
+            assert concepts[1].id == "1002"
+            assert concepts[1].get_metadata().body == "body2"
+
+    def test_concepts_are_removed_when_ontology_is_popped(self, context, service):
+        context.sheerka.om.push_ontology("new ontology")
+        res = service.define_new_concept(context, "foo", body="body")
+        assert service.get_by_id(res.value.metadata.id) is not NotFound
+
+        context.sheerka.om.pop_ontology(context)
+        assert service.get_by_id(res.value.metadata.id) is NotFound

tests/services/test_SheerkaEngine.py

@@ -0,0 +1,379 @@
+from typing import Callable
+
+import pytest
+
+from base import BaseTest
+from core.BuiltinConcepts import BuiltinConcepts
+from core.ExecutionContext import ExecutionContext, ExecutionContextActions
+from core.ReturnValue import ReturnValue
+from core.services.SheerkaEngine import SheerkaEngine
+from evaluators.CreateParserInput import CreateParserInput
+from evaluators.base_evaluator import AllReturnValuesEvaluator, BaseEvaluator, EvaluatorEvalResult, \
+    EvaluatorMatchResult, \
+    OneReturnValueEvaluator
+from helpers import _rvc
+
+ALL_STEPS = [
+    ExecutionContextActions.BEFORE_PARSING,
+    ExecutionContextActions.PARSING,
+    ExecutionContextActions.AFTER_PARSING,
+    ExecutionContextActions.BEFORE_EVALUATION,
+    ExecutionContextActions.EVALUATION,
+    ExecutionContextActions.AFTER_EVALUATION
+]
+
+
+class OneReturnValueEvaluatorForTesting(OneReturnValueEvaluator):
+    def __init__(self, name,
+                 step: ExecutionContextActions,
+                 priority: int,
+                 enabled=True,
+                 match: bool | Callable = True,
+                 match_context=None,
+                 eval_result: list[ReturnValue] = None,
+                 eval_eaten: list[ReturnValue] = None):
+        super().__init__(name, step, priority, enabled)
+        self.matches_delegate = match
+        self.matches_context = match_context
+        self.eval_result = eval_result
+        self.eval_eaten = eval_eaten
+
+    def matches(self, context: ExecutionContext, return_value: ReturnValue) -> EvaluatorMatchResult:
+        # if status is a bool, use it
+        # otherwise, it's a delegate, so apply to return_value
+        status = self.matches_delegate if \
+            isinstance(self.matches_delegate, bool) else \
+            self.matches_delegate(return_value)
+        return EvaluatorMatchResult(status, self.matches_context)
+
+    def eval(self, context: ExecutionContext,
+             evaluation_context: object,
+             return_value: ReturnValue) -> EvaluatorEvalResult:
+
+        # make sure to correctly set up the parent when the return value is modified
+        if self.eval_result:
+            for ret_val in self.eval_result:
+                if ret_val != return_value:
+                    ret_val.parents = [return_value]
+
+        return EvaluatorEvalResult(self.eval_result, self.eval_eaten or [return_value])
+
+
+class AllReturnValuesEvaluatorForTesting(AllReturnValuesEvaluator):
+    def __init__(self, name,
+                 step: ExecutionContextActions,
+                 priority: int,
+                 enabled=True,
+                 match: bool | Callable = True,
+                 match_context=None,
+                 eval_result: list[ReturnValue] = None,
+                 eval_eaten: list[ReturnValue] = None):
+        super().__init__(name, step, priority, enabled)
+        self.matches_delegate = match
+        self.matches_context = match_context
+        self.eval_result = eval_result
+        self.eval_eaten = eval_eaten
+
+    def matches(self, context: ExecutionContext, return_values: list[ReturnValue]) -> EvaluatorMatchResult:
+        # if status is a bool, use it
+        # otherwise, it's a delegate, so apply to return_value
+        status = self.matches_delegate if \
+            isinstance(self.matches_delegate, bool) else \
+            self.matches_delegate(return_values)
+        return EvaluatorMatchResult(status, self.matches_context)
+
+    def eval(self, context: ExecutionContext,
+             evaluation_context: object,
+             return_values: list[ReturnValue]) -> EvaluatorEvalResult:
+
+        # make sure to correctly set up the parent when the return value is modified
+        if self.eval_result:
+            for ret_val in self.eval_result:
+                ret_val.parents = return_values
+
+        return EvaluatorEvalResult(self.eval_result, self.eval_eaten or return_values)
+
+
+class TestSheerkaEngine(BaseTest):
+    @pytest.fixture()
+    def service(self, sheerka):
+        return SheerkaEngine(sheerka)
+
+    def test_i_can_compute_execution_plan(self, service):
+        assert service.compute_execution_plan([]) == {}
+
+        e1 = BaseEvaluator("eval1", ExecutionContextActions.BEFORE_EVALUATION, 5)
+        e2 = BaseEvaluator("eval2", ExecutionContextActions.BEFORE_EVALUATION, 5)
+        e3 = BaseEvaluator("eval3", ExecutionContextActions.BEFORE_EVALUATION, 10)
+        e4 = BaseEvaluator("eval4", ExecutionContextActions.EVALUATION, 10)
+        e5 = BaseEvaluator("eval5", ExecutionContextActions.AFTER_EVALUATION, 10, enabled=False)
+        res = service.compute_execution_plan([e1, e2, e3, e4, e5])
+        assert res == {ExecutionContextActions.BEFORE_EVALUATION: {5: [e1, e2], 10: [e3]},
+                       ExecutionContextActions.EVALUATION: {10: [e4]}}
+
+    def test_i_can_call_execute(self, sheerka, context, service):
+        service.execution_plan = {ExecutionContextActions.BEFORE_EVALUATION: {50: [CreateParserInput()]}}
+        start = [ReturnValue("TestSheerkaEngine", True, sheerka.newn(BuiltinConcepts.USER_INPUT, command="1 + 1"))]
+
+        ret = service.execute(context, start, [ExecutionContextActions.BEFORE_EVALUATION])
+        assert len(ret) == 1
+        ret = ret[0]
+        assert isinstance(ret, ReturnValue)
+        assert ret.who == CreateParserInput.NAME
+        assert ret.status is True
+        assert ret.parents == start
+
+    def test_that_return_values_is_unchanged_when_no_evaluator(self, context, service):
+        service.execution_plan = {}
+        start = [_rvc("foo")]
+
+        ret = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+
+        assert ret == start
+
+    def test_steps_are_executed_in_correct_order(self, context, service):
+        # properly init the service
+        _ = OneReturnValueEvaluatorForTesting
+        evaluators = [
+            _("eval1", ExecutionContextActions.AFTER_PARSING, 21, match=False),
+            _("eval2", ExecutionContextActions.BEFORE_EVALUATION, 5, match=False),
+            _("eval3", ExecutionContextActions.AFTER_EVALUATION, 12, match=False),
+            _("eval4", ExecutionContextActions.EVALUATION, 99, match=False),
+            _("eval5", ExecutionContextActions.BEFORE_PARSING, 5, match=False),
+            _("eval6", ExecutionContextActions.PARSING, 25, match=False),
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        # init test variables
+        start = [_rvc("foo")]
+        service.execute(context, start, ALL_STEPS)
+        # to check what happened, look at the execution context children
+        executed_steps = [ec.action_context["step"] for ec in context.get_children(level=1)]
+        assert executed_steps == ALL_STEPS
+
+    def test_higher_priority_evaluators_are_executed_first(self, context, service):
+        # properly init the service
+        _ = OneReturnValueEvaluatorForTesting
+        evaluators = [
+            _("eval1", ExecutionContextActions.EVALUATION, 20, match=False),
+            _("eval2", ExecutionContextActions.EVALUATION, 5, match=False),
+            _("eval3", ExecutionContextActions.EVALUATION, 20, match=False),
+            _("eval4", ExecutionContextActions.EVALUATION, 99, match=False),
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [_rvc("foo")]
+        service.execute(context, start, [ExecutionContextActions.EVALUATION])
+
+        # to check what happened, look at the execution context children
+        evaluators_executed = [ec.action_context["evaluator"] for ec in context.get_children() if
+                               "evaluator" in ec.action_context]
+        assert evaluators_executed == ["eval4", "eval1", "eval3", "eval2"]
+
+    def test_evaluation_loop_stops_when_no_modification(self, context, service):
+        rv_foo, rv_bar = _rvc("foo"), _rvc("bar")  # rv => ReturnValue
+        # properly init the service
+        _ = OneReturnValueEvaluatorForTesting
+        evaluators = [
+            _("eval1",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda r: context.sheerka.isinstance(r.value, "foo"),
+              eval_result=[rv_bar])
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [rv_foo]
+        service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        children = [ec for ec in context.get_children() if ec.action == ExecutionContextActions.EVALUATING_ITERATION]
+        assert len(children) == 2
+
+    def test_eval_is_not_called_if_match_fails_for_one_return(self, context, service):
+        # properly init the service
+        _ = OneReturnValueEvaluatorForTesting
+        evaluators = [
+            _("eval1",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda r: context.sheerka.isinstance(r.value, "foo"),
+              eval_result=[_rvc("bar")])
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [_rvc("baz")]
+        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        assert res == start
+
+        # check what happen in details
+        exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children()))
+        evaluation_trace = exec_context.values["evaluation"]
+        assert evaluation_trace == [{"item": start[0], "match": False}]
+
+    def test_eval_is_called_if_match_succeed_for_one_return(self, context, service):
+        # properly init the service
+        _ = OneReturnValueEvaluatorForTesting
+        evaluators = [
+            _("eval1",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda r: context.sheerka.isinstance(r.value, "foo"),
+              eval_result=[_rvc("bar")])
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [_rvc("foo")]
+        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        assert res == [_rvc("bar")]
+        assert res[0].parents == start
+
+        # check what happen in details
+        exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children()))
+        evaluation_trace = exec_context.values["evaluation"]
+        assert evaluation_trace == [{"item": start[0], "match": True, "new": res, "eaten": start}]
+
+    def test_all_item_are_processed_during_one_return(self, context, service):
+        rv_foo, rv_bar, rv_baz, rv_qux = _rvc("foo"), _rvc("bar"), _rvc("baz"), _rvc("qux")  # rv => ReturnValue
+
+        # properly init the service
+        _ = OneReturnValueEvaluatorForTesting
+        evaluators = [
+            _("eval1",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda r: context.sheerka.isinstance(r.value, "foo"),
+              eval_result=[rv_qux])
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [rv_bar, rv_foo, rv_baz]
+        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        assert res == [rv_bar, rv_qux, rv_baz]  # We must keep the order ! rv_qux replaces rv_foo
+        assert res[0].parents is None
+        assert res[1].parents == [rv_foo]
+        assert res[2].parents is None
+
+        # check what happen in details
+        exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children()))
+        evaluation_trace = exec_context.values["evaluation"]
+        assert evaluation_trace == [{"item": rv_bar, "match": False},
+                                    {"item": rv_foo, "match": True, "new": [rv_qux], "eaten": [rv_foo]},
+                                    {"item": rv_baz, "match": False}]
+
+    def test_evaluators_with_the_same_priority_do_not_compete_with_each_other_one_return(self, context, service):
+        rv_foo, rv_bar, rv_baz, rv_qux = _rvc("foo"), _rvc("bar"), _rvc("baz"), _rvc("qux")  # rv => ReturnValue
+
+        # properly init the service
+        # both evaluator want to eat 'foo'
+        _ = OneReturnValueEvaluatorForTesting
+        evaluators = [
+            _("eval1",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda r: context.sheerka.isinstance(r.value, "foo"),
+              eval_result=[rv_bar]),
+            _("eval2",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda r: context.sheerka.isinstance(r.value, "foo"),
+              eval_result=[rv_baz])
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [rv_qux, rv_foo, rv_qux]
+        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        assert res == [rv_qux, rv_bar, rv_baz, rv_qux]  # they both eat it !
+        assert res[1].parents == [rv_foo]
+        assert res[2].parents == [rv_foo]
+
+    def test_evaluators_with_higher_priority_take_precedence_one_return(self, context, service):
+        rv_foo, rv_bar, rv_baz = _rvc("foo"), _rvc("bar"), _rvc("baz")  # rv => ReturnValue
+
+        # properly init the service
+        # both evaluator want to eat 'foo'
+        _ = OneReturnValueEvaluatorForTesting
+        evaluators = [
+            _("eval1",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda r: context.sheerka.isinstance(r.value, "foo"),
+              eval_result=[rv_bar]),
+            _("eval2",
+              ExecutionContextActions.EVALUATION,
+              30,
+              match=lambda r: context.sheerka.isinstance(r.value, "foo"),
+              eval_result=[rv_baz])
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [rv_foo]
+        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        assert res == [rv_baz]
+        assert res[0].parents == start
+
+    def test_evaluator_matches_is_called_before_eval_for_all_return(self, context, service):
+        # properly init the service
+        _ = AllReturnValuesEvaluatorForTesting
+        evaluators = [
+            _("eval1",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda r: context.sheerka.isinstance(r[0].value, "foo"),
+              eval_result=[_rvc("bar")])
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [_rvc("baz")]
+        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        assert res == start
+
+        start = [_rvc("foo")]
+        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        assert res == [_rvc("bar")]
+        assert res[0].parents == start
+
+    def test_eval_is_not_call_if_match_fails_for_all_return(self, context, service):
+        rv_foo, rv_bar, rv_baz = _rvc("foo"), _rvc("bar"), _rvc("baz")  # rv => ReturnValue
+
+        # properly init the service
+        _ = AllReturnValuesEvaluatorForTesting
+        evaluators = [
+            _("eval1",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda lst: context.sheerka.isinstance(lst[0].value, "foo"),
+              eval_result=[rv_bar])
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [rv_baz, rv_foo]  # foo is not the first in the list
+        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        assert res == start
+
+        # check what happen in details
+        exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children()))
+        evaluation_trace = exec_context.values["evaluation"]
+        assert evaluation_trace == {"match": False}
+
+    def test_eval_is_called_if_match_succeed_for_all_return(self, context, service):
+        rv_foo, rv_bar, rv_baz = _rvc("foo"), _rvc("bar"), _rvc("baz")  # rv => ReturnValue
+        # properly init the service
+        _ = AllReturnValuesEvaluatorForTesting
+        evaluators = [
+            _("eval1",
+              ExecutionContextActions.EVALUATION,
+              20,
+              match=lambda lst: context.sheerka.isinstance(lst[0].value, "foo"),
+              eval_result=[rv_bar])
+        ]
+        service.execution_plan = service.compute_execution_plan(evaluators)
+
+        start = [rv_foo, rv_baz]
+        res = service.execute(context, start, [ExecutionContextActions.EVALUATION])
+        assert res == [rv_bar]
+        assert res[0].parents == start
+
+        children = list(context.get_children())
+        # check what happen in details
+        exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children()))
+        evaluation_trace = exec_context.values["evaluation"]
+        assert evaluation_trace == {"match": True, "new": res, "eaten": start}

tests/sheerkapickle/test_SheerkaPickler.py

@@ -0,0 +1,183 @@
+import logging
+
+import pytest
+
+from base import BaseTest
+from common.global_symbols import NoFirstToken, NotFound, NotInit, Removed
+from helpers import get_concept, get_concepts
+from ontologies.SheerkaOntologyManager import SheerkaOntologyManager
+from parsers.tokenizer import Keywords
+from sheerkapickle import tags
+from sheerkapickle.sheerkaplicker import SheerkaPickler
+from sheerkapickle.sheerkaunpickler import SheerkaUnpickler
+
+
+class Obj:
+    def __init__(self, a, b, c):
+        self.a = a
+        self.b = b
+        self.c = c
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, Obj):
+            return False
+
+        return self.a == other.a and self.b == other.b and self.c == other.c
+
+    def __hash__(self):
+        return hash((self.a, self.b, self.c))
+
+
+class TestSheerkaPickler(BaseTest):
+
+    @pytest.mark.parametrize("obj, expected", [
+        (1, 1),
+        (3.14, 3.14),
+        ("a string", "a string"),
+        (True, True),
+        (None, None),
+        ([1, 3.14, "a string"], [1, 3.14, "a string"]),
+        ((1, 3.14, "a string"), {tags.TUPLE: [1, 3.14, "a string"]}),
+        ({1}, {tags.SET: [1]}),
+        ({"a": "a", "b": 3.14, "c": True}, {"a": "a", "b": 3.14, "c": True}),
+        ({1: "a", 2: 3.14, 3: True}, {1: "a", 2: 3.14, 3: True}),
+        ([1, [3.14, "a string"]], [1, [3.14, "a string"]]),
+        ([1, (3.14, "a string")], [1, {tags.TUPLE: [3.14, "a string"]}]),
+        ([], []),
+        (Keywords.DEF, {tags.ENUM: 'parsers.tokenizer.Keywords.DEF'}),
+    ])
+    def test_i_can_flatten_and_restore_primitives(self, sheerka, obj, expected):
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == expected
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    @pytest.mark.parametrize("obj, expected", [
+        (NotInit, {tags.CUSTOM: NotInit.value}),
+        (NotFound, {tags.CUSTOM: NotFound.value}),
+        (Removed, {tags.CUSTOM: Removed.value}),
+        (NoFirstToken, {tags.CUSTOM: NoFirstToken.value}),
+    ])
+    def test_i_can_flatten_and_restore_custom_types(self, sheerka, obj, expected):
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == expected
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_can_flatten_and_restore_instances(self, sheerka):
+        obj1 = Obj(1, "b", True)
+        obj2 = Obj(3.14, ("a", "b"), obj1)
+
+        flatten = SheerkaPickler(sheerka).flatten(obj2)
+        assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                           'a': 3.14,
+                           'b': {'_sheerka/tuple': ['a', 'b']},
+                           'c': {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                                 'a': 1,
+                                 'b': 'b',
+                                 'c': True}}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj2
+
+    def test_i_can_manage_circular_reference(self, sheerka):
+        obj1 = Obj(1, "b", True)
+        obj1.c = obj1
+
+        flatten = SheerkaPickler(sheerka).flatten(obj1)
+        assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                           'a': 1,
+                           'b': 'b',
+                           'c': {'_sheerka/id': 0}}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded.a == obj1.a
+        assert decoded.b == obj1.b
+        assert decoded.c == decoded
+
+    def test_i_can_flatten_obj_with_new_props(self, sheerka):
+        # property 'z' is not part of the `Obj` definition
+        obj = Obj(1, "b", True)
+        obj.z = "new prop"
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                           'a': 1,
+                           'b': 'b',
+                           'c': True,
+                           'z': "new prop"}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_cannot_correctly_flatten_compiled_and_generator(self, sheerka):
+        obj = Obj((i for i in range(3)), compile("a + b", "<src>", mode="eval"), None)
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+
+        assert isinstance(flatten["a"], str)
+        assert flatten["a"].startswith("<generator object")
+        assert isinstance(flatten["b"], str)
+        assert flatten["b"].startswith("<code object")
+        assert flatten["c"] is None
+
+    @pytest.mark.parametrize("obj, expected", [
+        ({None: "a"}, {'null': "a"}),
+        ({Keywords.DEF: "a"}, {'parsers.tokenizer.Keywords.DEF': 'a'}),
+        ({(1, 2): "a"}, {(1, 2): "a"}),
+    ])
+    def test_i_can_manage_specific_keys_in_dictionaries(self, sheerka, obj, expected):
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == expected
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    @pytest.mark.skip("Concepts are not fully working")
+    def test_i_can_use_concept_as_dictionary_key(self, sheerka, context):
+        concept = get_concepts(context, "foo", use_sheerka=True)[0]
+
+        obj = {concept: "a"}
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        assert flatten == {'c:foo|1001:': 'a'}
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_can_manage_references(self, sheerka):
+        foo = Obj("foo", "bar", "baz")
+        obj = [Keywords.DEF, foo, Keywords.WHERE, Keywords.DEF, foo]
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+
+        assert flatten == [{'_sheerka/enum': 'parsers.tokenizer.Keywords.DEF'},
+                           {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj',
+                            'a': 'foo',
+                            'b': 'bar',
+                            'c': 'baz'},
+                           {'_sheerka/enum': 'parsers.tokenizer.Keywords.WHERE'},
+                           {'_sheerka/id': 0},
+                           {'_sheerka/id': 1}]
+
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == obj
+
+    def test_i_do_not_encode_logger(self, sheerka):
+        logger = logging.getLogger("log_name")
+        logger2 = logging.getLogger("log_name2")
+        obj = Obj("foo", logger, {"a": logger, "b": logger2})
+
+        flatten = SheerkaPickler(sheerka).flatten(obj)
+        decoded = SheerkaUnpickler(sheerka).restore(flatten)
+        assert decoded == Obj("foo", None, {"a": None, "b": None})
+
+    def test_ontology_are_not_serialized(self, sheerka, context):
+        om = SheerkaOntologyManager(sheerka, "mem://").freeze()
+        ontology = om.push_ontology("new ontology")
+
+        flatten = SheerkaPickler(sheerka).flatten(ontology)
+        assert flatten == "__ONTOLOGY:new ontology__"

tests/sheerkapickle/test_sheerka_handlers.py

@@ -0,0 +1,325 @@
+import pytest
+
+import sheerkapickle
+from base import BaseTest
+from core.concept import Concept
+
+
+def set_full_serialization(concept):
+    concept.get_metadata().full_serialization = True
+    return concept
+
+
+@pytest.mark.skip("Handler are not implemented")
+class TestSheerkaPickleHandler(BaseTest):
+
+    def test_i_can_encode_decode_unknown_concept_metadata(self, sheerka):
+        concept = set_full_serialization(Concept(name="foo", key="my_key"))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.key": "my_key"}'
+        assert decoded == concept
+
+        concept = set_full_serialization(Concept("foo", is_builtin=True, is_unique=True))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.is_builtin": true, "meta.is_unique": true}'
+
+        concept = set_full_serialization(Concept("foo", body="my_body"))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "my_body"}'
+
+        concept = set_full_serialization(Concept("foo", pre="my_pre"))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.pre": "my_pre"}'
+
+        concept = set_full_serialization(Concept("foo", post="my_post"))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.post": "my_post"}'
+
+        concept = set_full_serialization(Concept("foo", where="my_where"))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.where": "my_where"}'
+
+        concept = set_full_serialization(Concept("foo").def_var("a", "value_a").def_var("b", "value_b"))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.variables": [["a", "value_a"], ["b", "value_b"]], "values": [["a", {"_sheerka/custom": "**NotInit**"}], ["b", {"_sheerka/id": 1}]]}'
+
+        concept = Concept("foo").init_key()
+        sheerka.define_new_concept(self.get_context(sheerka), concept)
+        concept.get_metadata().full_serialization = True
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.key": "foo", "meta.id": "1001"}'
+
+    def test_i_can_encode_decode_unknown_concept_values(self):
+        sheerka = self.get_sheerka()
+
+        concept = set_full_serialization(Concept("foo"))
+        concept.set_value(ConceptParts.PRE, 10)  # an int
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#pre#", 10]]}'
+
+        concept = set_full_serialization(Concept("foo"))
+        concept.set_value(ConceptParts.POST, 'a string')  # an string
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#post#", "a string"]]}'
+
+        concept = set_full_serialization(Concept("foo"))
+        concept.set_value(ConceptParts.WHERE, ['a string', 3.14])  # a list
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#where#", ["a string", 3.14]]]}'
+
+        concept = set_full_serialization(Concept("foo"))
+        concept.set_value(ConceptParts.WHERE, ('a string', 3.14))  # a tuple
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#where#", {"_sheerka/tuple": ["a string", 3.14]}]]}'
+
+        concept = set_full_serialization(Concept("foo"))
+        concept.set_value(ConceptParts.BODY, set_full_serialization(Concept("foo", body="foo_body")))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["#body#", {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "meta.body": "foo_body"}]]}'
+
+    def test_i_can_encode_decode_unknown_concept_variables(self):
+        sheerka = self.get_sheerka()
+
+        concept = set_full_serialization(Concept("foo"))
+        concept.set_value("a", "value_a")  # string
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["a", "value_a"]]}'
+
+        concept = set_full_serialization(Concept("foo"))
+        concept.set_value("a", 10)  # int
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["a", 10]]}'
+
+        concept = set_full_serialization(Concept("foo"))
+        concept.set_value("a", set_full_serialization(Concept("bar")))  # another concept
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["a", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}'
+
+        concept = set_full_serialization(Concept("foo"))
+        concept.set_value("a", "a").set_value("b", "b")  # at least two variables
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "foo", "values": [["a", "a"], ["b", "b"]]}'
+
+    def test_i_can_encode_decode_known_concepts(self):
+        sheerka = self.get_sheerka()
+
+        ref_concept = Concept("my_name", True, True, "my_key", "my_body", "my_where", "my_pre", "my_post", "my_def")
+        ref_concept.def_var("a", "value_a").def_var("b", "value_b")
+
+        sheerka.define_new_concept(self.get_context(sheerka), ref_concept)
+
+        to_string = sheerkapickle.encode(sheerka, ref_concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ref_concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": "1001"}'
+
+        # same test, modify a value and check if this modification is correctly saved
+        concept = Concept().update_from(sheerka.get_by_id(ref_concept.id))
+        concept.set_value(ConceptParts.BODY, set_full_serialization(Concept("bar")))
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": "1001", "values": [["#body#", {"_sheerka/obj": "core.concept.Concept", "meta.name": "bar"}]]}'
+
+    def test_i_can_manage_reference_of_the_same_object(self):
+        sheerka = self.get_sheerka()
+
+        concept_ref = set_full_serialization(Concept("foo"))
+
+        concept = set_full_serialization(Concept("bar"))
+        concept.set_value(ConceptParts.PRE, concept_ref)
+        concept.set_value(ConceptParts.BODY, concept_ref)
+
+        to_string = sheerkapickle.encode(sheerka, concept)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == concept
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "meta.name": "bar", "values": [["#pre#", {"_sheerka/obj": "core.concept.Concept", "meta.name": "foo"}], ["#body#", {"_sheerka/id": 1}]]}'
+
+    def test_i_can_encode_decode_user_input(self):
+        sheerka = self.get_sheerka()
+
+        user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body="my_text", user_name="my_user_name")
+
+        to_string = sheerkapickle.encode(sheerka, user_input)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == user_input
+        assert to_string == f'{{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "{self.user_input_id}"], "user_name": "my_user_name", "text": "my_text"}}'
+
+    def test_i_can_encode_decode_user_input_when_tokens(self):
+        sheerka = self.get_sheerka()
+
+        text = "I have 'a complicated' 10 text"
+        tokens = list(Tokenizer(text))
+        user_input = sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens, user_name="my_user_name")
+
+        to_string = sheerkapickle.encode(sheerka, user_input)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="my_user_name")
+        assert to_string == f'{{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "{self.user_input_id}"], "user_name": "my_user_name", "text": "{text}"}}'
+
+    def test_i_can_encode_decode_return_value(self):
+        sheerka = self.get_sheerka()
+
+        ret_val = sheerka.ret("who", True, 10)
+
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ret_val
+        assert to_string == f'{{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "{self.return_value_id}"], "who": "who", "status": true, "value": 10}}'
+
+    def test_i_can_encode_decode_return_value_with_parent(self):
+        sheerka = self.get_sheerka()
+
+        ret_val = sheerka.ret("who", True, 10)
+        ret_val_parent = sheerka.ret("parent_who", True, "10")
+        ret_val.parents = [ret_val_parent, ret_val_parent]
+
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ret_val
+        assert decoded.parents == ret_val.parents
+        id_str = f', "concept/id": ["__RETURN_VALUE", "{self.return_value_id}"]'
+        parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]'
+        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}'
+
+    def test_i_can_encode_decode_return_values_with_complex_body(self):
+        sheerka = self.get_sheerka()
+
+        ret_val = sheerka.ret("who", True, set_full_serialization(Concept("foo", body="bar")))
+
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == ret_val
+
+    def test_i_can_encode_decode_return_values_from_concepts_parsers_or_evaluators(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+        ret_val = sheerka.ret(foo, True, 10)
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.ret("c:1001:", True, 10)
+
+        ret_val = sheerka.ret(DefConceptParser(), True, 10)
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.ret("parsers.DefConcept", True, 10)
+
+        ret_val = sheerka.ret(ConceptEvaluator(), True, 10)
+        to_string = sheerkapickle.encode(sheerka, ret_val)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == sheerka.ret("evaluators.Concept", True, 10)
+
+    def test_i_can_encode_decode_execution_context(self):
+        sheerka = self.get_sheerka()
+        c = Concept("foo").def_var("a")
+        context = ExecutionContext("who", Event("xxx"), sheerka, BuiltinConcepts.EVALUATE_CONCEPT, c, "my desc")
+        input_list = [ReturnValueConcept("who", True, 10), ReturnValueConcept("who2", False, 20)]
+        context.inputs = {"a": input_list, "b": set_full_serialization(Concept("foo"))}
+        context.values = {"c": input_list, "d": set_full_serialization(Concept("bar"))}
+        context.obj = set_full_serialization(Concept("baz"))
+        context.push("who3", BuiltinConcepts.EVALUATING_CONCEPT, c, desc="sub_child1")
+        context.push("who4", BuiltinConcepts.EVALUATING_ATTRIBUTE, "a", desc="sub_child2")
+
+        to_string = sheerkapickle.encode(sheerka, context)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+        assert decoded == context
+
+    def test_complicated_execution_context(self):
+        sheerka = self.get_sheerka(skip_builtins_in_db=False)
+
+        text = "def concept one as 1"
+        execution_context = ExecutionContext("s", Event(), sheerka, BuiltinConcepts.NOP, None, f"Evaluating '{text}'")
+        user_input = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="n"))
+        reduce_requested = sheerka.ret("s", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+
+        steps = [
+            BuiltinConcepts.BEFORE_PARSING,
+            BuiltinConcepts.PARSING,
+            BuiltinConcepts.AFTER_PARSING,
+            BuiltinConcepts.BEFORE_EVALUATION,
+            BuiltinConcepts.EVALUATION,
+            BuiltinConcepts.AFTER_EVALUATION
+        ]
+
+        ret = sheerka.execute(execution_context, [user_input, reduce_requested], steps)
+        execution_context.add_values(return_values=ret)
+
+        to_string = sheerkapickle.encode(sheerka, execution_context)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+
+        return_value = decoded.values["return_values"][0].value
+        assert sheerka.isinstance(return_value, BuiltinConcepts.NEW_CONCEPT)
+
+    def test_encode_simple_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo = set_full_serialization(Concept("foo"))
+        to_string = sheerkapickle.encode(sheerka, foo)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+
+        assert decoded == foo
+
+    def test_i_can_encode_decode_rule(self):
+        sheerka = self.get_sheerka()
+
+        rule = Rule("print", "my rule", "True", "Hello world")
+        rule.metadata.id = "1"
+
+        to_string = sheerkapickle.encode(sheerka, rule)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+
+        assert to_string == '{"_sheerka/obj": "core.rule.Rule", "rule/id": "1", "name": "my rule", "predicate": "True", "action_type": "print", "action": "Hello world"}'
+        assert decoded == rule
+
+    def test_i_can_encode_decode_dynamic_concept(self):
+        sheerka, context, foo = self.init_concepts("foo", global_truth=True, create_new=True)
+        sheerka.set_attr(context, foo, "attr", "attr_value")
+        sheerka.set_property(context, foo, "prop", "prop_value", all_concepts=True)
+
+        foo_instance = sheerka.new(foo)
+        dynamic_foo = sheerka.new_dynamic(foo_instance,
+                                          "SUFFIX",
+                                          "new_name",
+                                          props={"new_prop": "value"},
+                                          attrs={"new_attr": "value"})
+
+        to_string = sheerkapickle.encode(sheerka, dynamic_foo)
+        decoded = sheerkapickle.decode(sheerka, to_string)
+
+        assert decoded == dynamic_foo
+        assert to_string == '{"_sheerka/obj": "core.concept.Concept", "concept/id": "1001-SUFFIX", "meta.name": "new_name", "meta.key": "new_name", "meta.props": {"prop": "prop_value", "new_prop": "value"}, "meta.id": "1001-SUFFIX", "values": [["new_attr", "value"]]}'

tests/test_client.py

@@ -1,5 +1,3 @@
-import json
-
 from fastapi import HTTPException
 from starlette import status
 
@@ -7,92 +5,73 @@ from client import SheerkaClient, parse_arguments
 from mockserver import MockServer
 
 
-def test_i_can_start_with_a_default_hostname():
-    parsed = parse_arguments([])
+# @pytest.mark.skip("too long")
+class TestSheerkaClient:
+    def test_i_can_start_with_a_default_hostname(self):
+        parsed = parse_arguments([])
 
-    assert parsed.hostname == "http://localhost"
-    assert parsed.port == 56356
+        assert parsed.hostname == "http://localhost"
+        assert parsed.port == 56356
 
+    def test_i_can_override_hostname_and_port(self):
+        parsed = parse_arguments(["new_host", "--port", "1515"])
 
-def test_i_can_override_hostname_and_port():
-    parsed = parse_arguments(["new_host", "--port", "1515"])
+        assert parsed.hostname == "new_host"
+        assert parsed.port == 1515
 
-    assert parsed.hostname == "new_host"
-    assert parsed.port == 1515
+        parsed = parse_arguments(["new_host", "-p", "1515"])
 
-    parsed = parse_arguments(["new_host", "-p", "1515"])
+        assert parsed.hostname == "new_host"
+        assert parsed.port == 1515
 
-    assert parsed.hostname == "new_host"
-    assert parsed.port == 1515
+    def test_i_can_provide_user_and_password(self):
+        parsed = parse_arguments(["--username", "my_user", "--password", "my_password"])
+        assert parsed.username == "my_user"
+        assert parsed.password == "my_password"
 
+        parsed = parse_arguments(["-u", "my_user", "-P", "my_password"])
+        assert parsed.username == "my_user"
+        assert parsed.password == "my_password"
 
-def test_i_can_provide_user_and_password():
-    parsed = parse_arguments(["--username", "my_user", "--password", "my_password"])
-    assert parsed.username == "my_user"
-    assert parsed.password == "my_password"
-
-    parsed = parse_arguments(["-u", "my_user", "-P", "my_password"])
-    assert parsed.username == "my_user"
-    assert parsed.password == "my_password"
-
-
-def test_i_can_manage_when_no_server():
-    client = SheerkaClient("http://localhost", 80)
-    res = client.check_url()
-
-    assert res.status is False
-    assert res.message == "Connection refused."
-
-
-def test_i_can_manage_when_resource_is_not_found():
-    with MockServer([]):
-        client = SheerkaClient("http://localhost", 5000)
+    def test_i_can_manage_when_no_server(self):
+        client = SheerkaClient("http://localhost", 80)
         res = client.check_url()
 
-        assert not res.status
-        assert res.message == '{"detail":"Not Found"}'
+        assert res.status is False
+        assert res.message == "Connection refused."
 
+    def test_i_can_manage_when_resource_is_not_found(self):
+        with MockServer([]):
+            client = SheerkaClient("http://localhost", 5000)
+            res = client.check_url()
 
-def test_i_can_connect_to_a_server():
-    with MockServer([{
-        "path": "/",
-        "response": "Hello world"
-    }]):
-        client = SheerkaClient("http://localhost", 5000)
-        res = client.check_url()
-        assert res.status
-        assert res.message == '"Hello world"'
+            assert not res.status
+            assert res.message == '{"detail":"Not Found"}'
 
+    def test_i_can_connect_to_a_server(self):
+        with MockServer([{
+            "path": "/",
+            "response": "Hello world"
+        }]):
+            client = SheerkaClient("http://localhost", 5000)
+            res = client.check_url()
+            assert res.status
+            assert res.message == '"Hello world"'
 
-def test_i_can_authenticate_with_valid_credentials():
-    with MockServer([{
-        "path": "/",
-        "response": "Hello world"
-    }, {
-        "method": "post",
-        "path": "/token",
-        "response": {"access_token": "xxxx", "token_type": "bearer"}
-    }]):
-        client = SheerkaClient("http://localhost", 5000)
-        res = client.connect("valid_username", "valid_password")
-        assert res.status
-        assert res.message == "Connected as valid_username"
-
-
-def test_i_can_manage_when_authentication_fails():
-    with MockServer([{
-        "path": "/",
-        "response": "Hello world"
-    }, {
-        "method": "post",
-        "path": "/token",
-        "exception": HTTPException(
-            status_code=status.HTTP_401_UNAUTHORIZED,
-            detail="Incorrect username or password",
-            headers={"WWW-Authenticate": "Bearer"},
-        )
-    }]):
-        client = SheerkaClient("http://localhost", 5000)
-        res = client.connect("username", "wrong_password")
-        assert not res.status
-        assert res.message == 'Incorrect username or password'
+    def test_i_can_manage_when_authentication_fails(self):
+        with MockServer([{
+            "path": "/",
+            "response": "Hello world"
+        }, {
+            "method": "post",
+            "path": "/token",
+            "exception": HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Incorrect username or password",
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+        }]):
+            client = SheerkaClient("http://localhost", 5000)
+            res = client.connect("username", "wrong_password")
+            assert not res.status
+            assert res.message == 'Incorrect username or password'

tests/test_helpers.py

@@ -0,0 +1,223 @@
+import pytest
+
+from common.global_symbols import NotInit
+from core.concept import Concept, ConceptMetadata, DefinitionType
+from helpers import GetNextId, get_concept, get_concepts, get_metadata, get_metadatas
+
+
+def test_i_can_get_default_value_when_get_metadata():
+    metadata = get_metadata()
+    assert metadata.id is None
+    assert metadata.name is None
+    assert metadata.name is None
+    assert metadata.body is None
+    assert metadata.id is None
+    assert metadata.key is None
+    assert metadata.where is None
+    assert metadata.pre is None
+    assert metadata.post is None
+    assert metadata.ret is None
+    assert metadata.definition is None
+    assert metadata.definition_type == DefinitionType.DEFAULT
+    assert metadata.desc is None
+    assert metadata.props == {}
+    assert metadata.variables == tuple()
+    assert metadata.parameters == []
+    assert metadata.bound_body is None
+    assert metadata.is_builtin is False
+    assert metadata.is_unique is False
+    assert metadata.autouse is False
+
+
+def test_i_can_use_shortcut_to_declare_variables():
+    metadata = get_metadata(variables=(("var1", NotInit), ("var2", "value")))
+    assert metadata.variables == (("var1", NotInit), ("var2", "value"))  # default behaviour
+
+    metadata = get_metadata(variables=[("var1", NotInit), ("var2", "value")])
+    assert metadata.variables == (("var1", NotInit), ("var2", "value"))  # lists are transformed into tuples
+
+    metadata = get_metadata(variables=["var1", "var2"])
+    assert metadata.variables == (("var1", NotInit), ("var2", NotInit))  # expanded
+
+
+def test_i_can_clone():
+    metadata = ConceptMetadata(
+        "id",
+        "name",
+        "key",
+        True,
+        True,
+        "body",
+        "where",
+        "pre",
+        "post",
+        "ret",
+        "definition",
+        DefinitionType.BNF,
+        "desc",
+        True,
+        "bound_body",
+        {"prop": "value"},
+        (("variable", "value"),),
+        ("p1",),
+        "digest",
+        ("all_attr",),
+    )
+
+    clone = metadata.clone()
+    for attr, value in vars(metadata).items():
+        clone_value = getattr(clone, attr)
+        assert clone_value == value
+
+
+def test_i_can_override_values_when_i_clone_metadata():
+    metadata = get_metadata()
+    assert metadata.clone(name="new_name").name == "new_name"
+    assert metadata.clone(body="new_body").body == "new_body"
+    assert metadata.clone(key="new_key").key == "new_key"
+    assert metadata.clone(where="new_where").where == "new_where"
+    assert metadata.clone(pre="new_pre").pre == "new_pre"
+    assert metadata.clone(post="new_post").post == "new_post"
+    assert metadata.clone(ret="new_ret").ret == "new_ret"
+    assert metadata.clone(definition="new_definition").definition == "new_definition"
+    assert metadata.clone(definition_type="new_definition_type").definition_type == "new_definition_type"
+    assert metadata.clone(desc="new_desc").desc == "new_desc"
+    assert metadata.clone(props="new_props").props == "new_props"
+    assert metadata.clone(variables="new_variables").variables == "new_variables"
+    assert metadata.clone(parameters="new_parameters").parameters == "new_parameters"
+    assert metadata.clone(bound_body="new_bound_body").bound_body == "new_bound_body"
+    assert metadata.clone(is_builtin="new_is_builtin").is_builtin == "new_is_builtin"
+    assert metadata.clone(is_unique="new_is_unique").is_unique == "new_is_unique"
+    assert metadata.clone(autouse="new_autouse").autouse == "new_autouse"
+    assert metadata.clone(digest="new_digest").digest == "new_digest"
+    assert metadata.clone(all_attrs="new_all_attrs").all_attrs == "new_all_attrs"
+
+
+def test_i_cannot_change_the_id_when_cloning():
+    with pytest.raises(TypeError):
+        metadata = get_metadata()
+        metadata.clone(id="new_id")
+
+
+def test_i_can_auto_init():
+    next_id = GetNextId()
+    metadata = get_metadata("a plus b", body="a + b", variables=["a", "b"]).auto_init(next_id)
+
+    assert metadata.name == "a plus b"
+    assert metadata.id == "1001"
+    assert metadata.key == "__var__0 plus __var__1"
+    assert metadata.all_attrs == ('#where#', '#pre#', '#post#', '#body#', '#ret#', 'a', 'b')
+    assert metadata.is_unique is False
+    assert metadata.is_builtin is False
+    assert metadata.definition_type is DefinitionType.DEFAULT
+    assert metadata.digest == '426d88b1b928a421366c12fb283267b89610cbfb9efb470813ea8b5ba37a2013'
+
+
+def test_sequences_are_incremented_when_multiples_call():
+    next_id = GetNextId()
+    assert get_metadata("foo").auto_init(next_id).id == "1001"
+    assert get_metadata("bar").auto_init(next_id).id == "1002"
+
+
+def test_i_can_get_multiple_metadatas():
+    res = get_metadatas("foo", get_metadata("bar", body="body"))
+
+    assert len(res) == 2
+
+    metadata = res[0]
+    assert isinstance(metadata, ConceptMetadata)
+    assert metadata.name == "foo"
+    assert metadata.body is None
+    assert metadata.key is None
+    assert metadata.id is None
+
+    metadata = res[1]
+    assert isinstance(metadata, ConceptMetadata)
+    assert metadata.name == "bar"
+    assert metadata.body == "body"
+    assert metadata.key is None
+    assert metadata.id is None
+
+
+def test_i_can_get_multiple_already_initialized_metadatas():
+    res = get_metadatas("foo", get_metadata("bar", body="body"), next_id=GetNextId())
+
+    assert len(res) == 2
+
+    metadata = res[0]
+    assert isinstance(metadata, ConceptMetadata)
+    assert metadata.name == "foo"
+    assert metadata.body is None
+    assert metadata.key == "foo"
+    assert metadata.id == "1001"
+
+    metadata = res[1]
+    assert isinstance(metadata, ConceptMetadata)
+    assert metadata.name == "bar"
+    assert metadata.body == "body"
+    assert metadata.key == "bar"
+    assert metadata.id == "1002"
+
+
+def test_i_can_get_a_concept():
+    foo = get_concept("foo", variables=("var1",))
+
+    assert isinstance(foo, Concept)
+    assert foo.name == "foo"
+    assert foo.key is None
+    assert foo.id is None
+    assert foo.all_attrs() == ('#where#', '#pre#', '#post#', '#body#', '#ret#', 'var1')
+
+
+def test_i_can_request_basic_initialization_when_getting_a_concept():
+    next_id = GetNextId()
+    foo = get_concept("foo", variables=("var1",), sequence=next_id)
+
+    assert foo.name == "foo"
+    assert foo.key == "foo"
+    assert foo.id == "1001"
+    assert foo.all_attrs() == ('#where#', '#pre#', '#post#', '#body#', '#ret#', 'var1')
+
+
+def test_i_can_get_multiple_concepts(context):
+    next_id = GetNextId()
+
+    foo, bar, baz = get_concepts(context,
+                                 "foo",
+                                 "bar",
+                                 get_concept("baz", definition="baz var1", variables=("var1",)),
+                                 sequence=next_id)
+    assert foo.name == "foo"
+    assert foo.id == "1001"
+    assert foo.key == "foo"
+    assert bar.name == "bar"
+    assert bar.id == "1002"
+    assert bar.key == "bar"
+    assert baz.name == "baz"
+    assert baz.id == "1003"
+    assert baz.key == "baz __var__0"
+
+
+def test_i_can_get_multiple_concepts_using_sheerka(sheerka, context):
+    foo, bar, baz = get_concepts(context,
+                                 "foo",
+                                 "bar",
+                                 get_concept("baz", definition="baz var1", variables=("var1",)),
+                                 use_sheerka=True)
+    assert foo.name == "foo"
+    assert foo.id == "1001"
+    assert foo.key == "foo"
+    assert bar.name == "bar"
+    assert bar.id == "1002"
+    assert bar.key == "bar"
+    assert baz.name == "baz"
+    assert baz.id == "1003"
+    assert baz.key == "baz __var__0"
+    assert baz.get_value("var1") is NotInit
+
+    # the concepts are defined in Sheerka, so we can instantiate them
+    baz2 = sheerka.newn("baz", var1="value for var1")
+    assert baz2.name == "baz"
+    assert baz2.id == "1003"
+    assert baz2.key == "baz __var__0"
+    assert baz2.get_value("var1") == "value for var1"