From e41094f9089eaf48cdc6a5363738ad09452b0c8d Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Mon, 8 May 2023 17:50:28 +0200 Subject: [PATCH] Fixed #8 Fixed #12 Fixed #13 Fixed #14 --- Makefile | 23 + README.md | 22 +- pyproject.toml | 2 + requirements.txt | 10 + src/caching/BaseCache.py | 442 +++++ src/caching/Cache.py | 40 + src/caching/CacheManager.py | 342 ++++ src/caching/DictionaryCache.py | 88 + src/caching/FastCache.py | 71 + src/caching/IncCache.py | 24 + src/caching/ListCache.py | 66 + src/caching/ListIfNeededCache.py | 134 ++ src/caching/SetCache.py | 122 ++ src/caching/__init__.py | 0 src/client.py | 21 +- src/common/__init__.py | 0 src/common/global_symbols.py | 66 + src/common/utils.py | 293 ++++ src/core/BuiltinConcepts.py | 7 + src/core/ErrorContext.py | 35 + src/core/Event.py | 69 + src/core/ExecutionContext.py | 181 +- src/core/ReturnValue.py | 32 + src/core/Sheerka.py | 324 +++- src/core/__init__.py | 0 src/core/concept.py | 201 +++ src/core/global_symbols.py | 31 - src/core/services/BaseService.py | 49 + src/core/services/SheerkaConceptManager.py | 343 ++++ src/core/services/SheerkaEngine.py | 223 +++ src/core/services/__init__.py | 0 src/core/utils.py | 32 - src/evaluators/CreateParserInput.py | 30 + src/evaluators/__init__.py | 0 src/evaluators/base_evaluator.py | 73 + src/ontologies/Exceptions.py | 37 + src/ontologies/SheerkaOntologyManager.py | 534 ++++++ src/ontologies/__init__.py | 0 src/parsers/ParserInput.py | 19 + src/parsers/__init__.py | 0 src/parsers/tokenizer.py | 569 ++++++ src/sdp/__init__.py | 0 src/sdp/readme.md | 24 + src/sdp/sheerkaDataProvider.py | 81 +- src/sdp/sheerkaDataProviderIO.py | 6 +- src/sdp/sheerkaSerializer.py | 78 +- src/server/__init__.py | 0 src/server/authentication.py | 6 +- src/server/main.py | 41 +- src/sheerkapickle/__init__.py | 7 + src/sheerkapickle/handlers.py | 197 +++ src/sheerkapickle/sheerka_handlers.py | 240 +++ src/sheerkapickle/sheerkaplicker.py | 163 ++ src/sheerkapickle/sheerkaunpickler.py | 129 ++ src/sheerkapickle/tags.py | 6 + src/sheerkapickle/utils.py | 101 ++ tests/__init__.py | 0 tests/base.py | 35 + tests/caching/__init__.py | 18 + tests/caching/test_CacheManager.py | 288 ++++ tests/caching/test_DictionaryCache.py | 165 ++ tests/caching/test_FastCache.py | 111 ++ tests/caching/test_IncCache.py | 87 + tests/caching/test_ListCache.py | 281 +++ tests/caching/test_ListIfNeededCache.py | 648 +++++++ tests/caching/test_SetCache.py | 540 ++++++ tests/caching/test_cache.py | 512 ++++++ tests/common/__init__.py | 0 tests/common/test_utils.py | 122 ++ tests/conftest.py | 84 + tests/core/__init__.py | 0 tests/core/test_concept.py | 103 ++ tests/core/test_execution_context.py | 151 ++ tests/core/test_sheerka.py | 36 + tests/evaluators/__init__.py | 0 tests/evaluators/test_CreateParserInput.py | 23 + tests/helpers.py | 377 ++++ tests/ontologies/__init__.py | 0 .../ontologies/test_SheerkaOntoloyManager.py | 1522 +++++++++++++++++ tests/parsers/__init__.py | 0 tests/parsers/test_parser_input.py | 14 + tests/parsers/test_tokenizer.py | 211 +++ tests/sdp/__init__.py | 0 tests/sdp/test_sheerkaDataProvider.py | 16 +- tests/sdp/test_sheerkaSerializer.py | 2 +- tests/server/__init__.py | 0 tests/server/test_server.py | 19 + tests/services/__init__.py | 0 tests/services/test_ConceptManager.py | 186 ++ tests/services/test_SheerkaEngine.py | 379 ++++ tests/sheerkapickle/__init__.py | 0 tests/sheerkapickle/test_SheerkaPickler.py | 183 ++ tests/sheerkapickle/test_sheerka_handlers.py | 325 ++++ tests/test_client.py | 133 +- tests/test_helpers.py | 223 +++ 95 files changed, 12168 insertions(+), 260 deletions(-) create mode 100644 Makefile create mode 100644 pyproject.toml create mode 100644 src/caching/BaseCache.py create mode 100644 src/caching/Cache.py create mode 100644 src/caching/CacheManager.py create mode 100644 src/caching/DictionaryCache.py create mode 100644 src/caching/FastCache.py create mode 100644 src/caching/IncCache.py create mode 100644 src/caching/ListCache.py create mode 100644 src/caching/ListIfNeededCache.py create mode 100644 src/caching/SetCache.py create mode 100644 src/caching/__init__.py create mode 100644 src/common/__init__.py create mode 100644 src/common/global_symbols.py create mode 100644 src/common/utils.py create mode 100644 src/core/BuiltinConcepts.py create mode 100644 src/core/ErrorContext.py create mode 100644 src/core/Event.py create mode 100644 src/core/ReturnValue.py create mode 100644 src/core/__init__.py create mode 100644 src/core/concept.py delete mode 100644 src/core/global_symbols.py create mode 100644 src/core/services/BaseService.py create mode 100644 src/core/services/SheerkaConceptManager.py create mode 100644 src/core/services/SheerkaEngine.py create mode 100644 src/core/services/__init__.py delete mode 100644 src/core/utils.py create mode 100644 src/evaluators/CreateParserInput.py create mode 100644 src/evaluators/__init__.py create mode 100644 src/evaluators/base_evaluator.py create mode 100644 src/ontologies/Exceptions.py create mode 100644 src/ontologies/SheerkaOntologyManager.py create mode 100644 src/ontologies/__init__.py create mode 100644 src/parsers/ParserInput.py create mode 100644 src/parsers/__init__.py create mode 100644 src/parsers/tokenizer.py create mode 100644 src/sdp/__init__.py create mode 100644 src/sdp/readme.md create mode 100644 src/server/__init__.py create mode 100644 src/sheerkapickle/__init__.py create mode 100644 src/sheerkapickle/handlers.py create mode 100644 src/sheerkapickle/sheerka_handlers.py create mode 100644 src/sheerkapickle/sheerkaplicker.py create mode 100644 src/sheerkapickle/sheerkaunpickler.py create mode 100644 src/sheerkapickle/tags.py create mode 100644 src/sheerkapickle/utils.py create mode 100644 tests/__init__.py create mode 100644 tests/base.py create mode 100644 tests/caching/__init__.py create mode 100644 tests/caching/test_CacheManager.py create mode 100644 tests/caching/test_DictionaryCache.py create mode 100644 tests/caching/test_FastCache.py create mode 100644 tests/caching/test_IncCache.py create mode 100644 tests/caching/test_ListCache.py create mode 100644 tests/caching/test_ListIfNeededCache.py create mode 100644 tests/caching/test_SetCache.py create mode 100644 tests/caching/test_cache.py create mode 100644 tests/common/__init__.py create mode 100644 tests/common/test_utils.py create mode 100644 tests/conftest.py create mode 100644 tests/core/__init__.py create mode 100644 tests/core/test_concept.py create mode 100644 tests/core/test_execution_context.py create mode 100644 tests/core/test_sheerka.py create mode 100644 tests/evaluators/__init__.py create mode 100644 tests/evaluators/test_CreateParserInput.py create mode 100644 tests/helpers.py create mode 100644 tests/ontologies/__init__.py create mode 100644 tests/ontologies/test_SheerkaOntoloyManager.py create mode 100644 tests/parsers/__init__.py create mode 100644 tests/parsers/test_parser_input.py create mode 100644 tests/parsers/test_tokenizer.py create mode 100644 tests/sdp/__init__.py create mode 100644 tests/server/__init__.py create mode 100644 tests/server/test_server.py create mode 100644 tests/services/__init__.py create mode 100644 tests/services/test_ConceptManager.py create mode 100644 tests/services/test_SheerkaEngine.py create mode 100644 tests/sheerkapickle/__init__.py create mode 100644 tests/sheerkapickle/test_SheerkaPickler.py create mode 100644 tests/sheerkapickle/test_sheerka_handlers.py create mode 100644 tests/test_helpers.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ffc7f31 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +.PHONY: test + +test: + pytest + +coverage: + coverage run --source=src -m pytest + coverage html + +clean: + rm -rf build + rm -rf htmlcov + rm -rf .coverage + rm -rf docs/build + rm -rf docs/source/_build + rm -rf prof + rm -rf tests/prof + rm -rf tests/build + rm -rf Untitled*.ipynb + rm -rf .ipynb_checkpoints + find . -name '.pytest_cache' -exec rm -rf {} + + find . -name '__pycache__' -exec rm -rf {} + + find . -name 'debug.txt' -exec rm -rf {} + \ No newline at end of file diff --git a/README.md b/README.md index 2a78234..62e4e88 100644 --- a/README.md +++ b/README.md @@ -8,4 +8,24 @@ My personnal AI ```shell cd src uvicorn server:app --reload -``` \ No newline at end of file +``` + +## To start the client + +```shell +python $DEV_HOME/src/client.py --username --password +``` + +## to test + +```shell +pytest +``` + +## To run the coverage +```shell +coverage run --source=src -m pytest + +coverage report +coverage html +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4695789 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.pytest.ini_options] +pythonpath = "src tests" diff --git a/requirements.txt b/requirements.txt index 0f7e9b1..9ad5704 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,26 @@ anyio==3.6.2 +arrow==1.2.3 attrs==22.2.0 bcrypt==4.0.1 certifi==2022.12.7 cffi==1.15.1 charset-normalizer==3.0.1 click==8.1.3 +coverage==7.1.0 cryptography==39.0.0 ecdsa==0.18.0 exceptiongroup==1.1.0 fastapi==0.89.1 h11==0.14.0 +httpcore==0.16.3 httptools==0.5.0 +httpx==0.23.3 idna==3.4 iniconfig==2.0.0 +Jinja2==3.1.2 +jinja2-time==0.2.0 +make==0.1.6.post2 +MarkupSafe==2.1.2 oauthlib==3.2.2 packaging==23.0 passlib==1.7.4 @@ -22,12 +30,14 @@ pyasn1==0.4.8 pycparser==2.21 pydantic==1.10.4 pytest==7.2.0 +python-dateutil==2.8.2 python-dotenv==0.21.0 python-jose==3.3.0 python-multipart==0.0.5 PyYAML==6.0 requests==2.28.2 requests-oauthlib==1.3.1 +rfc3986==1.5.0 rsa==4.9 six==1.16.0 sniffio==1.3.0 diff --git a/src/caching/BaseCache.py b/src/caching/BaseCache.py new file mode 100644 index 0000000..5430221 --- /dev/null +++ b/src/caching/BaseCache.py @@ -0,0 +1,442 @@ +from threading import RLock + +from common.global_symbols import NotFound +from common.utils import sheerka_deepcopy + +MAX_INITIALIZED_KEY = 100 + + +class BaseCache: + """ + An in memory FIFO cache object + When the max_size is reach the first element that was put is removed + When you put the same key twice, the previous element is overridden + """ + + def __init__(self, max_size: int = None, default=NotFound, extend_exists=None, alt_sdp_get=None, sdp=None): + self._cache = {} + self._max_size = max_size + self._default = default # default value to return when key is not found. It can be a callable of key + self._extend_exists = extend_exists # search in remote + self._alt_sdp_get = alt_sdp_get # How to get the value when called by alt_sdp + self._sdp = sdp # current instance of SheerkaDataProvider + self._lock = RLock() + self._current_size = 0 + self._initialized_keys = set() # to keep the list of the keys already requested (using get()) + self._is_cleared = False # indicate that clear() was called + + self.to_add = set() + self.to_remove = set() + + # Explanation on _initialized_keys + # everytime you try to get an item, its key is added to _initialized_keys + # If the item is found, the entru is i + + def __len__(self): + """ + Return the number of items in the cache + :return: + """ + with self._lock: + return self._current_size + + def __contains__(self, key: str): + with self._lock: + return key in self._cache + + def __iter__(self): + with self._lock: + keys = self._cache.copy() + yield from keys + + def __next__(self): + return next(iter(self._cache)) + + def __repr__(self): + return f"{self.__class__.__name__}(size={self._current_size}, #keys={len(self._cache)})" + + def configure(self, max_size: int = None, default=NotFound, extend_exists=None, alt_sdp_get=None, sdp=None): + if max_size is not None: + self._max_size = max_size + + if default is not NotFound: + self._default = default + + if extend_exists is not None: + self._extend_exists = extend_exists + + if alt_sdp_get is not None: + self._alt_sdp_get = alt_sdp_get + + if sdp is not None: + self._sdp = sdp + + return self + + def auto_configure(self, cache_name: str): + """ + Convenient way to configure the cache + :param cache_name: + :return: + """ + self._default = lambda sdp, key: sdp.get(cache_name, key) + self._extend_exists = lambda sdp, key: sdp.exists(cache_name, key) + self._alt_sdp_get = lambda sdp, key: sdp.alt_get(cache_name, key) # by default, same than get + + return self + + def disable_default(self): + self._default = (lambda sdp, key: NotFound) if self._sdp else (lambda key: NotFound) + + def put(self, key: str, value: object, alt_sdp=None): + """ + Add a new entry in cache + :param key: + :param value: + :param alt_sdp: + :return: + """ + with self._lock: + if self._put(key, value, alt_sdp): + self._current_size += 1 + + def get(self, key: str, alt_sdp=None): + """ + Retrieve an entry from the cache + If the entry does not exist, will use the 'default' value or delegate + :param key: + :param alt_sdp: if not found in cache._sdp, look in other repositories + :return: + """ + with self._lock: + return self._get(key, alt_sdp) + + def alt_get(self, key: str): + """ + Alternate way to get an entry, from concept cache + This is mainly used for IncCache, in order to get the value without increasing it + It used for another cache, it must return the value from key WITHOUT modifying the state of the cache + :param key: + :return: + """ + with self._lock: + return self._alt_get(key) + + def get_all(self): + """ + Retrieve all items already in cache + This method does not fetch in the remoter repository + :return: + """ + with self._lock: + return self._cache.values() + + def inner_get(self, key: str): + return self._cache[key] + + def update(self, old_key: str, old_value: object, new_key: str, new_value: object, alt_sdp=None): + """ + Update an entry in the cache + :param old_key: key of the previous version of the entry + :param old_value: previous version of the entry + :param new_key: key of the entry + :param new_value: new value + :param alt_sdp: new value + :return: + """ + with self._lock: + self._update(old_key, old_value, new_key, new_value, alt_sdp) + + def delete(self, key: str, value: object = None, alt_sdp=None): + with self._lock: + try: + self._sync(key) + self._delete(key, value, alt_sdp) + return True + except KeyError: + return False + + def populate(self, populate_function: callable, get_key_function: callable, reset_events=False): + """ + Initialise the cache with a bunch of data + :param populate_function: iterable that produces item + :param get_key_function: how to compute the key of an item + :param reset_events: if TRUE, events are not updated + :return: + """ + with self._lock: + if reset_events: + to_add_copy = self.to_add.copy() + to_remove_copy = self.to_remove.copy() + + for item in (populate_function(self._sdp) if self._sdp else populate_function()): + self.put(get_key_function(item), item) + + if reset_events: + self.to_add = to_add_copy + self.to_remove = to_remove_copy + + def force_value(self, key: str, value: object): + """ + Force a value into a key without raising any event + """ + with self._lock: + self._cache[key] = value + + def remove_initialized_key(self, key: str): + """ + When a value is requested by alt_sdp, we should not keep track of the request + As the outcome is not known + """ + with self._lock: + self._initialized_keys.remove(key) + + def has(self, key: str): + """ + Return True if the key is in the cache + Never use extend_exist + :param key: + :return: + """ + with self._lock: + return key in self._cache + + def exists(self, key: str): + """ + Return True if the key is in the cache + Can use extend_exist + :param key: + :return: + """ + with self._lock: + if key in self._cache: + return True + + if self._extend_exists: + return self._extend_exists(self._sdp, key) if self._sdp else self._extend_exists(key) + else: + return False + + def evict(self, nb_items: int): + """ + Remove nb_items from the cache, using the replacement policy + :return: + """ + with self._lock: + nb_items = self._current_size if self._current_size < nb_items else nb_items + to_remove = [] + iter_cache = iter(self._cache) + try: + while nb_items > 0: + key = next(iter_cache) + if key in self.to_add or key in self.to_remove: + continue # cannot remove an item that is not yet committed + else: + to_remove.append(key) + nb_items -= 1 + except StopIteration: + pass + + for key in to_remove: + del (self._cache[key]) + try: + self._initialized_keys.remove(key) + except KeyError: + pass + + self._current_size -= len(to_remove) + + return len(to_remove) + + def evict_by_key(self, predicate: callable): + """ + Remove entries that matches the predicate + :param predicate: + :return: + """ + to_delete = [] + with self._lock: + for key in self._cache: + if predicate(key): + to_delete.append(key) + + for key in to_delete: + del (self._cache[key]) + try: + self._initialized_keys.remove(key) + except KeyError: + pass + + self._current_size -= len(to_delete) + + return len(to_delete) + + def clear(self, set_is_cleared: bool = True): + with self._lock: + # Seems that remote sdp is not correctly updated + self._cache.clear() + self._current_size = 0 + self._initialized_keys.clear() + self.to_add.clear() + self.to_remove.clear() + if set_is_cleared: + self._is_cleared = True + + def dump(self): + with self._lock: + return { + "current_size": self._current_size, + "cache": self._cache.copy() + } + + def copy(self): + with self._lock: + return self._cache.copy() + + def init_from_dump(self, dump: dict): + with self._lock: + self._current_size = dump["current_size"] + self._cache = dump["cache"].copy() + return self + + def reset_events(self): + with self._lock: + self.to_add.clear() + self.to_remove.clear() + self._is_cleared = False + + def reset_initialized_keys(self): + """ + Use when an ontology is put back. Reset all the previous requests as alt_sdp is a new one + """ + with self._lock: + self._initialized_keys.clear() + + def is_cleared(self): + with self._lock: + return self._is_cleared + + def clone(self): + return type(self)(self._max_size, self._default, self._extend_exists, self._alt_sdp_get, self._sdp) + + def test_only_reset(self): + """ + Clears the cache, but does not set is_cleared to True + It's a convenient way to clear the cache without altering alt_sdp behaviour + """ + self.clear(set_is_cleared=False) + + def _sync(self, *keys): + # KSI 2020-12-29. DO not try to use alt_sdp here + # Sync must only sync with the current sdp + for key in keys: + if key not in self._initialized_keys and callable(self._default): + # to keep sync with the remote repo is needed + # first check self._initialized_keys to prevent infinite loop + self.get(key) + + def _add_to_add(self, key: str): + """ + Adds the key to the list of recently added keys + :param key: + :type key: + :return: + :rtype: + """ + self.to_add.add(key) + try: + self.to_remove.remove(key) + except KeyError: + pass + + def _add_to_remove(self, key: str): + """ + Adds the key to the list of recently removed keys + :param key: + :type key: + :return: + :rtype: + """ + self.to_remove.add(key) + try: + self.to_add.remove(key) + except KeyError: + pass + + def _get(self, key: str, alt_sdp=None): + try: + value = self._cache[key] + except KeyError: + if len(self._initialized_keys) == MAX_INITIALIZED_KEY: + self._initialized_keys.clear() + if callable(self._default): + if key in self._initialized_keys: + # it means that we have already asked the repository + return NotFound + + simple_copy = True + + # first, tries to use the default value + value = self._default(self._sdp, key) if self._sdp else self._default(key) + if value is NotFound and alt_sdp and not self._is_cleared: + # try in the alternate (Sheerka) Data Provider + value = self._alt_sdp_get(alt_sdp, key) + simple_copy = False # in the case, make sure to make a deep copy + + if value is not NotFound: + value = value if simple_copy else sheerka_deepcopy(value) + self._cache[key] = value + + # update _current_size + if isinstance(value, (list, set)): + self._current_size += len(value) + else: + self._current_size += 1 + + if self._max_size and self._current_size > self._max_size: + self.evict(self._current_size - self._max_size) + + # else 'value' remains NotFound + else: + value = self._default + self._initialized_keys.add(key) + + return value + + def _alt_get(self, key: str): + return self._get(key) # by default, point to _get + + def _put(self, key: str, value: object, alt_sdp): + """ + To be defined in subclass + :param key: + :type key: + :param value: + :type value: + :param alt_sdp: + :type alt_sdp: + :return: + :rtype: + """ + pass + + def _update(self, old_key: str, old_value: object, new_key: str, new_value: object, alt_sdp): + """ + To be defined in subclass + :param old_key: + :type old_key: + :param old_value: + :type old_value: + :param new_key: + :type new_key: + :param new_value: + :type new_value: + :param alt_sdp: + :type alt_sdp: + :return: + :rtype: + """ + pass + + def _delete(self, key: str, value: object, alt_sdp): + raise NotImplementedError("_delete BaseCache") diff --git a/src/caching/Cache.py b/src/caching/Cache.py new file mode 100644 index 0000000..cf8cd79 --- /dev/null +++ b/src/caching/Cache.py @@ -0,0 +1,40 @@ +from caching.BaseCache import BaseCache +from common.global_symbols import Removed + + +class Cache(BaseCache): + """ + An in memory FIFO cache object + When the max_size is reach the first element that was put is removed + When you put the same key twice, the previous element is overridden + """ + + def _put(self, key: str, value: object, alt_sdp): + res = key not in self._cache + self._cache[key] = value + self._add_to_add(key) + return res + + def _update(self, old_key: str, old_value: object, new_key: str, new_value: object, alt_sdp): + self._cache[new_key] = new_value + self._add_to_add(new_key) + + if new_key != old_key: + self._sync(old_key) + if not self._is_cleared and alt_sdp and self._extend_exists and self._extend_exists(alt_sdp, old_key): + self._cache[old_key] = Removed + self._add_to_add(old_key) + self._current_size += 1 + else: + del (self._cache[old_key]) + self._add_to_remove(old_key) + + def _delete(self, key: str, value: object, alt_sdp): + if not self._is_cleared and alt_sdp and self._extend_exists and self._extend_exists(alt_sdp, key): + self._cache[key] = Removed + self._add_to_add(key) + # do not decrease self._current_size as 'Removed' takes on slot + else: + del (self._cache[key]) + self._add_to_remove(key) + self._current_size -= 1 diff --git a/src/caching/CacheManager.py b/src/caching/CacheManager.py new file mode 100644 index 0000000..3e55beb --- /dev/null +++ b/src/caching/CacheManager.py @@ -0,0 +1,342 @@ +from dataclasses import dataclass, field +from threading import RLock +from typing import Callable + +from caching.BaseCache import BaseCache +from common.global_symbols import NotFound +from core.concept import Concept, ConceptMetadata + + +@dataclass +class MultipleEntryError(Exception): + """ + Exception raised when trying to alter an entry with multiple element + without giving the origin of the element + """ + + key: str + + +@dataclass +class ConceptNotFound(Exception): + """ + Thrown when you try to remove a concept that is not found + """ + concept: object + + +@dataclass +class CacheDefinition: + cache: BaseCache + use_ref: bool + get_key: Callable[[ConceptMetadata], str] | None = field(repr=False) + persist: bool = True + + +class CacheManager: + """ + Single class to manage all the caches + """ + + def __init__(self, sdp=None): + """ + Manager for all the caches. + Make the link between the cache and the SheerkaDataProvider + :param sdp: + :type sdp: + """ + self.sdp = sdp + self.caches: dict[str, CacheDefinition] = {} # dict {cache_name: CacheDefinition} + self.concept_caches = [] + self.is_dirty = False # to indicate that the value of a cache has changed + + self._lock = RLock() + + def register_concept_cache(self, name, cache, get_key, use_ref): + """ + Special caches to manage concept definition + They store concepts metadata, using specific index + For example, you may declare an index to store metadata by id, by key or by whatever you need. + + It a convenient way to manage indexes for concepts definitions + Underneath, there are simple `Cache` objects. + :param name: + :param cache: + :param get_key: + :param use_ref: + :return: + """ + with self._lock: + if self.sdp: + cache.configure(sdp=self.sdp) + self.caches[name] = CacheDefinition(cache, use_ref, get_key) + self.concept_caches.append(name) + + def register_cache(self, name, cache, persist=True, use_ref=False): + """ + Define which type of cache along with how to compute the key + :param name: + :param cache: + :param persist: + :param use_ref: + :return: + """ + with self._lock: + if self.sdp: + cache.configure(sdp=self.sdp) + + self.caches[name] = CacheDefinition(cache, use_ref, None, persist) + + def add_concept(self, metadata: ConceptMetadata, alt_sdp=None): + """ + We need multiple indexes to retrieve a concept + So the new concept is dispatched into multiple caches + :param metadata: + :param alt_sdp: if not found in self.sdp, look in other repositories + :return: + """ + with self._lock: + for name in self.concept_caches: + cache_def = self.caches[name] + key = cache_def.get_key(metadata) + if key is None: + raise KeyError("") + cache_def.cache.put(key, metadata, alt_sdp) + + self.is_dirty = True + + def update_concept(self, old: ConceptMetadata, new: ConceptMetadata, alt_sdp=None): + """ + Update a concept. + :param old: old version of the concept + :param new: new version of the concept + :param alt_sdp: if not found in self.sdp, look in other repositories + :return: + """ + with self._lock: + for cache_name in self.concept_caches: + cache_def = self.caches[cache_name] + + old_key = cache_def.get_key(old) + new_key = cache_def.get_key(new) + cache_def.cache.update(old_key, old, new_key, new, alt_sdp=alt_sdp) + + self.is_dirty = True + + def remove_concept(self, concept: ConceptMetadata, alt_sdp=None): + """ + Remove a concept from all caches + :param concept: + :param alt_sdp: if not found in self.sdp, look in other repositories + :return: + """ + with self._lock: + # the first concept cache must the one where all concept are unique + # eg it has to be the concept by id + ref_cache_def = self.caches[self.concept_caches[0]] + concept_id = ref_cache_def.get_key(concept) + ref_concept = ref_cache_def.cache.get(concept_id) + + if ref_concept is NotFound and alt_sdp: + ref_concept = alt_sdp.get(self.concept_caches[0], concept_id) + + if ref_concept is NotFound: + raise ConceptNotFound(concept) + + for cache_name in self.concept_caches: + cache_def = self.caches[cache_name] + key = cache_def.get_key(ref_concept) + cache_def.cache.delete(key, ref_concept, alt_sdp=alt_sdp) + + self.is_dirty = True + + def get(self, cache_name, key, alt_sdp=None): + """ + From concept cache, get an entry + :param cache_name: + :param key: + :param alt_sdp: if not found in self.sdp, look in other repositories + :return: + """ + with self._lock: + return self.caches[cache_name].cache.get(key, alt_sdp) + + def alt_get(self, cache_name, key): + """ + Alternate way to get an entry, from concept cache + This is mainly used for IncCache, in order to get the value without increasing it + :param cache_name: + :param key: + :return: + """ + with self._lock: + return self.caches[cache_name].cache.alt_get(key) + + def put(self, cache_name, key, value, alt_sdp=None): + """ + Add to a cache + :param cache_name: + :param key: + :param value: + :param alt_sdp: if not found in self.sdp, look in other repositories + :return: + """ + with self._lock: + self.caches[cache_name].cache.put(key, value, alt_sdp) + self.is_dirty = True + + def delete(self, cache_name, key, value=None, alt_sdp=None): + """ + Delete an entry from the cache + :param cache_name: + :param key: + :param value: + :param alt_sdp: if not found in self.sdp, look in other repositories + :return: + """ + with self._lock: + if self.caches[cache_name].cache.delete(key, value, alt_sdp): + self.is_dirty = True + + def get_inner_cache(self, cache_name) -> BaseCache: + """ + Return the BaseCache object + :param cache_name: + :return: + """ + with self._lock: + return self.caches[cache_name].cache + + def copy(self, cache_name) -> dict: + """ + get a copy the content of the whole cache as a dictionary + :param self: + :param cache_name: + :return: + """ + return self.caches[cache_name].cache.copy() + + def populate(self, cache_name, populate_function, get_key_function, reset_events=False): + """ + Populate a specific cache with a bunch of items + :param cache_name: + :param populate_function: how to get the items + :param get_key_function: how to get the key, out of an item + :param reset_events: reset to_add and to_remove events after populate + :return: + """ + with self._lock: + self.caches[cache_name].cache.populate(populate_function, get_key_function, reset_events) + + def force_value(self, cache_name, key, value): + """ + Update the content of the cache, but does not raise any event + """ + with self._lock: + self.caches[cache_name].cache.force_value(key, value) + + def remove_initialized_key(self, cache_name, key): + """ + + """ + with self._lock: + self.caches[cache_name].cache.remove_initialized_key(key) + + def has(self, cache_name, key): + """ + True if the value is in cache only. Never try to look in a remote repository + :param cache_name: + :param key: + :return: + """ + with self._lock: + return self.caches[cache_name].cache.has(key) + + def exists(self, cache_name, key): + """ + True if the value is in cache. + If not found, may search in a remote repository + :param cache_name: + :param key: + :return: + """ + + with self._lock: + return self.caches[cache_name].cache.exists(key) + + def commit(self, context): + """ + Persist all the caches into a physical persistence storage + :param context: + :return: + """ + + def update_full_serialisation(items, value): + # Take care, infinite recursion is not handled !! + if isinstance(items, (list, set, tuple)): + for item in items: + update_full_serialisation(item, value) + elif isinstance(items, dict): + for values in items.values(): + update_full_serialisation(values, value) + elif isinstance(items, Concept): + items.get_metadata().full_serialization = value + + with self._lock: + with self.sdp.get_transaction(context.event.get_digest()) as transaction: + for cache_name, cache_def in self.caches.items(): + if not cache_def.persist: + continue + + if cache_def.cache.is_cleared(): + transaction.clear(cache_name) + + for key in cache_def.cache.to_remove: + transaction.remove(cache_name, key) + + for key in cache_def.cache.to_add: + if key == "*self*": + transaction.add(cache_name, None, cache_def.cache.dump()["cache"]) + else: + to_save = cache_def.cache.inner_get(key) + update_full_serialisation(to_save, True) + transaction.add(cache_name, key, to_save, cache_def.use_ref) + update_full_serialisation(to_save, False) + + cache_def.cache.reset_events() + self.is_dirty = False + + def clear(self, cache_name=None, set_is_cleared=True): + with self._lock: + if cache_name: + self.caches[cache_name].cache.clear(set_is_cleared) + else: + for cache_def in self.caches.values(): + cache_def.cache.clear(set_is_cleared) + + def dump(self): + """ + For test purpose, dumps the whole content of the cache manager + :return: + """ + with self._lock: + res = {} + for cache_name, cache_def in self.caches.items(): + res[cache_name] = cache_def.cache.dump() + + return res + + def init_from_dump(self, dump): + with self._lock: + for cache_name, content in dump.items(): + if cache_name in self.caches: + self.caches[cache_name].cache.init_from_dump(content) + + return self + + def reset(self): + """For unit test speed enhancement""" + self.clear() + self.caches.clear() + self.concept_caches.clear() + self.is_dirty = False diff --git a/src/caching/DictionaryCache.py b/src/caching/DictionaryCache.py new file mode 100644 index 0000000..b6ddaac --- /dev/null +++ b/src/caching/DictionaryCache.py @@ -0,0 +1,88 @@ +from caching.BaseCache import MAX_INITIALIZED_KEY +from caching.Cache import BaseCache +from common.global_symbols import NotFound + + +class DictionaryCache(BaseCache): + """ + Kind of all or nothing dictionary database + You can get the values key by by + But when you want to put, you must put the whole database + For this reason, alt_sdp is not supported. The top ontology layer contains the whole database + """ + + def auto_configure(self, cache_name): + """ + Convenient way to configure the cache + :param cache_name: + :return: + """ + self._default = lambda sdp, key: sdp.get(cache_name) # retrieve the whole entry + self._extend_exists = None # not used + self._alt_sdp_get = None # not used + + return self + + def _get(self, key, alt_sdp=None): + """ + Management of the default is different + :param key: + :return: + """ + try: + return self._cache[key] + except KeyError: + if key in self._initialized_keys: + return NotFound + + if len(self._initialized_keys) == MAX_INITIALIZED_KEY: + self._initialized_keys.clear() + + self._initialized_keys.add(key) + + if callable(self._default): + default_values = self._default(self._sdp, key) if self._sdp else self._default(key) + else: + default_values = self._default + + if isinstance(default_values, dict): + self._cache.update(default_values) # update the whole cache dictionary to resync with remote sdp + + self._count_items() + return self._cache[key] if key in self._cache else NotFound + + def _put(self, key, value, alt_sdp): + """ + Adds a whole dictionary + :param key: True to append, False to reset + :param value: dictionary + :param alt_sdp: NOT SUPPORTED as the values from alt_sdp must be retrieved and computed BEFORE the put + :return: + """ + if not isinstance(key, bool): + raise KeyError + + if not isinstance(value, dict): + raise ValueError + + if key: # update the current cache + if self._cache is None: + self._cache = value.copy() + else: + self._cache.update(value) + else: # reset the current cache + self._cache = value + + self._count_items() + + # special meaning for to_add + self._add_to_add("*self*") + return False + + def _delete(self, key, value, alt_sdp): + raise NotImplementedError("_delete DictionaryCache") + + def _count_items(self): + self._current_size = 0 + for v in self._cache.values(): + self._current_size += len(v) if hasattr(v, "__len__") and not isinstance(v, str) else 1 diff --git a/src/caching/FastCache.py b/src/caching/FastCache.py new file mode 100644 index 0000000..c8bdffc --- /dev/null +++ b/src/caching/FastCache.py @@ -0,0 +1,71 @@ +from common.global_symbols import NotFound + + +class FastCache: + """ + Simplest LRU cache + """ + + def __init__(self, max_size=256, default=None): + self.max_size = max_size + self.cache = {} + self.lru = [] + self.default = default + self.calls = {} + + def __contains__(self, item): + return self.has(item) + + def __iter__(self): + yield from self.cache + + def __next__(self): + return next(iter(self.cache)) + + def __len__(self): + return len(self.cache) + + def put(self, key, value): + if len(self.cache) == self.max_size: + del self.cache[self.lru.pop(0)] + + if key in self.cache: + self.lru.remove(key) + + self.cache[key] = value + self.lru.append(key) + self.calls[key] = 0 + + def has(self, key): + return key in self.cache + + def get(self, key): + try: + res = self.cache[key] + self.calls[key] += 1 + return res + except KeyError: + if self.default: + value = self.default(key) + self.put(key, value) + return value + + return NotFound + + def evict_by_key(self, predicate): + to_remove = [] + + for k, v in self.cache.items(): + if predicate(k): + to_remove.append(k) + + for k in to_remove: + self.lru.remove(k) + del self.cache[k] + + def copy(self): + return self.cache.copy() + + def clear(self): + self.cache.clear() + self.lru.clear() \ No newline at end of file diff --git a/src/caching/IncCache.py b/src/caching/IncCache.py new file mode 100644 index 0000000..8b91b43 --- /dev/null +++ b/src/caching/IncCache.py @@ -0,0 +1,24 @@ +from caching.Cache import Cache +from common.global_symbols import NotFound, Removed + + +class IncCache(Cache): + """ + Increment the value of the key every time it's accessed + """ + + def _get(self, key, alt_sdp=None): + value = super()._get(key, alt_sdp=alt_sdp) + if value in (NotFound, Removed): + value = 0 + value += 1 + self._put(key, value, alt_sdp) + return value + + def _put(self, key, value, alt_sdp): + self._cache[key] = value + self._add_to_add(key) + return True + + def _alt_get(self, key): + return super()._get(key) # point to parent, not to self diff --git a/src/caching/ListCache.py b/src/caching/ListCache.py new file mode 100644 index 0000000..0d174e6 --- /dev/null +++ b/src/caching/ListCache.py @@ -0,0 +1,66 @@ +from caching.Cache import BaseCache +from common.global_symbols import NotFound, Removed +from common.utils import sheerka_deepcopy + + + +class ListCache(BaseCache): + """ + An in memory FIFO cache object + When the max_size is reach the first element that was put is removed + Items of this cache are list + """ + + def _put(self, key, value, alt_sdp): + if key in self._cache: + self._cache[key].append(value) + else: + self._sync(key) + + if key not in self._cache and alt_sdp and not self._is_cleared: + previous = self._alt_sdp_get(alt_sdp, key) + if previous not in (NotFound, Removed): + self._cache[key] = sheerka_deepcopy(previous) + + if key in self._cache: + self._cache[key].append(value) + else: + self._cache[key] = [value] + + self._add_to_add(key) + return True + + def _update(self, old_key, old_value, new_key, new_value, alt_sdp): + self._sync(old_key, new_key) + + if old_key not in self._cache and alt_sdp and not self._is_cleared: + # no value found in local cache or remote repository + # Use the values from alt_sdp + previous = self._alt_sdp_get(alt_sdp, old_key) + if previous in (NotFound, Removed): + raise KeyError(old_key) + + self._cache[old_key] = sheerka_deepcopy(previous) + self._current_size += len(previous) + + if old_key != new_key: + self._cache[old_key].remove(old_value) + if len(self._cache[old_key]) == 0: + if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, old_key): + self._cache[old_key] = Removed + self._add_to_add(old_key) + self._current_size += 1 + else: + del (self._cache[old_key]) + self._add_to_remove(old_key) + else: + self._add_to_add(old_key) + + self._put(new_key, new_value, alt_sdp) + self._add_to_add(new_key) + else: + for i in range(len(self._cache[new_key])): + if self._cache[new_key][i] == old_value: + self._cache[new_key][i] = new_value # avoid add and remove in dict + break # only the first one is affected + self._add_to_add(new_key) \ No newline at end of file diff --git a/src/caching/ListIfNeededCache.py b/src/caching/ListIfNeededCache.py new file mode 100644 index 0000000..f243280 --- /dev/null +++ b/src/caching/ListIfNeededCache.py @@ -0,0 +1,134 @@ +from caching.Cache import BaseCache +from common.global_symbols import NotFound, Removed +from common.utils import sheerka_deepcopy + + +class ListIfNeededCache(BaseCache): + """ + An in memory FIFO cache object + When the max_size is reach the first element that was put is removed + When you put the same key twice, you now have a list of two elements + """ + + def _put(self, key, value, alt_sdp): + if key in self._cache: + if isinstance(self._cache[key], list): + self._cache[key].append(value) + else: + self._cache[key] = value if self._cache[key] is Removed else [self._cache[key], value] + else: + self._sync(key) + + if key not in self._cache and alt_sdp and not self._is_cleared: + previous = self._alt_sdp_get(alt_sdp, key) + if previous not in (NotFound, Removed): + self._cache[key] = sheerka_deepcopy(previous) + + if key in self._cache: + if isinstance(self._cache[key], list): + self._cache[key].append(value) + else: + self._cache[key] = value if self._cache[key] is Removed else [self._cache[key], value] + else: + self._cache[key] = value + self._add_to_add(key) + return True + + def _update(self, old_key, old_value, new_key, new_value, alt_sdp): + + self._sync(old_key, new_key) + + if old_key not in self._cache and alt_sdp and not self._is_cleared: + # no value found in local cache or remote repository + # Use the values from alt_sdp + previous = self._alt_sdp_get(alt_sdp, old_key) + if previous in (NotFound, Removed): + raise KeyError(old_key) + + self._cache[old_key] = sheerka_deepcopy(previous) + self._current_size += len(previous) if isinstance(previous, list) else 1 + + if old_key != new_key: + if isinstance(self._cache[old_key], list): + self._cache[old_key].remove(old_value) + if len(self._cache[old_key]) == 1: + self._cache[old_key] = self._cache[old_key][0] + self._add_to_add(old_key) + else: + if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, old_key): + self._cache[old_key] = Removed + self._add_to_add(old_key) + self._current_size += 1 + else: + del (self._cache[old_key]) + self._add_to_remove(old_key) + + self._put(new_key, new_value, alt_sdp) + self._add_to_add(new_key) + else: + if isinstance(self._cache[new_key], list): + for i in range(len(self._cache[new_key])): + if self._cache[new_key][i] == old_value: + self._cache[new_key][i] = new_value # avoid add and remove in dict + break + else: + self._cache[new_key] = new_value + self._add_to_add(new_key) + + def _delete(self, key, value, alt_sdp): + if value is None: + # Remove the whole key + if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key): + if key in self._cache: + previous = self._cache[key] + if isinstance(previous, list): + self._current_size -= len(previous) + 1 + else: + self._current_size += 1 + + self._cache[key] = Removed + self._add_to_add(key) + else: + previous = self._cache[key] + self._current_size -= len(previous) if isinstance(previous, list) else 1 + del self._cache[key] + self._add_to_remove(key) + + else: + # Remove a single value + try: + previous = self._cache[key] + if isinstance(previous, list): + previous.remove(value) + self._cache[key] = previous[0] if len(previous) == 1 else previous + self._current_size -= 1 + self.to_add.add(key) + else: + if previous == value: + # I am about to delete the entry + if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key): + self._cache[key] = Removed + self.to_add.add(key) + # self._current_size -= 1 # Do not decrease size, as it's replaced by 'Removed' + else: + del self._cache[key] + self._current_size -= 1 + self.to_remove.add(key) + except KeyError as ex: + previous = self._alt_sdp_get(alt_sdp, key) if not self._is_cleared and alt_sdp else NotFound + if previous in (NotFound, Removed): + raise ex + + if isinstance(previous, list): + previous = sheerka_deepcopy(previous) + previous.remove(value) # raise an exception if value in not in the list + self._cache[key] = previous[0] if len(previous) == 1 else previous + self._current_size -= 1 + self.to_add.add(key) + else: + if previous == value: + self._cache[key] = Removed + self.to_add.add(key) + self._current_size -= 1 + + return True diff --git a/src/caching/SetCache.py b/src/caching/SetCache.py new file mode 100644 index 0000000..a654dc1 --- /dev/null +++ b/src/caching/SetCache.py @@ -0,0 +1,122 @@ +from caching.Cache import BaseCache +from common.global_symbols import NotFound, Removed +from common.utils import sheerka_deepcopy + + +class SetCache(BaseCache): + """ + An in memory FIFO cache object + When the max_size is reach the first element that was put is removed + + You can use the same key multiple times, but the elements under this key will be unique + When there are multiple elements, a python set is used + + >> self.put('key', 'value1') + >> assert {'value1'} == self.get('key') + >> self.put('key', 'value2') + >> assert {'value1', 'value2'} == self.get('key') + """ + + def _put(self, key, value, alt_sdp): + if key in self._cache: + if self._cache[key] is Removed: + self._cache[key] = {value} + elif value in self._cache[key]: + return False + else: + self._cache[key].add(value) + else: + self._sync(key) + + if key not in self._cache and alt_sdp and not self._is_cleared: + previous = self._alt_sdp_get(alt_sdp, key) + if previous not in (NotFound, Removed): + self._cache[key] = sheerka_deepcopy(previous) + + if key in self._cache: + if self._cache[key] == Removed: + self._cache[key] = {value} + else: + self._cache[key].add(value) + else: + self._cache[key] = {value} + + self._add_to_add(key) + return True + + def _update(self, old_key, old_value, new_key, new_value, alt_sdp): + self._sync(old_key, new_key) + + if old_key not in self._cache and alt_sdp and not self._is_cleared: + # no value found in local cache or remote repository + # Use the values from alt_sdp + previous = self._alt_sdp_get(alt_sdp, old_key) + if previous in (NotFound, Removed): + raise KeyError(old_key) + + self._cache[old_key] = sheerka_deepcopy(previous) + self._current_size += len(previous) + + if old_key != new_key: + if isinstance(self._cache[old_key], set): + self._cache[old_key].remove(old_value) + if len(self._cache[old_key]) == 0: + if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, old_key): + self._cache[old_key] = Removed + self._add_to_add(old_key) + self._current_size += 1 + else: + del (self._cache[old_key]) + self._add_to_remove(old_key) + else: + self._add_to_add(old_key) + + self._put(new_key, new_value, alt_sdp) + self._add_to_add(new_key) + else: + self._cache[new_key].remove(old_value) + self._put(new_key, new_value, alt_sdp) + self._add_to_add(new_key) + + def _delete(self, key, value, alt_sdp): + if value is None: + if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key): + self._current_size += 1 - len(self._cache[key]) if key in self._cache else 1 + self._cache[key] = Removed + self._add_to_add(key) + else: + self._current_size -= len(self._cache[key]) + del self._cache[key] + self._add_to_remove(key) + + else: + try: + self._cache[key].remove(value) + if len(self._cache[key]) == 0: + if not self._is_cleared and alt_sdp and self._extend_exists(alt_sdp, key): + self._cache[key] = Removed + self._add_to_add(key) + # self._current_size -= 1 # Do not decrease size, as it's replaced by 'Removed' + else: + del self._cache[key] + self._add_to_remove(key) + self._current_size -= 1 + else: + self._add_to_add(key) + self._current_size -= 1 + except KeyError as ex: + previous = self._alt_sdp_get(alt_sdp, key) if not self._is_cleared and alt_sdp else NotFound + if previous in (NotFound, Removed): + raise ex + + previous = sheerka_deepcopy(previous) + previous.remove(value) # will raise a KeyError if value is not in the set + if len(previous) == 0: + self._cache[key] = Removed + self._current_size += 1 + else: + self._cache[key] = previous + self._current_size += len(previous) + self._add_to_add(key) + + return True diff --git a/src/caching/__init__.py b/src/caching/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/client.py b/src/client.py index 9355c7e..826ca58 100644 --- a/src/client.py +++ b/src/client.py @@ -30,6 +30,7 @@ class SheerkaClient: self.url = f"{self.hostname}:{self.port}" if self.port else f"{self.hostname}" self.history_file = path.abspath(path.join(path.expanduser("~"), ".sheerka", "history.txt")) self.token = None + self.user = None def init_folder(self): root_path = path.dirname(self.history_file) @@ -56,7 +57,9 @@ class SheerkaClient: form_data = {"username": username, "password": password} res = requests.post(token_url, data=form_data) if res: - self.token = res.json()["access_token"] + as_json = res.json() + self.token = as_json["access_token"] + self.user = {"first_name": as_json["first_name"], "last_name": as_json["last_name"]} return TestResponse(True, f"Connected as {username}") else: self.token = None @@ -65,6 +68,14 @@ class SheerkaClient: self.token = None return TestResponse(False, str(ex)) + def help(self): + self.print_info("Basic commands:") + self.print_info(" clear") + self.print_info(" connect") + if not self.token: + self.print_error("You are not connected.") + self.print_info("You should use the command `connect('username', 'password'`)") + def run(self): while True: try: @@ -80,6 +91,10 @@ class SheerkaClient: prompt_toolkit.shortcuts.clear() continue + if _in == "help": + self.help() + continue + # allow reconnection m = connect_regex.match(_in) if m: @@ -92,9 +107,9 @@ class SheerkaClient: # Call Sheerka if self.token: headers = {"Authorization": f"Bearer {self.token}"} - response = requests.post(f"{self.url}/echo/{_in}", headers=headers) + response = requests.post(f"{self.url}/command/{_in}", headers=headers) else: - response = requests.post(f"{self.url}/echo/{_in}") + response = requests.post(f"{self.url}/command/{_in}") # read the response from the server if response: diff --git a/src/common/__init__.py b/src/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/common/global_symbols.py b/src/common/global_symbols.py new file mode 100644 index 0000000..ffc6de0 --- /dev/null +++ b/src/common/global_symbols.py @@ -0,0 +1,66 @@ +EVENT_CONCEPT_PRECEDENCE_MODIFIED = "evt_cp_m" +EVENT_RULE_PRECEDENCE_MODIFIED = "evt_rp_m" +EVENT_CONTEXT_DISPOSED = "evt_ctx_d" +EVENT_USER_INPUT_EVALUATED = "evt_ui_e" +EVENT_CONCEPT_CREATED = "evt_c_c" +EVENT_CONCEPT_MODIFIED = "evt_c_m" +EVENT_CONCEPT_DELETED = "evt_c_d" +EVENT_CONCEPT_ID_DELETED = "evt_c_id_d" +EVENT_RULE_CREATED = "evt_r_c" +EVENT_RULE_DELETED = "evt_r_d" +EVENT_RULE_ID_DELETED = "evt_r_id_d" +EVENT_ONTOLOGY_CREATED = "evt_o_c" +EVENT_ONTOLOGY_DELETED = "evt_o_d" + +VARIABLE_PREFIX = "__var__" + + +class CustomType: + """ + Base class for custom types used in Sheerka + A custom type is a type that has only one instance across the application and have a semantic meaning + For example the type 'None' is a singleton which have a semantic meaning. + We need to define others in Sheerka + """ + + def __init__(self, value): + self.value = value + + def __repr__(self): + return self.value + + def __eq__(self, other): + return isinstance(other, CustomType) and self.value == other.value + + def __hash__(self): + return hash(self.value) + + +class NotInitType(CustomType): + def __init__(self): + super(NotInitType, self).__init__("**NotInit**") + + +class NotFoundType(CustomType): + """ + Using when an entry in not found in Cache or in sdp + """ + + def __init__(self): + super(NotFoundType, self).__init__("**NotFound**") + + +class RemovedType(CustomType): + def __init__(self): + super(RemovedType, self).__init__("**Removed**") + + +class NoFirstTokenType(CustomType): + def __init__(self): + super(NoFirstTokenType, self).__init__("**NoFirstToken**") + + +NotInit = NotInitType() +NotFound = NotFoundType() +Removed = RemovedType() +NoFirstToken = NoFirstTokenType() diff --git a/src/common/utils.py b/src/common/utils.py new file mode 100644 index 0000000..cda0832 --- /dev/null +++ b/src/common/utils.py @@ -0,0 +1,293 @@ +import importlib +import pkgutil +from copy import deepcopy + +from common.global_symbols import CustomType + + +def get_class(qname): + """ + Loads a class from its full qualified name + :param qname: + :return: + """ + parts = qname.split('.') + module = ".".join(parts[:-1]) + m = __import__(module) + for comp in parts[1:]: + m = getattr(m, comp) + return m + + +def get_module(qname): + """ + Loads a module from its full qualified name + :param qname: + :return: + """ + parts = qname.split('.') + m = __import__(qname) + for comp in parts[1:]: + m = getattr(m, comp) + return m + + +def get_full_qualified_name(obj): + """ + Returns the full qualified name of a class (including its module name ) + :param obj: + :return: + """ + if obj.__class__ == type: + module = obj.__module__ + if module is None or module == str.__class__.__module__: + return obj.__name__ # Avoid reporting __builtin__ + else: + return module + '.' + obj.__name__ + else: + module = obj.__class__.__module__ + if module is None or module == str.__class__.__module__: + return obj.__class__.__name__ # Avoid reporting __builtin__ + else: + return module + '.' + obj.__class__.__name__ + + +def get_logger_name(name): + """ + Wrapper to fancy how the name of the loggers are returned + :param name: + :type name: + :return: + :rtype: + """ + if name is None: + return None + return name.split(".")[-1] + + +def import_module_and_sub_module(module_name): + """ + Import the module, and one sub level + :param module_name: + :return: + """ + mod = get_module(module_name) + for (module_loader, name, ispkg) in pkgutil.iter_modules(mod.__path__, module_name + "."): + importlib.import_module(name) + + +def get_sub_classes(package_name, base_class): + def _get_class(name): + modname, _, clsname = name.rpartition('.') + mod = importlib.import_module(modname) + cls = getattr(mod, clsname) + return cls + + base_class = _get_class(base_class) if isinstance(base_class, str) else base_class + all_class = set(base_class.__subclasses__()).union( + [s for c in base_class.__subclasses__() for s in get_sub_classes(package_name, c)]) + + # limit to the classes of the package + return [c for c in all_class if c.__module__.startswith(package_name)] + + +def sheerka_deepcopy(obj): + """ + Internal implementation of deepcopy that will eventually handle Concept's circular references + :param obj: + :return: + """ + from core.concept import Concept + already_seen = {} + + def copy_concept(c: Concept): + id_c = id(c) + if id_c in already_seen: + ref = already_seen[id_c] + if ref == '_##_REF_##_': + raise Exception("Circular Ref not managed yet!") + else: + return ref + + already_seen[id_c] = '_##_REF_##_' + + cls = type(c) + instance = cls() + # update the metadata + for prop_name, prop_value in vars(c.get_metadata()).items(): + if prop_name != "props": + setattr(instance.get_metadata(), prop_name, prop_value) + else: + setattr(instance.get_metadata(), prop_name, sheerka_deepcopy(prop_value)) + + # update the values + for prop_name, prop_value in c.values().items(): + setattr(instance, prop_name, prop_value) + + already_seen[id_c] = instance + return instance + + if isinstance(obj, CustomType): + return obj + elif isinstance(obj, dict): + res = {sheerka_deepcopy(k): sheerka_deepcopy(v) for k, v in obj.items()} + return res + elif isinstance(obj, list): + return [sheerka_deepcopy(item) for item in obj] + elif isinstance(obj, set): + return {sheerka_deepcopy(item) for item in obj} + elif isinstance(obj, tuple): + return tuple((sheerka_deepcopy(item) for item in obj)) + elif isinstance(obj, Concept): + return copy_concept(obj) + else: + return deepcopy(obj) + + +def str_concept(t, drop_name=None, prefix="c:"): + """ + The key,id identifiers of a concept are stored in a tuple + we want to return the key and the id, separated by a pipe + None value must be replaced by an empty string + + >>> assert str_concept(("key", "id")) == "c:key#id:" + >>> assert str_concept((None, "id")) == "c:#id:" + >>> assert str_concept(("key", None)) == "c:key:" + >>> assert str_concept((None, None)) == "" + >>> assert str_concept(Concept(name="foo", id="bar")) == "c:foo#bar:" + >>> assert str_concept(Concept(name="foo", id="bar"), drop_name=True) == "c:#bar:" + >>> assert str_concept(("key", "id"), prefix='r:') == "r:key|id:" + :param t: + :param drop_name: True if we only want the id (and not the key) + :param prefix: + :return: + """ + if isinstance(t, tuple): + name, id_ = t[0], t[1] + else: + name, id_ = t.name, t.id + + if name is None and id_ is None: + return "" + + result = prefix if (name is None or drop_name) else prefix + name + if id_: + result += "#" + id_ + return result + ":" + + +def unstr_concept(concept_repr, prefix='c:'): + """ + if concept_repr is like :c:key:id: + return the key and the id + >>> assert unstr_concept("c:key:") == ("key", None) + >>> assert unstr_concept("c:key#id:") == ("key", "id") + >>> assert unstr_concept("c:#id:") == ("None", "id") + >>> assert unstr_concept("c:key#:") == ("key", "None") + >>> assert unstr_concept("r:key#id:", prefix='r:') == ("key", "id") + >>> # Otherwise, return (None,None) + + :param concept_repr: + :return: + """ + if not (concept_repr and + isinstance(concept_repr, str) and + concept_repr.startswith(prefix) and + concept_repr.endswith(":")): + return None, None + + i = 2 + length = len(concept_repr) + key = "" + while i < length: + c = concept_repr[i] + i += 1 + if c in (":", "#"): + break + key += c + else: + return None, None + + if c == ":": + return key if key != "" and i == length else None, None + + c_id = "" + while i < length: + c = concept_repr[i] + i += 1 + if c == ":": + break + c_id += c + else: + return None, None + + if i != length: + return None, None + + return key if key != "" else None, c_id if c_id != "" else None + + +def compute_hash(obj): + """ + Helper to get the hash from collection + :param obj: + :type obj: + :return: + :rtype: + """ + try: + if isinstance(obj, (list, tuple)): + return hash(tuple([compute_hash(o) for o in obj])) + + if isinstance(obj, set): + return hash(tuple([compute_hash(o) for o in sorted(list(obj))])) + + if isinstance(obj, dict): + return hash(repr(obj)) + + return hash(obj) + except: + return 0 + + +def decode_enum(enum_repr: str): + """ + Tries to transform ClassName.Name into an enum + :param enum_repr: + :return: + """ + if not (enum_repr and isinstance(enum_repr, str)): + return None + + try: + idx = enum_repr.rindex(".") + if idx == len(enum_repr): + return None + + cls_name = enum_repr[:idx] + cls = get_class(cls_name) + name = enum_repr[idx + 1:] + return cls[name] + + except ValueError: + return None + + except TypeError: + return None + + +def to_dict(items, get_attr): + """ + Create a dictionary from a list when duplicates keys are merged into lists + :param items: + :type items: + :param get_attr: + :type get_attr: + :return: + :rtype: + """ + res = {} + for item in items: + res.setdefault(get_attr(item), []).append(item) + + return res diff --git a/src/core/BuiltinConcepts.py b/src/core/BuiltinConcepts.py new file mode 100644 index 0000000..32c60b3 --- /dev/null +++ b/src/core/BuiltinConcepts.py @@ -0,0 +1,7 @@ +class BuiltinConcepts: + SHEERKA = "__SHEERKA" + + NEW_CONCEPT = "__NEW_CONCEPT" + UNKNOWN_CONCEPT = "__UNKNOWN_CONCEPT" + USER_INPUT = "__USER_INPUT" + PARSER_INPUT = "__PARSER_INPUT" diff --git a/src/core/ErrorContext.py b/src/core/ErrorContext.py new file mode 100644 index 0000000..a8ad3fa --- /dev/null +++ b/src/core/ErrorContext.py @@ -0,0 +1,35 @@ +from common.utils import compute_hash +from core.ExecutionContext import ExecutionContext + + +class SheerkaException(Exception): + pass + + +class ErrorContext: + """ + This class represents the result of a data flow processing + """ + + def __init__(self, who: str, context: ExecutionContext, value: object = None): + self.who = who + self.context = context + self.value = value + self.parents = None + + def __repr__(self): + return f"Error(who={self.who}, context_id={self.context.long_id}, value={self.value})" + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, ErrorContext): + return False + + return self.who == other.who and \ + self.context.id == other.context.id and \ + self.value == other.value + + def __hash__(self): + return hash((self.who, self.context.id, compute_hash(self.value))) diff --git a/src/core/Event.py b/src/core/Event.py new file mode 100644 index 0000000..61a91e9 --- /dev/null +++ b/src/core/Event.py @@ -0,0 +1,69 @@ +import hashlib +from datetime import datetime + + +class Event(object): + """ + Class that represents something that modifies the state of the system + """ + + def __init__(self, message="", user_id="", date=None, parents=None): + self.user_id: str = user_id # id of the user that triggers the modification + self.date: datetime | None = date or datetime.now() # when + self.message: str = message # user input or whatever that modifies the system + self.parents: list[str] = parents # digest(s) of the parent(s) of this event + self._digest: str | None = None # digest of the event + + def __str__(self): + return f"{self.date.strftime('%d/%m/%Y %H:%M:%S')} {self.message}" + + def __repr__(self): + return f"{self.get_digest()[:12]} {self.message}" + + def get_digest(self): + """ + Returns the digest of the event + :return: sha256 of the event + """ + + if self._digest: + return self._digest + + if self.user_id == "": + # only possible during the unit test + # We use this little trick to speed up the unit test + self._digest = self.message[6:] if self.message.startswith("TEST::") else "xxx" + return self._digest + + if not isinstance(self.message, str): + raise NotImplementedError(f"message={self.message}") + + to_hash = f"Event:{self.user_id}{self.date}{self.message}{self.parents}".encode("utf-8") + self._digest = hashlib.sha256(to_hash).hexdigest() + return self._digest + + def to_dict(self): + return self.__dict__ + + def from_dict(self, as_dict): + self.user_id = as_dict["user_id"] + self.date = datetime.fromisoformat(as_dict["date"]) + self.message = as_dict["message"] + self.parents = as_dict["parents"] + self._digest = as_dict["_digest"] # freeze the digest + + def __eq__(self, other): + if id(self) == id(other): + return True + + if isinstance(other, Event): + return (self.user_id == other.user_id and + self.date == other.date and + self.message == other.message and + self.parents == other.parents) + + return False + + def __hash__(self): + return hash(self.get_digest()) + diff --git a/src/core/ExecutionContext.py b/src/core/ExecutionContext.py index 809a16d..c5315ff 100644 --- a/src/core/ExecutionContext.py +++ b/src/core/ExecutionContext.py @@ -1,2 +1,181 @@ +from __future__ import annotations + +import time + +from core.Event import Event + + +class ExecutionContextActions: + TESTING = "Testing" + INIT_SHEERKA = "Init Sheerka" + EVALUATE_USER_INPUT = "Evaluate user input" + EVALUATING_STEP = "Evaluating step" + EVALUATING_ITERATION = "Evaluating iteration" + BEFORE_PARSING = "Before parsing" + PARSING = "Parsing" + AFTER_PARSING = "After parsing" + BEFORE_EVALUATION = "Before evaluation" + EVALUATION = "Evaluation" + AFTER_EVALUATION = "After Evaluation" + + +class ContextHint: + REDUCE_CONCEPTS = "Reduce Concepts" # to tell the process to only keep the meaningful results + + +ids = {} # keep track of the next execution context id, for a given event id + + +def get_next_id(event_digest): + """ + For a given event, give the next id + :param event_digest: + :type event_digest: + :return: + :rtype: + """ + if event_digest in ids: + ids[event_digest] += 1 + else: + ids[event_digest] = 0 + return ids[event_digest] + + class ExecutionContext: - pass + """ + To keep track of the execution of a request + Note that the protected hints are working correctly only if the hint is added BEFORE the creation of the child + """ + + def __init__(self, + who: str, + event: Event, + sheerka, + action: ExecutionContextActions, + action_context: object, + desc: str = None, + logger=None, + global_hints=None, + protected_hints=None, + parent: ExecutionContext = None): + self._id = get_next_id(event.get_digest()) + self._parent = parent + self._children = [] + self._start = 0 # when the execution starts (to measure elapsed time) + self._stop = 0 # when the execution stops (to measure elapses time) + self._logger = logger + + self.who = who # who is asking + self.event = event # what was the (original) trigger + self.sheerka = sheerka # sheerka + self.action = action + self.action_context = action_context + self.desc = desc # human description of what is going on + + self.private_hints = set() + self.protected_hints = set() if protected_hints is None else protected_hints.copy() + self.global_hints = set() if global_hints is None else global_hints + + self.inputs = {} # what were the parameters of the execution context + self.values = {} # what was produced by the execution context + + def __repr__(self): + msg = f"ExecutionContext(who={self.who}, id={self._id}, action={self.action}, context={self.action_context}" + if self.desc: + msg += f", desc='{self.desc}'" + msg += ")" + return msg + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, ExecutionContext): + return False + + return self.long_id == other.long_id + + def __hash__(self): + return hash(self.long_id) + + @property + def long_id(self): + return f"{self.event.get_digest()}:{self._id}" + + @property + def id(self): + return self._id + + @property + def elapsed(self): + if self._start == 0: + return 0 + + return (self._stop if self._stop > 0 else time.time_ns()) - self._start + + @property + def elapsed_str(self): + nano_sec = self.elapsed + dt = nano_sec / 1e6 + return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s" + + def add_inputs(self, **kwargs): + """ + When entering stacking an ExecutionContext, list of variable that are worth to trace + :param kwargs: + :type kwargs: + :return: + :rtype: + """ + self.inputs.update(kwargs) + return self + + def add_values(self, **kwargs): + """ + When popping from an ExecutionContext, list of variable that are worth to trace + :param kwargs: + :type kwargs: + :return: + :rtype: + """ + self.values.update(kwargs) + return self + + def push(self, + who: str, + action: ExecutionContextActions, + action_context: object, + desc: str = None, + logger=None): + child = ExecutionContext( + who, + self.event, + self.sheerka, + action, + action_context, + desc, + logger or self._logger, + self.global_hints, + self.protected_hints, + self + ) + self._children.append(child) + return child + + def get_children(self, level=-1): + """ + recursively look for children + :return: + :rtype: + """ + for child in self._children: + yield child + if level != 1: + yield from child.get_children(level - 1) + + def __enter__(self): + self._start = time.time_ns() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._stop = time.time_ns() diff --git a/src/core/ReturnValue.py b/src/core/ReturnValue.py new file mode 100644 index 0000000..fb67e36 --- /dev/null +++ b/src/core/ReturnValue.py @@ -0,0 +1,32 @@ +from typing import Any + +from common.utils import compute_hash + + +class ReturnValue: + """ + This class represents the result of a data flow processing + """ + + def __init__(self, who: str = None, status: bool = None, value: Any = None, parents: list = None): + self.who = who + self.status = status + self.value = value + self.parents = parents + + def __repr__(self): + return f"ReturnValue(who={self.who}, status={self.status}, value={self.value})" + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, ReturnValue): + return False + + return self.who == other.who and \ + self.status == other.status and \ + self.value == other.value + + def __hash__(self): + return hash((self.who, self.status, compute_hash(self.value))) diff --git a/src/core/Sheerka.py b/src/core/Sheerka.py index d100674..8c54413 100644 --- a/src/core/Sheerka.py +++ b/src/core/Sheerka.py @@ -1,2 +1,324 @@ +import inspect +import logging +import sys +from dataclasses import dataclass +from operator import attrgetter +from os import path +from typing import Callable + +from caching.Cache import Cache +from caching.IncCache import IncCache +from common.utils import get_logger_name, get_sub_classes, import_module_and_sub_module +from core.BuiltinConcepts import BuiltinConcepts +from core.ErrorContext import ErrorContext +from core.Event import Event +from core.ExecutionContext import ContextHint, ExecutionContext, ExecutionContextActions +from core.ReturnValue import ReturnValue +from core.concept import Concept, ConceptMetadata +from ontologies.SheerkaOntologyManager import SheerkaOntologyManager +from server.authentication import User + +EXECUTE_STEPS = [ + ExecutionContextActions.BEFORE_PARSING, + ExecutionContextActions.PARSING, + ExecutionContextActions.AFTER_PARSING, + ExecutionContextActions.BEFORE_EVALUATION, + ExecutionContextActions.EVALUATION, + ExecutionContextActions.AFTER_EVALUATION +] + + +@dataclass +class SheerkaConfig: + """ + After each execution, persist the whole executions flow as a file + This file will be used by the debugger + """ + save_execution_context: bool = True + + +@dataclass +class SheerkaMethod: + """ + Wrapper to sheerka method, to indicate if it's safe to call + """ + name: str + service: str + method: Callable + has_side_effect: bool + + def __repr__(self): + return self.name + + def __hash__(self): + return hash((self.name, self.service)) + + class Sheerka: - pass + OBJECTS_IDS_ENTRY = "Objects_Ids" + CHICKEN_AND_EGG_CONCEPTS_ENTRY = "Chicken_And_Egg_Concepts" + + def __init__(self): + """ + Engine of the so called Sheerka + """ + self.name = "Sheerka" + self.om: SheerkaOntologyManager = None + self.config = SheerkaConfig() + self.during_initialisation = False + self.log = logging.getLogger(get_logger_name(__name__)) + self.init_log = logging.getLogger(get_logger_name("init." + __name__)) + + self.services = {} # sheerka plugins + self.evaluators = {} # cache for evaluators + + self.sheerka_methods = {} + self.methods_with_context = set() # only the names, the method is defined in sheerka_methods + self.global_context_hints = set() + + def bind_service_method(self, service_name, bound_method, can_modify_state, as_name=None, visible=True): + """ + Bind service method to sheerka instance for ease of use ? + :param service_name: + :param bound_method: + :param can_modify_state: Can update the state of Sheerka => can produce side_effect + :param as_name: give another name to the method + :param visible: make the method visible to Sheerka + :return: + """ + if as_name is None: + as_name = bound_method.__name__ + + if visible: + signature = inspect.signature(bound_method) + if len(signature.parameters) > 0 and list(signature.parameters.keys())[0] == "context": + self.methods_with_context.add(as_name) + self.sheerka_methods[as_name] = SheerkaMethod(as_name, service_name, bound_method, can_modify_state) + + setattr(self, bound_method.__name__, bound_method) + + def initialize(self, root_folder: str = None, **kwargs): + """ + Starting Sheerka + Loads the current configuration + Notes that when it's the first time, it also creates the needed working folders + :param root_folder: root configuration folder + :return: ReturnValue(Success or Error) + """ + + if root_folder is None: + root_folder = path.abspath(path.join(path.expanduser("~"), ".sheerka")) + + self.initialize_logging(False, root_folder) + + self.config.save_execution_context = kwargs.get("save_execution_context", self.config.save_execution_context) + + try: + self.init_log.info("Starting Sheerka") + self.during_initialisation = True + # from sheerkapickle.sheerka_handlers import initialize_pickle_handlers + # initialize_pickle_handlers() + + self.om = SheerkaOntologyManager(self, root_folder) + # self.builtin_cache, self.builtin_cache_by_class_name = self.get_builtins_classes_as_dict() + + self.initialize_caching() + self.initialize_evaluators() + self.initialize_services() + # self.initialize_builtin_evaluators() + # self.om.init_subscriptions() + + event = Event("Initializing Sheerka.", user_id=self.name) + self.om.save_event(event) + with ExecutionContext(self.name, + event, + self, + ExecutionContextActions.INIT_SHEERKA, + None, + desc="Initializing Sheerka.") as exec_context: + if self.om.current_sdp().first_time: + self.first_time_initialisation(exec_context) + + self.initialize_services_deferred(exec_context, self.om.current_sdp().first_time) + + res = ReturnValue(self.name, True, self.get_startup_config()) + + exec_context.add_values(return_values=res) + + if self.om.is_dirty(): + self.om.commit(exec_context) + + if self.config.save_execution_context: + self.om.save_execution_context(exec_context, is_admin=True) + + # append the other ontologies if needed + self.om.freeze() + self.initialize_ontologies(exec_context) + + # self.init_log.debug(f"Sheerka successfully initialized") + + except IOError as e: + res = ReturnValue(self.name, False, ErrorContext(self.name, exec_context, e)) + + finally: + self.during_initialisation = False + + return res + + @staticmethod + def initialize_logging(is_debug, root_folder): + if is_debug: + # log_format = "%(asctime)s %(name)s" + log_format = "[%(levelname)s] [%(name)s]" + log_format += " %(message)s" + log_level = logging.DEBUG + else: + log_format = "%(message)s" + log_level = logging.INFO + + logging.basicConfig(format=log_format, level=log_level, handlers=[logging.StreamHandler(sys.stdout)]) + logging.addLevelName(logging.ERROR, f"\033[1;41m%s\033[1;0m{logging.getLevelName(logging.ERROR)}") + + def initialize_ontologies(self, context): + ontologies = self.om.current_sdp().load_ontologies() + if not ontologies: + return + for ontology_name in list(reversed(ontologies))[1:]: + self.om.push_ontology(ontology_name, False) + # self.initialize_services_deferred(context, False) + + def first_time_initialisation(self, context): + pass + # self.record_var(context, self.name, "save_execution_context", self.save_execution_context) + + def initialize_caching(self): + + cache = IncCache().auto_configure(self.OBJECTS_IDS_ENTRY) + self.om.register_cache(self.OBJECTS_IDS_ENTRY, cache) + + cache = Cache().auto_configure(self.CHICKEN_AND_EGG_CONCEPTS_ENTRY) + self.om.register_cache(self.CHICKEN_AND_EGG_CONCEPTS_ENTRY, cache, persist=False) + + def initialize_services(self): + """ + Introspect to find services and bind them + :return: + """ + self.init_log.info("Initializing services") + + import_module_and_sub_module('core.services') + base_class = "core.services.BaseService.BaseService" + services = [service(self) for service in get_sub_classes("core.services", base_class)] + services.sort(key=attrgetter("order")) + for service in services: + if hasattr(service, "initialize"): + service.initialize() + self.services[service.NAME] = service + + self.init_log.info(f"{len(services)} service(s) found.") + + def initialize_services_deferred(self, context, is_first_time): + """ + Initialize part of services that may take some time or that need the execution context + :return: + """ + self.init_log.debug(f"Initializing services (deferred, {is_first_time=})") + for service in self.services.values(): + if hasattr(service, "initialize_deferred"): + service.initialize_deferred(context, is_first_time) + + def initialize_evaluators(self): + self.init_log.info("Initializing evaluators") + + base_class1 = "evaluators.base_evaluator.OneReturnValueEvaluator" + base_class2 = "evaluators.base_evaluator.AllReturnValuesEvaluator" + import_module_and_sub_module('evaluators') + + evaluators = [evaluator() for evaluator in get_sub_classes("evaluators", base_class1)] + \ + [evaluator() for evaluator in get_sub_classes("evaluators", base_class2)] + + self.evaluators = {e.NAME: e for e in evaluators} + self.init_log.info(f"{len(evaluators)} evaluator(s) found.") + + def bind_services_methods(self): + # init methods + # self.bind_service_method(self.name, self.test, False) + # self.bind_service_method(self.name, self.test_using_context, False) + # self.bind_service_method(self.name, self.test_dict, False) + # self.bind_service_method(self.name, self.test_error, False) + # self.bind_service_method(self.name, self.is_sheerka, False) + # self.bind_service_method(self.name, self.objvalue, False) + pass + + def get_startup_config(self): + """ + Return a dictionary with current configuration, used for initialization + :return: + :rtype: + """ + return { + "config": self.config.__dict__ + } + + def publish(self, context, topic, data=None): + """ + To be removed as it must be part of the EventManager service + :param context: + :type context: + :param topic: + :type topic: + :param data: + :type data: + :return: + :rtype: + """ + pass + + def evaluate_user_input(self, command: str, user: User): + self.log.info("Processing '%s' from '%s'", command, user.email) + + event = Event(command, user_id=user.email) + self.om.save_event(event) + with ExecutionContext(user.email, + event, + self, + ExecutionContextActions.EVALUATE_USER_INPUT, + command, + desc=f"Evaluating '{command}'", + global_hints=self.global_context_hints.copy()) as exec_context: + user_input = ReturnValue(self.name, True, self.newn(BuiltinConcepts.USER_INPUT, command=command)) + exec_context.private_hints.add(ContextHint.REDUCE_CONCEPTS) + + # KSI : 2023-04-30 + # Il me manque le execute et toute la classe SheerkaProcessUserInput + exec_context.add_inputs(user_input=user_input) + ret = self.execute(exec_context, [user_input], EXECUTE_STEPS) + exec_context.add_values(return_values=ret) + + if self.om.is_dirty(): + self.om.commit(exec_context) + + return ret + + def isinstance(self, a, b): + """ + Returns true if 'a' is a concept of type 'b' + Note that this function can be moved into ConceptManager + I keep it here for quick access + :param a: + :type a: + :param b: + :type b: + :return: + :rtype: + """ + if not isinstance(a, Concept): + return False + + if isinstance(b, (Concept, ConceptMetadata)): + return a.id == b.id + + if b.startswith("c:#"): + return a.id == b[3:-1] + + return a.key == b diff --git a/src/core/__init__.py b/src/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/concept.py b/src/core/concept.py new file mode 100644 index 0000000..0c86681 --- /dev/null +++ b/src/core/concept.py @@ -0,0 +1,201 @@ +from dataclasses import dataclass + +from common.global_symbols import NotFound, NotInit + + +class ConceptDefaultProps: + """ + Lists metadata that can contains some code + """ + WHERE = "#where#" + PRE = "#pre#" + POST = "#post#" + BODY = "#body#" + RET = "#ret#" + + +DefaultProps = [v for k, v in ConceptDefaultProps.__dict__.items() if not k.startswith("_")] + + +class DefinitionType: + DEFAULT = "Default" + BNF = "Bnf" + + +@dataclass +class ConceptMetadata: + """ + Static information of the Concept + + """ + id: str # unique identifier for a concept. The id will never be modified (but the key can) + name: str + key: str + is_builtin: bool + is_unique: bool + body: str # main method, can also be the value of the concept + where: str # condition to recognize variables in name + pre: str # list of preconditions before calling the main function + post: str # list of post conditions after calling the main function + ret: str # variable to return when a concept is recognized + definition: str # regex used to define the concept + definition_type: DefinitionType # definition can be done with something else than regex + desc: str # possible description for the concept + autouse: bool # indicates if eval must be automatically called on the concept once validated + bound_body: str # which property must be considered have default value for the concept + props: dict # hashmap of properties, values + variables: tuple # list of concept variables(tuple), with their default values + parameters: tuple # list of variables that are part of the name of the concept + digest: str = None + all_attrs: tuple = None + + +@dataclass +class ConceptRuntimeInfo: + """ + Dynamic information of the Concept + They are related to the instance of the concept + """ + is_evaluated: bool = False # True is the concept is evaluated by sheerka.eval_concept() + need_validation: bool = False # True if the properties of the concept need to be validated + recognized_by: str = None # RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME, RECOGNIZED_BY_KEY (from Sheerka.py) + + def copy(self): + return ConceptRuntimeInfo(self.is_evaluated, + self.need_validation, + self.recognized_by) + + +class Concept: + """ + Default concept object + A concept is the base object of our universe + Everything is a concept + """ + + def __init__(self, metadata: ConceptMetadata): + + self._metadata: ConceptMetadata = metadata + self._compiled = {} # cached ast for the where, pre, post and body parts and variables + self._compiled_context_hints = {} # context hints to use when evaluating compiled + self._bnf = None # compiled bnf expression + self._runtime_info = ConceptRuntimeInfo() # runtime settings for the concept + self._all_attrs = None + + def __repr__(self): + text = f"({self._metadata.id}){self._metadata.name}" + if self._metadata.pre: + text += f", #pre={self._metadata.pre}" + + for attr in [attr for attr in self.all_attrs() if not attr.startswith("#")]: + text += f", {attr}={self.get_value(attr)}" + + return text + + def __eq__(self, other): + # I don't want this test to be part of the recursion + # So let's just get rif ogf it + if not isinstance(other, Concept): + return False + + # I chose to use an iterative algorithm in order to be able to spot circular reference + # without inner functions. + # I also think that it's a better approach for a function that can be massively called + + stack = [self, other] + id_self = id(self) + + while stack: + right = stack.pop() + left = stack.pop() + + if id(left) == id(right): + return True + + # 1. in order for two concepts to be equal, they must have the same definition + # 2. They must have the same properties and variables + if left.get_definition_digest() != right.get_definition_digest(): + return False + + if left.all_attrs() != right.all_attrs(): + return False + + for attr in left.all_attrs(): + value = left.get_value(attr) + other_value = right.get_value(attr) + + if isinstance(value, Concept) and isinstance(other_value, Concept): + if id(value) == id_self or id(other_value) == id_self: + # infinite recursion detected + pass + else: + stack.extend([value, other_value]) + else: + if value != other_value: + return False + + return True + + def __hash__(self): + return self._metadata.digest + + @property + def id(self): + return self._metadata.id + + @property + def name(self): + return self._metadata.name + + @property + def key(self): + return self._metadata.key + + @property + def body(self): + return self.get_value(ConceptDefaultProps.BODY) + + @property + def str_id(self): + return f"c:#{self.id}:" if self.id else f"c:{self.name}:" + + def get_definition_digest(self): + return self._metadata.digest + + def all_attrs(self): + if self._all_attrs is None: + return self._metadata.all_attrs + + return self._all_attrs + + def get_metadata(self) -> ConceptMetadata: + return self._metadata + + def set_value(self, name: str, value: object): + """ + Set the resolved value of a metadata or a variable (not the metadata itself) + :param name: + :param value: + :return: + """ + setattr(self, name, value) + if name == self._metadata.bound_body: + setattr(self, ConceptDefaultProps.BODY, value) + elif self._metadata.bound_body and name == ConceptDefaultProps.BODY: + setattr(self, self._metadata.bound_body, value) + + return self + + def get_value(self, name: str): + """ + Gets the resolved value of a metadata + :param name: + :return: + """ + try: + return getattr(self, name) + except AttributeError: + return NotInit if name in self.all_attrs() else NotFound + + def get_runtime_info(self): + return self._runtime_info diff --git a/src/core/global_symbols.py b/src/core/global_symbols.py deleted file mode 100644 index 89631df..0000000 --- a/src/core/global_symbols.py +++ /dev/null @@ -1,31 +0,0 @@ -class CustomType: - """ - Base class for custom types used in Sheerka - A custom type is a type that has only one instance across the application and have a semantic meaning - For example the type 'None' is a singleton which have a semantic meaning. - We need to define others in Sheerka - """ - - def __init__(self, value): - self.value = value - - def __repr__(self): - return self.value - - def __eq__(self, other): - return isinstance(other, CustomType) and self.value == other.value - - def __hash__(self): - return hash(self.value) - - -class NotFoundType(CustomType): - """ - Using when an entry in not found in Cache or in sdp - """ - - def __init__(self): - super(NotFoundType, self).__init__("**NotFound**") - - -NotFound = NotFoundType() diff --git a/src/core/services/BaseService.py b/src/core/services/BaseService.py new file mode 100644 index 0000000..230c95d --- /dev/null +++ b/src/core/services/BaseService.py @@ -0,0 +1,49 @@ +from common.global_symbols import NotFound +from common.utils import sheerka_deepcopy +from core.Sheerka import Sheerka + + +class BaseService: + """ + Base class for services + """ + + def __init__(self, sheerka: Sheerka, order=999): + self.sheerka = sheerka + self.order = order # initialisation order. The lowest is initialized first + + def initialize(self): + """ + Adds cache or bind methods + :return: + """ + pass + + def state_properties(self): + pass + + def push_state(self, context): + """ + Use variable Manager to store the state of the service + """ + args = self.state_properties() + if args: + for prop_name in args: + self.sheerka.record_var(context, self.NAME, prop_name, sheerka_deepcopy(getattr(self, prop_name))) + + def pop_state(self): + """ + Use Variable Manager to restore the state of a service + :return: + """ + args = self.state_properties() + if args: + for prop_name in args: + if (value := self.sheerka.load_var(self.NAME, prop_name)) is not NotFound: + setattr(self, prop_name, value) + + def store_var(self, context, var_name): + """ + Store/record the value of an attribute + """ + self.sheerka.record_var(context, self.NAME, var_name, getattr(self, var_name)) diff --git a/src/core/services/SheerkaConceptManager.py b/src/core/services/SheerkaConceptManager.py new file mode 100644 index 0000000..76ee7a1 --- /dev/null +++ b/src/core/services/SheerkaConceptManager.py @@ -0,0 +1,343 @@ +import hashlib +import logging +from dataclasses import dataclass + +from caching.Cache import Cache +from caching.FastCache import FastCache +from caching.ListIfNeededCache import ListIfNeededCache +from common.global_symbols import NotFound, NotInit, VARIABLE_PREFIX +from common.utils import get_logger_name +from core.BuiltinConcepts import BuiltinConcepts +from core.ErrorContext import ErrorContext, SheerkaException +from core.ExecutionContext import ExecutionContext +from core.ReturnValue import ReturnValue +from core.concept import Concept, ConceptMetadata, DefaultProps, DefinitionType +from core.services.BaseService import BaseService +from parsers.tokenizer import TokenKind, Tokenizer, strip_tokens + +PROPERTIES_FOR_DIGEST = ("name", "key", + "definition", "definition_type", + "is_builtin", "is_unique", + "where", "pre", "post", "body", "ret", + "desc", "bound_body", "autouse", "props", "variables", "parameters") + + +@dataclass +class ConceptAlreadyDefined(SheerkaException): + concept: ConceptMetadata + already_defined_id: str + + +@dataclass +class InvalidBnf(SheerkaException): + bnf: str + + +@dataclass +class FirstItemError(SheerkaException): + pass + + +class ConceptManager(BaseService): + """ + The service is used for the administration of concepts + You can define new concept, modify or delete them + + There are also function to help retrieve them easily (like first token cache) + Already instantiated concepts are managed by the Memory service + """ + + NAME = "ConceptManager" + + USER_CONCEPTS_IDS = "User_Concepts_IDs" # incremented everytime a new concept is created + CONCEPTS_BY_ID_ENTRY = "ConceptManager:Concepts_By_ID" # to store all the concepts + CONCEPTS_BY_KEY_ENTRY = "ConceptManager:Concepts_By_Key" + CONCEPTS_BY_NAME_ENTRY = "ConceptManager:Concepts_By_Name" + CONCEPTS_BY_HASH_ENTRY = "ConceptManager:Concepts_By_Hash" # sto + + def __init__(self, sheerka): + super().__init__(sheerka, order=11) + + self.log = logging.getLogger(get_logger_name(__name__)) + self.init_log = logging.getLogger(get_logger_name("init." + __name__)) + self.bnf_expr_cache = FastCache() + + def initialize(self): + self.init_log.debug(f"Initializing ConceptManager, order={self.order}") + self.sheerka.bind_service_method(self.NAME, self.define_new_concept, True) + self.sheerka.bind_service_method(self.NAME, self.newn, True) + self.sheerka.bind_service_method(self.NAME, self.newi, True) + + register_concept_cache = self.sheerka.om.register_concept_cache + + # Cache of concept metadata, organized by id + cache = Cache().auto_configure(self.CONCEPTS_BY_ID_ENTRY) + register_concept_cache(self.CONCEPTS_BY_ID_ENTRY, cache, lambda c: c.id, True) + + cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_KEY_ENTRY) + register_concept_cache(self.CONCEPTS_BY_KEY_ENTRY, cache, lambda c: c.key, True) + + cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_NAME_ENTRY) + register_concept_cache(self.CONCEPTS_BY_NAME_ENTRY, cache, lambda c: c.name, True) + + cache = ListIfNeededCache().auto_configure(self.CONCEPTS_BY_HASH_ENTRY) + register_concept_cache(self.CONCEPTS_BY_HASH_ENTRY, cache, lambda c: c.digest, True) + + def initialize_deferred(self, context, is_first_time): + if is_first_time: + self.sheerka.om.put(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS, 1000) + + _ = self._create_builtin_concept + _(1, BuiltinConcepts.SHEERKA, desc="Sheerka") + _(2, BuiltinConcepts.NEW_CONCEPT, desc="On new concept creation", variables=("metadata",)) + _(3, BuiltinConcepts.UNKNOWN_CONCEPT, desc="Unknown concept", variables=("requested_name", "requested_id")) + _(4, BuiltinConcepts.USER_INPUT, desc="Any external input", variables=("command",)) + _(5, BuiltinConcepts.PARSER_INPUT, desc="tokenized input", variables=("pi",)) + + self.init_log.debug('%s builtin concepts created', + len(self.sheerka.om.current_cache_manager().concept_caches)) + + def define_new_concept(self, context: ExecutionContext, + name: str, + is_builtin: bool = False, # is the concept defined Sheerka + is_unique: bool = False, # is the concept a singleton + body: str = "", # return value of the concept + where: str = "", # condition to recognize variables in name + pre: str = "", # list of preconditions before calling the main function + post: str = "", # list of post conditions after calling the main function + ret: str = "", # variable to return when a concept is recognized + definition: str = "", # regex used to define the concept + definition_type: DefinitionType = DefinitionType.DEFAULT, + autouse: bool = False, # indicate if the concept must be automatically evaluated + bound_body: str = None, # + desc: str = "", # possible description for the concept + props: dict = None, # hashmap of default properties + variables: list = None, # list of concept variables(tuple), with their default values + parameters: list = None # list of variables that are part of the name of the concept + ) -> ReturnValue: + """ + Adds the definition of a new concept + :return: + :rtype: + """ + concept_key = self.create_concept_key(name, definition, variables) + concept_id = "waiting for id" + + metadata = ConceptMetadata( + concept_id, + name, + concept_key, + is_builtin, + is_unique, + body, + where, + pre, + post, + ret, + definition, + definition_type, + desc, + autouse, + bound_body, + props or {}, + variables or (), + parameters or (), + ) + + digest = self.compute_metadata_digest(metadata) + if self.sheerka.om.exists_in_current(self.CONCEPTS_BY_HASH_ENTRY, digest): + already_defined = self.sheerka.om.get(self.CONCEPTS_BY_HASH_ENTRY, digest) + error = ErrorContext(self.NAME, context, ConceptAlreadyDefined(metadata, already_defined.id)) + return ReturnValue(self.NAME, False, error) + + metadata.digest = digest + metadata.all_attrs = self.compute_all_attrs(variables) + + # bnf_expr = None + # if definition_type == DefinitionType.BNF: + # try: + # bnf_expr = self.compute_concept_bnf(definition) + # except InvalidBnf as ex: + # error = ErrorContext(self.NAME, context, ex) + # return ReturnValue(self.NAME, False, error) + + # try: + # first_item_res = self.recompute_first_items(context, None, [metadata]) + # except FirstItemError as ex: + # return ReturnValue(self.NAME, False, ex) + + # at this point everything is fine. let's get the id and save everything + om = self.sheerka.om + metadata.id = str(self.sheerka.om.get(self.sheerka.OBJECTS_IDS_ENTRY, self.USER_CONCEPTS_IDS)) + om.add_concept(metadata) + # self.update_first_items_caches(context, first_item_res) + # if bnf_expr: + # self.bnf_expr_cache.put(metadata.id, bnf_expr) + # # update references + # for ref in self.compute_references(bnf_expr): + # om.put(self.CONCEPTS_REFERENCES_ENTRY, ref, metadata.id) + + return ReturnValue(self.NAME, True, self.newn(BuiltinConcepts.NEW_CONCEPT, metadata=metadata)) + + def newn(self, concept_name: str, **kwargs): + """ + new_by_name + Creates and returns an instance of a new concept by its name + :param concept_name: + :type concept_name: + :param kwargs: + :type kwargs: + :return: + :rtype: + """ + metadata = self.get_by_name(concept_name) + if metadata is NotFound: + return self._inner_new(self.get_by_name(BuiltinConcepts.UNKNOWN_CONCEPT), requested_name=concept_name) + + if isinstance(metadata, list): + return [self._inner_new(m, **kwargs) for m in metadata] + + return self._inner_new(metadata, **kwargs) + + def newi(self, concept_id: str, **kwargs): + """ + new_by_id + Creates and returns an instance of a new concept by its id + :param concept_id: + :type concept_id: + :param kwargs: + :type kwargs: + :return: + :rtype: + """ + metadata = self.get_by_id(concept_id) + if metadata is NotFound: + return self._inner_new(self.get_by_name(BuiltinConcepts.UNKNOWN_CONCEPT), requested_id=concept_id) + return self._inner_new(metadata, **kwargs) + + def get_by_name(self, key: str): + """ + Returns a concept metadata, using its name + :param key: + :type key: + :return: + :rtype: + """ + return self.sheerka.om.get(self.CONCEPTS_BY_NAME_ENTRY, key) + + def get_by_id(self, concept_id: str): + """ + Returns a concept metadata, using its name + :param concept_id: + :type concept_id: + :return: + :rtype: + """ + return self.sheerka.om.get(self.CONCEPTS_BY_ID_ENTRY, concept_id) + + def get_by_key(self, key: str): + """ + Returns a concept metadata, using its name + :param key: + :type key: + :return: + :rtype: + """ + return self.sheerka.om.get(self.CONCEPTS_BY_KEY_ENTRY, key) + + @staticmethod + def compute_metadata_digest(metadata: ConceptMetadata): + """ + Compute once for all the digest of the definition of a concept + :param metadata: + :type metadata: + :return: + :rtype: + """ + as_dict = {p: getattr(metadata, p) for p in PROPERTIES_FOR_DIGEST} + return hashlib.sha256(f"{as_dict}".encode("utf-8")).hexdigest() + + @staticmethod + def compute_all_attrs(variables: tuple | None): + """ + Compute the list of available attributes for a concept + :param variables: + :return: + :rtype: + """ + all_attrs = DefaultProps.copy() + if variables: + all_attrs += [k for k, v in variables] + + return tuple(all_attrs) + + @staticmethod + def compute_concept_bnf(definition): + pass + + @staticmethod + def create_concept_key(name: str, definition: str | None, variables: tuple | None): + """ + Creates the key from the definition + :param name: + :type name: + :param definition: + :type definition: + :param variables: + :type variables: + :return: + :rtype: + """ + + definition_to_use = definition or name + tokens = list(Tokenizer(definition_to_use, yield_eof=False)) + + if variables is None or len(strip_tokens(tokens, True)) == 1: + variables_to_use = [] + else: + variables_to_use = [k for k, v in variables] + + parts = [] + for token in tokens: + if token.type == TokenKind.WHITESPACE: + continue + if token.value in variables_to_use: + parts.append(VARIABLE_PREFIX + str(variables_to_use.index(token.value))) + else: + parts.append(token.value) + + return " ".join(parts) + + def _create_builtin_concept(self, concept_id: int, name: str, desc: str, variables: tuple = ()): + variables_to_use = tuple((k, NotInit) for k in variables) + concept_key = self.create_concept_key(name, None, variables_to_use) + metadata = ConceptMetadata( + str(concept_id), + name, + concept_key, + True, + False, + "", + "", + "", + "", + "", + "", + DefinitionType.DEFAULT, + desc, + False, + variables[0] if variables else "", + {}, + variables_to_use, + variables, + ) + metadata.digest = self.compute_metadata_digest(metadata) + metadata.all_attrs = self.compute_all_attrs(variables_to_use) + self.sheerka.om.add_concept(metadata) + + @staticmethod + def _inner_new(_metadata_def: ConceptMetadata, **kwargs): + concept = Concept(_metadata_def) + for k, v in kwargs.items(): + concept.set_value(k, v) + return concept diff --git a/src/core/services/SheerkaEngine.py b/src/core/services/SheerkaEngine.py new file mode 100644 index 0000000..7a745c0 --- /dev/null +++ b/src/core/services/SheerkaEngine.py @@ -0,0 +1,223 @@ +from dataclasses import dataclass + +from common.utils import to_dict +from core.ExecutionContext import ExecutionContext, ExecutionContextActions +from core.ReturnValue import ReturnValue +from core.services.BaseService import BaseService +from evaluators.base_evaluator import AllReturnValuesEvaluator, BaseEvaluator, OneReturnValueEvaluator + + +@dataclass +class EvaluationPlan: + sorted_priorities: list[int] # list of available priorities + evaluators: dict[int, list[BaseEvaluator]] + + +class SheerkaEngine(BaseService): + """ + This service is used to process user input + It is responsible to parse and evaluate the information + It also holds the rule engine + """ + NAME = "Engine" + + def __init__(self, sheerka): + super().__init__(sheerka, order=15) + self.execution_plan = None # { ExecutionContextActions : { priority : [evaluators] }} + self.no_evaluation_plan = EvaluationPlan([], {}) + + def initialize(self): + self.execution_plan = self.compute_execution_plan(self.sheerka.evaluators.values()) + self.sheerka.bind_service_method(self.NAME, self.execute, True) + + def call_evaluators(self, + context: ExecutionContext, + return_values: list[ReturnValue], + step: ExecutionContextActions): + """ + Calls all evaluators defined for a given step + :param context: + :type context: + :param return_values: + :type return_values: + :param step: + :type step: + :return: + :rtype: + """ + + plan = self.get_evaluation_plan(context, step) + iteration = 0 + while True: + with context.push(self.NAME, + ExecutionContextActions.EVALUATING_ITERATION, + {"step": step, "iteration": iteration}, + desc=f"iteration #{iteration}") as iteration_context: + simple_digest = return_values.copy() + iteration_context.add_inputs(return_values=simple_digest) + + for priority in plan.sorted_priorities: + + return_values_copy = return_values.copy() + new_return_values = {} + return_values_to_delete = set() + for evaluator in plan.evaluators[priority]: + + sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})" + with iteration_context.push(self.NAME, + step, + {"step": step, + "iteration": iteration, + "evaluator": evaluator.name}, + desc=sub_context_desc) as evaluator_context: + evaluator_context.add_inputs(return_values=return_values_copy) + + # process evaluators that work on one simple return value at the time + if isinstance(evaluator, OneReturnValueEvaluator): + self.call_one_return_value_evaluator(evaluator_context, + evaluator, + return_values_copy, + new_return_values, + return_values_to_delete) + + # process evaluators that work on all return values + else: + self.call_all_return_values_evaluator(evaluator_context, + evaluator, + return_values_copy, + new_return_values, + return_values_to_delete) + + # Recreate the new return_value + # Try to keep the order of what replaces what + return_values = [] + for item in return_values_copy: + if item not in return_values_to_delete: + return_values.append(item) + if item in new_return_values: + return_values.extend(new_return_values[item]) + + iteration_context.add_values(return_values=return_values.copy()) + + iteration += 1 + if simple_digest == return_values: + # I can use a variable like 'has_changed', but I think that this comparison is explicit + # It explains that I stay in the loop if something was modified + break + + return return_values + + def execute(self, + context: ExecutionContext, + return_values: list[ReturnValue], + steps: list[ExecutionContextActions]): + """ + Runs the processing engine on the return_values + :param context: + :type context: + :param return_values: + :type return_values: + :param steps: + :type steps: + :return: + :rtype: + """ + for step in steps: + copy = return_values.copy() + with context.push(self.NAME, ExecutionContextActions.EVALUATING_STEP, {"step": step}) as sub_context: + sub_context.add_inputs(return_values=copy) + + return_values = self.call_evaluators(sub_context, return_values, step) + + sub_context.add_values(return_values=return_values) + sub_context.add_values(has_changed=(copy != return_values)) + + return return_values + + def get_evaluation_plan(self, context: ExecutionContext, step: ExecutionContextActions) -> EvaluationPlan: + if step not in self.execution_plan: + return self.no_evaluation_plan + + evaluators = self.execution_plan[step] + return EvaluationPlan(sorted(evaluators.keys(), reverse=True), evaluators) + + @staticmethod + def call_one_return_value_evaluator(context: ExecutionContext, + evaluator: OneReturnValueEvaluator, + return_values: list[ReturnValue], + new_return_values: dict[ReturnValue, list[ReturnValue]], + return_values_to_delete: set[ReturnValue]): + """ + + :param context: + :type context: + :param evaluator: + :type evaluator: + :param return_values: + :type return_values: + :param new_return_values: + :type new_return_values: + :param return_values_to_delete: + :type return_values_to_delete: + :return: + :rtype: + """ + context_trace = [] + for item in return_values: + debug = {"item": item} + context_trace.append(debug) + + m = evaluator.matches(context, item) + debug["match"] = m.status + + if m.status: + result = evaluator.eval(context, m.obj, item) + return_values_to_delete.update(result.eaten) + new_return_values.setdefault(item, []).extend(result.new) + debug["new"] = result.new + debug["eaten"] = result.eaten + + context.add_values(evaluation=context_trace) + + @staticmethod + def call_all_return_values_evaluator(context: ExecutionContext, + evaluator: AllReturnValuesEvaluator, + return_values: list[ReturnValue], + new_return_values: dict[ReturnValue, list[ReturnValue]], + return_values_to_delete: set[ReturnValue]): + """ + + :param context: + :type context: + :param evaluator: + :type evaluator: + :param return_values: + :type return_values: + :param new_return_values: + :type new_return_values: + :param return_values_to_delete: + :type return_values_to_delete: + :return: + :rtype: + """ + debug = {} + m = evaluator.matches(context, return_values) + debug["match"] = m.status + + if m.status: + result = evaluator.eval(context, m.obj, return_values) + return_values_to_delete.update(result.eaten) + new_return_values.setdefault(result.new[0].parents[0], []).extend(result.new) + debug["new"] = result.new + debug["eaten"] = result.eaten + + context.add_values(evaluation=debug) + + @staticmethod + def compute_execution_plan(evaluators): + evaluators = [e for e in evaluators if e.enabled] + by_step = to_dict(evaluators, lambda e: e.step) + for k, v in by_step.items(): + by_step[k] = to_dict(v, lambda e: e.priority) + + return by_step diff --git a/src/core/services/__init__.py b/src/core/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/utils.py b/src/core/utils.py deleted file mode 100644 index 6e32806..0000000 --- a/src/core/utils.py +++ /dev/null @@ -1,32 +0,0 @@ -def get_class(qname): - """ - Loads a class from its full qualified name - :param qname: - :return: - """ - parts = qname.split('.') - module = ".".join(parts[:-1]) - m = __import__(module) - for comp in parts[1:]: - m = getattr(m, comp) - return m - - -def get_full_qualified_name(obj): - """ - Returns the full qualified name of a class (including its module name ) - :param obj: - :return: - """ - if obj.__class__ == type: - module = obj.__module__ - if module is None or module == str.__class__.__module__: - return obj.__name__ # Avoid reporting __builtin__ - else: - return module + '.' + obj.__name__ - else: - module = obj.__class__.__module__ - if module is None or module == str.__class__.__module__: - return obj.__class__.__name__ # Avoid reporting __builtin__ - else: - return module + '.' + obj.__class__.__name__ diff --git a/src/evaluators/CreateParserInput.py b/src/evaluators/CreateParserInput.py new file mode 100644 index 0000000..c9ec92f --- /dev/null +++ b/src/evaluators/CreateParserInput.py @@ -0,0 +1,30 @@ +from core.BuiltinConcepts import BuiltinConcepts +from core.ErrorContext import ErrorContext +from core.ExecutionContext import ExecutionContext, ExecutionContextActions +from core.ReturnValue import ReturnValue +from evaluators.base_evaluator import EvaluatorEvalResult, EvaluatorMatchResult, OneReturnValueEvaluator +from parsers.ParserInput import ParserInput + + +class CreateParserInput(OneReturnValueEvaluator): + NAME = "CreateParserInput" + + def __init__(self): + super().__init__(self.NAME, ExecutionContextActions.BEFORE_EVALUATION, 50) + + def matches(self, context: ExecutionContext, return_value: ReturnValue) -> EvaluatorMatchResult: + if return_value.status and \ + context.sheerka.isinstance(return_value.value, BuiltinConcepts.USER_INPUT): + return EvaluatorMatchResult(True) + return EvaluatorMatchResult(False) + + def eval(self, context, evaluator_context, return_value): + parser_input = ParserInput(return_value.value.get_value("command")) + if parser_input.init(): + parser_input_concept = context.sheerka.newn(BuiltinConcepts.PARSER_INPUT, pi=parser_input) + new_ret_val = ReturnValue(self.NAME, True, parser_input_concept, parents=[return_value]) + return EvaluatorEvalResult([new_ret_val], [return_value]) + else: + error = ErrorContext(self.NAME, context, parser_input) + new_ret_val = ReturnValue(self.NAME, False, error, parents=[return_value]) + return EvaluatorEvalResult([new_ret_val], [return_value]) diff --git a/src/evaluators/__init__.py b/src/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/evaluators/base_evaluator.py b/src/evaluators/base_evaluator.py new file mode 100644 index 0000000..e26961a --- /dev/null +++ b/src/evaluators/base_evaluator.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass + +from core.ExecutionContext import ExecutionContext, ExecutionContextActions +from core.ReturnValue import ReturnValue + + +@dataclass +class EvaluatorMatchResult: + status: bool + obj: object = None + + +@dataclass +class EvaluatorEvalResult: + new: list[ReturnValue] = None + eaten: list[ReturnValue] = None + + +class BaseEvaluator: + """ + Base class to evaluate ReturnValues + """ + + def __init__(self, name, step: ExecutionContextActions, priority: int, enabled=True): + self.name = name + self.step = step + self.priority = priority + self.enabled = enabled + + def __repr__(self): + return f"{self.name} ({self.priority})" + + def __eq__(self, other): + if not isinstance(other, BaseEvaluator): + return False + + return self.name == other.name and \ + self.priority == other.priority and \ + self.step == other.step and \ + self.enabled == other.enabled + + def __hash__(self): + return hash((self.name, self.priority, self.step, self.enabled)) + + +class OneReturnValueEvaluator(BaseEvaluator): + """ + Evaluate one specific return value + """ + + def matches(self, context: ExecutionContext, + return_value: ReturnValue) -> EvaluatorMatchResult: + pass + + def eval(self, context: ExecutionContext, + evaluation_context: object, + return_value: ReturnValue) -> EvaluatorEvalResult: + pass + + +class AllReturnValuesEvaluator(BaseEvaluator): + """ + Evaluates the groups of ReturnValues + """ + + def matches(self, context: ExecutionContext, + return_values: list[ReturnValue]) -> EvaluatorMatchResult: + pass + + def eval(self, context: ExecutionContext, + evaluation_context: object, + return_values: list[ReturnValue]) -> EvaluatorEvalResult: + pass diff --git a/src/ontologies/Exceptions.py b/src/ontologies/Exceptions.py new file mode 100644 index 0000000..e8bd787 --- /dev/null +++ b/src/ontologies/Exceptions.py @@ -0,0 +1,37 @@ +class OntologyManagerFrozen(Exception): + """ + Raised when you try to add a cache manager while the ontology manager is frozen + """ + pass + + +class OntologyManagerNotFrozen(Exception): + """ + Raised when you try to push or pop a cache manager while the ontology manager is not frozen + """ + pass + + +class OntologyManagerCannotPopLatest(Exception): + """ + Raised when you try pop the latest cache manager + """ + pass + + +class OntologyAlreadyExists(Exception): + """ + When the ontology exists AND is not the top layer + """ + + def __init__(self, name): + self.name = name + + +class OntologyNotFound(Exception): + """ + During revert(), when the requested ontology does not exist + """ + + def __init__(self, ontology): + self.ontology = ontology diff --git a/src/ontologies/SheerkaOntologyManager.py b/src/ontologies/SheerkaOntologyManager.py new file mode 100644 index 0000000..7bd5d66 --- /dev/null +++ b/src/ontologies/SheerkaOntologyManager.py @@ -0,0 +1,534 @@ +from caching.Cache import Cache +from caching.CacheManager import CacheManager +from caching.DictionaryCache import DictionaryCache +from caching.SetCache import SetCache +from common.global_symbols import EVENT_CONCEPT_ID_DELETED, \ + EVENT_RULE_ID_DELETED, NotFound, \ + Removed +from ontologies.Exceptions import OntologyAlreadyExists, OntologyManagerCannotPopLatest, OntologyManagerFrozen, \ + OntologyManagerNotFrozen, OntologyNotFound +from sdp.sheerkaDataProvider import SheerkaDataProvider + + +class AlternateSdp: + """ + Other Cache managers that are linked together + """ + + def __init__(self, ontologies): + self.names = [o.name for o in ontologies] + self.cache_managers = [o.cache_manager for o in ontologies] + + def get(self, cache_name: str, key: str): + last = len(self.cache_managers) - 1 + for i, cache_manager in enumerate(self.cache_managers): + value = cache_manager.get(cache_name, key) + if value is not NotFound: + return value + + if i != last: + # forget than the key was requested + cache_manager.remove_initialized_key(cache_name, key) + + return NotFound + + def alt_get(self, cache_name: str, key: str): + last = len(self.cache_managers) - 1 + for i, cache_manager in enumerate(self.cache_managers): + value = cache_manager.alt_get(cache_name, key) + if value is not NotFound: + return value + + if i != last: + # forget than the key was requested + cache_manager.remove_initialized_key(cache_name, key) + + return NotFound + + def exists(self, cache_name: str, key: str): + for cache_manager in self.cache_managers: + if cache_manager.exists(cache_name, key): + return True + return False + + +class Ontology: + """ + an Ontology is + * a CacheManager (which is linked to a sdp) + * and a way to get to the next Ontologies (as AlternateSdp) + """ + + def __init__(self, name, depth, cache_manager: CacheManager, alt_sdp: AlternateSdp | None): + self.name = name + self.depth = depth + self.cache_manager = cache_manager + self.alt_sdp = alt_sdp + self.concepts_attributes = None + + def __repr__(self): + return f"Ontology('{self.name}')" + + def __eq__(self, other): + if not isinstance(other, Ontology): + return False + + return self.name == other.name and self.depth == other.depth + + def __hash__(self): + return hash((self.name, self.depth)) + + +class SheerkaOntologyManager: + ROOT_ONTOLOGY_NAME = "__default__" + SELF_CACHE_MANAGER = "__ontology_manager__" # cache to store SheerkaOntologyManager info + CONCEPTS_BY_ONTOLOGY_ENTRY = "ConceptsByOntologyEntry" + RULES_BY_ONTOLOGY_ENTRY = "RulesByOntologyEntry" + ONTOLOGY_BY_CONCEPT_ENTRY = "OntologyByConceptEntry" + ONTOLOGY_BY_RULE_ENTRY = "OntologyByRuleEntry" + + def __init__(self, sheerka, root_folder): + self.sheerka = sheerka + self.root_folder = root_folder + self.frozen = False + + # We create the first cache manager + # For the time being, this there is no AltSdp yet ! + ref_cache_manager = CacheManager(sdp=SheerkaDataProvider(root_folder, self.sheerka)) + self.ontologies = [Ontology(self.ROOT_ONTOLOGY_NAME, 0, ref_cache_manager, None)] + + # Ontology manager also needs to keep track of its own stuff + # So we create a separate sdp + internal_sdp = SheerkaDataProvider(root_folder, self.sheerka, self.SELF_CACHE_MANAGER) + self.internal_cache_manager = CacheManager(sdp=internal_sdp) + + # add cache to track all the concepts defined under a given ontology + # key : name of the ontology + # value : sets of all concepts id + cache = SetCache(max_size=None).auto_configure(self.CONCEPTS_BY_ONTOLOGY_ENTRY) + self.internal_cache_manager.register_cache(self.CONCEPTS_BY_ONTOLOGY_ENTRY, cache) + + # add cache to track all the rules defined under a given ontology + # key : name of the ontology + # value : sets of all rules id + cache = SetCache(max_size=None).auto_configure(self.RULES_BY_ONTOLOGY_ENTRY) + self.internal_cache_manager.register_cache(self.RULES_BY_ONTOLOGY_ENTRY, cache) + + # add cache to track where the concept is created (under which ontology) + # key : concept id + # value : name of the ontology + cache = Cache(max_size=None).auto_configure(self.ONTOLOGY_BY_CONCEPT_ENTRY) + self.internal_cache_manager.register_cache(self.ONTOLOGY_BY_CONCEPT_ENTRY, cache) + + # add cache to track where the rule is created (under which ontology) + # key : rule id + # value : name of the ontology + cache = Cache(max_size=None).auto_configure(self.ONTOLOGY_BY_RULE_ENTRY) + self.internal_cache_manager.register_cache(self.ONTOLOGY_BY_RULE_ENTRY, cache) + + @property + def ontologies_names(self): + return [o.name for o in self.ontologies] + + def freeze(self): + """ + Once frozen, you can no longer register cache + It's to ensure consistency between ontologies + + The same caches must exist for all ontologies + if an information is not found in a cache manager, we can request to the parent ontology. + This is possible only if the same cache exists in the parent ontology. + + To ensure that. + 1. You `register_cache()` all the caches + 2. You call `freeze()` + + You won't be able to `add` or `push` new ontologies if the `freeze()` is not called + :return: + :rtype: + """ + self.frozen = True + return self + + def push_ontology(self, name) -> Ontology: + """ + Add an ontology layer + :param name: name of the layer + :param cache_only: + """ + if not self.frozen: + raise OntologyManagerNotFrozen() + + # pseudo clone cache manager + cache_manager = CacheManager(sdp=self.get_sdp(name)) + for cache_name, cache_def in self.current_cache_manager().caches.items(): + clone = cache_def.cache.clone() + if cache_name in self.current_cache_manager().concept_caches: + cache_manager.register_concept_cache(cache_name, clone, cache_def.get_key, cache_def.use_ref) + else: + cache_manager.register_cache(cache_name, clone, cache_def.persist, cache_def.use_ref) + + # Dictionary cache special treatment + if isinstance(clone, DictionaryCache): + clone.put(False, cache_def.cache.copy()) # only a shadow copy for now + clone.reset_events() + + alt_sdp = AlternateSdp(self.ontologies) + new_ontology = Ontology(name, len(self.ontologies), cache_manager, alt_sdp) + self.ontologies.insert(0, new_ontology) + return new_ontology + + def pop_ontology(self, context): + """ + Remove the top ontology layer + """ + if not self.frozen: + raise OntologyManagerNotFrozen() + + if len(self.ontologies) == 1: + raise OntologyManagerCannotPopLatest() + + # remove concepts and rules tracking for the ontology to pop + ontology_name = self.current_ontology().name + concepts = self.internal_cache_manager.get(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name) + if concepts is not NotFound: + for concept in concepts: + self.sheerka.publish(context, EVENT_CONCEPT_ID_DELETED, concept) + self.internal_cache_manager.delete(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept) + self.internal_cache_manager.delete(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name) + + rules = self.internal_cache_manager.get(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name) + if rules is not NotFound: + for rule in rules: + self.sheerka.publish(context, EVENT_RULE_ID_DELETED, rule) + self.internal_cache_manager.delete(self.ONTOLOGY_BY_RULE_ENTRY, rule) + self.internal_cache_manager.delete(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name) + + return self.ontologies.pop(0) + + def add_ontology(self, ontology: Ontology): + """ + Put back a previously created ontology + :param ontology: how to get the items + """ + if not self.frozen: + raise OntologyManagerNotFrozen() + + ontology.alt_sdp = AlternateSdp(self.ontologies) + self.ontologies.insert(0, ontology) + for cache_def in ontology.cache_manager.caches.values(): + cache_def.cache.reset_initialized_keys() + + return self + + def revert_ontology(self, context, ontology) -> Ontology: + """ + Pop every ontology until the requested one is found. + The requested one is also popped + :param context: + :type context: + :param ontology: + :type ontology: + :return: + :rtype: + """ + if ontology not in self.ontologies: + raise OntologyNotFound(ontology) + + while self.current_ontology() != ontology: + self.pop_ontology(context) + self.pop_ontology(context) + + return self.current_ontology() + + def get_ontology(self, name=None): + """ + Return the first ontology with the corresponding name + When no name is given, return the top ontology + """ + if name is None: + return self.ontologies[0] + + for ontology in self.ontologies: + if ontology.name == name: + return ontology + + raise KeyError(name) + + def save_ontologies_names(self): + self.current_sdp().save_ontologies(self.ontologies_names) + + def already_on_top(self, name): + """ + Returns True if the ontology 'name' is already on the top + Raises a OntologyAlreadyExists exception if the ontology exists, but not at the top + """ + if self.ontologies[0].name == name: + return True + + if name in self.ontologies_names: + raise OntologyAlreadyExists(name) + + return False + + def record_sheerka_state(self): + """ + The current ontology can keep extra information + """ + pass + # # TODO persist these information ? + # self.current_ontology().concepts_attributes = copy_concepts_attrs() + + def reset_sheerka_state(self): + pass + # if self.current_ontology().concepts_attributes is not None: + # load_concepts_attrs(self.current_ontology().concepts_attributes) + + def current_cache_manager(self) -> CacheManager: + return self.ontologies[0].cache_manager + + def current_sdp(self) -> SheerkaDataProvider: + return self.ontologies[0].cache_manager.sdp + + def current_ontology(self) -> Ontology: + return self.ontologies[0] + + def register_concept_cache(self, name, cache, get_key, use_ref): + """ + Define which type of cache along with how to compute the key + :param name: + :param cache: + :param get_key: + :param use_ref: + :return: + """ + if self.frozen: + raise OntologyManagerFrozen + + return self.current_cache_manager().register_concept_cache(name, cache, get_key, use_ref) + + def register_cache(self, name, cache, persist=True, use_ref=False): + """ + Define which type of cache along with how to compute the key + :param name: + :param cache: + :param persist: + :param use_ref: + :return: + """ + if self.frozen: + raise OntologyManagerFrozen + + return self.current_cache_manager().register_cache(name, cache, persist, use_ref) + + def add_concept(self, concept): + """ + We need multiple indexes to retrieve a concept + So the new concept is dispatched into multiple caches + :param concept: + :return: + """ + self.current_cache_manager().add_concept(concept, self.ontologies[0].alt_sdp) + + # update internal states + self._on_concept_created(concept) + + def update_concept(self, old, new): + """ + Update a concept. + :param old: old version of the concept + :param new: new version of the concept + :return: + """ + self.current_cache_manager().update_concept(old, new, self.ontologies[0].alt_sdp) + + # update internal states + self._on_concept_deleted(old) + self._on_concept_created(new) + + def remove_concept(self, concept): + """ + Remove a concept from all caches + :param concept: + :return: + """ + self.current_cache_manager().remove_concept(concept, self.ontologies[0].alt_sdp) + + # update internal states + self._on_concept_deleted(concept) + + def get(self, cache_name, key): + """ + Browses the ontologies, looking for the data 'key' in entry 'cache_name' + If a value is found in a low level cache, updates the top level one + :param cache_name: + :param key: + :return: + """ + value = self.current_cache_manager().get(cache_name, key, self.ontologies[0].alt_sdp) + return NotFound if value is Removed else value + + def exists(self, cache_name, key): + """ + Browses the ontologies to check if the data 'key' is defined in entry 'cache_name' + :param cache_name: + :param key: + :return: + """ + for ontology in self.ontologies: + if ontology.cache_manager.exists(cache_name, key): + return True + + return False + + def exists_in_current(self, cache_name, key): + """ + Checks if the keys exists in the current ontology only + :param cache_name: + :type cache_name: + :param key: + :type key: + :return: + :rtype: + """ + return self.current_cache_manager().exists(cache_name, key) + + def list(self, entry, cache_only=False): + """ + list all entries + """ + return list(self.get_all(entry, cache_only).values()) + + def get_all(self, entry, cache_only=False): + """ + Return all from all ontologies + First look in sdp, then override with the cache, for all ontologies + :param entry: cache name / sdp entry + :param cache_only: Do no fetch data from remote sdp + """ + res = {} + for ontology in reversed(self.ontologies): + + if not cache_only: + # get values from sdp + values = ontology.cache_manager.sdp.get(entry) + if values is Removed: + res.clear() + + elif values is not NotFound: + for k, v in values.items(): + if v is Removed: + del res[k] + else: + res[k] = v + + # override with the values from cache + try: + cache = ontology.cache_manager.get_inner_cache(entry) + + if cache.is_cleared(): + res.clear() + + for k in cache: + v = cache.alt_get(k) # Do not use get(), because of IncCache() + if v is Removed: + del res[k] + else: + res[k] = v + + except KeyError: + pass + + return res + + def put(self, cache_name, key, value): + """ + Add to a cache + :param cache_name: + :param key: + :param value: + :return: + """ + return self.current_cache_manager().put(cache_name, key, value, self.ontologies[0].alt_sdp) + + def delete(self, cache_name, key, value=None): + """ + Delete an entry + :param cache_name: + :param key: + :param value: + :return: + """ + return self.current_cache_manager().delete(cache_name, key, value, self.ontologies[0].alt_sdp) + + def populate(self, cache_name, populate_function, get_key_function, reset_events=False, all_ontologies=False): + """ + Populate a specific cache with a bunch of items + :param cache_name: + :param populate_function: how to get the items + :param get_key_function: how to get the key, out of an item + :param reset_events: reset the to_add and to_remove events after the populate + :param all_ontologies: populate all ontology layers + :return: + """ + self.current_cache_manager().populate(cache_name, populate_function, get_key_function, reset_events) + if all_ontologies: + for ontology in self.ontologies[1:]: + ontology.cache_manager.populate(cache_name, populate_function, get_key_function, reset_events) + + def copy(self, cache_name): + """ + get a copy the content of the top ontology layer + :param self: + :param cache_name: + :return: + """ + return self.current_cache_manager().caches[cache_name].cache.copy() + + def commit(self, context): + """ + Persist all the caches into a physical persistence storage + :param context: + :return: + """ + self.internal_cache_manager.commit(context) + return self.current_cache_manager().commit(context) + + def clear(self, cache_name=None): + return self.current_cache_manager().clear(cache_name) + + def get_sdp(self, name=None): + """ + Return new instance of SheerkaDataProvider + """ + if name: + return SheerkaDataProvider(self.root_folder, self.sheerka, name) + else: + return self.current_sdp() + + def save_event(self, event): + return self.current_sdp().save_event(event) + + def save_execution_context(self, execution_context, is_admin): + return self.current_sdp().save_execution_context(execution_context, is_admin) + + def is_dirty(self): + return self.current_cache_manager().is_dirty + + def _on_concept_created(self, concept): + self.internal_cache_manager.put(self.CONCEPTS_BY_ONTOLOGY_ENTRY, self.current_ontology().name, concept.id) + self.internal_cache_manager.put(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id, self.current_ontology().name) + + def _on_concept_deleted(self, concept): + ontology_name = self.internal_cache_manager.get(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id) + self.internal_cache_manager.delete(self.CONCEPTS_BY_ONTOLOGY_ENTRY, ontology_name, concept.id) + self.internal_cache_manager.delete(self.ONTOLOGY_BY_CONCEPT_ENTRY, concept.id) + + def _on_rule_created(self, rule): + self.internal_cache_manager.put(self.RULES_BY_ONTOLOGY_ENTRY, self.current_ontology().name, rule.id) + self.internal_cache_manager.put(self.ONTOLOGY_BY_RULE_ENTRY, rule.id, self.current_ontology().name) + + def _on_rule_deleted(self, rule): + ontology_name = self.internal_cache_manager.get(self.ONTOLOGY_BY_RULE_ENTRY, rule.id) + self.internal_cache_manager.delete(self.RULES_BY_ONTOLOGY_ENTRY, ontology_name, rule.id) + self.internal_cache_manager.delete(self.ONTOLOGY_BY_RULE_ENTRY, rule.id) diff --git a/src/ontologies/__init__.py b/src/ontologies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/parsers/ParserInput.py b/src/parsers/ParserInput.py new file mode 100644 index 0000000..80216e1 --- /dev/null +++ b/src/parsers/ParserInput.py @@ -0,0 +1,19 @@ +from parsers.tokenizer import Tokenizer + + +class ParserInput: + def __init__(self, text, yield_oef=True): + self.original_text = text + self.yield_oef = yield_oef + self.tokens = None + self.exception = None + + def init(self) -> bool: + try: + # the eof if forced, but will not be yield if not set to. + self.tokens = list(Tokenizer(self.original_text, yield_eof=True)) + return True + except Exception as ex: + self.tokens = None + self.exception = ex + return False diff --git a/src/parsers/__init__.py b/src/parsers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/parsers/tokenizer.py b/src/parsers/tokenizer.py new file mode 100644 index 0000000..0f42c18 --- /dev/null +++ b/src/parsers/tokenizer.py @@ -0,0 +1,569 @@ +from dataclasses import dataclass, field +from enum import Enum + +from common.global_symbols import VARIABLE_PREFIX + + +class TokenKind(Enum): + EOF = "eof" + WHITESPACE = "whitespace" + NEWLINE = "newline" + IDENTIFIER = "identifier" + CONCEPT = "concept" + RULE = "rule" + EXPR = "expression" + STRING = "string" + NUMBER = "number" + LPAR = "lpar" + RPAR = "rpar" + LBRACKET = "lbracket" + RBRACKET = "rbracket" + LBRACE = "lbrace" + RBRACE = "rbrace" + PLUS = "plus" + MINUS = "minus" + STAR = "star" + SLASH = "slash" + PERCENT = "percent" + COMMA = "comma" # , + SEMICOLON = "semicolon" # ; + COLON = "colon" # : + DOT = "dot" # . + QMARK = "qmark" + VBAR = "vbar" + AMPER = "amper" + EQUALS = "=" + AT = "at" + BACK_QUOTE = "bquote" # ` + BACK_SLASH = "bslash" # \ + CARAT = "carat" # ^ + DOLLAR = "dollar" # $ + EURO = "dollar" # € + STERLING = "steling" # £ + EMARK = "emark" # ! + GREATER = "greater" # > + LESS = "less" # < + HASH = "HASH" # # + TILDE = "tilde" # ~ + UNDERSCORE = "underscore" # _ + DEGREE = "degree" # ° + QUOTE = "'" # ' + WORD = "word" + EQUALSEQUALS = "==" + STARSTAR = "**" + SLASHSLASH = "//" + VAR_DEF = "concept variable" # __var__ + REGEX = "regex" # r'xxx' or r\"xxx\" or r|xxx| or r/xxx/ but not r:xxx: which means rules + + +@dataclass() +class Token: + type: TokenKind + value: object + index: int + line: int + column: int + + _strip_quote: str = field(default=None, repr=False, compare=False, hash=None) + _str_value: str = field(default=None, repr=False, compare=False, hash=None) + _repr_value: str = field(default=None, repr=False, compare=False, hash=None) + + def __repr__(self): + return f"Token({self.repr_value})" + + @property + def strip_quote(self): + if self._strip_quote: + return self._strip_quote + + if self.type in (TokenKind.STRING, TokenKind.REGEX): + self._strip_quote = self.value[1:-1] + else: + self._strip_quote = self.value + return self._strip_quote + + @property + def str_value(self): + if self._str_value: + return self._str_value + + self._str_value = self.to_str(False) + return self._str_value + + @property + def repr_value(self): + if self._repr_value: + return self._repr_value + + if self.type == TokenKind.EOF: + self._repr_value = "" + elif self.type == TokenKind.WHITESPACE: + self._repr_value = "" if self.value == "" else "" if self.value[0] == "\t" else "" + elif self.type == TokenKind.NEWLINE: + self._repr_value = "" + elif self.type == TokenKind.CONCEPT: + self._repr_value = str_concept(self.value) + elif self.type == TokenKind.RULE: + self._repr_value = str_concept(self.value, prefix="r:") + else: + self._repr_value = self.str_value + return self._repr_value + + @staticmethod + def is_whitespace(token): + return token and token.type == TokenKind.WHITESPACE + + def to_str(self, strip_quote): + if strip_quote and self.type == TokenKind.STRING: + return self.value[1:-1] + elif self.type == TokenKind.CONCEPT: + return str_concept(self.value) + elif self.type == TokenKind.RULE: + return str_concept(self.value, prefix="r:") + elif self.type == TokenKind.REGEX: + return "r" + self.value + else: + return str(self.value) + + def clone(self): + return Token(self.type, self.value, self.index, self.line, self.column) + + +@dataclass() +class LexerError(Exception): + message: str + text: str + index: int + line: int + column: int + + +class Keywords(Enum): + DEF = "def" + CONCEPT = "concept" + RULE = "rule" + FROM = "from" + BNF = "bnf" + AS = "as" + WHERE = "where" + PRE = "pre" + POST = "post" + ISA = "isa" + RET = "ret" + WHEN = "when" + PRINT = "print" + THEN = "then" + AUTO_EVAL = "auto_eval" + DEF_VAR = "def_var" + + +class Tokenizer: + """ + Class that can iterate on the tokens + """ + + def __init__(self, text, yield_eof=True, parse_word=False): + self.text = text + self.text_len = len(text) + self.column = 1 + self.line = 1 + self.i = 0 + self.yield_eof = yield_eof + self.parse_word = parse_word + + def __iter__(self): + + while self.i < self.text_len: + c = self.text[self.i] + if c == "+": + if self.i + 1 < self.text_len and self.text[self.i + 1].isdigit(): + number = self.eat_number(self.i) + yield Token(TokenKind.NUMBER, number, self.i, self.line, self.column) + self.i += len(number) + self.column += len(number) + else: + yield Token(TokenKind.PLUS, "+", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "-": + if self.i + 1 < self.text_len and self.text[self.i + 1].isdigit(): + number = self.eat_number(self.i) + yield Token(TokenKind.NUMBER, number, self.i, self.line, self.column) + self.i += len(number) + self.column += len(number) + else: + yield Token(TokenKind.MINUS, "-", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "_": + if self.i + 7 < self.text_len and \ + self.text[self.i: self.i + 7] == VARIABLE_PREFIX and \ + self.text[self.i + 7].isdigit(): + number = self.eat_number(self.i + 7) + yield Token(TokenKind.VAR_DEF, VARIABLE_PREFIX + number, self.i, self.line, self.column) + self.i += 7 + len(number) + self.column += 7 + len(number) + elif self.i + 1 < self.text_len and (self.text[self.i + 1].isalpha() or self.text[self.i + 1] == "_"): + identifier = self.eat_identifier(self.i) + yield Token(TokenKind.IDENTIFIER, identifier, self.i, self.line, self.column) + self.i += len(identifier) + self.column += len(identifier) + else: + yield Token(TokenKind.UNDERSCORE, "_", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "/": + if self.i + 1 < self.text_len and self.text[self.i + 1] == "/": + yield Token(TokenKind.SLASHSLASH, "//", self.i, self.line, self.column) + self.i += 2 + self.column += 2 + else: + yield Token(TokenKind.SLASH, "/", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "*": + if self.i + 1 < self.text_len and self.text[self.i + 1] == "*": + yield Token(TokenKind.STARSTAR, "**", self.i, self.line, self.column) + self.i += 2 + self.column += 2 + else: + yield Token(TokenKind.STAR, "*", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "{": + yield Token(TokenKind.LBRACE, "{", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "}": + yield Token(TokenKind.RBRACE, "}", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "(": + yield Token(TokenKind.LPAR, "(", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == ")": + yield Token(TokenKind.RPAR, ")", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "[": + yield Token(TokenKind.LBRACKET, "[", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "]": + yield Token(TokenKind.RBRACKET, "]", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "=": + if self.i + 1 < self.text_len and self.text[self.i + 1] == "=": + yield Token(TokenKind.EQUALSEQUALS, "==", self.i, self.line, self.column) + self.i += 2 + self.column += 2 + else: + yield Token(TokenKind.EQUALS, "=", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == " " or c == "\t": + whitespace = self.eat_whitespace(self.i) + yield Token(TokenKind.WHITESPACE, whitespace, self.i, self.line, self.column) + self.i += len(whitespace) + self.column += len(whitespace) + elif c == ",": + yield Token(TokenKind.COMMA, ",", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == ".": + yield Token(TokenKind.DOT, ".", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == ";": + yield Token(TokenKind.SEMICOLON, ";", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == ":": + yield Token(TokenKind.COLON, ":", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "?": + yield Token(TokenKind.QMARK, "?", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "|": + yield Token(TokenKind.VBAR, "|", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "&": + yield Token(TokenKind.AMPER, "&", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "<": + yield Token(TokenKind.LESS, "<", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == ">": + yield Token(TokenKind.GREATER, ">", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "!": + yield Token(TokenKind.EMARK, "!", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "`": + yield Token(TokenKind.BACK_QUOTE, "`", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "\\": + yield Token(TokenKind.BACK_SLASH, "\\", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "^": + yield Token(TokenKind.CARAT, "^", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "$": + yield Token(TokenKind.DOLLAR, "$", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "€": + yield Token(TokenKind.EURO, "€", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "£": + yield Token(TokenKind.STERLING, "£", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "#": + yield Token(TokenKind.HASH, "#", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "°": + yield Token(TokenKind.DEGREE, "°", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "~": + yield Token(TokenKind.TILDE, "~", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "%": + yield Token(TokenKind.PERCENT, "%", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "\n" or c == "\r": + newline = self.eat_newline(self.i) + yield Token(TokenKind.NEWLINE, newline, self.i, self.line, self.column) + self.i += len(newline) + self.column = 1 + self.line += 1 + elif c == "c" and self.i + 1 < self.text_len and self.text[self.i + 1] == ":": + name, id, length = self.eat_concept(self.i + 2, self.line, self.column + 2) + yield Token(TokenKind.CONCEPT, (name, id), self.i, self.line, self.column) + self.i += length + 2 + self.column += length + 2 + elif c == "r" and self.i + 1 < self.text_len and self.text[self.i + 1] == ":": + name, id, length = self.eat_concept(self.i + 2, self.line, self.column + 2) + yield Token(TokenKind.RULE, (name, id), self.i, self.line, self.column) + self.i += length + 2 + self.column += length + 2 + elif c == "r" and self.i + 1 < self.text_len and self.text[self.i + 1] in "'\"|/": + string, newlines, column_index = self.eat_string(self.i + 1, self.line, self.column) + yield Token(TokenKind.REGEX, string, self.i, self.line, self.column) # quotes are kept + self.i += len(string) + 1 + self.column = column_index # 1 if newlines > 0 else self.column + len(string) + self.line += newlines + elif self.parse_word and (c.isalpha() or c.isdigit()): + word = self.eat_word(self.i) + yield Token(TokenKind.WORD, word, self.i, self.line, self.column) + self.i += len(word) + self.column += len(word) + elif c.isalpha(): + identifier = self.eat_identifier(self.i) + yield Token(TokenKind.IDENTIFIER, identifier, self.i, self.line, self.column) + self.i += len(identifier) + self.column += len(identifier) + elif c.isdigit(): + number = self.eat_number(self.i) + yield Token(TokenKind.NUMBER, number, self.i, self.line, self.column) + self.i += len(number) + self.column += len(number) + elif c == "'" and self.i > 0 and self.text[self.i - 1] != " ": + yield Token(TokenKind.QUOTE, "'", self.i, self.line, self.column) + self.i += 1 + self.column += 1 + elif c == "'" or c == '"': + string, newlines, column_index = self.eat_string(self.i, self.line, self.column) + yield Token(TokenKind.STRING, string, self.i, self.line, self.column) # quotes are kept + self.i += len(string) + self.column = column_index # 1 if newlines > 0 else self.column + len(string) + self.line += newlines + else: + raise LexerError(f"Unknown token '{c}'", self.text, self.i, self.line, self.column) + + if self.yield_eof: + yield Token(TokenKind.EOF, "", self.i, self.line, self.column) + + def eat_concept(self, start, line, column): + key, id, buffer = None, None, "" + i = start + processing_key = True + + while i < self.text_len: + + c = self.text[i] + if c == "\n": + raise LexerError(f"New line in concept name", self.text[start:i], i, line, column + i - start) + + if c == ":": + if processing_key: + key = buffer if buffer else None + else: + id = buffer if buffer else None + i += 1 # eat the colon + break + + if c == "#": + key = buffer if buffer else None + buffer = "" + processing_key = False + i += 1 + continue + + buffer += c + i += 1 + else: + raise LexerError(f"Missing ending colon", self.text[start:i], i, line, column + i - start) + + if (key, id) == (None, None): + raise LexerError(f"Concept identifiers not found", "", start, line, column) + + return key, id, i - start + + def eat_whitespace(self, start): + result = self.text[start] + i = start + 1 + while i < self.text_len: + c = self.text[i] + if c == " " or c == "\t": + result += c + i += 1 + else: + break + + return result + + def eat_newline(self, start): + if start + 1 == self.text_len: + return self.text[start] + + current = self.text[start] + next = self.text[start + 1] + if current == "\n" and next == "\r" or current == "\r" and next == "\n": + return current + next + + return current + + def eat_identifier(self, start): + result = self.text[start] + i = start + 1 + while i < self.text_len: + c = self.text[i] + if c.isalpha() or c == "_" or c == "-" or c.isdigit(): + result += c + i += 1 + else: + break + + return result + + def eat_number(self, start): + result = self.text[start] + i = start + 1 + while i < self.text_len: + c = self.text[i] + if c.isdigit() or c == ".": + result += c + i += 1 + else: + break + + return result + + def eat_string(self, start_index, start_line, start_column): + quote = self.text[start_index] + result = self.text[start_index] + lines_count = 0 + column_index = start_column + 1 + + i = start_index + 1 + escape = False + # newline = None + while i < self.text_len: + c = self.text[i] + result += c + i += 1 + column_index += 1 + + if c == "\n": + lines_count += 1 + column_index = 1 + + if c == "\\": + escape = True + elif c == quote and not escape: + break + else: + escape = False + + # # add trailing new line if needed + # if newline: + # lines_count += 1 + # column_index = 1 + + if result[-1] != quote or len(result) == 1: + raise LexerError("Missing Trailing quote", result, i, start_line + lines_count, + 1 if lines_count > 0 else start_column + len(result)) + + return result, lines_count, column_index + + def eat_word(self, start): + """ + Word is an alphanum (no space) + :param start: + :return: + """ + result = self.text[start] + i = start + 1 + while i < self.text_len: + c = self.text[i] + if c.isalpha() or c.isdigit(): + result += c + i += 1 + else: + break + + return result + + +def strip_tokens(tokens, strip_eof=False): + """ + Remove the starting and trailing spaces and newline + """ + if tokens is None: + return None + + start = 0 + length = len(tokens) + while start < length and tokens[start].type in (TokenKind.WHITESPACE, TokenKind.NEWLINE): + start += 1 + + if start == length: + return [] + + end_tokens = (TokenKind.WHITESPACE, TokenKind.NEWLINE, TokenKind.EOF) \ + if strip_eof \ + else (TokenKind.WHITESPACE, TokenKind.NEWLINE) + + end = length - 1 + while end > 0 and tokens[end].type in end_tokens: + end -= 1 + + return tokens[start: end + 1] diff --git a/src/sdp/__init__.py b/src/sdp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/sdp/readme.md b/src/sdp/readme.md new file mode 100644 index 0000000..c968a79 --- /dev/null +++ b/src/sdp/readme.md @@ -0,0 +1,24 @@ +# How to serialize ? + +## General rule +- 1 byte : type of object code +- int : version of the encoder +- data : can be the json representation of the object + +### Current supported types +- J : Json object +- P : pickle +- E : events +- S : state +- X : executionContext +- M : context metadata + +Not verified yet ! + +- C : concept (with history management) +- D : concept definitions (no history management) +- R : executionContext ('R' stands for Result or ReturnValue, no history management) +- O : ServiceObj (from pickle) +- M : MemoryObject (using SheerkaPickle) +- X : Rule (from sheerkaPickle, 'X' stands for nothing, I am running out of meaningful letters) +- T : CustomType \ No newline at end of file diff --git a/src/sdp/sheerkaDataProvider.py b/src/sdp/sheerkaDataProvider.py index 206f798..33b6597 100644 --- a/src/sdp/sheerkaDataProvider.py +++ b/src/sdp/sheerkaDataProvider.py @@ -8,9 +8,10 @@ from os import path from threading import RLock from typing import Callable +from common.utils import get_logger_name +from core.Event import Event from core.ExecutionContext import ExecutionContext -from core.Sheerka import Sheerka -from core.global_symbols import NotFound +from common.global_symbols import NotFound from sdp.sheerkaDataProviderIO import SheerkaDataProviderIO from sdp.sheerkaSerializer import Serializer, SerializerContext @@ -27,72 +28,6 @@ def json_default_converter(o): return o.isoformat() -class Event(object): - """ - Class that represents something that modifies the state of the system - """ - - def __init__(self, message="", user_id="", date=None, parents=None): - self.user_id: str = user_id # id of the user that triggers the modification - self.date: datetime | None = date or datetime.now() # when - self.message: str = message # user input or whatever that modifies the system - self.parents: list[str] = parents # digest(s) of the parent(s) of this event - self._digest: str | None = None # digest of the event - - def __str__(self): - return f"{self.date.strftime('%d/%m/%Y %H:%M:%S')} {self.message}" - - def __repr__(self): - return f"{self.get_digest()[:12]} {self.message}" - - def get_digest(self): - """ - Returns the digest of the event - :return: sha256 of the event - """ - - if self._digest: - return self._digest - - if self.user_id == "": - # only possible during the unit test - # We use this little trick to speed up the unit test - self._digest = self.message[6:] if self.message.startswith("TEST::") else "xxx" - return self._digest - - if not isinstance(self.message, str): - raise NotImplementedError(f"message={self.message}") - - to_hash = f"Event:{self.user_id}{self.date}{self.message}{self.parents}".encode("utf-8") - self._digest = hashlib.sha256(to_hash).hexdigest() - return self._digest - - def to_dict(self): - return self.__dict__ - - def from_dict(self, as_dict): - self.user_id = as_dict["user_id"] - self.date = datetime.fromisoformat(as_dict["date"]) - self.message = as_dict["message"] - self.parents = as_dict["parents"] - self._digest = as_dict["_digest"] # freeze the digest - - def __eq__(self, other): - if id(self) == id(other): - return True - - if isinstance(other, Event): - return (self.user_id == other.user_id and - self.date == other.date and - self.message == other.message and - self.parents == other.parents) - - return False - - def __hash__(self): - return hash(self.get_digest()) - - class State: """ Class that represents the state of the system (dictionary of all known entries) @@ -118,7 +53,7 @@ class SheerkaDataProviderTransaction: Note that Transaction within Transaction is not supported """ - def __init__(self, sdp, event): + def __init__(self, sdp, event: Event): self.sdp: SheerkaDataProvider = sdp self.event: Event = event self.state: State | None = None @@ -181,6 +116,7 @@ class SheerkaDataProviderTransaction: pass def __exit__(self, exc_type, exc_val, exc_tb): + # TODO: Manage exception in order to rollback self.state.parents = [] if self.snapshot is None else [self.snapshot] self.state.events = [self.event_digest] self.state.date = datetime.now() @@ -207,10 +143,11 @@ class SheerkaDataProvider: REF_PREFIX = "##REF##:" def __init__(self, root=None, sheerka=None, name="__default__"): - self.log = logging.getLogger(__name__) - self.init_log = logging.getLogger("init." + __name__) - self.init_log.debug("Initializing sdp.") + self.log = logging.getLogger(get_logger_name(__name__)) + self.init_log = logging.getLogger(get_logger_name("init." + __name__)) + self.init_log.debug(f"Initializing sdp '{name}'") + from core.Sheerka import Sheerka self.sheerka: Sheerka = sheerka self.io = SheerkaDataProviderIO.get(root) self.first_time = self.io.first_time diff --git a/src/sdp/sheerkaDataProviderIO.py b/src/sdp/sheerkaDataProviderIO.py index 9a482c5..6e3ae50 100644 --- a/src/sdp/sheerkaDataProviderIO.py +++ b/src/sdp/sheerkaDataProviderIO.py @@ -3,13 +3,15 @@ import logging import os from os import path +from common.utils import get_logger_name + class SheerkaDataProviderIO: def __init__(self, root): self.root = root - self.log = logging.getLogger(__name__) - self.init_log = logging.getLogger("init." + __name__) + self.log = logging.getLogger(get_logger_name(__name__)) + self.init_log = logging.getLogger(get_logger_name("init." + __name__)) def exists(self, file_path): pass diff --git a/src/sdp/sheerkaSerializer.py b/src/sdp/sheerkaSerializer.py index decd5be..95bdeb6 100644 --- a/src/sdp/sheerkaSerializer.py +++ b/src/sdp/sheerkaSerializer.py @@ -7,9 +7,12 @@ import struct from dataclasses import dataclass from enum import Enum -from core.utils import get_class, get_full_qualified_name +from common.global_symbols import CustomType, NotFound, NotInit, Removed +from common.utils import get_class, get_full_qualified_name, get_logger_name +# https://stackoverflow.com/questions/15721363/preserve-python-tuples-with-json + def json_default_converter(o): """ Default formatter for json @@ -24,7 +27,7 @@ def json_default_converter(o): if isinstance(o, Enum): return o.name - raise Exception(f"Cannot serialize object '{o}', class='{o.__class__.__name__}'") + # raise Exception(f"Cannot serialize object '{o}', class='{o.__class__.__name__}'") # In debug mode, just # # with open("json_encoding_error.txt", "a") as f: @@ -47,21 +50,23 @@ class Serializer: HISTORY = "##history##" def __init__(self): - self.log = logging.getLogger(__name__) - self.init_log = logging.getLogger("init." + __name__) + self.log = logging.getLogger(get_logger_name(__name__)) + self.init_log = logging.getLogger(get_logger_name("init." + __name__)) self.init_log.debug("Initializing serializers") self._cache = [] # add builtin serializers self.register(EventSerializer()) self.register(StateSerializer()) + self.register(ExecutionContextSerializer()) + self.register(ConceptMetadataSerializer()) + self.register(CustomTypeSerializer()) # self.register(ConceptSerializer()) # self.register(DictionarySerializer()) - # self.register(ExecutionContextSerializer()) # self.register(MemoryObjectSerializer()) # before ServiceObjSerializer # self.register(ServiceObjSerializer()) # self.register(RuleSerializer()) - # self.register(CustomTypeSerializer()) + # def register(self, serializer): """ @@ -204,12 +209,35 @@ class JsonSerializer(BaseSerializer): return obj +class CustomTypeSerializer(BaseSerializer): + def __init__(self): + BaseSerializer.__init__(self, "T", 1) + + def matches(self, obj): + return isinstance(obj, CustomType) + + def dump(self, stream, obj, context): + stream.write(obj.value.encode("utf-8")) + stream.seek(0) + return stream + + def load(self, stream, context): + value = stream.read().decode("utf-8") + if value == NotInit.value: + return NotInit + elif value == NotFound.value: + return NotFound + elif value == Removed.value: + return Removed + raise NotImplemented(f"CustomTypeSerializer.load({value})") + + class EventSerializer(BaseSerializer): def __init__(self): BaseSerializer.__init__(self, "E", 1) def matches(self, obj): - return get_full_qualified_name(obj) == "sdp.sheerkaDataProvider.Event" + return get_full_qualified_name(obj) == "core.Event.Event" def dump(self, stream, obj, context): stream.write(json.dumps(obj.to_dict(), default=json_default_converter).encode("utf-8")) @@ -231,6 +259,21 @@ class StateSerializer(PickleSerializer): "S", 1) + +class ExecutionContextSerializer(JsonSerializer): + CLASS_NAME = "core.ExecutionContext.ExecutionContext" + + def __init__(self): + super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "X", 1) + + +class ConceptMetadataSerializer(PickleSerializer): + CLASS_NAME = "core.concept.ConceptMetadata" + + def __init__(self): + super().__init__(lambda obj: get_full_qualified_name(obj) == self.CLASS_NAME, "M", 1) + +# # # # class JsonSerializer(BaseSerializer): @@ -359,24 +402,3 @@ class StateSerializer(PickleSerializer): # super().__init__(lambda obj: isinstance(obj, Rule), "X", 1) # # -# class CustomTypeSerializer(BaseSerializer): -# def __init__(self): -# BaseSerializer.__init__(self, "T", 1) -# -# def matches(self, obj): -# return isinstance(obj, CustomType) -# -# def dump(self, stream, obj, context): -# stream.write(obj.value.encode("utf-8")) -# stream.seek(0) -# return stream -# -# def load(self, stream, context): -# value = stream.read().decode("utf-8") -# if value == NotInit.value: -# return NotInit -# elif value == NotFound.value: -# return NotFound -# elif value == Removed.value: -# return Removed -# raise NotImplemented(f"CustomTypeSerializer.load({value})") diff --git a/src/server/__init__.py b/src/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server/authentication.py b/src/server/authentication.py index b9ac675..2e5cb31 100644 --- a/src/server/authentication.py +++ b/src/server/authentication.py @@ -19,7 +19,8 @@ oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") fake_users_db = { "kodjo": { "username": "kodjo", - "full_name": "Kodjo Sossouvi", + "firstname": "Kodjo", + "lastname": "Sossouvi", "email": "kodjo.sossouvi@gmail.com", "hashed_password": "$2b$12$fb9jW7QUZ9KIEAAtVmWMEOGtehKy9FafUr7Zfrsb3ZMhsBbzZs7SC", # password is kodjo "disabled": False, @@ -42,7 +43,8 @@ class User(BaseModel): """ username: str email: str | None = None - full_name: str | None = None + firstname: str | None = None + lastname: str | None = None disabled: bool | None = None diff --git a/src/server/main.py b/src/server/main.py index bbc0fb3..2feb129 100644 --- a/src/server/main.py +++ b/src/server/main.py @@ -4,12 +4,32 @@ import uvicorn from fastapi import Depends, FastAPI, HTTPException from fastapi.security import OAuth2PasswordRequestForm from starlette import status +from starlette.middleware.cors import CORSMiddleware from constants import CLIENT_OPERATION_QUIT, EXIT_COMMANDS, SHEERKA_PORT +from core.Sheerka import Sheerka from server.authentication import ACCESS_TOKEN_EXPIRE_MINUTES, User, authenticate_user, create_access_token, \ fake_users_db, get_current_active_user app = FastAPI() +origins = [ + "http://localhost:56426", + "http://localhost:5173", +] + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# NEED TO FIND A WAY TO INIT SHEERKA within the __name__ == "__main__" section +# As of now, if we do that, sheerka is not properly initialized using the command +# 'uvicorn server.main:app' from the shell +sheerka = Sheerka() +sheerka.initialize() @app.get("/") @@ -34,7 +54,10 @@ async def login(form_data: OAuth2PasswordRequestForm = Depends()): access_token_expires = timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES) access_token = create_access_token(data={"sub": user.username}, expires_delta=access_token_expires) - return {"access_token": access_token, "token_type": "bearer"} + return {"first_name": user.firstname, + "last_name": user.lastname, + "access_token": access_token, + "token_type": "bearer"} @app.post("/echo/{message}", status_code=status.HTTP_200_OK, response_model=dict) @@ -62,5 +85,21 @@ async def echo(message: str, current_user: User = Depends(get_current_active_use } +@app.post("/command/{message}", status_code=status.HTTP_200_OK, response_model=dict) +async def command(message: str, current_user: User = Depends(get_current_active_user)) -> dict: + if message in EXIT_COMMANDS: + return { + "status": True, + "response": "Take care.", + "command": CLIENT_OPERATION_QUIT + } + res = sheerka.evaluate_user_input(message, current_user) + return { + "status": res[0].status, + "response": res[0].value.body, + "command": None + } + + if __name__ == "__main__": uvicorn.run("server.main:app", port=SHEERKA_PORT, log_level="info") diff --git a/src/sheerkapickle/__init__.py b/src/sheerkapickle/__init__.py new file mode 100644 index 0000000..6e75f6f --- /dev/null +++ b/src/sheerkapickle/__init__.py @@ -0,0 +1,7 @@ +from .sheerkaplicker import encode +from .sheerkaunpickler import decode + +__all__ = ('encode', 'decode') + +# register built-in handlers +__import__('sheerkapickle.handlers', level=0) diff --git a/src/sheerkapickle/handlers.py b/src/sheerkapickle/handlers.py new file mode 100644 index 0000000..7c043ee --- /dev/null +++ b/src/sheerkapickle/handlers.py @@ -0,0 +1,197 @@ +import re +import threading +import uuid + +from sheerkapickle import utils + + +class ToReduce: + def __init__(self, predicate, get_value): + self.predicate = predicate + self.get_value = get_value + + +class SheerkaRegistry(object): + + def __init__(self): + self._handlers = {} + self._base_handlers = {} + + def get(self, cls_or_name: str, default=None): + """ + :param cls_or_name: the type or its fully qualified name + :param default: default value, if no matching handler is not found + + Looks up a handler by type reference or its fully + qualified name. If a direct match + is not found, the search is performed over all + handlers registered with base=True. + """ + handler = self._handlers.get(cls_or_name) + # attempt to find a base class + if handler is None and utils.is_type(cls_or_name): + for cls, base_handler in self._base_handlers.items(): + if issubclass(cls_or_name, cls): + return base_handler + return default if handler is None else handler + + def register(self, cls, handler: 'BaseHandler' = None, base: bool = False): + """Register the custom handler for a class + + :param cls: The custom object class to handle + :param handler: The custom handler class (if + None, a decorator wrapper is returned) + :param base: Indicates whether the handler should + be registered for all subclasses + + This function can be also used as a decorator + by omitting the `handler` argument:: + + @jsonpickle.handlers.register(Foo, base=True) + class FooHandler(jsonpickle.handlers.BaseHandler): + pass + + """ + if handler is None: + def _register(handler_cls): + self.register(cls, handler=handler_cls, base=base) + return handler_cls + + return _register + if not utils.is_type(cls): + raise TypeError('{!r} is not a class/type'.format(cls)) + # store both the name and the actual type for the ugly cases like + # _sre.SRE_Pattern that cannot be loaded back directly + self._handlers[utils.importable_name(cls)] = \ + self._handlers[cls] = handler + if base: + # only store the actual type for subclass checking + self._base_handlers[cls] = handler + + def unregister(self, cls): + self._handlers.pop(cls, None) + self._handlers.pop(utils.importable_name(cls), None) + self._base_handlers.pop(cls, None) + + +registry = SheerkaRegistry() +register = registry.register +unregister = registry.unregister +get = registry.get + + +class BaseHandler(object): + + def __init__(self, sheerka, context): + """ + Initialize a new handler to handle a registered type. + + :Parameters: + - `context`: reference to pickler/unpickler + + """ + self.sheerka = sheerka + self.context = context + + def __call__(self, sheerka, context): + """This permits registering either Handler instances or classes + + :Parameters: + - `context`: reference to pickler/unpickler + """ + self.sheerka = sheerka + self.context = context + return self + + def flatten(self, obj: object, data: dict): + """ + Flatten `obj` into a json-friendly form and write result to `data`. + + :param object obj: The object to be serialized. + :param dict data: A partially filled dictionary which will contain the + json-friendly representation of `obj` once this method has + finished. + """ + raise NotImplementedError('You must implement flatten() in %s' % + self.__class__) + + def new(self, data: dict): + raise NotImplementedError('You must implement new() in %s' % + self.__class__) + + def restore(self, data: dict, instance: object): + """ + Restore an object of the registered type from the json-friendly + representation `obj` and return it. + """ + raise NotImplementedError('You must implement restore() in %s' % + self.__class__) + + @classmethod + def handles(self, cls: str): + """ + Register this handler for the given class. Suitable as a decorator, + e.g.:: + + @MyCustomHandler.handles + class MyCustomClass: + def __reduce__(self): + ... + """ + registry.register(cls, self) + return cls + + +class RegexHandler(BaseHandler): + """Flatten _sre.SRE_Pattern (compiled regex) objects""" + + def flatten(self, obj, data): + data['pattern'] = obj.pattern + return data + + def new(self, data): + return re.compile(data['pattern']) + + def restore(self, data, instance): + return instance + + +RegexHandler.handles(type(re.compile(''))) + + +class UUIDHandler(BaseHandler): + """Serialize uuid.UUID objects""" + + def flatten(self, obj, data): + data['hex'] = obj.hex + return data + + def new(self, data): + return uuid.UUID(data['hex']) + + def restore(self, data, instance): + return instance + + +UUIDHandler.handles(uuid.UUID) + + +class LockHandler(BaseHandler): + """Serialize threading.Lock objects""" + + def flatten(self, obj, data): + data['locked'] = obj.locked() + return data + + def new(self, data): + lock = threading.Lock() + if data.get('locked', False): + lock.acquire() + return lock + + def restore(self, data, instance): + return instance + + +_lock = threading.Lock() +LockHandler.handles(_lock.__class__) diff --git a/src/sheerkapickle/sheerka_handlers.py b/src/sheerkapickle/sheerka_handlers.py new file mode 100644 index 0000000..99a3dd4 --- /dev/null +++ b/src/sheerkapickle/sheerka_handlers.py @@ -0,0 +1,240 @@ +from common.global_symbols import NotInit +from core.concept import Concept +from sheerkapickle.handlers import BaseHandler, registry + +default_concept = Concept() +default_concept_values = default_concept.values() +CONCEPT_ID = "concept/id" +RULE_ID = "rule/id" + + +class ConceptHandler(BaseHandler): + + def flatten(self, obj: Concept, data): + pickler = self.context + sheerka = self.sheerka + + if obj.get_metadata().full_serialization: + ref = default_concept + ref_values = default_concept_values + else: + ref = sheerka.get_by_id(obj.id, allow_dynamic=True) + ref_values = ref.values() + data[CONCEPT_ID] = obj.id + + # transform metadata + for name in CONCEPT_PROPERTIES_TO_SERIALIZE: + value = getattr(obj.get_metadata(), name) + ref_value = getattr(ref.get_metadata(), name) + if value != ref_value: + value_to_use = [list(t) for t in value] if name == "variables" else value + data["meta." + name] = pickler.flatten(value_to_use) + + # # transform values + for name, value in obj.values().items(): + if name not in ref_values or value != ref_values[name]: + if "values" not in data: + data["values"] = [] + data["values"].append((pickler.flatten(name), pickler.flatten(value))) + + return data + + def new(self, data): + sheerka = self.sheerka + if CONCEPT_ID in data: + return sheerka.new((None, data[CONCEPT_ID]), allow_dynamic=True) + else: + return Concept() + + def restore(self, data, instance): + pickler = self.context + + for key, value in data.items(): + if key.startswith("_sheerka/") or key == CONCEPT_ID: + continue + + resolved_value = pickler.restore(data[key]) + + if key.startswith("meta."): + # get metadata + resolved_prop = key[5:] + if resolved_prop == "variables": + for prop_name, prop_value in resolved_value: + instance.def_var(prop_name, prop_value) + else: + setattr(instance.get_metadata(), resolved_prop, resolved_value) + elif key == "values": + # get properties + for prop_name, prop_value in resolved_value: + instance.set_value(prop_name, NotInit if prop_value is None else prop_value) + else: + raise Exception("Sanity check as it's not possible yet") + + # instance.freeze_definition_hash() + return instance + + +class UserInputHandler(ConceptHandler): + + def flatten(self, obj: UserInputConcept, data): + data[CONCEPT_ID] = (obj.key, obj.id) + data["user_name"] = obj.user_name + data["text"] = core.utils.get_text_from_tokens(obj.text) if isinstance(obj.text, list) else \ + obj.text.as_text() if isinstance(obj.text, ParserInput) else \ + obj.text + return data + + def new(self, data): + return UserInputConcept.__new__(UserInputConcept) + + def restore(self, data, instance): + instance.__init__(data["text"], data["user_name"]) + instance.get_metadata().key = data[CONCEPT_ID][0] + instance.get_metadata().id = data[CONCEPT_ID][1] + instance.freeze_definition_hash() + return instance + + +class ReturnValueHandler(BaseHandler): + + def flatten(self, obj: ReturnValueConcept, data): + pickler = self.context + + data[CONCEPT_ID] = (obj.key, obj.id) + data["who"] = f"c:{obj.who.id}:" if isinstance(obj.who, Concept) else \ + obj.who.name if isinstance(obj.who, (BaseParser, BaseEvaluator)) else \ + obj.who + data["status"] = obj.status + data["value"] = pickler.flatten(obj.value) + if obj.parents: + data["parents"] = pickler.flatten(obj.parents) + return data + + def new(self, data): + return ReturnValueConcept.__new__(ReturnValueConcept) + + def restore(self, data, instance): + pickler = self.context + + instance.__init__(data["who"], data["status"], pickler.restore(data["value"])) + if "parents" in data: + instance.parents = pickler.restore(data["parents"]) + + instance.get_metadata().key = data[CONCEPT_ID][0] + instance.get_metadata().id = data[CONCEPT_ID][1] + instance.freeze_definition_hash() + return instance + + +class SheerkaHandler(ConceptHandler): + + # def flatten(self, obj: BuiltinConcepts, data): + # return ConceptHandler().flatten()data + + def new(self, data): + return self.sheerka + + # def restore(self, data, instance): + # return instance + + +class ExecutionContextHandler(BaseHandler): + + def flatten(self, obj, data): + pickler = self.context + + for prop in CONTEXT_PROPERTIES_TO_SERIALIZE: + if prop in ("who", "action", "action_context"): + value = str(getattr(obj, prop)) + else: + value = getattr(obj, prop) + + if value is not None: + data[prop] = pickler.flatten(value) + + return data + + def new(self, data): + return ExecutionContext(data["who"], None, self.sheerka, BuiltinConcepts.NOP, None) + + def restore(self, data, instance): + pickler = self.context + + for prop in CONTEXT_PROPERTIES_TO_SERIALIZE: + if prop not in data or prop == "who": + continue + setattr(instance, prop, pickler.restore(data[prop])) + + return instance + + +class RuleContextHandler(BaseHandler): + + def flatten(self, obj, data): + data[RULE_ID] = obj.id + data["name"] = obj.metadata.name + data["predicate"] = obj.metadata.predicate + data["action_type"] = obj.metadata.action_type + data["action"] = obj.metadata.action + return data + + def new(self, data): + return Rule(data["action_type"], data["name"], data["predicate"], data["action"]) + + def restore(self, data, instance): + instance.metadata.id = data[RULE_ID] + return instance + + +class PythonNodeHandler(BaseHandler): + + def flatten(self, obj, data): + pickler = self.context + + data["source"] = obj.source + data["ast_str"] = obj.ast_str + data["objects"] = pickler.flatten(obj.objects) + return data + + def new(self, data): + return PythonNode.__new__(PythonNode) + + def restore(self, data, instance): + pickler = self.context + + instance.__init__(data["source"], objects=pickler.restore(data["objects"])) + instance.ast_str = data["ast_str"] + return instance + + +class DefRuleNodeHandler(BaseHandler): + def flatten(self, obj, data): + pickler = self.context + + data["tokens"] = pickler.flatten(obj.tokens) + data["name"] = pickler.flatten(obj.name) + return data + + def new(self, data): + return DefRuleNode.__new__(DefRuleNode) + + def restore(self, data, instance): + pickler = self.context + + instance.__init__(data["source"], objects=pickler.restore(data["objects"])) + instance.tokens = pickler.restore(data["tokens"]) + instance.name = pickler.restore(data["name"]) + return instance + + +def initialize_pickle_handlers(): + registry.register(Concept, ConceptHandler, True) + registry.register(UserInputConcept, UserInputHandler, True) + registry.register(ReturnValueConcept, ReturnValueHandler, True) + registry.register(Sheerka, SheerkaHandler, True) + registry.register(ExecutionContext, ExecutionContextHandler, True) + registry.register(Rule, RuleContextHandler, True) + registry.register(PythonNode, PythonNodeHandler, True) + registry.register(DefRuleNode, DefRuleNodeHandler, True) + registry.register(DefExecRuleNode, DefRuleNodeHandler, True) # TODO: fix inheritance that does not work + registry.register(DefFormatRuleNode, DefRuleNodeHandler, True) # TODO: fix inheritance that does not work \ No newline at end of file diff --git a/src/sheerkapickle/sheerkaplicker.py b/src/sheerkapickle/sheerkaplicker.py new file mode 100644 index 0000000..0c82508 --- /dev/null +++ b/src/sheerkapickle/sheerkaplicker.py @@ -0,0 +1,163 @@ +import json +from logging import Logger + +from common.utils import get_full_qualified_name, str_concept +from core.concept import Concept +from sheerkapickle import utils, tags, handlers + + +def encode(sheerka, obj): + pickler = SheerkaPickler(sheerka) + flatten = pickler.flatten(obj) + return json.dumps(flatten) + + +class ToReduce: + def __init__(self, predicate, get_value): + self.predicate = predicate + self.get_value = get_value + + +class SheerkaPickler: + """ + Json sheerkapickle + Inspired by jsonpickle (https://github.com/jsonpickle/jsonpickle) + which failed to work in my environment + """ + + def __init__(self, sheerka): + self.ids = {} + self.objs = [] + self.id_count = -1 + self.sheerka = sheerka + self.to_reduce = [] + + self.to_reduce.append(ToReduce(lambda o: isinstance(o, Logger), lambda o: None)) + # from parsers.BaseParser import BaseParser + # from evaluators.BaseEvaluator import BaseEvaluator + from ontologies.SheerkaOntologyManager import Ontology + # from core.sheerka.Sheerka import SheerkaMethod + # self.to_reduce.append(ToReduce(lambda o: isinstance(o, (BaseParser, BaseEvaluator)), lambda o: o.name)) + # self.to_reduce.append(ToReduce(lambda o: isinstance(o, ParserInput), lambda o: o.as_text())) + self.to_reduce.append(ToReduce(lambda o: isinstance(o, Ontology), lambda o: f"__ONTOLOGY:{o.name}__")) + # self.to_reduce.append(ToReduce(lambda o: isinstance(o, SheerkaMethod), lambda o: o.name)) + + def flatten(self, obj): + if utils.is_to_discard(obj): + return str(obj) + + if utils.is_primitive(obj): + return obj + + if utils.is_custom_type(obj): + return self._flatten_custom_type(obj) + + if utils.is_type(obj): + return str(obj) + + if utils.is_tuple(obj): + return {tags.TUPLE: [self.flatten(v) for v in obj]} + + if utils.is_set(obj): + return {tags.SET: [self.flatten(v) for v in obj]} + + if utils.is_list(obj): + return [self.flatten(v) for v in obj] + + if utils.is_dictionary(obj): + return self._flatten_dict(obj) + + if utils.is_enum(obj): + return self._flatten_enum(obj) + + if utils.is_object(obj): + return self._flatten_obj_instance(obj) + + raise Exception(f"Cannot flatten '{obj}'") + + def _flatten_dict(self, obj): + data = {} + for k, v in obj.items(): + if k is None: + k_str = "null" + elif utils.is_enum(k): + k_str = get_full_qualified_name(k) + "." + k.name + elif isinstance(k, Concept): + k_str = str_concept(k) + else: + k_str = k + + data[k_str] = self.flatten(v) + + return data + + def _flatten_enum(self, obj): + # check if the object was already seen + exists, _id = self.exist(obj) + if exists: + return {tags.ID: _id} + else: + self.id_count = self.id_count + 1 + self.ids[id(obj)] = self.id_count + self.objs.append(obj) + + data = {} + class_name = get_full_qualified_name(obj) + data[tags.ENUM] = class_name + "." + obj.name + return data + + def _flatten_obj_instance(self, obj): + for reduce in self.to_reduce: + if reduce.predicate(obj): + return reduce.get_value(obj) + + # check if the object was already seen + exists, _id = self.exist(obj) + if exists: + return {tags.ID: _id} + else: + self.id_count = self.id_count + 1 + self.ids[id(obj)] = self.id_count + self.objs.append(obj) + + # flatten + data = {} + cls = obj.__class__ if hasattr(obj, '__class__') else type(obj) + class_name = utils.importable_name(cls) + data[tags.OBJECT] = class_name + + handler = handlers.get(class_name) + if handler is not None: + return handler(self.sheerka, self).flatten(obj, data) + + if hasattr(obj, "__dict__"): + for k, v in obj.__dict__.items(): + data[k] = self.flatten(v) + + return data + + def _flatten_custom_type(self, obj): + # check if the object was already seen + exists, _id = self.exist(obj) + if exists: + return {tags.ID: _id} + else: + self.id_count = self.id_count + 1 + self.ids[id(obj)] = self.id_count + self.objs.append(obj) + + return {tags.CUSTOM: obj.value} + + def exist(self, obj): + try: + v = self.ids[id(obj)] + return True, v + except KeyError: + return False, None + + # def exist(self, obj): + # for k, v in self.ids.items(): + # if k == id(obj): + # return True, v + # + # return False, None \ No newline at end of file diff --git a/src/sheerkapickle/sheerkaunpickler.py b/src/sheerkapickle/sheerkaunpickler.py new file mode 100644 index 0000000..c76701a --- /dev/null +++ b/src/sheerkapickle/sheerkaunpickler.py @@ -0,0 +1,129 @@ +import json + +from common.global_symbols import NoFirstToken, NotFound, NotInit, Removed +from common.utils import decode_enum, get_class, unstr_concept +from sheerkapickle import tags, utils, handlers + + +def decode(sheerka, obj): + return SheerkaUnpickler(sheerka).restore(json.loads(obj)) + + +class SheerkaUnpickler: + def __init__(self, sheerka): + self.sheerka = sheerka + self.objs = [] + + def restore(self, obj): + if has_tag(obj, tags.ID): + return self._restore_id(obj) + + if has_tag(obj, tags.TUPLE): + return self._restore_tuple(obj) + + if has_tag(obj, tags.CUSTOM): + return self._restore_custom(obj) + + if has_tag(obj, tags.SET): + return self._restore_set(obj) + + if has_tag(obj, tags.ENUM): + return self._restore_enum(obj) + + if has_tag(obj, tags.OBJECT): + return self._restore_obj(obj) + + if utils.is_list(obj): + return self._restore_list(obj) + + if utils.is_dictionary(obj): + return self._restore_dict(obj) + + return obj + + def _restore_list(self, obj): + return [self.restore(v) for v in obj] + + def _restore_tuple(self, obj): + return tuple([self.restore(v) for v in obj[tags.TUPLE]]) + + def _restore_custom(self, obj): + if obj[tags.CUSTOM] == NotInit.value: + instance = NotInit + elif obj[tags.CUSTOM] == NotFound.value: + instance = NotFound + elif obj[tags.CUSTOM] == Removed.value: + instance = Removed + elif obj[tags.CUSTOM] == NoFirstToken.value: + instance = NoFirstToken + else: + raise KeyError(f"unknown {obj[tags.CUSTOM]}") + + self.objs.append(instance) + return instance + + def _restore_set(self, obj): + return set([self.restore(v) for v in obj[tags.SET]]) + + def _restore_enum(self, obj): + instance = decode_enum(obj[tags.ENUM]) + self.objs.append(instance) + return instance + + def _restore_dict(self, obj): + data = {} + for k, v in obj.items(): + resolved_key = self._resolve_key(k) + data[resolved_key] = self.restore(v) + return data + + def _restore_id(self, obj): + try: + return self.objs[obj[tags.ID]] + except IndexError: + pass + + def _restore_obj(self, obj): + handler = handlers.get(obj[tags.OBJECT]) + + if handler: + handler = handler(self.sheerka, self) + instance = handler.new(obj) + self.objs.append(instance) + instance = handler.restore(obj, instance) + else: + # KSI 202011: Hack because Property is removed + # To suppress asap + if obj[tags.OBJECT] == "core.concept.Property": + return self.restore(obj["value"]) + + cls = get_class(obj[tags.OBJECT]) + instance = cls.__new__(cls) + self.objs.append(instance) + + for k, v in obj.items(): + if k == tags.OBJECT: + continue + value = self.restore(v) + setattr(instance, k, value) + + return instance + + def _resolve_key(self, key): + + if key == "null": + return None + + concept_key, concept_id = unstr_concept(key) + if concept_key is not None: + return self.sheerka.new((concept_key, concept_id)) if concept_id else self.sheerka.new(concept_key) + + as_enum = decode_enum(key) + if as_enum is not None: + return as_enum + + return key + + +def has_tag(obj, tag): + return type(obj) is dict and tag in obj \ No newline at end of file diff --git a/src/sheerkapickle/tags.py b/src/sheerkapickle/tags.py new file mode 100644 index 0000000..afd51b1 --- /dev/null +++ b/src/sheerkapickle/tags.py @@ -0,0 +1,6 @@ +ID = "_sheerka/id" +TUPLE = "_sheerka/tuple" +SET = "_sheerka/set" +OBJECT = "_sheerka/obj" +ENUM = "_sheerka/enum" +CUSTOM = "_sheerka/custom" diff --git a/src/sheerkapickle/utils.py b/src/sheerkapickle/utils.py new file mode 100644 index 0000000..83a1e64 --- /dev/null +++ b/src/sheerkapickle/utils.py @@ -0,0 +1,101 @@ +import base64 +import types +from enum import Enum + +from common.global_symbols import CustomType + +class_types = (type,) +PRIMITIVES = (str, bool, type(None), int, float) + + +def is_type(obj): + """Returns True is obj is a reference to a type. + """ + # use "isinstance" and not "is" to allow for metaclasses + return isinstance(obj, class_types) + + +def is_enum(obj): + return isinstance(obj, Enum) + + +def is_custom_type(obj): + return isinstance(obj, CustomType) + + +def is_object(obj): + """Returns True is obj is a reference to an object instance.""" + + return (isinstance(obj, object) and + not isinstance(obj, (type, + types.FunctionType, + types.BuiltinFunctionType, + types.GeneratorType))) + + +def is_to_discard(obj): + return isinstance(obj, (types.GeneratorType, types.CodeType)) + + +def is_primitive(obj): + return isinstance(obj, PRIMITIVES) + + +def is_dictionary(obj): + return isinstance(obj, dict) + + +def is_list(obj): + return isinstance(obj, list) + + +def is_set(obj): + return isinstance(obj, set) + + +def is_bytes(obj): + return isinstance(obj, bytes) + + +def is_tuple(obj): + return isinstance(obj, tuple) + + +def is_class(obj): + return isinstance(obj, type) + + +def b64encode(data): + """ + Encode binary data to ascii text in base64. Data must be bytes. + """ + return base64.b64encode(data).decode('ascii') + + +def translate_module_name(module): + """Rename builtin modules to a consistent module name. + + Prefer the more modern naming. + + This is used so that references to Python's `builtins` module can + be loaded in both Python 2 and 3. We remap to the "__builtin__" + name and unmap it when importing. + + Map the Python2 `exceptions` module to `builtins` because + `builtins` is a superset and contains everything that is + available in `exceptions`, which makes the translation simpler. + + See untranslate_module_name() for the reverse operation. + """ + lookup = dict(__builtin__='builtins', exceptions='builtins') + return lookup.get(module, module) + + +def importable_name(cls): + """ + Fully qualified name (prefixed by builtin when needed) + """ + # Use the fully-qualified name if available (Python >= 3.3) + name = getattr(cls, '__qualname__', cls.__name__) + module = translate_module_name(cls.__module__) + return '{}.{}'.format(module, name) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/base.py b/tests/base.py new file mode 100644 index 0000000..9727d6a --- /dev/null +++ b/tests/base.py @@ -0,0 +1,35 @@ +import os +import shutil +from os import path + +import pytest + +from core.Sheerka import Sheerka +from sdp.sheerkaDataProvider import SheerkaDataProvider + + +class BaseTest: + @pytest.fixture() + def sdp(self) -> SheerkaDataProvider: + return SheerkaDataProvider("mem://", name="test") + + +class UsingFileBasedSheerka(BaseTest): + TESTS_ROOT_DIRECTORY = path.abspath("../build/tests") + SHEERKA_ROOT_DIR = os.path.join(TESTS_ROOT_DIRECTORY, ".sheerka") + + @pytest.fixture(scope="class") + def sheerka_fb(self): + """ + the default fixture to get Sheerka is overriden + :return: + :rtype: + """ + # first, make sure to create a fresh environment + if path.exists(self.SHEERKA_ROOT_DIR): + shutil.rmtree(self.SHEERKA_ROOT_DIR) + + # create the new Sheerka instance + sheerka = Sheerka() + sheerka.initialize(root_folder=self.SHEERKA_ROOT_DIR) + return sheerka diff --git a/tests/caching/__init__.py b/tests/caching/__init__.py new file mode 100644 index 0000000..8aba1e2 --- /dev/null +++ b/tests/caching/__init__.py @@ -0,0 +1,18 @@ +class FakeSdp: + def __init__(self, /, get_value=None, extend_exists=None, get_alt_value=None, populate=None): + self.get_value = get_value + self.extend_exists = extend_exists + self.populate_function = populate + self.get_alt_value = get_alt_value + + def get(self, cache_name, key): + return self.get_value(cache_name, key) + + def exists(self, cache_name, key): + return self.extend_exists(cache_name, key) + + def alt_get(self, cache_name, key): + return self.get_alt_value(cache_name, key) + + def populate(self): + return self.populate_function() if callable(self.populate_function) else self.populate_function diff --git a/tests/caching/test_CacheManager.py b/tests/caching/test_CacheManager.py new file mode 100644 index 0000000..2a13293 --- /dev/null +++ b/tests/caching/test_CacheManager.py @@ -0,0 +1,288 @@ +from dataclasses import dataclass + +import pytest + +from base import BaseTest +from caching.Cache import Cache +from caching.CacheManager import CacheManager, ConceptNotFound +from caching.ListIfNeededCache import ListIfNeededCache +from common.global_symbols import NotFound +from helpers import get_metadata +from tests.caching import FakeSdp + + +@dataclass +class Obj: + key: str + value: str + + +class TestCacheManager(BaseTest): + + def test_i_can_push_into_sdp(self, context, sdp): + cache_manager = CacheManager(sdp) + cache_manager.register_cache("test", Cache(), persist=True) + cache_manager.put("test", "key", "value") + + cache_manager.commit(context) + assert sdp.exists("test", "key") + + def test_sdp_given_to_the_cache_manager_can_be_used_by_the_cache(self, context, sdp): + """ + When the sdp is given to the cache_manager, the 'Cache' object can use the lambda with two parameters + :param context: + :type context: + :return: + :rtype: + """ + cache_manager = CacheManager(sdp) + cache = Cache(default=lambda _sdp, k: _sdp.get("test", k)) + cache_manager.register_cache("test", cache) + + with cache_manager.sdp.get_transaction(context.event) as transaction: + transaction.add("test", "key", "value") + + assert cache_manager.get("test", "key") == "value" + + def test_cache_can_use_auto_configure_when_sdp_is_given_to_the_cache_manager(self, context, sdp): + """ + When sdp is given to the cache manager, + Cache can be configured using auto_configure as the sdp will be provided by the cache_manager + :param context: + :type context: + :return: + :rtype: + """ + with sdp.get_transaction(context.event) as transaction: + transaction.add("test", "key", "value") + + cache_manager = CacheManager(sdp) + cache = Cache().auto_configure("test") + cache_manager.register_cache("test", cache) + + assert cache_manager.get("test", "key") == "value" + + def test_i_can_get_value_when_the_sdp_is_given_to_the_cache(self, context, sdp): + """ + If the sdp is not given to the cache manager, the cache will need to explicitly provide the sdp + during its configuration + :param context: + :type context: + :return: + :rtype: + """ + with sdp.get_transaction(context.event) as transaction: + transaction.add("test", "key", "value") + + cache_manager = CacheManager() + cache = Cache(default=lambda k: sdp.get("test", k)) + cache_manager.register_cache("test", cache) + + assert cache_manager.get("test", "key") == "value" + + def test_i_can_get_value_from_alt_sdp(self, sdp): + """ + When nothing is found in cache and in sdp, we use alternate sdp + :param sdp: + :type sdp: + :return: + :rtype: + """ + cache_manager = CacheManager(sdp) + cache_manager.register_cache("test", Cache().auto_configure("test")) + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !") + assert cache_manager.get("test", "key", alt_sdp=alt_sdp) == "value found !" + + def test_i_can_commit_simple_cache(self, context, sdp): + cache_manager = CacheManager(sdp) + cache_manager.register_cache("test", Cache().auto_configure("test")) + cache = cache_manager.caches["test"].cache + + cache_manager.put("test", "key", "value") + + cache_manager.commit(context) + + cache.clear() + assert cache_manager.sdp.get("test", "key") == "value" + assert cache.get("key") == "value" + + cache.update("key", "value", "key", "another_value") + cache_manager.commit(context) + assert cache_manager.sdp.get("test", "key") == "another_value" + + cache.update("key", "another_value", "key2", "another_value") + cache_manager.commit(context) + assert cache_manager.sdp.get("test", "key") is NotFound + assert cache_manager.sdp.get("test", "key2") == "another_value" + + # sanity check + # sdp 'test' has value, but sdp '__default__' does not + assert cache_manager.sdp.name == "test" + assert cache_manager.sdp.state.data == {'test': {'key2': 'another_value'}} + + def test_i_can_use_concept_cache(self): + cache_manager = CacheManager() + cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True) + cache_manager.register_concept_cache("name", ListIfNeededCache(), lambda c: c.name, True) + + # caches are correctly created + assert cache_manager.concept_caches == ["id", "name"] + assert "id" in cache_manager.caches + assert "name" in cache_manager.caches + + # caches are correctly updated on insertion + meta1 = get_metadata(id="1", name="foo") + meta2 = get_metadata(id="2", name="bar") + meta3 = get_metadata(id="3", name="foo") + + for metadata in meta1, meta2, meta3: + cache_manager.add_concept(metadata) + + assert cache_manager.get_inner_cache("id").copy() == {"1": meta1, "2": meta2, "3": meta3} + assert cache_manager.get_inner_cache("name").copy() == {"foo": [meta1, meta3], "bar": meta2} + + # caches are correctly updated on modification + meta3prime = get_metadata(id="3", name="bar") + cache_manager.update_concept(meta3, meta3prime) + + assert cache_manager.get_inner_cache("id").copy() == {"1": meta1, "2": meta2, "3": meta3prime} + assert cache_manager.get_inner_cache("name").copy() == {"foo": meta1, "bar": [meta2, meta3prime]} + + # caches are correctly updated on removal + cache_manager.remove_concept(meta3prime) + assert cache_manager.get_inner_cache("id").copy() == {"1": meta1, "2": meta2} + assert cache_manager.get_inner_cache("name").copy() == {"foo": meta1, "bar": meta2} + + def test_i_cannot_remove_a_concept_that_does_not_exists(self): + cache_manager = CacheManager() + cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True) + cache_manager.register_concept_cache("key", ListIfNeededCache(), lambda c: c.key, True) + + meta1 = get_metadata(id="1", name="foo") + + with pytest.raises(ConceptNotFound) as ex: + cache_manager.remove_concept(meta1) + + assert ex.value.concept == meta1 + + def test_nothing_is_sent_to_sdp_if_persist_is_false(self, context, sdp): + cache_manager = CacheManager(sdp) + cache_manager.register_cache("test", Cache(), persist=False) + + cache_manager.put("test", "key", "value") + cache_manager.commit(context) + + value = sdp.get("test", "key") + assert value is NotFound + + def test_i_can_delete_from_cache_manager(self, context, sdp): + cache_manager = CacheManager(sdp) + cache_manager.register_cache("test", Cache(), persist=True) + cache_manager.put("test", "key", "value") + cache_manager.commit(context) + + # sanity check + assert cache_manager.get("test", "key") == "value" + assert sdp.get("test", "key") == "value" + + # I remove but I don't commit + cache_manager.delete("test", "key") + assert cache_manager.get("test", "key") is NotFound + assert sdp.get("test", "key") == "value" + + # commit + cache_manager.commit(context) + assert cache_manager.get("test", "key") is NotFound + assert sdp.get("test", "key") is NotFound + + def test_i_can_get_the_inner_cache(self): + cache_manager = CacheManager() + cache = Cache() + cache_manager.register_cache("test", cache) + + inner_cache = cache_manager.get_inner_cache("test") + assert id(inner_cache) == id(cache) + + def test_i_can_get_a_copy_of_a_cache(self): + cache_manager = CacheManager() + cache = Cache() + cache.put("key1", "value1") + cache.put("key2", "value2") + cache_manager.register_cache("test", cache) + + copy = cache_manager.copy("test") + + assert isinstance(copy, dict) + assert copy == {"key1": "value1", "key2": "value2"} + + def test_i_can_populate(self): + cache_manager = CacheManager() + cache = Cache() + cache_manager.register_cache("test", cache) + + obj1, obj2 = Obj("key1", "value1"), Obj("key2", "value2") + cache_manager.populate("test", lambda: [obj1, obj2], lambda o: o.key) + + assert cache_manager.copy("test") == {"key1": obj1, "key2": obj2} + + def test_has(self, context, sdp): + cache_manager = CacheManager(sdp) + cache_manager.register_cache("test", Cache().auto_configure("test")) + + with cache_manager.sdp.get_transaction(context.event) as transaction: + transaction.add("test", "key_from_sdp", "value") + + cache_manager.put("test", "key_in_cache", "value") + + assert cache_manager.has("test", "key_in_cache") + assert not cache_manager.has("test", "key_from_sdp") + + def test_exist(self, context, sdp): + cache_manager = CacheManager(sdp) + cache_manager.register_cache("test", Cache().auto_configure("test")) + + with cache_manager.sdp.get_transaction(context.event) as transaction: + transaction.add("test", "key_from_sdp", "value") + + cache_manager.put("test", "key_in_cache", "value") + + assert cache_manager.exists("test", "key_in_cache") + assert cache_manager.exists("test", "key_from_sdp") + + def test_i_can_clear_a_single_cache(self, context, sdp): + cache_manager = CacheManager(sdp) + cache_manager.register_cache("test", Cache().auto_configure("test")) + cache_manager.put("test", "key1", "value1") + cache_manager.put("test", "key2", "value2") + + cache_manager.commit(context) + + cache_manager.clear("test") + assert cache_manager.copy("test") == {} + assert sdp.exists("test", "key1") + assert sdp.exists("test", "key2") + + cache_manager.commit(context) + assert cache_manager.copy("test") == {} + assert not sdp.exists("test", "key1") + assert not sdp.exists("test", "key2") + + def test_i_can_clear_all_caches(self, sdp): + cache_manager = CacheManager(sdp) + cache_manager.register_cache("test1", Cache()) + cache_manager.register_cache("test2", Cache()) + cache_manager.put("test1", "key", "value") + cache_manager.put("test2", "key", "value") + + cache_manager.clear() + assert cache_manager.copy("test1") == {} + assert cache_manager.copy("test2") == {} + + def test_i_cannot_add_null_keys_into_concept_cache(self): + cache_manager = CacheManager() + cache_manager.register_concept_cache("id", Cache(), lambda c: c.id, True) + + with pytest.raises(KeyError): + meta1 = get_metadata(name="foo") + cache_manager.add_concept(meta1) diff --git a/tests/caching/test_DictionaryCache.py b/tests/caching/test_DictionaryCache.py new file mode 100644 index 0000000..acb2363 --- /dev/null +++ b/tests/caching/test_DictionaryCache.py @@ -0,0 +1,165 @@ +import pytest + +from base import BaseTest +from caching.DictionaryCache import DictionaryCache +from common.global_symbols import NotFound +from tests.caching import FakeSdp + + +class TestDictionaryCache(BaseTest): + @pytest.mark.parametrize('key', [None, "str_value", 0, 1.0]) + def test_key_must_be_true_or_false(self, key): + cache = DictionaryCache() + + # key must be true False + with pytest.raises(KeyError): + cache.put("key", key) + + def test_value_must_be_a_dictionary(self): + cache = DictionaryCache() + + with pytest.raises(ValueError): + cache.put(True, "value") + + with pytest.raises(ValueError): + cache.put(False, "value") + + def test_i_can_put_and_retrieve_value_from_dictionary_cache(self): + cache = DictionaryCache() + + entry = {"key": "value", "key2": ["value21", "value22"]} + cache.put(False, entry) + assert len(cache) == 3 + assert id(cache._cache) == id(entry) + assert cache.get("key") == "value" + assert cache.get("key2") == ["value21", "value22"] + + # I can append values + cache.put(True, {"key": "another_value", "key3": "value3"}) + assert len(cache) == 4 + assert cache.get("key") == "another_value" + assert cache.get("key2") == ["value21", "value22"] + assert cache.get("key3") == "value3" + + # I can reset + entry = {"key": "value", "key2": ["value21", "value22"]} + cache.put(False, entry) + assert len(cache) == 3 + assert id(cache._cache) == id(entry) + assert cache.get("key") == "value" + assert cache.get("key2") == ["value21", "value22"] + assert cache.get("key3") is NotFound + + assert cache.copy() == {'key': 'value', 'key2': ['value21', 'value22']} + + def test_i_can_get_a_value_that_does_not_exist_without_compromising_the_cache(self): + cache = DictionaryCache() + cache.put(False, {"key": "value"}) + + assert cache.get("key2") is NotFound + assert cache.copy() == {"key": "value"} + + def test_i_can_append_to_a_dictionary_cache_even_if_it_is_new(self): + cache = DictionaryCache() + + entry = {"key": "value", "key2": ["value21", "value22"]} + cache.put(True, entry) + assert len(cache) == 3 + assert id(cache._cache) != id(entry) + assert cache.get("key") == "value" + assert cache.get("key2") == ["value21", "value22"] + + def test_exists_in_dictionary_cache(self): + cache = DictionaryCache() + assert not cache.exists("key") + + cache.put(True, {"key": "value"}) + assert cache.exists("key") + + def test_default_for_dictionary_cache(self): + cache = DictionaryCache(default={"key": "value", "key2": "value2"}) + + # cache is fully set when the value is found + assert cache.get("key") == "value" + assert cache.copy() == {"key": "value", "key2": "value2"} + + # cache is fully set when the value is not found + cache.test_only_reset() + assert cache.get("key3") is NotFound + assert cache.copy() == {"key": "value", "key2": "value2"} + + # cache is not corrupted when value is found + cache.put(True, {"key3": "value3", "key4": "value4"}) + assert cache.get("key3") == "value3" + assert cache.copy() == {"key": "value", "key2": "value2", "key3": "value3", "key4": "value4"} + + # cache is not corrupted when value is not found + cache._cache["key"] = "another value" # operation that is normally not possible + assert cache.get("key5") is NotFound + assert cache.copy() == {"key": "value", "key2": "value2", "key3": "value3", "key4": "value4"} + + def test_default_callable_for_dictionary_cache(self): + cache = DictionaryCache(default=lambda k: {"key": "value", "key2": "value2"}) + + assert cache.get("key") == "value" + assert "key2" in cache + assert len(cache) == 2 + + cache.clear() + assert cache.get("key3") is NotFound + assert len(cache) == 2 + assert "key" in cache + assert "key2" in cache + + def test_default_callable_with_internal_sdp_for_dictionary_cache(self): + cache = DictionaryCache(default=lambda sdp, key: sdp.get("cache_name", key), + sdp=FakeSdp(lambda entry, k: {"key": "value", "key2": "value2"})) + + assert cache.get("key") == "value" + assert "key2" in cache + assert len(cache) == 2 + + cache.clear() + assert cache.get("key3") is NotFound + assert len(cache) == 2 + assert "key" in cache + assert "key2" in cache + + def test_dictionary_cache_cannot_be_null(self): + cache = DictionaryCache(default=lambda k: NotFound) + assert cache.get("key") is NotFound + assert cache._cache == {} + + cache = DictionaryCache(default=NotFound) + assert cache.get("key") is NotFound + assert cache._cache == {} + + cache = DictionaryCache(default=lambda k: None) + assert cache.get("key") is NotFound + assert cache._cache == {} + + cache = DictionaryCache(default=None) + assert cache.get("key") is NotFound + assert cache._cache == {} + + def test_auto_configure_retrieves_the_whole_remote_repository(self, sdp, context): + cache = DictionaryCache(sdp=sdp).auto_configure("test") + with sdp.get_transaction(context.event) as transaction: + transaction.add("test", "key1", "value1") + transaction.add("test", "key2", "value2") + + # when call for a value that is not in the cache, Dictionary cache is configured to retrieve the repo + cache.get("value") + + assert cache.copy() == {'key1': 'value1', 'key2': 'value2'} + + def test_we_do_no_go_twice_in_repo_when_not_found(self, sdp, context): + cache = DictionaryCache(sdp=sdp).auto_configure("test") + + assert cache.get("key") is NotFound + + # now add a value in remote repo + with sdp.get_transaction(context.event) as transaction: + transaction.add("test", "key", "value") + + assert cache.get("key") is NotFound # the key was previously requested diff --git a/tests/caching/test_FastCache.py b/tests/caching/test_FastCache.py new file mode 100644 index 0000000..ad87579 --- /dev/null +++ b/tests/caching/test_FastCache.py @@ -0,0 +1,111 @@ +from caching.FastCache import FastCache +from common.global_symbols import NotFound + + +def test_i_can_put_an_retrieve_values(): + cache = FastCache() + cache.put("key", "value") + + assert cache.get("key") == "value" + assert cache.cache == {"key": "value"} + assert cache.lru == ["key"] + + +def test_i_can_put_and_retrieve_multiple_items(): + cache = FastCache() + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key3", "value3") + + assert cache.cache == {"key1": "value1", "key2": "value2", "key3": "value3"} + assert cache.lru == ["key1", "key2", "key3"] + + +def test_i_the_least_used_is_remove_first(): + cache = FastCache(3) + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key3", "value3") + + cache.put("key4", "value4") + assert cache.cache == {"key4": "value4", "key2": "value2", "key3": "value3"} + assert cache.lru == ["key2", "key3", "key4"] + + cache.put("key5", "value5") + assert cache.cache == {"key4": "value4", "key5": "value5", "key3": "value3"} + assert cache.lru == ["key3", "key4", "key5"] + + +def test_i_can_put_the_same_key_several_times(): + cache = FastCache() + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key1", "value3") + + assert cache.cache == {"key1": "value3", "key2": "value2"} + assert cache.lru == ["key2", "key1"] + + +def test_none_is_returned_when_not_found(): + cache = FastCache() + assert cache.get("foo") is NotFound + + +def test_i_can_evict_by_key(): + cache = FastCache() + cache.put("key1", "value1") + cache.put("to_keep1", "to_keep_value1") + cache.put("key2", "value2") + cache.put("to_keep2", "to_keep_value2") + cache.put("key3", "value3") + cache.put("to_keep3", "to_keep_value3") + + cache.evict_by_key(lambda k: k.startswith("key")) + assert cache.cache == {"to_keep1": "to_keep_value1", + "to_keep2": "to_keep_value2", + "to_keep3": "to_keep_value3"} + + assert cache.lru == ["to_keep1", "to_keep2", "to_keep3"] + + +def test_i_can_get_default_value(): + cache = FastCache(max_size=3, default=lambda key: key + 1) + + assert cache.get(1) == 2 + assert cache.get(2) == 3 + assert cache.get(3) == 4 + assert cache.get(4) == 5 + + assert cache.cache == {2: 3, 3: 4, 4: 5} # only 3 values + + +def test_i_can_iter_on_entries(): + cache = FastCache() + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key3", "value3") + + res = [] + for k in cache: + assert k in cache + res.append(k) + + assert res == ["key1", "key2", "key3"] + + +def test_i_can_count(): + cache = FastCache() + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key3", "value3") + + assert len(cache) == 3 + + +def test_i_can_copy(): + cache = FastCache() + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key3", "value3") + + assert cache.copy() == {"key1": "value1", "key2": "value2", "key3": "value3"} diff --git a/tests/caching/test_IncCache.py b/tests/caching/test_IncCache.py new file mode 100644 index 0000000..43cfaef --- /dev/null +++ b/tests/caching/test_IncCache.py @@ -0,0 +1,87 @@ +from base import BaseTest +from caching.IncCache import IncCache +from common.global_symbols import NotFound, Removed +from tests.caching import FakeSdp + + +class FakeIncSdp: + """ + FakeIncSdp has two levels of "ontology" + if a value of + """ + def __init__(self, init_value1, init_value2): + self.level1 = IncCache() + self.level1.put("key", init_value1) + + self.level2 = IncCache() + self.level2.put("key", init_value2) + + def alt_get(self, cache_name, key): + for cache in [self.level1, self.level2]: + value = cache.alt_get(key) + if value is not NotFound: + return value + + return NotFound + + +class TestIncCache(BaseTest): + def test_i_can_put_and_retrieve_values_from_inc_cache(self): + cache = IncCache() + + assert cache.get("key") == 1 + assert cache.get("key") == 2 + assert cache.get("key") == 3 + assert cache.get("key2") == 1 + assert cache.get("key2") == 2 + + cache.put("key", 100) + assert cache.get("key") == 101 + + assert cache.copy() == {'key': 101, 'key2': 2} + + def test_i_can_alt_get(self): + cache = IncCache() + + assert cache.get("key") == 1 + assert cache.get("key") == 2 + assert cache.alt_get("key") == 2 + assert cache.alt_get("key") == 2 + assert cache.get("key") == 3 + + def test_current_cache_takes_precedence_over_alt_sdp(self): + cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + assert cache.get("key") == 1 + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: 10) + assert cache.get("key", alt_sdp=alt_sdp) == 2 + + def test_remote_repository_takes_precedence_over_alt_sdp(self): + cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: 5)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: 10) + assert cache.get("key", alt_sdp=alt_sdp) == 6 + assert cache.get("key", alt_sdp=alt_sdp) == 7 # then we use the value from the cache + + def test_i_can_take_value_from_alt_sdp(self): + cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeIncSdp(10, NotFound) + assert cache.get("key", alt_sdp=alt_sdp) == 11 + assert cache.get("key", alt_sdp=alt_sdp) == 12 # then we use the value from the cache + + def test_i_can_get_when_alt_sdp_and_cache_is_cleared(self): + cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + cache.clear() + + alt_sdp = FakeIncSdp(10, NotFound) + assert cache.get("key", alt_sdp=alt_sdp) == 1 + assert cache.get("key", alt_sdp=alt_sdp) == 2 # then we use the value from the cache + + def test_i_can_manage_when_the_value_from_alt_sdp_is_removed(self): + cache = IncCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeIncSdp(Removed, 10) + assert cache.get("key", alt_sdp=alt_sdp) == 1 + assert cache.get("key", alt_sdp=alt_sdp) == 2 # then we use the value from the cache \ No newline at end of file diff --git a/tests/caching/test_ListCache.py b/tests/caching/test_ListCache.py new file mode 100644 index 0000000..803fc47 --- /dev/null +++ b/tests/caching/test_ListCache.py @@ -0,0 +1,281 @@ +import pytest + +from base import BaseTest +from caching.ListCache import ListCache +from common.global_symbols import NotFound, Removed +from tests.caching import FakeSdp + + +class TestListCache(BaseTest): + + def test_i_can_put_and_retrieve_value_from_list_cache(self): + cache = ListCache() + + cache.put("key", "value") + assert cache.get("key") == ["value"] + assert len(cache) == 1 + + cache.put("key", "value2") # we can append to this list + assert cache.get("key") == ["value", "value2"] + assert len(cache) == 2 + + cache.put("key2", "value") + assert cache.get("key2") == ["value"] + assert len(cache) == 3 + + # duplicates are allowed + cache.put("key", "value") + assert cache.get("key") == ["value", "value2", "value"] + assert len(cache) == 4 + + assert cache.copy() == {'key': ['value', 'value2', 'value'], 'key2': ['value']} + + def test_i_can_put_in_list_cache_when_alt_sdp_returns_values(self): + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])) + assert cache.get("key") == ["value1", "value2"] + + cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed)) + assert cache.get("key3") == ["value1"] + + def test_i_can_put_in_list_cache_when_alt_sdp_returns_values_and_cache_is_cleared(self): + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + cache.clear() + + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])) + assert cache.get("key") == ["value2"] + + cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed)) + assert cache.get("key3") == ["value1"] + + def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_list_cache(self): + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key", "value1") + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx")) + assert cache.get("key") == ["value1", "value2"] + + def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_list_cache(self): + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: ["value1"])).auto_configure("cache_name") + + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx")) + assert cache.get("key") == ["value1", "value2"] + + def test_i_can_get_when_alt_sdp(self): + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.get("key", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1"])) + assert cache.get("key") == ["value1"] + + def test_i_can_update_from_list_cache(self): + cache = ListCache() + + cache.put("key", "value") + cache.put("key", "value2") + cache.put("key", "value") + cache.update("key", "value", "key", "another value") + + assert len(cache._cache) == 1 + assert len(cache) == 3 + assert cache.get("key") == ["another value", "value2", "value"] # only the first one is affected + + cache.update("key", "value2", "key2", "value2") + assert len(cache._cache) == 2 + assert len(cache) == 3 + assert cache.get("key") == ["another value", "value"] + assert cache.get("key2") == ["value2"] + + cache.update("key2", "value2", "key3", "value2") + assert len(cache._cache) == 2 + assert len(cache) == 3 + assert cache.get("key") == ["another value", "value"] + assert cache.get("key3") == ["value2"] + assert cache.get("key2") is NotFound + + with pytest.raises(KeyError): + cache.update("wrong key", "value", "key", "value") + + def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self): + cache = ListCache(default=lambda sdp, key: sdp.get("cache_name", key), + extend_exists=lambda sdp, key: sdp.exists("cache_name", key), + sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)) + + cache.put("key", "value") + cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True)) + + assert cache.get("key") == ["new_value"] + + def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self): + # There is nothing in cache or remote repository. + # We must ust the value from alt_sdp + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + previous_value = ["old_1", "old_2", "value"] + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True, + get_alt_value=lambda cache_name, key: previous_value) + cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp) + assert cache.get("key") == ["old_1", "old_2", "new_value"] + assert previous_value == ["old_1", "old_2", "value"] + + def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self): + # keys are different + # make sure that current cache take precedence over alt_sdp + # In this test, the values from alt_sdp are never used + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: ["xxx1"] if key == "key1" else NotFound) + + # one values in 'key1' + cache.put("key1", "old_1") + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == Removed + assert cache.get("key2") == ["new_value"] + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # Multiple values in 'key1' + cache.clear() + cache.put("key1", "old_1") + cache.put("key1", "old_2") + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == ["old_2"] + assert cache.get("key2") == ["new_value"] + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self): + # keys are different + # make sure that current repo take precedence over alt_sdp + remote = FakeSdp(get_value=lambda cache_name, key: ["old_1"] if key == "key1" else NotFound) + cache = ListCache(sdp=remote).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: ["xxx1"] if key == "key1" else NotFound) + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == Removed + assert cache.get("key2") == ["new_value"] + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # Multiple values in 'key1' + remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2"] if key == "key1" else NotFound) + cache = ListCache(sdp=remote).auto_configure("cache_name") + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == ["old_2"] + assert cache.get("key2") == ["new_value"] + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self): + # keys are different + # No value found in cache or remote repository, + # Will use values from alt_sdp + # The old key is the same, so it has to be marked as Removed + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + # one values in 'key1' + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: ["old_1"] if key == "key1" else NotFound) + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == Removed + assert cache.get("key2") == ["new_value"] + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # Multiple values in 'key1' + cache.test_only_reset() + old_values = ["old_1", "old_2"] + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound) + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == ["old_2"] + assert cache.get("key2") == ["new_value"] + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + assert old_values == ["old_1", "old_2"] # not modified + + def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self): + # If a value exists in destination key, either in local cache or remote repository + # It takes precedence + # If no value is found, we must use the value from alt_sdp + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: ["xxx2"] if key == "key2" else NotFound) + cache.put("key1", "source_value") + cache.put("key2", "old_value") + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == ['old_value', 'new_value'] + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + + def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self): + # If a value exists in destination key, either in local cache or remote repository + # It takes precedence + # If no value is found, we must use the value from alt_sdp + remote_repo = FakeSdp(get_value=lambda cache_name, key: ["old_value"] if key == "key2" else NotFound) + cache = ListCache(sdp=remote_repo).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: ["xxx2"] if key == "key2" else NotFound) + cache.put("key1", "source_value") + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == ['old_value', 'new_value'] + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + + def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self): + # If a value exists in destination key, either in local cache or remote repository + # It takes precedence + # If no value is found, we must use the value from alt_sdp + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key1", "source_value") + previous_values = ["old_1", "old_2"] + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound) + + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == ["old_1", "old_2", 'new_value'] + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + assert previous_values == ["old_1", "old_2"] # not modified + + def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self): + cache = ListCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]) + cache.clear() + + with pytest.raises(KeyError): + cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp) + + with pytest.raises(KeyError): + cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp) + + def test_default_is_called_before_updating_list_cache(self): + cache = ListCache(default=lambda k: NotFound) + with pytest.raises(KeyError): + cache.update("old_key", "old_value", "new_key", "new_value") + + cache = ListCache(default=lambda k: ["old_value", "other old value"]) + cache.update("old_key", "old_value", "old_key", "new_value") + assert cache.get("old_key") == ["new_value", "other old value"] + + cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else NotFound) + cache.update("old_key", "old_value", "new_key", "new_value") + assert cache.get("old_key") == ["other old value"] + assert cache.get("new_key") == ["new_value"] + + cache = ListCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else ["other new"]) + cache.update("old_key", "old_value", "new_key", "new_value") + assert cache.get("old_key") == ["other old value"] + assert cache.get("new_key") == ["other new", "new_value"] \ No newline at end of file diff --git a/tests/caching/test_ListIfNeededCache.py b/tests/caching/test_ListIfNeededCache.py new file mode 100644 index 0000000..9593a0a --- /dev/null +++ b/tests/caching/test_ListIfNeededCache.py @@ -0,0 +1,648 @@ +import pytest + +from base import BaseTest +from caching.ListIfNeededCache import ListIfNeededCache +from common.global_symbols import NotFound, Removed +from tests.caching import FakeSdp + + +class TestListIfNeededCache(BaseTest): + def test_i_can_put_and_retrieve_value_from_list_if_needed_cache(self): + cache = ListIfNeededCache() + + cache.put("key", "value") + assert cache.get("key") == "value" + + # second time with the same key creates a list + cache.put("key", "value2") + assert cache.get("key") == ["value", "value2"] + assert len(cache) == 2 + + # third time, we now have a list + cache.put("key", "value3") + assert cache.get("key") == ["value", "value2", "value3"] + assert len(cache) == 3 + + # other keys are not affected + cache.put("key2", "value") + assert cache.get("key2") == "value" + assert len(cache) == 4 + + # duplicates are allowed + cache.put("key", "value") + assert cache.get("key") == ["value", "value2", "value3", "value"] + assert len(cache) == 5 + + def test_i_can_put_in_list_if_need_cache_when_alt_sdp_returns_values(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "value1")) + assert cache.get("key") == ["value1", "value2"] + + cache.put("key2", "value3", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2"])) + assert cache.get("key2") == ["value1", "value2", "value3"] + + cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed)) + assert cache.get("key3") == "value1" + + def test_i_can_put_in_list_if_need__cache_when_alt_sdp_returns_values_and_cache_is_cleared(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + cache.clear() + + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "value1")) + assert cache.get("key") == "value2" + + cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed)) + assert cache.get("key3") == "value1" + + def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_list_if_needed_cache(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key", "value1") + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx")) + assert cache.get("key") == ["value1", "value2"] + + def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_list_if_needed_cache(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: "value1")).auto_configure("cache_name") + + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx")) + assert cache.get("key") == ["value1", "value2"] + + def test_i_can_update_from_list_if_needed_cache(self): + cache = ListIfNeededCache() + + cache.put("key", "value") + cache.put("key", "value2") + cache.put("key", "value") + + # only the first 'value' is affected + cache.update("key", "value", "key", "another value") + assert len(cache._cache) == 1 + assert len(cache) == 3 + assert cache.get("key") == ["another value", "value2", "value"] + + # change the key + cache.update("key", "value2", "key2", "value2") + assert len(cache._cache) == 2 + assert len(cache) == 3 + assert cache.get("key") == ["another value", "value"] + assert cache.get("key2") == "value2" + + # rename the newly created key + cache.update("key2", "value2", "key3", "value2") + assert len(cache._cache) == 2 + assert len(cache) == 3 + assert cache.get("key") == ["another value", "value"] + assert cache.get("key3") == "value2" + assert cache.get("key2") is NotFound + + # from list to single item and vice versa + cache.update("key", "value", "key3", "value") + assert len(cache._cache) == 2 + assert len(cache) == 3 + assert cache.get("key") == "another value" # 'key' is no longer a list + assert cache.get("key3") == ["value2", "value"] # 'key3' is now a list + assert cache.get("key2") is NotFound + + with pytest.raises(KeyError): + cache.update("wrong key", "value", "key", "value") + + def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key", "value") + cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True)) + + assert cache.get("key") == "new_value" + + def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + # one value in alt_sdp + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True, + get_alt_value=lambda cache_name, key: "old_value") + cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp) + assert cache.get("key") == "new_value" + + # multiple values in alt_sdp + cache.test_only_reset() + previous_value = ["old_1", "old_2", "value"] + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True, + get_alt_value=lambda cache_name, key: previous_value) + cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp) + assert cache.get("key") == ["old_1", "old_2", "new_value"] + assert previous_value == ["old_1", "old_2", "value"] + + def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self): + # keys are different + # make sure that current cache take precedence over alt_sdp + # In this test, the values from alt_sdp are never used + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: "xxx1" if key == "key1" else NotFound) + + # one values in 'key1' + cache.put("key1", "old_1") + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == Removed + assert cache.get("key2") == "new_value" + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # Multiple values in 'key1' + cache.clear() + cache.put("key1", "old_1") + cache.put("key1", "old_2") + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == "old_2" + assert cache.get("key2") == "new_value" + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # List of values in 'key1' + cache.clear() + cache.put("key1", "old_1") + cache.put("key1", "old_2") + cache.put("key1", "old_3") + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == ["old_2", "old_3"] + assert cache.get("key2") == "new_value" + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self): + # keys are different + # make sure that current repo take precedence over alt_sdp + remote = FakeSdp(get_value=lambda cache_name, key: "old_1" if key == "key1" else NotFound) + cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: "xxx1" if key == "key1" else NotFound) + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == Removed + assert cache.get("key2") == "new_value" + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # Multiple values in 'key1' + remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2"] if key == "key1" else NotFound) + cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name") + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == "old_2" + assert cache.get("key2") == "new_value" + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # List of values in 'key1' + remote = FakeSdp(get_value=lambda cache_name, key: ["old_1", "old_2", "old_3"] if key == "key1" else NotFound) + cache = ListIfNeededCache(sdp=remote).auto_configure("cache_name") + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == ["old_2", "old_3"] + assert cache.get("key2") == "new_value" + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self): + # keys are different + # No value found in cache or remote repository, + # Will use values from alt_sdp + # The old key is the same, so it has to be marked as Removed + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + # one values in 'key1' + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: "old_1" if key == "key1" else NotFound) + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == Removed + assert cache.get("key2") == "new_value" + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # Multiple values in 'key1' + cache.test_only_reset() + old_values = ["old_1", "old_2"] + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound) + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == "old_2" + assert cache.get("key2") == "new_value" + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + assert old_values == ["old_1", "old_2"] # not modified + + # List of values in 'key1' + cache.test_only_reset() + old_values = ["old_1", "old_2", "old_3"] + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound) + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == ["old_2", "old_3"] + assert cache.get("key2") == "new_value" + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + assert old_values == ["old_1", "old_2", "old_3"] # not modified + + def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self): + # If a value exists in destination key, either in local cache or remote repository + # It take precedence + # If no value is found, we must use the value from alt_sdp + + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound) + cache.put("key1", "source_value") + cache.put("key2", "old_value") + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == ['old_value', 'new_value'] + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + + def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self): + # If a value exists in destination key, either in local cache or remote repository + # It take precedence + # If no value is found, we must use the value from alt_sdp + remote_repo = FakeSdp(get_value=lambda cache_name, key: "old_value" if key == "key2" else NotFound) + cache = ListIfNeededCache(sdp=remote_repo).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound) + cache.put("key1", "source_value") + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == ['old_value', 'new_value'] + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + + def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self): + # If a value exists in destination key, either in local cache or remote repository + # It take precedence + # If no value is found, we must use the value from alt_sdp + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + # one value in 'key2' + cache.put("key1", "source_value") + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: "old_value" if key == "key2" else NotFound) + + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == ['old_value', 'new_value'] + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + + # Multiple values in 'key2' + cache.test_only_reset() + cache.put("key1", "source_value") + previous_values = ["old_1", "old_2"] + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound) + + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == ["old_1", "old_2", 'new_value'] + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + assert previous_values == ["old_1", "old_2"] # not modified + + def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1") + cache.clear() + + with pytest.raises(KeyError): + cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp) + + with pytest.raises(KeyError): + cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp) + + def test_default_is_called_before_updating_list_if_needed_cache(self): + cache = ListIfNeededCache(default=lambda k: NotFound) + with pytest.raises(KeyError): + cache.update("old_key", "old_value", "new_key", "new_value") + + cache = ListIfNeededCache(default=lambda k: "old_value") + cache.update("old_key", "old_value", "old_key", "new_value") + assert cache.get("old_key") == "new_value" + + cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"]) + cache.update("old_key", "old_value", "old_key", "new_value") + assert cache.get("old_key") == ["new_value", "other old value"] + + cache = ListIfNeededCache(default=lambda k: ["old_value", "other old value"] if k == "old_key" else NotFound) + cache.update("old_key", "old_value", "new_key", "new_value") + assert cache.get("old_key") == "other old value" + assert cache.get("new_key") == "new_value" + + def test_i_can_delete_key_and_values(self): + cache = ListIfNeededCache() + cache.put("key", "value1") + cache.put("key", "value11") + cache.put("key2", "value2") + cache.put("key2", "value22") + cache.put("key2", "value222") + cache.put("key3", "value3") + cache.put("key3", "value33") + cache.put("key4", "value4") + cache.reset_events() + + assert len(cache) == 8 + + # I can remove a whole key + cache.delete("key") + assert cache.get("key") is NotFound + assert len(cache) == 6 + assert cache.to_remove == {"key"} + assert cache.to_add == set() + + # I can remove an element while a list is remaining + cache.reset_events() + cache.delete("key2", "value22") + assert cache.get("key2") == ["value2", "value222"] + assert len(cache) == 5 + assert cache.to_add == {"key2"} + assert cache.to_remove == set() + + # I can remove an element while a single element is remaining + cache.reset_events() + cache.delete("key3", "value33") + assert cache.get("key3") == "value3" + assert len(cache) == 4 + assert cache.to_add == {"key3"} + assert cache.to_remove == set() + + # I can remove an element while nothing remains + cache.reset_events() + cache.delete("key4", "value4") + assert cache.get("key4") is NotFound + assert len(cache) == 3 + assert cache.to_remove == {"key4"} + assert cache.to_add == set() + + # I do not remove when the value is not the same + cache.reset_events() + cache.delete("key3", "value33") # value33 was already remove + assert cache.get("key3") == "value3" + assert len(cache) == 3 + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_key_from_cache(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + cache.put("key", "value") + + cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + assert cache.copy() == {"key": Removed} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_value_from_cache(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + cache.put("key", "value") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value="value", alt_sdp=alt_sdp) + assert cache.copy() == {"key": Removed} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_cache_and_then_put_back(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + cache.put("key", "value") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value="value", alt_sdp=alt_sdp) # remove all values + cache.put("key", "value") + + assert cache.copy() == {"key": "value"} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_one_value(self): + # There is a value in alt_cache_manager, + # But this, there are remaining values in current cache after deletion + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + cache.put("key", "value") + cache.put("key", "value2") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value="value", alt_sdp=alt_sdp) + assert cache.copy() == {"key": "value2"} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_values(self): + # There is a value in alt_cache_manager, + # But this, there are remaining values in current cache after deletion + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + cache.put("key", "value") + cache.put("key", "value2") + cache.put("key", "value3") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value="value", alt_sdp=alt_sdp) + assert cache.copy() == {"key": ['value2', 'value3']} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure( + "cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value=None, alt_sdp=alt_sdp) + assert cache.copy() == {"key": Removed} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: "value")).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value="value", alt_sdp=alt_sdp) + assert cache.copy() == {"key": Removed} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository_and_then_put_back(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure( + "cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value=None, alt_sdp=alt_sdp) # remove all values + cache.put("key", "value") + + assert cache.copy() == {"key": "value"} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_remaining_one_value(self): + # There is a value in alt_cache_manager, + # But this, there are remaining values in current cache after deletion + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: ["value1", "value2"])).auto_configure( + "cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value="value1", alt_sdp=alt_sdp) + assert cache.copy() == {"key": "value2"} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_key_from_alt_sdp(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, the key is empty + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1, value2"], + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value=None, alt_sdp=alt_sdp) + assert cache.copy() == {"key": Removed} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, the key is empty + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1", + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value="value1", alt_sdp=alt_sdp) + assert cache.copy() == {"key": Removed} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_and_then_put_back(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, the key is empty + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value1", + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value="value1", alt_sdp=alt_sdp) + cache.put("key", "value") + + assert cache.copy() == {"key": "value"} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_one_value_remaining(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, one value remains in the cache + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2"], + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value="value1", alt_sdp=alt_sdp) + assert cache.copy() == {"key": "value2"} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_multiple_values_remaining(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, one value remains in the cache + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1", "value2", "value3"], + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value="value1", alt_sdp=alt_sdp) + assert cache.copy() == {"key": ["value2", "value3"]} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_an_already_removed_value_from_alt_sdp(self): + # alt_cache_manager is used because no value in cache or in remote repository + # But the alternate sdp returns Removed, which means that previous value was deleted + # It's like there is nothing to delete + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed, + extend_exists=lambda cache_name, key: False) + + cache.delete("key", value="value1", alt_sdp=alt_sdp) + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_deleting_an_entry_that_does_not_exist_is_not_an_error(self): + cache = ListIfNeededCache() + cache.put("key", "value1") + + cache.reset_events() + cache.delete("key3") + assert len(cache) == 1 + assert cache.to_add == set() + assert cache.to_remove == set() + + cache.delete("key3", "value") + assert len(cache) == 1 + assert cache.to_add == set() + assert cache.to_remove == set() + + cache.delete("key", "value2") + assert len(cache) == 1 + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value", + extend_exists=lambda cache_name, key: True) + + cache.clear() + cache.delete("key", value=None, alt_sdp=alt_sdp) + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + cache.delete("key", value="value", alt_sdp=alt_sdp) + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository(self): + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: Removed)).auto_configure("cache_name") + cache.put("key", "value") + + assert cache.copy() == {"key": "value"} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository_from_alt_sdp(self): + # The key is removed in the sub layers + # We can put it back + cache = ListIfNeededCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed, + extend_exists=lambda cache_name, key: False) + + cache.put("key", "value", alt_sdp=alt_sdp) + + assert cache.copy() == {"key": "value"} + assert cache.to_remove == set() + assert cache.to_add == {"key"} \ No newline at end of file diff --git a/tests/caching/test_SetCache.py b/tests/caching/test_SetCache.py new file mode 100644 index 0000000..3ad8f87 --- /dev/null +++ b/tests/caching/test_SetCache.py @@ -0,0 +1,540 @@ +import pytest + +from base import BaseTest +from caching.SetCache import SetCache +from common.global_symbols import NotFound, Removed +from tests.caching import FakeSdp + + +class TestSetCache(BaseTest): + + def test_i_can_put_and_retrieve_values_from_set_cache(self): + cache = SetCache() + + cache.put("key", "value") + assert cache.get("key") == {"value"} + assert len(cache) == 1 + + # we can add to this set + cache.put("key", "value2") + assert cache.get("key") == {"value", "value2"} + assert len(cache) == 2 + + # other keys are not affected + cache.put("key2", "value") + assert cache.get("key2") == {"value"} + assert len(cache) == 3 + + # duplicates are removed + cache.put("key", "value") + assert cache.get("key") == {"value", "value2"} + assert len(cache) == 3 + + assert cache.copy() == {'key': {'value', 'value2'}, 'key2': {'value'}} + + def test_i_can_put_in_set_cache_when_alt_sdp_returns_values(self): + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: {"value1"})) + assert cache.get("key") == {"value1", "value2"} + + cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed)) + assert cache.get("key3") == {"value1"} + + def test_i_can_put_in_set_cache_when_alt_sdp_returns_values_and_cache_is_cleared(self): + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + cache.clear() + + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: {"value1"})) + assert cache.get("key") == {"value2"} + + cache.put("key3", "value1", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: Removed)) + assert cache.get("key3") == {"value1"} + + def test_current_cache_take_precedence_over_alt_sdp_when_i_put_data_in_set_cache(self): + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key", "value1") + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx")) + assert cache.get("key") == {"value1", "value2"} + + def test_current_sdp_take_precedence_over_alt_sdp_when_i_put_data_in_set_cache(self): + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: {"value1"})).auto_configure("cache_name") + + cache.put("key", "value2", alt_sdp=FakeSdp(get_alt_value=lambda cache_name, key: "xxx")) + assert cache.get("key") == {"value1", "value2"} + + def test_i_can_update_from_set_cache(self): + cache = SetCache() + + cache.put("key", "value") + cache.put("key", "value2") + cache.update("key", "value", "key", "another value") + + assert len(cache._cache) == 1 + assert len(cache) == 2 + assert cache.get("key") == {"another value", "value2"} + + cache.update("key", "value2", "key2", "value2") + assert len(cache._cache) == 2 + assert len(cache) == 2 + assert cache.get("key") == {"another value"} + assert cache.get("key2") == {"value2"} + + cache.update("key", "another value", "key3", "another value") + assert len(cache._cache) == 2 + assert len(cache) == 2 + assert cache.get("key") is NotFound + assert cache.get("key2") == {"value2"} + assert cache.get("key3") == {"another value"} + + with pytest.raises(KeyError): + cache.update("wrong key", "value", "key", "value") + + def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same(self): + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key", "value") + cache.update("key", "value", "key", "new_value", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + + assert cache.get("key") == {"new_value"} + + def test_i_can_update_when_alt_sdp_from_cache_keys_are_the_same_but_nothing_in_cache(self): + # There is nothing in cache or remote repository. + # We must ust the value from alt_sdp + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + previous_value = {"old_1", "old_2", "value"} + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: True, + get_alt_value=lambda cache_name, key: previous_value) + cache.update("key", "value", "key", "new_value", alt_sdp=alt_sdp) + assert cache.get("key") == {"old_1", "old_2", "new_value"} + assert previous_value == {"old_1", "old_2", "value"} + + def test_i_can_update_when_alt_sdp_from_cache_keys_are_different(self): + # keys are different + # make sure that current cache take precedence over alt_sdp + # In this test, the values from alt_sdp are never used + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: {"xxx1"} if key == "key1" else NotFound) + + # one values in 'key1' + cache.put("key1", "old_1") + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == Removed + assert cache.get("key2") == {"new_value"} + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # Multiple values in 'key1' + cache.clear() + cache.put("key1", "old_1") + cache.put("key1", "old_2") + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == {"old_2"} + assert cache.get("key2") == {"new_value"} + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + def test_i_can_update_when_alt_sdp_from_repository_keys_are_different(self): + # keys are different + # make sure that current repo take precedence over alt_sdp + remote = FakeSdp(get_value=lambda cache_name, key: {"old_1"} if key == "key1" else NotFound) + cache = SetCache(sdp=remote).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: {"xxx1"} if key == "key1" else NotFound) + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == Removed + assert cache.get("key2") == {"new_value"} + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # Multiple values in 'key1' + remote = FakeSdp(get_value=lambda cache_name, key: {"old_1", "old_2"} if key == "key1" else NotFound) + cache = SetCache(sdp=remote).auto_configure("cache_name") + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == {"old_2"} + assert cache.get("key2") == {"new_value"} + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + def test_i_can_update_when_alt_sdp_from_alt_sdp_keys_are_different_one_value(self): + # keys are different + # No value found in cache or remote repository, + # Will use values from alt_sdp + # The old key is the same, so it has to be marked as Removed + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + # one values in 'key1' + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: {"old_1"} if key == "key1" else NotFound) + + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == Removed + assert cache.get("key2") == {"new_value"} + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + + # Multiple values in 'key1' + cache.test_only_reset() + old_values = {"old_1", "old_2"} + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key1", + get_alt_value=lambda cache_name, key: old_values if key == "key1" else NotFound) + cache.update("key1", "old_1", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == {"old_2"} + assert cache.get("key2") == {"new_value"} + assert cache.to_add == {"key2", "key1"} + assert cache.to_remove == set() + assert old_values == {"old_1", "old_2"} # not modified + + def test_i_can_update_when_alt_sdp_cache_take_precedence_for_destination_key(self): + # If a value exists in destination key, either in local cache or remote repository + # It take precedence + # If no value is found, we must use the value from alt_sdp + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: {"xxx2"} if key == "key2" else NotFound) + cache.put("key1", "source_value") + cache.put("key2", "old_value") + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == {'old_value', 'new_value'} + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + + def test_i_can_update_when_alt_sdp_repository_take_precedence_for_destination_key(self): + # If a value exists in destination key, either in local cache or remote repository + # It take precedence + # If no value is found, we must use the value from alt_sdp + remote_repo = FakeSdp(get_value=lambda cache_name, key: {"old_value"} if key == "key2" else NotFound) + cache = SetCache(sdp=remote_repo).auto_configure("cache_name") + + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: "xxx2" if key == "key2" else NotFound) + cache.put("key1", "source_value") + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == {'old_value', 'new_value'} + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + + def test_i_can_update_when_alt_sdp_use_alt_sdp_when_no_destination_value_found(self): + # If a value exists in destination key, either in local cache or remote repository + # It take precedence + # If no value is found, we must use the value from alt_sdp + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + cache.put("key1", "source_value") + previous_values = {"old_1", "old_2"} + alt_sdp = FakeSdp(extend_exists=lambda cache_name, key: key == "key2", + get_alt_value=lambda cache_name, key: previous_values if key == "key2" else NotFound) + + cache.update("key1", "source_value", "key2", "new_value", alt_sdp=alt_sdp) + assert cache.get("key1") == NotFound + assert cache.get("key2") == {"old_1", "old_2", 'new_value'} + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key1"} + assert previous_values == {"old_1", "old_2"} # not modified + + def test_i_can_update_when_alt_sdp_and_cache_is_cleared(self): + cache = SetCache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: ["value1"]) + cache.clear() + + with pytest.raises(KeyError): + cache.update("key", "value1", "key", "value2", alt_sdp=alt_sdp) + + with pytest.raises(KeyError): + cache.update("key", "value1", "key2", "value2", alt_sdp=alt_sdp) + + def test_default_is_called_before_updating_set_cache(self): + cache = SetCache(default=lambda k: NotFound) + with pytest.raises(KeyError): + cache.update("old_key", "old_value", "new_key", "new_value") + + cache = SetCache(default=lambda k: {"old_value", "other old value"}) + cache.update("old_key", "old_value", "old_key", "new_value") + assert cache.get("old_key") == {"new_value", "other old value"} + + cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else NotFound) + cache.update("old_key", "old_value", "new_key", "new_value") + assert cache.get("old_key") == {"other old value"} + assert cache.get("new_key") == {"new_value"} + + cache = SetCache(default=lambda k: {"old_value", "other old value"} if k == "old_key" else {"other new"}) + cache.update("old_key", "old_value", "new_key", "new_value") + assert cache.get("old_key") == {"other old value"} + assert cache.get("new_key") == {"other new", "new_value"} + + def test_i_can_delete_values_from_set_cache(self): + cache = SetCache() + cache.put("key", "value1") + cache.put("key", "value2") + cache.reset_events() + + cache.delete("key", "fake_value") + assert cache.get("key") == {"value1", "value2"} + assert len(cache) == 2 + assert cache.to_add == set() + assert cache.to_remove == set() + + cache.delete("key", "value1") + assert cache.get("key") == {"value2"} + assert cache.to_add == {"key"} + assert len(cache) == 1 + + cache.delete("key", "value2") + assert cache.get("key") is NotFound + assert cache.to_remove == {"key"} + assert len(cache) == 0 + + def test_i_can_delete_key_from_set_cache(self): + cache = SetCache() + cache.put("key", "value1") + cache.put("key", "value2") + + cache.delete("key") + assert cache.get("key") is NotFound + assert cache.to_remove == {"key"} + assert len(cache) == 0 + + def test_i_can_delete_a_key_that_does_not_exists(self): + cache = SetCache() + cache.delete("key") + + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_key_from_cache(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + cache.put("key", "value") + + cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + assert cache.copy() == {"key": Removed} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_cache(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + cache.put("key", "value") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value="value", alt_sdp=alt_sdp) + assert cache.copy() == {"key": Removed} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_cache_and_then_put_back(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + cache.put("key", "value") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "xxx", extend_exists=lambda cache_name, key: True) + cache.delete("key", value="value", alt_sdp=alt_sdp) + cache.put("key", "value") + + assert cache.copy() == {"key": {"value"}} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_cache_remaining_values(self): + # There is a value in alt_cache_manager, + # But there is a value in the current cache after deletion + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + cache.put("key", "value1") + cache.put("key", "value2") + + cache.delete("key", value="value1", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + assert cache.copy() == {"key": {"value2"}} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_key_from_remote_repository(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name") + + cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + assert cache.copy() == {"key": Removed} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value"})).auto_configure("cache_name") + + cache.delete("key", value="value", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + assert cache.copy() == {"key": Removed} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_and_then_put_back(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name") + + cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + cache.put("key", "value") + + assert cache.copy() == {"key": {"value"}} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_remote_repository_remaining_values(self): + # There is a value in alt_cache_manager, + # But there is a value in the current cache after deletion + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: {"value1", "value2"})).auto_configure("cache_name") + + cache.delete("key", value="value1", alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + assert cache.copy() == {"key": {"value2"}} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_key_from_alt_sdp(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, the key is empty + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1, value2"}, + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value=None, alt_sdp=alt_sdp) + assert cache.copy() == {"key": Removed} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, the key is empty + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1"}, + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value="value1", alt_sdp=alt_sdp) + assert cache.copy() == {"key": Removed} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_and_then_put_back(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, the key is empty + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1"}, + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value="value1", alt_sdp=alt_sdp) + cache.put("key", "value") + + assert cache.copy() == {"key": {"value"}} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_delete_when_alt_sdp_a_value_from_alt_sdp_one_value_remaining(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, the key is empty + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1", "value2"}, + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value="value1", alt_sdp=alt_sdp) + assert cache.copy() == {"key": {"value2"}} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_key_that_does_not_exist_from_alt_sdp(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, the key is empty + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=(lambda cache_name, key: {"value1", "value2"} if key == "key" else NotFound), + extend_exists=lambda cache_name, key: key == "key") + + cache.delete("key2", value=None, alt_sdp=alt_sdp) + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_a_value_that_does_not_exist_from_alt_sdp(self): + # alt_cache_manager is used because no value in cache or in remote repository + # After value deletion, the key is empty + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value1", "value2"}, + extend_exists=lambda cache_name, key: True) + + cache.delete("key", value="value4", alt_sdp=alt_sdp) + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_an_already_removed_value_from_alt_sdp(self): + # alt_cache_manager is used because no value in cache or in remote repository + # But the alternate sdp returns Removed, which means that previous value was deleted + # It's like there is nothing to delete + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed, + extend_exists=lambda cache_name, key: False) + + cache.delete("key", value="value1", alt_sdp=alt_sdp) + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_i_can_delete_when_alt_sdp_and_cache_is_cleared(self): + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: {"value"}, + extend_exists=lambda cache_name, key: True) + + cache.clear() + cache.delete("key", value=None, alt_sdp=alt_sdp) + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + cache.delete("key", value="value", alt_sdp=alt_sdp) + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository(self): + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: Removed)).auto_configure("cache_name") + cache.put("key", "value") + + assert cache.copy() == {"key": {"value"}} + assert cache.to_remove == set() + assert cache.to_add == {"key"} + + def test_i_can_add_when_alt_sdp_from_a_removed_remote_repository_from_alt_sdp(self): + # The key is removed in the sub layers + # We can put it back + cache = SetCache(sdp=FakeSdp(get_value=lambda entry, k: NotFound)).auto_configure("cache_name") + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: Removed, + extend_exists=lambda cache_name, key: False) + + cache.put("key", "value", alt_sdp=alt_sdp) + + assert cache.copy() == {"key": {"value"}} + assert cache.to_remove == set() + assert cache.to_add == {"key"} diff --git a/tests/caching/test_cache.py b/tests/caching/test_cache.py new file mode 100644 index 0000000..434221c --- /dev/null +++ b/tests/caching/test_cache.py @@ -0,0 +1,512 @@ +import pytest + +from caching.BaseCache import BaseCache, MAX_INITIALIZED_KEY +from caching.Cache import Cache +from caching.DictionaryCache import DictionaryCache +from caching.IncCache import IncCache +from caching.ListCache import ListCache +from caching.ListIfNeededCache import ListIfNeededCache +from caching.SetCache import SetCache +from common.global_symbols import NotFound, Removed +from tests.caching import FakeSdp + + +class TestCache: + def test_i_can_configure(self): + cache = Cache() + cache.configure(max_size=256, + default="default_delegate", + extend_exists="extend_exists_delegate", + alt_sdp_get="alt_sdp_delegate", + sdp=FakeSdp()) + + # Caution, in this test, I initialize default, extend_exists and alt_get_delegate with string + # to simplify the test, but in real usage, they are lambda + # default = lambda sdp, key: sdp.get(cache_name, key) or lambda key: func(key) + # extend_exists = lambda sdp, key: sdp.exists(cache_name, key) or lambda key: func(key) + # alt_sdp_get = lambda sdp, key: sdp.alt_get(cache_name, key) + + assert cache._max_size == 256 + assert cache._default == "default_delegate" + assert cache._extend_exists == "extend_exists_delegate" + assert cache._alt_sdp_get == "alt_sdp_delegate" + assert cache._sdp is not None + + def test_i_can_auto_configure(self): + sdp = FakeSdp(get_value=lambda cache_name, key: key + 1 if cache_name == "cache_name" else NotFound, + extend_exists=lambda cache_name, key: True if cache_name == "cache_name" else False, + get_alt_value=lambda cache_name, key: key + 2 if cache_name == "cache_name" else NotFound) + + cache = Cache(sdp=sdp).auto_configure("cache_name") + assert cache._default(cache._sdp, 10) == 11 + assert cache._extend_exists(cache._sdp, 10) is True + assert cache._alt_sdp_get(cache._sdp, 10) == 12 + + cache = Cache(sdp=sdp).auto_configure("another_cache") + assert cache._default(cache._sdp, 10) == NotFound + assert cache._extend_exists(cache._sdp, 10) is False + assert cache._alt_sdp_get(cache._sdp, 10) == NotFound + + def test_i_can_get_an_retrieve_value_from_cache(self): + cache = Cache() + cache.put("key", "value") + assert cache.get("key") == "value" + assert len(cache) == 1 + + cache.put("key", "another value") # another value in the cache replace the previous one + assert cache.get("key") == "another value" + assert len(cache) == 1 + + cache.put("key2", "value2") # another key + assert cache.get("key2") == "value2" + assert len(cache) == 2 + assert cache.copy() == {"key": "another value", "key2": "value2"} + + def test_i_do_not_evict_when_put(self): + """ + It's because we evict on get() + :return: + :rtype: + """ + maxsize = 5 + cache = Cache(max_size=5) + + for key in range(maxsize + 2): + cache.put(str(key), key) + + assert len(cache) == maxsize + 2 + assert cache.copy() == { + "0": 0, + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + } + + def test_i_can_evict_when_get(self): + maxsize = 5 + cache = Cache(max_size=5, default=lambda k: int(k)) + + for key in range(maxsize + 2): + cache.get(str(key)) + + assert len(cache) == maxsize + assert cache.copy() == { + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + } + + def test_i_do_not_evict_when_items_are_not_committed(self): + maxsize = 5 + cache = Cache(max_size=5, default=lambda k: k) + + for key in range(maxsize + 2): + cache.put(str(key), key) + + assert len(cache) == maxsize + 2 + + cache.get("-1") + assert len(cache) == maxsize + 2 + assert cache.copy() == { + "0": 0, + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + } + + def test_i_can_get_a_value_from_alt_sdp(self): + cache = Cache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !") + assert cache.get("key", alt_sdp=alt_sdp) == "value found !" + + # The value is now in cache + assert cache.copy() == {'key': 'value found !'} + + def test_i_cannot_get_a_value_from_alt_sdp_when_cache_is_cleared(self): + cache = Cache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name") + cache.clear() + + alt_sdp = FakeSdp(get_alt_value=lambda cache_name, key: "value found !") + assert cache.get("key", alt_sdp=alt_sdp) is NotFound + assert cache.copy() == {} + + def test_i_can_get_default_value_from_simple_cache(self): + cache = Cache() + assert cache.get("key") is NotFound + + cache = Cache(default=10) + assert cache.get("key") == 10 + assert "key" not in cache # default value are not put in cache + + cache = Cache(default=lambda key: key + "_not_found") + assert cache.get("key") == "key_not_found" + assert "key" in cache # default callable are put in cache + + cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key), + sdp=FakeSdp(get_value=lambda entry, key: key + "_not_found")) + assert cache.get("key") == "key_not_found" + assert "key" in cache # default callable are put in cache + + def test_i_do_not_ask_the_remote_repository_twice(self): + nb_request = [] + + cache = Cache(default=lambda key: nb_request.append("requested")) + assert cache.get("key") is None + assert cache.get("key") is None + assert len(nb_request) == 1 + + def test_i_can_update_from_simple_cache(self): + cache = Cache() + cache.put("key", "value") + cache.update("key", "value", "key", "new_value") + + assert len(cache._cache) == 1 + assert len(cache) == 1 + assert cache.get("key") == "new_value" + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + cache.reset_events() + cache.update("key", "new_value", "another_key", "another_value") + assert len(cache._cache) == 1 + assert len(cache) == 1 + assert cache.get("key") is NotFound + assert cache.get("another_key") == "another_value" + assert cache.to_add == {"another_key"} + assert cache.to_remove == {"key"} + + with pytest.raises(KeyError): + cache.update("wrong key", "value", "key", "value") + + def test_i_can_update_when_alt_sdp_same_keys(self): + cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key), + extend_exists=lambda sdp, key: sdp.exists("cache_name", key), + sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)) + + cache.put("key", "value") + cache.update("key", "value", "key", "new_value", FakeSdp(extend_exists=lambda cache_name, key: True)) + + assert cache.get("key") == "new_value" + + def test_i_can_update_when_alt_sdp_different_keys(self): + cache = Cache(default=lambda sdp, key: sdp.get("cache_name", key), + extend_exists=lambda sdp, key: sdp.exists("cache_name", key), + sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)) + + cache.put("key", "value") + cache.update("key", "value", "key2", "value2", FakeSdp(extend_exists=lambda cache_name, key: True)) + + assert cache.get("key2") == "value2" + assert cache.get("key") == Removed + assert cache.to_add == {"key", "key2"} + assert cache.to_remove == set() + + @pytest.mark.parametrize("cache", [ + Cache(), ListCache(), ListIfNeededCache(), SetCache(), IncCache() + ]) + def test_i_can_manage_cache_events(self, cache: BaseCache): + cache.put("key", "value") + + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + cache.update("key", "value", "key", "another value") + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + cache.update("key", "another value", "key2", "value2") + assert cache.to_add == {"key2"} + assert cache.to_remove == {"key"} + + cache.update("key2", "value2", "key", "value") + assert cache.to_add == {"key"} + assert cache.to_remove == {"key2"} + + @pytest.mark.parametrize("cache", [ + ListCache(), SetCache(), ListIfNeededCache() + ]) + def test_i_can_manage_list_and_set_cache_events(self, cache): + cache.put("key", "value") + cache.put("key", "value2") + + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + cache.update("key", "value", "key", "another value") + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + cache.update("key", "value2", "key2", "value2") + assert cache.to_add == {"key", "key2"} + assert cache.to_remove == set() + + cache.update("key", "another value", "key3", "another value") + assert cache.to_add == {"key2", "key3"} + assert cache.to_remove == {"key"} + + @pytest.mark.parametrize("cache", [ + Cache(), ListCache(), SetCache(), ListIfNeededCache(), IncCache() + ]) + def test_exists(self, cache): + assert not cache.exists("key") + cache.put("key", "value") + assert cache.exists("key") + + def test_exists_extend(self): + cache = Cache(extend_exists=lambda k: True if k == "special_key" else False) + assert not cache.exists("key") + assert cache.exists("special_key") + + def test_i_can_extend_exists_when_internal_sdp(self): + cache = Cache(extend_exists=lambda sdp, k: True if k == "special_key" else False, sdp=FakeSdp) + assert not cache.exists("key") + assert cache.exists("special_key") + + @pytest.mark.parametrize("cache, default, new_value, expected", [ + (ListCache(), lambda k: NotFound, "value", ["value"]), + (ListCache(), lambda k: ["value"], "value", ["value", "value"]), + (ListIfNeededCache(), lambda k: NotFound, "value", "value"), + (ListIfNeededCache(), lambda k: "value", "value1", ["value", "value1"]), + (ListIfNeededCache(), lambda k: ["value1", "value2"], "value1", ["value1", "value2", "value1"]), + (SetCache(), lambda k: NotFound, "value", {"value"}), + (SetCache(), lambda k: {"value"}, "value", {"value"}), + (SetCache(), lambda k: {"value1"}, "value2", {"value1", "value2"}), + ]) + def test_default_is_called_before_put_to_keep_in_sync(self, cache, default, new_value, expected): + cache.configure(default=default) + cache.put("key", new_value) + + assert cache.get("key") == expected + + def test_default_is_called_before_updating_simple_cache(self): + cache = Cache(default=lambda k: NotFound) + with pytest.raises(KeyError): + cache.update("old_key", "old_value", "new_key", "new_value") + + cache = Cache(default=lambda k: "old_value") + cache.update("old_key", "old_value", "new_key", "new_value") + assert cache.get("new_key") == "new_value" + + def test_i_can_delete_an_entry_from_cache(self): + cache = Cache() + cache.put("key", "value") + + assert cache.get("key") == "value" + cache.delete("key") + assert cache.get("key") is NotFound + assert cache.to_remove == {"key"} + + def test_i_can_delete_when_entry_is_only_in_db(self): + cache = Cache(default=lambda k: "value" if k == 'key' else NotFound) + + cache.delete("another_key") + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + cache.delete("key") + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == {"key"} + + def test_i_can_delete_an_entry_from_cache_when_alt_sdp_and_value_in_cache(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = Cache(extend_exists=lambda sdp, k: sdp.exists("cache_name", k)) + cache.put("key", "value") + + cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + assert cache.copy() == {"key": Removed} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_an_entry_from_cache_when_alt_sdp_when_in_remote_repository(self): + # There is a value in alt_cache_manager, + # No remaining value in current cache after deletion + # The key must be flagged as Removed + cache = Cache(default=lambda k: "value", extend_exists=lambda sdp, k: sdp.exists("cache_name", k)) + + cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + assert cache.copy() == {"key": Removed} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_i_can_delete_an_entry_from_cache_when_alt_sdp_and_no_value_in_cache_or_remote_repository(self): + # alt_cache_manager is used when no value found + cache = Cache(default=lambda sdp, k: sdp.get("cache_name", k), + extend_exists=lambda sdp, k: sdp.exists("cache_name", k), + sdp=FakeSdp(get_value=lambda entry, k: NotFound)) + + cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: True)) + assert cache.copy() == {"key": Removed} + assert cache.to_add == {"key"} + assert cache.to_remove == set() + + def test_no_error_when_deleting_a_key_that_does_not_exists_when_alt_sdp(self): + # alt_cache_manager is used when no value found + cache = Cache(default=lambda sdp, k: sdp.get("cache_name", k), + extend_exists=lambda sdp, k: sdp.exists("cache_name", k), + sdp=FakeSdp(get_value=lambda entry, k: NotFound)) + + cache.delete("key", value=None, alt_sdp=FakeSdp(extend_exists=lambda cache_name, key: False)) + assert cache.copy() == {} + assert cache.to_add == set() + assert cache.to_remove == set() + + def test_initialized_key_is_removed_when_the_entry_is_found(self): + caches = [Cache(), ListCache(), ListIfNeededCache(), SetCache()] + + for cache in caches: + cache.put("key", "value") + cache.get("key") + + assert len(cache._initialized_keys) == 0 + + cache = IncCache() + cache.put("key", 10) + cache.get("key") + assert len(cache._initialized_keys) == 0 + + def test_initialized_keys_are_reset_when_max_length_is_reached(self): + cache = Cache() + for i in range(MAX_INITIALIZED_KEY): + cache.get(str(i)) + + assert len(cache._initialized_keys) == MAX_INITIALIZED_KEY + + cache.get(str(MAX_INITIALIZED_KEY + 1)) + assert len(cache._initialized_keys) == 1 + + def test_i_can_populate(self): + items = [("1", "1"), ("2", "2"), ("3", "3")] + cache = Cache() + + cache.populate(lambda: items, lambda item: item[0]) + + assert len(cache) == 3 + assert cache.get("1") == ("1", "1") + assert cache.get("2") == ("2", "2") + assert cache.get("3") == ("3", "3") + + assert cache.to_add == {"1", "2", "3"} + assert cache.to_remove == set() + + def test_i_can_populate_using_internal_sdp(self): + items = [("1", "1"), ("2", "2"), ("3", "3")] + cache = Cache(sdp=FakeSdp(populate=items)) + + cache.populate(lambda sdp: sdp.populate(), lambda item: item[0]) + + assert len(cache) == 3 + assert cache.get("1") == ("1", "1") + assert cache.get("2") == ("2", "2") + assert cache.get("3") == ("3", "3") + + assert cache.to_add == {"1", "2", "3"} + assert cache.to_remove == set() + + def test_i_can_reset_the_event_after_populate(self): + items = [("1", "1"), ("2", "2"), ("3", "3")] + cache = Cache() + cache.to_add = {"some_value"} + cache.to_remove = {"some_other_value"} + + cache.populate(lambda: items, lambda item: item[0], reset_events=True) + + assert len(cache) == 3 + assert cache.copy() == {"1": ("1", "1"), + "2": ("2", "2"), + "3": ("3", "3")} + assert cache.to_add == {"some_value"} + assert cache.to_remove == {"some_other_value"} + + def test_i_can_get_all(self): + items = [("1", "1"), ("2", "2"), ("3", "3")] + cache = Cache() + + cache.populate(lambda: items, lambda item: item[0]) + + res = cache.get_all() + assert len(res) == 3 + assert list(res) == [('1', '1'), ('2', '2'), ('3', '3')] + + def test_i_can_clone_cache(self): + cache = Cache(max_size=256, + default=lambda sdp, key: sdp.get("cache_name", key), + extend_exists=False, + alt_sdp_get=lambda sdp, key: sdp.alt_get("cache_name", key), + sdp=FakeSdp(get_value=lambda entry, key: key + "_not_found")) + cache.put("key1", "value1") + cache.put("key2", "value2") + + clone = cache.clone() + assert type(cache) == type(clone) + assert clone._max_size == cache._max_size + assert clone._default == cache._default + assert clone._extend_exists == cache._extend_exists + assert clone._alt_sdp_get == cache._alt_sdp_get + assert clone._sdp == cache._sdp + assert clone._cache == {} # value are not copied + assert clone._initialized_keys == set() + assert clone._current_size == 0 + assert clone.to_add == set() + assert clone.to_remove == set() + + clone.configure(sdp=FakeSdp(lambda entry, key: key + " found !")) + + assert cache.get("key3") == "key3_not_found" + assert clone.get("key3") == "key3 found !" + + @pytest.mark.parametrize("cache", [ + Cache(), + DictionaryCache(), + IncCache(), + ListCache(), + ListIfNeededCache() + ]) + def test_i_can_clone_all_caches(self, cache): + clone = cache.clone() + assert type(clone) == type(cache) + + def test_sanity_check_on_list_if_needed_cache(self): + cache = ListIfNeededCache() + clone = cache.clone() + + clone.put("key", "value1") + clone.put("key", "value2") + + assert clone.get("key") == ["value1", "value2"] + + def test_i_can_clear_when_alt_sdp(self): + cache = Cache().auto_configure("cache_name") + + cache.put("key1", "value1") + cache.put("key2", "value2") + + cache.clear() + + assert cache.copy() == {} + assert cache._is_cleared + + def test_i_can_iter_on_the_content(self): + cache = Cache() + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.put("key3", "value3") + + res = [] + for k in cache: + assert k in cache + res.append(k) + + assert res == ["key1", "key2", "key3"] diff --git a/tests/common/__init__.py b/tests/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/common/test_utils.py b/tests/common/test_utils.py new file mode 100644 index 0000000..74c5a5f --- /dev/null +++ b/tests/common/test_utils.py @@ -0,0 +1,122 @@ +from dataclasses import dataclass + +import pytest + +from common.utils import decode_enum, get_class, to_dict, str_concept, unstr_concept +from helpers import get_concept +from parsers.tokenizer import Keywords, Token, TokenKind + + +@dataclass +class Obj: + prop1: str + prop2: str + + def __hash__(self): + return hash((self.prop1, self.prop1)) + + def __eq__(self, other): + if not isinstance(other, Obj): + return False + + return self.prop1 == other.prop1 and self.prop2 == other.prop2 + + +@dataclass +class Obj2: + prop1: object + prop2: object + + +def get_tokens(lst): + res = [] + for e in lst: + if e == " ": + res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0)) + elif e == "\n": + res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0)) + elif e == "": + res.append(Token(TokenKind.EOF, "\n", 0, 0, 0)) + else: + res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0)) + + return res + + +def test_i_can_get_class(): + # example of classes that should be in the result + create_parser_input = get_class("evaluators.CreateParserInput.CreateParserInput") + + assert isinstance(create_parser_input, type) + + +@pytest.mark.parametrize("text, expected_key, expected_id", [ + (None, None, None), + (10, None, None), + ("", None, None), + ("xxx", None, None), + ("c:", None, None), + ("c:key", None, None), + ("c:key:", "key", None), + ("c:key#id", None, None), + ("c:key#id:", "key", "id"), + ("c:#id:", None, "id"), + ("c:key#:", "key", None), + ("c:key#id:x", None, None), + ("c:one: plus c:two:", None, None), + ("c:one#id: plus c:two:", None, None), +]) +def test_i_can_unstr_concept(text, expected_key, expected_id): + k, i = unstr_concept(text) + assert k == expected_key + assert i == expected_id + + +@pytest.mark.parametrize("text, expected_key, expected_id", [ + ("r:key:", "key", None), + ("r:key#id:", "key", "id"), +]) +def test_i_can_unstr_concept_rules(text, expected_key, expected_id): + k, i = unstr_concept(text, prefix="r:") + assert k == expected_key + assert i == expected_id + + +def test_i_can_str_concept(): + assert str_concept(("key", "id")) == "c:key#id:" + assert str_concept((None, "id")) == "c:#id:" + assert str_concept(("key", None)) == "c:key:" + assert str_concept((None, None)) == "" + assert str_concept(("key", "id"), drop_name=True) == "c:#id:" + + concept = get_concept("foo") + assert str_concept(concept) == "c:foo:" + + concept.get_metadata().id = "1001" + assert str_concept(concept) == "c:foo#1001:" + assert str_concept(concept, drop_name=True) == "c:#1001:" + + assert str_concept(("key", "id"), prefix='r:') == "r:key#id:" + + +@pytest.mark.parametrize("text, expected", [ + (None, None), + (10, None), + ("", None), + ("xxx", None), + ("xxx.", None), + ("xxx.yyy", None), + ("parsers.tokenizer.Keywords.CONCEPT", Keywords.CONCEPT), +]) +def test_i_can_decode_enum(text, expected): + actual = decode_enum(text) + assert actual == expected + + +@pytest.mark.parametrize("items, expected", [ + ([], {}), + ([Obj("a", "1"), Obj("a", "2"), Obj("b", "3")], {"a": [Obj("a", "1"), Obj("a", "2")], + "b": [Obj("b", "3")]}), +]) +def test_i_can_to_dict(items, expected): + assert to_dict(items, lambda obj: obj.prop1) == expected diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7094130 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,84 @@ +import pytest + +from helpers import GetNextId + + +@pytest.fixture(scope="session") +def sheerka(): + from core.Sheerka import Sheerka + + sheerka = Sheerka() + sheerka.initialize("mem://") + return sheerka + + +@pytest.fixture(scope="module", autouse=True) +def on_new_module(sheerka, request): + """ + For each new module, make sure to create a new ontology + Remove it at the end of the module + :param sheerka: + :type sheerka: + :param request: + :type request: + :return: + :rtype: + """ + from core.Event import Event + from core.ExecutionContext import ExecutionContext, ExecutionContextActions + module_name = request.module.__name__.split(".")[-1] + context = ExecutionContext("test", + Event(message=f"Executing module {module_name}"), + sheerka, + ExecutionContextActions.TESTING, + None) + + ontology = sheerka.om.push_ontology(module_name) + yield + sheerka.om.revert_ontology(context, ontology) + + +@pytest.fixture(scope="function") +def context(sheerka): + from core.Event import Event + from core.ExecutionContext import ExecutionContext, ExecutionContextActions + + return ExecutionContext("test", + Event(message=""), + sheerka, + ExecutionContextActions.TESTING, + None) + + +@pytest.fixture() +def next_id(): + return GetNextId() + + +class TestUsingFileBasedSheerka: + @pytest.fixture(scope="class") + def sheerka(self): + sheerka = Sheerka() + sheerka.initialize() + return sheerka + + +class NewOntology: + """ + For some test who may need to declare the same concepts across the tests + """ + from core.ExecutionContext import ExecutionContext + + def __init__(self, context: ExecutionContext, name="current_test"): + self.sheerka = context.sheerka + self.context = context + self.name = name + self.ontology = None + + def __enter__(self): + self.ontology = self.sheerka.om.push_ontology(self.name) + return self.ontology + + def __exit__(self, exc_type, exc_val, exc_tb): + self.sheerka.om.revert_ontology(self.context, self.ontology) + return False diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/test_concept.py b/tests/core/test_concept.py new file mode 100644 index 0000000..5977720 --- /dev/null +++ b/tests/core/test_concept.py @@ -0,0 +1,103 @@ +from common.global_symbols import NotFound, NotInit +from core.concept import ConceptDefaultProps +from helpers import GetNextId, get_concept + + +def test_i_can_retrieve_concept_properties(): + foo = get_concept("a plus b", "a + b", variables=("a", "b"), id="1001") + + assert foo.name == "a plus b" + assert foo.id == "1001" + assert foo.str_id == "c:#1001:" + assert foo.all_attrs() == ('#where#', '#pre#', '#post#', '#body#', '#ret#', 'a', 'b') + assert foo.get_definition_digest() == "3a2cfcda8ffd0d99a7f8c7d2f1ffc4a99fc96162f3be7b9875f30751d3691af6" + + # sanity check to make sure that 'get_concept' works as expected + assert foo.get_metadata().variables == (("a", NotInit), ("b", NotInit)) + + +def test_i_can_set_and_get_value(): + foo = get_concept("foo", variables=["a"]) + foo.set_value("a", "some value") + assert foo.get_value("a") == "some value" + + +def test_i_can_set_and_get_value_from_bound_attr(): + foo = get_concept("foo", variables=["a"], bound_body="a") + + foo.set_value("a", "some value") + assert foo.get_value(ConceptDefaultProps.BODY) == "some value" + + foo.set_value(ConceptDefaultProps.BODY, "another value") + assert foo.get_value("a") == "another value" + + +def test_i_can_test_concept_equality(): + foo1 = get_concept("foo", "a + b", variables=["a", "b"], id=5) + foo2 = get_concept("foo", "a + b", variables=["a", "b"], id=6) + foo1.set_value("a", 10).set_value("b", 20) + foo2.set_value("a", 10).set_value("b", 20) + + assert foo1 == foo2 + + +def test_i_can_detect_when_concepts_are_not_equal(): + foo1 = get_concept("foo", "a + b", variables=["a", "b"], id=5) + foo2 = get_concept("foo", "a + b", variables=["a", "b"], id=6) + foo1.set_value("a", 10).set_value("b", 20) + foo2.set_value("a", 10).set_value("b", 25) + + assert foo1 != foo2 + + +def test_i_can_test_concept_equality_in_case_of_infinite_recursion(): + foo1 = get_concept("foo", "a + b", variables=["a"], id=5) + foo2 = get_concept("foo", "a + b", variables=["a"], id=6) + + # foo1 and foo2 are equals + assert foo1 == foo2 + + foo1.set_value("a", foo1) + foo2.set_value("a", foo2) + assert foo1 == foo2 + + foo1.set_value("a", foo2) + foo2.set_value("a", foo1) + assert foo1 == foo2 + + +def test_i_can_test_concept_equality_in_case_of_infinite_recursion_with_more_than_two_concepts(): + foo1 = get_concept("foo", "a + b", variables=["a"], id=5) + foo2 = get_concept("foo", "a + b", variables=["a"], id=6) + foo3 = get_concept("foo", "a + b", variables=["a"], id=7) + + foo1.set_value("a", foo2) + foo2.set_value("a", foo3) + foo3.set_value("a", foo1) + assert foo1 == foo2 + + foo1.set_value("a", foo2) + foo2.set_value("a", foo3) + foo3.set_value("a", foo3) + assert foo1 == foo2 + + +def test_i_cannot_get_an_attribute_which_is_not_defined(): + next_id = GetNextId() + foo = get_concept("add a b", definition="add", variables=["a", "b"], sequence=next_id) + + assert foo.get_value("a") is NotInit + assert foo.get_value("b") is NotInit + assert foo.get_value("c") is NotFound + + +def test_i_can_repr_a_concept(): + next_id = GetNextId() + foo = get_concept("foo", sequence=next_id) + assert repr(foo) == "(1001)foo" + + bar = get_concept("bar", pre="is an int", sequence=next_id) + assert repr(bar) == "(1002)bar, #pre=is an int" + + baz = get_concept("baz", definition="add a b", variables=["a", "b"], sequence=next_id) + assert repr(baz) == "(1003)baz, a=**NotInit**, b=**NotInit**" diff --git a/tests/core/test_execution_context.py b/tests/core/test_execution_context.py new file mode 100644 index 0000000..b399fd0 --- /dev/null +++ b/tests/core/test_execution_context.py @@ -0,0 +1,151 @@ +from core.Event import Event +from core.ExecutionContext import ExecutionContext, ExecutionContextActions + + +def test_i_can_create_execution_context(sheerka): + event = Event("myEvent", "fake_userid") + context1 = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value1", "my desc") + + assert context1.who == "who" + assert context1.event == event + assert context1.sheerka == sheerka + assert context1.action == ExecutionContextActions.TESTING + assert context1.action_context == "value1" + assert context1.desc == "my desc" + assert context1.id == 0 + assert context1.long_id == f"{event.get_digest()}:{context1.id}" + + +def test_i_can_push(sheerka): + event = Event("test") + context = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value") + with context.push("pusher", ExecutionContextActions.PARSING, "action_context", "my desc") as sub_context: + assert sub_context.who == "pusher" + assert sub_context.event == event + assert sub_context.sheerka == sheerka + assert sub_context.action == ExecutionContextActions.PARSING + assert sub_context.action_context == "action_context" + assert sub_context.desc == "my desc" + assert sub_context.id == context.id + 1 + + +def test_i_can_increment_ids(sheerka): + # The id of an execution context is linked to the event + # If the event is the same, the id is incremented + + event = Event("TEST::myEvent", "fake_userid") + context1 = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value") + context2 = context1.push("who1", ExecutionContextActions.TESTING, "value1") + context3 = context2.push("who2", ExecutionContextActions.TESTING, "value2") + context4 = context1.push("who1", ExecutionContextActions.TESTING, "value3") + context5 = ExecutionContext("who", event, sheerka, ExecutionContextActions.TESTING, "value4") + + assert context1.id == 0 + assert context2.id == 1 + assert context3.id == 2 + assert context4.id == 3 + assert context5.id == 4 + + event2 = Event("TEST::myEvent2", "fake_userid") + context6 = ExecutionContext("who", event2, sheerka, ExecutionContextActions.TESTING, "value") + assert context6.id == 0 + + +def test_i_can_manage_global_hints(context): + context2 = context.push("pusher", ExecutionContextActions.TESTING, None) + context3 = context2.push("pusher", ExecutionContextActions.TESTING, None) + context4 = context3.push("pusher", ExecutionContextActions.TESTING, None) + context5 = context.push("pusher", ExecutionContextActions.TESTING, None) + + context.global_hints.add("new_hint") + assert context.global_hints == {"new_hint"} + assert context2.global_hints == {"new_hint"} + assert context3.global_hints == {"new_hint"} + assert context4.global_hints == {"new_hint"} + assert context5.global_hints == {"new_hint"} + + context4.global_hints.add("another_hint") + assert context.global_hints == {"new_hint", "another_hint"} + assert context2.global_hints == {"new_hint", "another_hint"} + assert context3.global_hints == {"new_hint", "another_hint"} + assert context4.global_hints == {"new_hint", "another_hint"} + assert context5.global_hints == {"new_hint", "another_hint"} + + +def test_i_can_manage_protected_hint(context): + # Note that protected hint only works if the hint is added BEFORE the creation of the child + context.protected_hints.add("new_hint") + context2 = context.push("pusher", ExecutionContextActions.TESTING, None) + context3 = context2.push("pusher", ExecutionContextActions.TESTING, None) + context3.protected_hints.add("another_hint") + context4 = context3.push("pusher", ExecutionContextActions.TESTING, None) + context5 = context.push("pusher", ExecutionContextActions.TESTING, None) + + assert context.protected_hints == {"new_hint"} + assert context2.protected_hints == {"new_hint"} + assert context3.protected_hints == {"new_hint", "another_hint"} + assert context4.protected_hints == {"new_hint", "another_hint"} + assert context5.protected_hints == {"new_hint"} + + +def test_i_can_manage_private_hints(context): + context.private_hints.add("new_hint") + context2 = context.push("pusher", ExecutionContextActions.TESTING, None) + context3 = context2.push("pusher", ExecutionContextActions.TESTING, None) + context3.private_hints.add("another_hint") + context4 = context3.push("pusher", ExecutionContextActions.TESTING, None) + context5 = context.push("pusher", ExecutionContextActions.TESTING, None) + + assert context.private_hints == {"new_hint"} + assert context2.private_hints == set() + assert context3.private_hints == {"another_hint"} + assert context4.private_hints == set() + assert context5.private_hints == set() + + +def test_i_can_keep_track_of_children(context): + context2 = context.push("pusher", ExecutionContextActions.TESTING, None) + context3 = context.push("pusher", ExecutionContextActions.TESTING, None) + context4 = context2.push("pusher2", ExecutionContextActions.TESTING, None) + + assert len(context._children) == 2 + assert len(context2._children) == 1 + assert len(context3._children) == 0 + assert len(context4._children) == 0 + + +def test_i_can_get_children(context): + context1 = context.push("child 1", ExecutionContextActions.TESTING, None) + context2 = context.push("child 2", ExecutionContextActions.TESTING, None) + context3 = context.push("child 3", ExecutionContextActions.TESTING, None) + context21 = context2.push("child 21", ExecutionContextActions.TESTING, None) + context22 = context2.push("child 22", ExecutionContextActions.TESTING, None) + context211 = context21.push("child 211", ExecutionContextActions.TESTING, None) + context31 = context3.push("child 31", ExecutionContextActions.TESTING, None) + + assert list(context1.get_children()) == [] + + assert list(context.get_children()) == [ + context1, + context2, + context21, + context211, + context22, + context3, + context31 + ] + + assert list(context.get_children(level=1)) == [ + context1, + context2, + context3 + ] + + assert list(context.get_children(level=2)) == [ + context1, + context2, + context21, + context22, + context3, + context31, + ] diff --git a/tests/core/test_sheerka.py b/tests/core/test_sheerka.py new file mode 100644 index 0000000..2d3c7cf --- /dev/null +++ b/tests/core/test_sheerka.py @@ -0,0 +1,36 @@ +from os import path + +from base import UsingFileBasedSheerka +from helpers import get_concept, get_concepts, get_file_content + + +class TestSheerka(UsingFileBasedSheerka): + + def test_i_can_initialize_sheerka(self, sheerka_fb): + sheerka = sheerka_fb + assert path.exists(self.SHEERKA_ROOT_DIR) + + last_event_path = path.join(self.SHEERKA_ROOT_DIR, "LAST_EVENT") + assert path.exists(last_event_path) + + last_event_digest = get_file_content(last_event_path) + last_event_folder = path.join(self.SHEERKA_ROOT_DIR, "events", last_event_digest[:24], last_event_digest) + assert path.exists(last_event_folder) + assert path.exists(last_event_folder + "_admin_context") + + assert len(sheerka.services) > 0 + assert len(sheerka.evaluators) > 0 + + # add test to validate that we can access bind methods + + def test_i_can_use_isinstance(self, sheerka, context): + foo, bar = get_concepts(context, "foo", "bar", use_sheerka=True) + assert sheerka.isinstance(foo, foo.key) + assert sheerka.isinstance(foo, foo.str_id) + assert sheerka.isinstance(foo, foo) + assert sheerka.isinstance(foo, foo.get_metadata()) + + assert not sheerka.isinstance(foo, bar.key) + assert not sheerka.isinstance(foo, bar.str_id) + assert not sheerka.isinstance(foo, bar) + assert not sheerka.isinstance(foo, bar.get_metadata()) diff --git a/tests/evaluators/__init__.py b/tests/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/evaluators/test_CreateParserInput.py b/tests/evaluators/test_CreateParserInput.py new file mode 100644 index 0000000..1390422 --- /dev/null +++ b/tests/evaluators/test_CreateParserInput.py @@ -0,0 +1,23 @@ +import pytest + +from base import BaseTest +from core.BuiltinConcepts import BuiltinConcepts +from evaluators.CreateParserInput import CreateParserInput +from helpers import _rv, _rvf + + +class TestCreateParserInput(BaseTest): + + @pytest.fixture() + def evaluator(self, sheerka): + return sheerka.evaluators[CreateParserInput.NAME] + + def test_i_can_match(self, sheerka, context, evaluator): + ret_val = _rv(sheerka.newn(BuiltinConcepts.USER_INPUT, command="hello sheerka")) + assert evaluator.matches(context, ret_val).status is True + + ret_val = _rv(sheerka.newn(BuiltinConcepts.UNKNOWN_CONCEPT)) # it responds to USER_INPUT only + assert evaluator.matches(context, ret_val).status is False + + ret_val = _rvf(sheerka.newn(BuiltinConcepts.USER_INPUT, command="hello sheerka")) # status should be true + assert evaluator.matches(context, ret_val).status is False diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..0f846bd --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,377 @@ +from common.global_symbols import NotInit +from core.ExecutionContext import ExecutionContext +from core.ReturnValue import ReturnValue +from core.concept import Concept, ConceptMetadata, DefinitionType +from core.services.SheerkaConceptManager import ConceptManager + + +class GetNextId: + def __init__(self): + self.seq = 1000 + + def next(self): + self.seq += 1 + return self.seq + + +def get_concept(name=None, body=None, + id=None, + key=None, + where=None, + pre=None, + post=None, + ret=None, + definition=None, + definition_type=None, + desc=None, + props=None, + variables=None, + parameters=None, + bound_body=None, + is_builtin=False, + is_unique=False, + autouse=False, + sequence=None) -> Concept: + """ + Create a Concept objet + Caution : 'id' and 'key' are not initialized + + :param name: + :type name: + :param body: + :type body: + :param id: + :type id: + :param key: + :type key: + :param where: + :type where: + :param pre: + :type pre: + :param post: + :type post: + :param ret: + :type ret: + :param definition: + :type definition: + :param definition_type: + :type definition_type: + :param desc: + :type desc: + :param props: + :type props: + :param variables: + :type variables: + :param parameters: + :type parameters: + :param bound_body: + :type bound_body: + :param is_builtin: + :type is_builtin: + :param is_unique: + :type is_unique: + :param autouse: + :type autouse: + :param sequence: + :type sequence: + :return: + :rtype: + """ + metadata = get_metadata( + name, body, + id, + key, + where, + pre, + post, + ret, + definition, + definition_type, + desc, + props, + variables, + parameters, + bound_body, + is_builtin, + is_unique, + autouse + ) + if sequence: + metadata.auto_init(sequence) + else: + metadata.digest = ConceptManager.compute_metadata_digest(metadata) + metadata.all_attrs = ConceptManager.compute_all_attrs(metadata.variables) + return Concept(metadata) + + +def get_metadata(name=None, body=None, + id=None, + key=None, + where=None, + pre=None, + post=None, + ret=None, + definition=None, + definition_type=DefinitionType.DEFAULT, + desc=None, + props=None, + variables=None, + parameters=None, + bound_body=None, + is_builtin=False, + is_unique=False, + autouse=False, + digest=None, + all_attrs=None): + new_variables = [] + if variables: + for v in variables: + if isinstance(v, tuple): + new_variables.append(v) + else: + new_variables.append((v, NotInit)) + + return ConceptMetadata( + id, + name, + key, + is_builtin, + is_unique, + body, + where, + pre, + post, + ret, + definition, + definition_type, + desc, + autouse, + bound_body, + props or {}, + tuple(new_variables), + parameters or [], + digest, + all_attrs, + ) + + +def metadata_auto_init(self: ConceptMetadata, sequence) -> ConceptMetadata: + """ + Helper function for the unit tests. + This method will be added to the `ConceptMetadata` to ease the writing of the unit tests + It properly initializes the ConceptMetadata + :param self: + :type self: + :param sequence: + :type sequence: + :return: + :rtype: + """ + if not self.id: + self.id = str(sequence.next()) + if not self.key: + self.key = ConceptManager.create_concept_key(self.name, self.definition, self.variables) + if not self.is_unique: + self.is_unique = False + if not self.is_builtin: + self.is_builtin = False + if not self.definition_type: + self.definition_type = DefinitionType.DEFAULT + if not self.all_attrs: + self.all_attrs = ConceptManager.compute_all_attrs(self.variables) + if not self.digest: + self.digest = ConceptManager.compute_metadata_digest(self) + + # Note that I do not automatically update the digest as I don't want to make unnecessary computations + + return self + + +def metadata_clone(self: ConceptMetadata, name=None, body=None, + key=None, + where=None, + pre=None, + post=None, + ret=None, + definition=None, + definition_type=None, + desc=None, + props=None, + variables=None, + parameters=None, + bound_body=None, + is_builtin=None, + is_unique=None, + autouse=None, + digest=None, + all_attrs=None) -> ConceptMetadata: + """ + Helper function for the unit tests. + This method will be added to the `ConceptMetadata` to ease the writing of the unit tests + It clones a ConceptMetadata, but can override some attributes if requested + :param self: + :type self: + :param name: + :type name: + :param body: + :type body: + :param key: + :type key: + :param where: + :type where: + :param pre: + :type pre: + :param post: + :type post: + :param ret: + :type ret: + :param definition: + :type definition: + :param definition_type: + :type definition_type: + :param desc: + :type desc: + :param props: + :type props: + :param variables: + :type variables: + :param parameters: + :type parameters: + :param bound_body: + :type bound_body: + :param is_builtin: + :type is_builtin: + :param is_unique: + :type is_unique: + :param autouse: + :type autouse: + :param digest: + :type digest: + :param all_attrs: + :type all_attrs: + :return: + :rtype: + """ + return ConceptMetadata( + id=self.id, + name=self.name if name is None else name, + body=self.body if body is None else body, + key=self.key if key is None else key, + where=self.where if where is None else where, + pre=self.pre if pre is None else pre, + post=self.post if post is None else post, + ret=self.ret if ret is None else ret, + definition=self.definition if definition is None else definition, + definition_type=self.definition_type if definition_type is None else definition_type, + desc=self.desc if desc is None else desc, + props=self.props if props is None else props, + variables=self.variables if variables is None else variables, + parameters=self.parameters if parameters is None else parameters, + bound_body=self.bound_body if bound_body is None else bound_body, + is_builtin=self.is_builtin if is_builtin is None else is_builtin, + is_unique=self.is_unique if is_unique is None else is_unique, + autouse=self.autouse if autouse is None else autouse, + digest=self.digest if digest is None else digest, + all_attrs=self.all_attrs if all_attrs is None else all_attrs, + ) + + +# Helpers functions for unit tests +setattr(ConceptMetadata, 'auto_init', metadata_auto_init) +setattr(ConceptMetadata, 'clone', metadata_clone) + + +def get_metadatas(*args, **kwargs): + as_metadatas = [arg if isinstance(arg, ConceptMetadata) else get_metadata(arg) for arg in args] + next_id = kwargs.get("next_id", None) + if next_id: + for metadata in as_metadatas: + metadata_auto_init(metadata, next_id) + + return as_metadatas + + +def get_concepts(context: ExecutionContext, *concepts, **kwargs) -> list[Concept]: + """ + Simple and quick way to get initialize concepts for a test + :param sheerka: + :type sheerka: + :param context: + :type context: + :param concepts: + :type concepts: + :param kwargs: + :type kwargs: + :return: + :rtype: + """ + res = [] + use_sheerka = kwargs.pop("use_sheerka", False) + sequence = kwargs.pop("sequence", None) + for c in concepts: + if use_sheerka: + c = define_new_concept(context, c) + elif isinstance(c, str): + c = get_concept(c) + + if sequence: + c.get_metadata().auto_init(sequence) + + res.append(c) + + return res + + +def define_new_concept(context: ExecutionContext, c: str | Concept) -> Concept: + sheerka = context.sheerka + if isinstance(c, str): + retval = sheerka.define_new_concept(context, c) + else: + metadata = c.get_metadata() + retval = sheerka.define_new_concept(context, + metadata.name, + metadata.is_builtin, + metadata.is_unique, + metadata.body, + metadata.where, + metadata.pre, + metadata.post, + metadata.ret, + metadata.definition, + metadata.definition_type, + metadata.autouse, + metadata.bound_body, + metadata.desc, + metadata.props, + metadata.variables, + metadata.parameters) + + assert retval.status + concept = sheerka.newn(retval.value.metadata.name) + return concept + + +def get_file_content(file_name): + with open(file_name) as f: + return f.read() + + +def _rv(value, who="Test"): + return ReturnValue(who=who, status=True, value=value) + + +def _rvc(concept_name, who="Test"): + next_id = GetNextId() + concept = get_concept(concept_name, sequence=next_id) + return ReturnValue(who=who, status=True, value=concept) + + +def _rvf(value, who="Test"): + """ + Return Value False + :param value: + :type value: + :return: + :rtype: + """ + return ReturnValue(who=who, status=False, value=value) diff --git a/tests/ontologies/__init__.py b/tests/ontologies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/ontologies/test_SheerkaOntoloyManager.py b/tests/ontologies/test_SheerkaOntoloyManager.py new file mode 100644 index 0000000..5186a54 --- /dev/null +++ b/tests/ontologies/test_SheerkaOntoloyManager.py @@ -0,0 +1,1522 @@ +from dataclasses import dataclass + +import pytest + +from base import BaseTest +from caching.Cache import Cache +from caching.DictionaryCache import DictionaryCache +from caching.IncCache import IncCache +from caching.ListCache import ListCache +from caching.ListIfNeededCache import ListIfNeededCache +from common.global_symbols import NotFound, Removed +from helpers import get_metadata, get_metadatas +from ontologies.Exceptions import OntologyAlreadyExists, OntologyManagerCannotPopLatest, OntologyManagerFrozen, \ + OntologyManagerNotFrozen, OntologyNotFound +from ontologies.SheerkaOntologyManager import Ontology, SheerkaOntologyManager + + +@dataclass +class DummyObj: + key: str + value: object + + +class TestSheerkaOntology(BaseTest): + @pytest.fixture() + def manager(self, sheerka): + return SheerkaOntologyManager(sheerka, "mem://") + + @staticmethod + def init_by_id_and_by_name(manager): + """ + initialize a manager with concept caches for by id and by name + :param manager: + :type manager: + :return: + :rtype: + """ + cache = Cache().auto_configure("by_id") + manager.register_concept_cache("by_id", cache, lambda obj: obj.id, use_ref=True) + cache = ListIfNeededCache().auto_configure("by_key") + manager.register_concept_cache("by_key", cache, lambda obj: obj.key, use_ref=True) + manager.freeze() + return manager + + def test_i_can_create_ontology_manager(self, manager): + assert len(manager.ontologies) == 1 + assert manager.ontologies_names == [SheerkaOntologyManager.ROOT_ONTOLOGY_NAME] + + # current cache manager and current sdp point to the top of the list + assert id(manager.current_cache_manager()) == id(manager.ontologies[0].cache_manager) + assert id(manager.current_sdp()) == id(manager.ontologies[0].cache_manager.sdp) + + # No cache defined by default + assert manager.current_cache_manager().caches == {} + + def test_i_can_register_a_cache_and_get_data(self, context, manager): + cache = Cache() + manager.register_cache("test", cache) + + assert id(cache._sdp) == id(manager.current_sdp()) # sdp is automatically added to the new Cache + + manager.put("test", "key", "value") + assert manager.get("test", "key") == "value" + assert manager.current_sdp().get("test", "key") == NotFound # not yet committed + + manager.commit(context) + assert manager.get("test", "key") == "value" + assert manager.current_sdp().get("test", "key") == "value" + + def test_i_can_no_longer_register_cache_once_ontology_is_frozen(self, manager): + manager.freeze() + + with pytest.raises(OntologyManagerFrozen): + manager.register_cache("test", Cache()) + + with pytest.raises(OntologyManagerFrozen): + manager.register_concept_cache("test", Cache(), lambda obj: obj.key, True) + + def test_i_cannot_push_ontology_if_not_frozen(self, manager): + with pytest.raises(OntologyManagerNotFrozen): + manager.push_ontology("new_ontology") + + def test_i_can_push_ontology_from_simple_caches(self, manager): + """ + Once registered, the same cache are created every time a new ontology is added or pushed + :param manager: + :type manager: + :return: + :rtype: + """ + manager.register_cache("Cache", Cache(), persist=True, use_ref=True) + manager.register_cache("DictionaryCache", DictionaryCache(), persist=True, use_ref=False) + manager.register_cache("ListIfNeededCache", ListIfNeededCache(), persist=False, use_ref=False) + manager.register_cache("ListCache", ListCache(), persist=False, use_ref=True) + manager.register_cache("IncCache", IncCache(), False) + manager.freeze() + + manager.push_ontology("new_ontology") + + assert len(manager.ontologies) == 2 + ref_cache_manager = manager.ontologies[1].cache_manager + assert ref_cache_manager.sdp.name == "__default__" + + cache_manager_0 = manager.ontologies[0].cache_manager + assert len(cache_manager_0.caches) == 5 + assert cache_manager_0.concept_caches == [] + + # check that the definition of the newly created caches are the same thant the ref + for cache_name in ("Cache", "DictionaryCache", "ListIfNeededCache", "ListCache", "IncCache"): + cache_def = cache_manager_0.caches[cache_name] + assert cache_def.persist == ref_cache_manager.caches[cache_name].persist + assert cache_def.use_ref == ref_cache_manager.caches[cache_name].use_ref + assert type(cache_def.cache) == type(ref_cache_manager.caches[cache_name].cache) + assert cache_def.cache._sdp.name == "new_ontology" + + def test_i_can_push_multiple_ontologies(self, manager): + manager.freeze() + manager.push_ontology("ontology 1") + manager.push_ontology("ontology 2") + + assert len(manager.ontologies) == 3 + assert manager.ontologies[0].name == "ontology 2" + assert manager.ontologies[0].depth == 2 + + assert manager.ontologies[1].name == "ontology 1" + assert manager.ontologies[1].depth == 1 + + assert manager.ontologies[2].name == SheerkaOntologyManager.ROOT_ONTOLOGY_NAME + assert manager.ontologies[2].depth == 0 + + def test_i_can_push_ontology_from_concept_caches(self, manager): + """ + Same test than before, but for cache definitions + :param manager: + :type manager: + :return: + :rtype: + """ + self.init_by_id_and_by_name(manager) + + manager.push_ontology("new_ontology") + assert len(manager.ontologies) == 2 + + ref_cache_manager = manager.ontologies[1].cache_manager + cache_manager_0 = manager.ontologies[0].cache_manager + + assert len(cache_manager_0.caches) == 2 + assert cache_manager_0.concept_caches == ref_cache_manager.concept_caches + assert cache_manager_0.sdp.name == "new_ontology" + + for cache_name in ("by_key", "by_id"): + cache_def = cache_manager_0.caches[cache_name] + assert cache_def.persist == ref_cache_manager.caches[cache_name].persist + assert cache_def.use_ref == ref_cache_manager.caches[cache_name].use_ref + assert type(cache_def.cache) == type(ref_cache_manager.caches[cache_name].cache) + assert cache_def.cache._sdp.name == "new_ontology" + + def test_i_can_get_database_value(self, context, manager): + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value") + + assert not manager.current_cache_manager().has("cache_name", "key") + assert manager.get("cache_name", "key") == "value" + assert manager.current_cache_manager().has("cache_name", "key") + + def test_i_cannot_pop_ontology_when_not_frozen(self, context, manager): + with pytest.raises(OntologyManagerNotFrozen): + manager.pop_ontology(context) + + def test_i_cannot_pop_the_latest_cache_manager(self, context, manager): + """ + Cannot pop when there is only one ontology left + :param manager: + :type manager: + :param context: + :type context: + :return: + :rtype: + """ + manager.freeze() + with pytest.raises(OntologyManagerCannotPopLatest): + manager.pop_ontology(context) + + def test_i_can_pop_ontology(self, context, manager): + manager.freeze() + + manager.push_ontology("ontology1") + manager.push_ontology("ontology2") + manager.push_ontology("ontology3") + assert len(manager.ontologies) == 4 + + manager.pop_ontology(context) + assert len(manager.ontologies) == 3 + + manager.pop_ontology(context) + manager.pop_ontology(context) + with pytest.raises(OntologyManagerCannotPopLatest): + manager.pop_ontology(context) + + def test_i_can_revert_ontology(self, context, manager): + manager.freeze() + + manager.push_ontology("ontology1") + ontology = manager.push_ontology("ontology2") + manager.push_ontology("ontology3") + manager.push_ontology("ontology4") + + manager.revert_ontology(context, ontology) + assert len(manager.ontologies) == 2 + assert manager.current_ontology().name == "ontology1" + + def test_i_cannot_revert_when_ontology_is_already_popped(self, context, manager): + manager.freeze() + + manager.push_ontology("ontology1") + ontology = manager.push_ontology("ontology2") + manager.pop_ontology(context) + + with pytest.raises(OntologyNotFound) as err: + manager.revert_ontology(context, ontology) + + assert err.value.ontology == ontology + + def test_i_cannot_revert_when_ontology_that_does_not_exists(self, context, manager): + manager.freeze() + manager.push_ontology("ontology1") + + ontology = Ontology("fake", 0, manager.current_cache_manager(), None) + with pytest.raises(OntologyNotFound) as err: + manager.revert_ontology(context, ontology) + + assert err.value.ontology == ontology + + def test_i_can_push_ontology_to_override_values(self, context, manager): + manager.register_cache("cache_name", Cache()) + manager.freeze() + + # sanity check + manager.put("cache_name", "key", "value1") + assert manager.get("cache_name", "key") == "value1" + + # push an ontology and override the value + manager.push_ontology("new ontology") + manager.put("cache_name", "key", "value2") + assert manager.get("cache_name", "key") == "value2" + + # The new value is discarded when the ontology is removed + manager.pop_ontology(context) + assert manager.get("cache_name", "key") == "value1" + + def test_i_cannot_get_values_from_parent_ontologies_if_default_parameter_is_not_set(self, context, manager): + """ + If this test, the property called `default` of the `Cache` is not defined. + The `default` must be defined and be a callable in order to make the magic happen + :param context: + :type context: + :param manager: + :type manager: + :return: + :rtype: + """ + manager.register_cache("cache_name", Cache()) + manager.freeze() + manager.put("cache_name", "key", "value1") + assert manager.get("cache_name", "key") == "value1" + + # push an ontology and try to get the value + manager.push_ontology("new ontology") + assert manager.get("cache_name", "key") is NotFound + + def test_i_can_get_values_from_parent_ontologies_if_default_parameter_is_set(self, context, manager): + """ + `default` and `alt_sdp_get` must be defined in order to allow parent request + There is no special constraint for that. It's just that I don't want to check every possible cases + `default` and `alt_sdp_get` will always be defined in Sheerka, so it's useless to code situations where + there are not. + :param context: + :type context: + :param manager: + :type manager: + :return: + :rtype: + """ + manager.register_cache("cache_name", Cache( + default=lambda sdp, key: sdp.get("cache_name", key), + alt_sdp_get=lambda sdp, key: sdp.get("cache_name", key))) + + manager.freeze() + manager.put("cache_name", "key", "value1") + assert manager.get("cache_name", "key") == "value1" + + # push an ontology and try to get the value + manager.push_ontology("new ontology") + assert manager.get("cache_name", "key") == "value1" + + def test_i_can_access_values_after_push_and_pop(self, context, manager): + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + # put value in DB, but not in cache + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value1") + assert not manager.current_cache_manager().has("cache_name", "key") # value not in cache + assert manager.current_sdp().exists("cache_name", "key") # but value is in DB + + # add an ontology layer, and put in DB again, not in cache + manager.push_ontology("new ontology") + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value2") + + # At this point, the value is in DB, but not in cache + assert not manager.current_cache_manager().has("cache_name", "key") + assert manager.get("cache_name", "key") == "value2" # taken from the current ontology + assert manager.current_cache_manager().has("cache_name", "key") + + # sanity check + # Let's check sdp values + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {'key': 'value2'}} + assert manager.ontologies[1].cache_manager.sdp.state.data == {'cache_name': {'key': 'value1'}} + + # remove a layer + manager.pop_ontology(context) + assert not manager.current_cache_manager().has("cache_name", "key") # value is no longer in cache + assert manager.get("cache_name", "key") == "value1" + assert manager.current_cache_manager().has("cache_name", "key") # it's now in cache + + # sanity check + # Let's check sdp values + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {'key': 'value1'}} + + def test_i_can_add_ontology(self, context, manager): + """ + In this test, I put back (using `SheerkaOntologyManager.add()`) a previously created ontology + :param context: + :type context: + :param manager: + :type manager: + :return: + :rtype: + """ + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + # init the system + # key1 is in the cache and in sdp + # key2 is only in sdp + manager.push_ontology("new ontology") + manager.put("cache_name", "key1", "value1") + manager.commit(context) + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key2", "value2") + + # call key3 to check. This time there is no value, but there will be later + assert manager.get("cache_name", "key3") is NotFound + + # now remove the ontology + new_ontology = manager.pop_ontology(context) + + # add another ontology, with its own values + manager.push_ontology("another ontology") + manager.put("cache_name", "key1", "value1_from_another") + manager.put("cache_name", "key2", "value2_from_another") + manager.put("cache_name", "key3", "value3_from_another") + manager.commit(context) + + # put back the ontology on top + manager.add_ontology(new_ontology) + + assert manager.get("cache_name", "key1") == "value1" + assert manager.get("cache_name", "key2") == "value2" + assert manager.get("cache_name", "key3") == "value3_from_another" + + def test_i_can_get_ontology(self, manager): + manager.freeze() + + manager.push_ontology("name1") + manager.push_ontology("name2") + manager.push_ontology("name3") + + assert manager.get_ontology("name2").name == "name2" + assert manager.get_ontology().name == "name3" + + with pytest.raises(KeyError): + assert manager.get_ontology("name4") + + def test_i_can_manage_multiple_ontology_layers(self, context, manager): + manager.register_cache("cache_name", Cache(default=lambda sdp, key: sdp.get("cache_name", key))) + manager.freeze() + + # default layer + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value1") + + # add an ontology layer + manager.push_ontology("new ontology") + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value2") + + # add an ontology layer + manager.push_ontology("another ontology") + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value3") + + # add an ontology layer + manager.push_ontology("fourth ontology") + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value4") + + assert manager.get("cache_name", "key") == "value4" + + manager.pop_ontology(context) + assert manager.get("cache_name", "key") == "value3" + + manager.pop_ontology(context) + assert manager.get("cache_name", "key") == "value2" + + manager.pop_ontology(context) + assert manager.get("cache_name", "key") == "value1" + + def test_i_have_access_to_sub_layers_values(self, context, manager): + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + # default layer + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value") + + # add ontology layers + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + + # I can get the low level value + assert manager.get("cache_name", "key") == "value" + + # check that the value is copied on the top level cache + assert manager.current_cache_manager().has("cache_name", "key") + assert not manager.ontologies[1].cache_manager.has("cache_name", "key") # not the top level + assert manager.ontologies[2].cache_manager.has("cache_name", "key") # the data comes from it + + def test_i_can_get_value_from_all_layers(self, context, manager): + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + manager.put("cache_name", "key", "value") + + # add ontology layers + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + + assert manager.get("cache_name", "key") == "value" + + manager.pop_ontology(context) + assert manager.get("cache_name", "key") == "value" + + manager.pop_ontology(context) + assert manager.get("cache_name", "key") == "value" + + def test_i_can_only_get_top_layer_values_when_dictionary_cache(self, context, manager): + manager.register_cache("cache_name", DictionaryCache().auto_configure("cache_name")) + manager.freeze() + + manager.put("cache_name", False, {"key": "value"}) # add some values in default layer + + # add some other values in another layer + manager.push_ontology("new ontology") + manager.put("cache_name", False, {"key1": "value1"}) + + assert manager.get("cache_name", "key") is NotFound # other layer are not visible + assert manager.get("cache_name", "key1") == "value1" # I can only see the current layer + + # I still can use get all + assert manager.get_all("cache_name") == {"key": "value", "key1": "value1"} + + # I can get back my values after pop + manager.pop_ontology(context) + assert manager.copy("cache_name") == {"key": "value"} + + def test_dictionary_caches_values_are_copied_when_a_new_ontology_is_pushed(self, manager): + manager.register_cache("cache_name", DictionaryCache().auto_configure("cache_name")) + manager.freeze() + + manager.put("cache_name", False, {"key": "value"}) # add some values in default layer + manager.push_ontology("new ontology") + + assert manager.copy("cache_name") == {"key": "value"} + assert manager.current_cache_manager().get_inner_cache("cache_name").to_add == set() + assert manager.current_cache_manager().get_inner_cache("cache_name").to_remove == set() + + def test_initialized_key_are_correctly_managed_when_multiple_layers(self, context, manager): + manager.register_cache("c_name", Cache().auto_configure("c_name")) + manager.freeze() + + manager.put("c_name", "key", "value") + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("c_name", "key2", "value2") # not in cache + + # add ontology layers + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + manager.push_ontology("last ontology") + + manager.get("c_name", "key") # == "value" but we don't care + assert manager.ontologies[0].cache_manager.caches["c_name"].cache._initialized_keys == {"key"} + assert manager.ontologies[1].cache_manager.caches["c_name"].cache._initialized_keys == set() + assert manager.ontologies[2].cache_manager.caches["c_name"].cache._initialized_keys == set() + assert manager.ontologies[3].cache_manager.caches["c_name"].cache._initialized_keys == set() + + manager.get("c_name", "key2") # == "value2" but we don't care + assert manager.ontologies[0].cache_manager.caches["c_name"].cache._initialized_keys == {"key", "key2"} + assert manager.ontologies[1].cache_manager.caches["c_name"].cache._initialized_keys == set() + assert manager.ontologies[2].cache_manager.caches["c_name"].cache._initialized_keys == set() + assert manager.ontologies[3].cache_manager.caches["c_name"].cache._initialized_keys == {"key2"} + + manager.get("c_name", "no_key") # is NotFound but we don't care + assert manager.ontologies[0].cache_manager.caches["c_name"].cache._initialized_keys == {"key", "key2", "no_key"} + assert manager.ontologies[1].cache_manager.caches["c_name"].cache._initialized_keys == set() + assert manager.ontologies[2].cache_manager.caches["c_name"].cache._initialized_keys == set() + assert manager.ontologies[3].cache_manager.caches["c_name"].cache._initialized_keys == {"key2", "no_key"} + + def test_i_cannot_get_a_value_that_does_not_exists(self, manager): + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + # add ontology layers + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + + assert manager.get("cache_name", "key") is NotFound + + def test_i_cannot_get_a_value_that_is_removed(self, manager): + manager.register_cache("cache_name", Cache()) + manager.freeze() + + manager.put("cache_name", "key", "value") + + # add ontology layers + manager.push_ontology("new ontology") + manager.put("cache_name", "key", Removed) + + assert manager.get("cache_name", "key") is NotFound + + def test_i_can_remove_an_entry(self, manager): + manager.register_cache("cache_name", Cache()) + manager.freeze() + + manager.put("cache_name", "key", "value") + + # add ontology layers + manager.push_ontology("new ontology") + manager.delete("cache_name", "key") # remove the entry in this ontology + + assert manager.get("cache_name", "key") is NotFound + + # sanity check, the value still exist + assert manager.ontologies[0].cache_manager.caches["cache_name"].cache.copy() == {} + assert manager.ontologies[1].cache_manager.caches["cache_name"].cache.copy() == {'key': 'value'} + + def test_i_cannot_get_value_that_is_removed_in_sub_level(self, manager): + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + manager.put("cache_name", "key", "value") # value exists + + # add ontology layer + manager.push_ontology("new ontology") + manager.put("cache_name", "key", Removed) # value is removed + + # add another layer + manager.push_ontology("another ontology") # no indication + + assert manager.get("cache_name", "key") is NotFound + + # check that the cache of the top level ontology is updated + assert manager.current_cache_manager().caches["cache_name"].cache.copy() == {"key": Removed} + + def test_i_can_test_if_a_value_exists(self, context, manager): + manager.register_cache("cache_name", Cache(extend_exists=lambda sdp, key: sdp.exists("cache_name", key))) + manager.freeze() + + # default layer + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value") + + # add ontology layers + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + + # I can get the low level value + assert manager.exists("cache_name", "key") + + # check that the value is not in cache (only in the low level database) + assert not manager.current_cache_manager().has("cache_name", "key") + assert not manager.ontologies[1].cache_manager.has("cache_name", "key") + assert not manager.ontologies[2].cache_manager.has("cache_name", "key") + + def test_i_can_test_if_the_ontology_has_an_entry(self, context, manager): + manager.register_cache("cache_name", Cache(extend_exists=lambda sdp, key: sdp.exists("cache_name", key))) + manager.freeze() + + # default layer + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value") + + # add ontology layers + manager.push_ontology("new ontology") + + assert manager.exists("cache_name", "key") # it can be seen by the exists() + assert not manager.exists_in_current("cache_name", "key") # but not by the has + + # add the entry in the current sdp + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value") + assert manager.exists_in_current("cache_name", "key") + + # add the entry only in cache + manager.put("cache_name", "key2", "value") + assert manager.exists_in_current("cache_name", "key") # it can be seen + + def test_i_can_check_that_a_value_does_not_exist(self, manager): + + manager.register_cache("cache_name", Cache(extend_exists=lambda sdp, key: sdp.exists("cache_name", key))) + manager.freeze() + + # add ontology layers + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + + assert not manager.exists("cache_name", "key") + + def test_i_can_list_from_multiple_ontologies(self, context, manager): + + manager.register_cache("cache_name", Cache()) + manager.freeze() + + # default layer + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key1", DummyObj("key1", "value1")) + + manager.push_ontology("new ontology") + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key2", DummyObj("key2", "value2")) + transaction.add("cache_name", "key1", DummyObj("key1", "value11")) # key1 is modified + + manager.push_ontology("another ontology") + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key3", DummyObj("key3", "value3")) + + assert manager.list("cache_name") == [DummyObj("key1", "value11"), + DummyObj("key2", "value2"), + DummyObj("key3", "value3")] + + def test_i_can_list_from_multiple_ontologies_even_if_they_are_not_all_filled(self, context, manager): + + manager.register_cache("cache_name", Cache()) + manager.freeze() + + # default layer + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key1", DummyObj("key1", "value1")) + + manager.push_ontology("new ontology") + # nothing in this ontology + + manager.push_ontology("another ontology") + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key3", DummyObj("key3", "value3")) + + assert manager.list("cache_name") == [DummyObj("key1", "value1"), + DummyObj("key3", "value3")] + + def test_i_can_list_when_no_items(self, manager): + + manager.register_cache("cache_name", Cache()) + manager.freeze() + + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + + assert manager.list("cache_name") == [] + + def test_i_can_put_entry(self, context, manager): + manager.register_cache("cache_name", Cache()) + manager.freeze() + + # default ontology + manager.put("cache_name", "key", "value") + manager.commit(context) + + assert manager.get("cache_name", "key") == "value" + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {'key': 'value'}} + + # add an ontology layer + manager.push_ontology("new ontology") + manager.put("cache_name", "key", "value2") + manager.commit(context) + assert manager.get("cache_name", "key") == "value2" + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {'key': 'value2'}} + assert manager.ontologies[1].cache_manager.sdp.state.data == {'cache_name': {'key': 'value'}} + + def test_i_can_put_in_a_list_cache(self, context, manager): + # in this test, sub layers have values. + # We need to check that those values are not lost when adding a new element + + manager.register_cache("cache_name", ListCache().auto_configure("cache_name")) + manager.freeze() + + # default ontology + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", ["val1"]) + + # add an ontology layer + manager.push_ontology("new ontology") + manager.put("cache_name", "key", "val2") + manager.commit(context) + + assert manager.get("cache_name", "key") == ["val1", "val2"] + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {'key': ['val1', 'val2']}} + assert manager.ontologies[1].cache_manager.sdp.state.data == {'cache_name': {'key': ['val1']}} + + # and I can keep adding in another layer + manager.push_ontology("another ontology") + manager.put("cache_name", "key", "val3") + manager.commit(context) + + assert manager.get("cache_name", "key") == ["val1", "val2", "val3"] + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {'key': ['val1', 'val2', 'val3']}} + assert manager.ontologies[1].cache_manager.sdp.state.data == {'cache_name': {'key': ['val1', 'val2']}} + assert manager.ontologies[2].cache_manager.sdp.state.data == {'cache_name': {'key': ['val1']}} + + def test_i_can_remove_an_entry_that_is_only_in_db(self, context, manager): + + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + # default ontology + # value in DB but not in cache + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value") + + manager.delete("cache_name", "key") + manager.commit(context) + + assert manager.get("cache_name", "key") is NotFound + + # sanity check, the entry is removed + assert manager.ontologies[0].cache_manager.caches["cache_name"].cache.copy() == {} + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {}} + + def test_i_can_remove_when_value_is_in_low_level(self, context, manager): + # In this test, there is a value in a lower level ontology + # After calling delete(), the value is no longer accessible, but not deleted + + manager.register_cache("cache_name", Cache(default=lambda sdp, key: sdp.get("cache_name", key), + extend_exists=lambda sdp, key: sdp.exists("cache_name", key))) + manager.freeze() + + # default ontology + # value in DB but not in cache + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value") + + # add an ontology layer + manager.push_ontology("new ontology") + manager.delete("cache_name", "key", "value") + manager.commit(context) + + assert manager.get("cache_name", "key") is NotFound + # sanity check, the entry is removed + assert manager.ontologies[0].cache_manager.caches["cache_name"].cache.copy() == {"key": Removed} + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {"key": Removed}} + + assert manager.ontologies[1].cache_manager.caches["cache_name"].cache.copy() == {} + assert manager.ontologies[1].cache_manager.sdp.state.data == {'cache_name': {"key": "value"}} + + # The entry still exists in lower ontology + manager.pop_ontology(context) + assert manager.get("cache_name", "key") == "value" + + def test_i_can_remove_when_value_is_in_both_low_and_current_level(self, context, manager): + # In this test, there is a value is in a lower level ontology and in the current ontology + # After calling delete(), the value is no longer accessible, but not deleted + + manager.register_cache("cache_name", Cache(default=lambda sdp, key: sdp.get("cache_name", key), + extend_exists=lambda sdp, key: sdp.exists("cache_name", key))) + manager.freeze() + + # default ontology + # value in DB but not in cache + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value") + + # add an ontology layer + manager.push_ontology("new ontology") + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value2") + + manager.delete("cache_name", "key", "value") + manager.commit(context) + + assert manager.get("cache_name", "key") is NotFound + # sanity check, the entry is removed + assert manager.ontologies[0].cache_manager.caches["cache_name"].cache.copy() == {"key": Removed} + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {"key": Removed}} + + assert manager.ontologies[1].cache_manager.caches["cache_name"].cache.copy() == {} + assert manager.ontologies[1].cache_manager.sdp.state.data == {'cache_name': {"key": "value"}} + + # The entry still exists in lower ontology + manager.pop_ontology(context) + assert manager.get("cache_name", "key") == "value" + + def test_i_can_remove_when_value_is_not_low_level(self, context, manager): + # In this test, there is a value is only in the current level + # The value is deleted + + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + # add an ontology layer + # so that the value does not exist in the lower level ontology + manager.push_ontology("new ontology") + + # value in DB but not in cache + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", "value") + + manager.delete("cache_name", "key") + manager.commit(context) + + assert manager.get("cache_name", "key") is NotFound + + # sanity check, the entry is removed + assert manager.ontologies[0].cache_manager.caches["cache_name"].cache.copy() == {} + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {}} + + def test_i_can_remove_list_if_needed_when_value_is_in_low_level(self, context, manager): + # In this test, there are multiple values in a low level ontology + # We remove only one, + # So the top level ontology must be a copy minus the removed value + + cache = ListIfNeededCache().auto_configure("cache_name") + + manager.register_cache("cache_name", cache) + manager.freeze() + + # default ontology + # value in DB but not in cache + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("cache_name", "key", ["value", "value2"]) + + # add ontology layers + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + manager.delete("cache_name", "key", "value") + manager.commit(context) + + assert manager.get("cache_name", "key") == "value2" + # sanity check, the entry is removed + assert manager.ontologies[0].cache_manager.caches["cache_name"].cache.copy() == {"key": "value2"} + assert manager.ontologies[0].cache_manager.sdp.state.data == {'cache_name': {"key": "value2"}} + + assert manager.ontologies[1].cache_manager.caches["cache_name"].cache.copy() == {} + assert manager.ontologies[1].cache_manager.sdp.state.data == {} + + assert manager.ontologies[2].cache_manager.caches["cache_name"].cache.copy() == {"key": ["value", "value2"]} + assert manager.ontologies[2].cache_manager.sdp.state.data == {'cache_name': {"key": ["value", "value2"]}} + + # The entry still exists in lower ontology + manager.pop_ontology(context) + manager.pop_ontology(context) + assert manager.get("cache_name", "key") == ["value", "value2"] + + def test_i_can_add_concept_default_layer(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo").auto_init(next_id) + + manager.add_concept(foo) + manager.commit(context) + + assert manager.get("by_key", foo.key) == foo + assert manager.get("by_id", foo.id) == foo + + assert manager.current_sdp().get("by_key", foo.key) == foo + assert manager.current_sdp().get("by_id", foo.id) == foo + + def test_i_can_add_concept_in_top_layer(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo").auto_init(next_id) + + # add an ontology layer + manager.push_ontology("new ontology") + + manager.add_concept(foo) + manager.commit(context) + + assert manager.get("by_key", foo.key) == foo + assert manager.get("by_id", foo.id) == foo + + assert manager.current_sdp().get("by_key", foo.key) == foo + assert manager.current_sdp().get("by_id", foo.id) == foo + + # sanity check + assert list(manager.ontologies[0].cache_manager.sdp.state.data.keys()) == ['by_id', 'by_key'] + assert manager.ontologies[1].cache_manager.sdp.state.data == {} + + def test_i_can_add_the_concepts_with_the_same_key_from_different_layers(self, context, manager, next_id): + foo1 = get_metadata("foo x", body="x + 1", variables=["x"]).auto_init(next_id) + foo2 = get_metadata("foo x", body="x + 1", variables=["x"]).auto_init(next_id) + + cache = ListIfNeededCache().auto_configure("by_key") + manager.register_concept_cache("by_key", cache, lambda obj: obj.key, use_ref=True) + manager.freeze() + + manager.add_concept(foo1) + manager.commit(context) + + manager.push_ontology("new ontology") + manager.add_concept(foo2) + manager.commit(context) + + assert manager.current_sdp().get("by_key", foo1.key) == [foo1, foo2] + + def test_i_can_update_concept_in_default_layer(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo", body="body").auto_init(next_id) + + manager.add_concept(foo) + + modified = foo.clone(body="new body") + assert foo != modified + + manager.update_concept(foo, modified) + manager.commit(context) + + assert manager.get("by_key", foo.key) == modified + assert manager.get("by_id", foo.id) == modified + + def test_i_can_update_concept_in_top_layer(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo").auto_init(next_id) + + manager.add_concept(foo) + manager.commit(context) + + # add an ontology layer + manager.push_ontology("new ontology") + + modified = foo.clone(body="new body") + assert foo != modified + + manager.update_concept(foo, modified) + manager.commit(context) + + assert manager.get("by_key", foo.key) == modified + assert manager.get("by_id", foo.id) == modified + + # sanity check. + # make sure that the previous values are kept + assert manager.ontologies[0].cache_manager.sdp.get('by_key', foo.key) == modified + assert manager.ontologies[0].cache_manager.sdp.get('by_id', foo.id) == modified + assert manager.ontologies[1].cache_manager.sdp.get('by_key', foo.key) == foo + assert manager.ontologies[1].cache_manager.sdp.get('by_id', foo.id) == foo + + # so I can get the old values when I pop ontology + manager.pop_ontology(context) + assert manager.get("by_key", foo.key) == foo + assert manager.get("by_id", foo.id) == foo + + def test_i_can_update_when_concept_in_both_top_and_bottom_layers(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo").auto_init(next_id) + + manager.add_concept(foo) + manager.commit(context) + + # add an ontology layer + # and modify the concept + # The database is updated, but not the internal cache + manager.push_ontology("new ontology") + modified1 = foo.clone(body="new body") + assert foo != modified1 + with manager.current_sdp().get_transaction(context.event) as transaction: + transaction.add("by_key", foo.key, modified1) + transaction.add("by_id", foo.id, modified1) + + # modify the top layer a second time + modified2 = foo.clone(body="body", pre="True") + manager.update_concept(foo, modified2) + manager.commit(context) + + assert manager.get("by_key", foo.key) == modified2 + assert manager.get("by_id", foo.id) == modified2 + + # sanity check. + # make sure that the previous values are kept + # sanity check + assert manager.ontologies[0].cache_manager.sdp.get('by_key', foo.key) == modified2 + assert manager.ontologies[0].cache_manager.sdp.get('by_id', foo.id) == modified2 + assert manager.ontologies[1].cache_manager.sdp.get('by_key', foo.key) == foo + assert manager.ontologies[1].cache_manager.sdp.get('by_id', foo.id) == foo + + # so I can get the old values when I pop ontology + manager.pop_ontology(context) + assert manager.get("by_key", foo.key) == foo + assert manager.get("by_id", foo.id) == foo + + def test_i_can_update_when_the_key_changes(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo", id="1", key="my_key") + + # create an entry + manager.add_concept(foo) + manager.commit(context) + + # add a new layer, and modify the concept + manager.push_ontology("new ontology") + modified = foo.clone(key="another key") + manager.update_concept(foo, modified) + manager.commit(context) + + assert manager.get("by_id", modified.id) == modified + assert manager.get("by_key", modified.key) == modified + assert manager.get("by_key", foo.key) == NotFound + + # sanity + assert manager.ontologies[0].cache_manager.sdp.get('by_key', foo.key) == Removed + assert manager.ontologies[0].cache_manager.sdp.get('by_key', modified.key) == modified + assert manager.ontologies[0].cache_manager.sdp.get('by_id', foo.id) == modified + assert manager.ontologies[1].cache_manager.sdp.get('by_key', foo.key) == foo + assert manager.ontologies[1].cache_manager.sdp.get('by_key', modified.key) == NotFound + assert manager.ontologies[1].cache_manager.sdp.get('by_id', foo.id) == foo + + def test_i_can_update_when_key_changes_and_there_are_lists(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo, foo2, bar = get_metadatas("foo", get_metadata("foo", body="x"), "bar", next_id=next_id) + + # create entries + manager.add_concept(foo) + manager.add_concept(foo2) + manager.add_concept(bar) + manager.commit(context) + + # add a new layer, and modify the concept + manager.push_ontology("new ontology") + modified = foo.clone(key="bar") + manager.update_concept(foo, modified) + manager.commit(context) + + assert manager.get("by_id", modified.id) == modified + assert manager.get("by_key", modified.key) == [bar, modified] + assert manager.get("by_key", foo.key) == foo2 + + # sanity check + assert manager.ontologies[0].cache_manager.sdp.get('by_key', foo.key) == foo2 + assert manager.ontologies[0].cache_manager.sdp.get('by_key', modified.key) == [bar, modified] + assert manager.ontologies[0].cache_manager.sdp.get('by_id', foo.id) == modified + assert manager.ontologies[1].cache_manager.sdp.get('by_key', foo.key) == [foo, foo2] + assert manager.ontologies[1].cache_manager.sdp.get('by_key', modified.key) == bar + assert manager.ontologies[1].cache_manager.sdp.get('by_id', foo.id) == foo + + def test_i_can_remove_concept_from_default_layer(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo").auto_init(next_id) + + manager.add_concept(foo) + manager.commit(context) + + manager.remove_concept(foo) + manager.commit(context) + + assert manager.get("by_id", foo.id) == NotFound + assert manager.get("by_key", foo.key) == NotFound + + # sanity check + assert manager.current_sdp().get("by_key") == {} + assert manager.current_sdp().get("by_id") == {} + + def test_i_can_remove_concept_from_top_layer(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo").auto_init(next_id) + + manager.add_concept(foo) + manager.commit(context) + + # add a new layer, and remove the concept + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + + manager.remove_concept(foo) + manager.commit(context) + + assert manager.get("by_id", foo.id) == NotFound + assert manager.get("by_key", foo.key) == NotFound + + # sanity check + assert manager.current_sdp().get("by_id") == {foo.id: Removed} + assert manager.current_sdp().get("by_key") == {foo.key: Removed} + assert manager.ontologies[1].cache_manager.sdp.get("by_id") == NotFound + assert manager.ontologies[1].cache_manager.sdp.get("by_key") == NotFound + assert manager.ontologies[2].cache_manager.sdp.get("by_id") == {foo.id: foo} + assert manager.ontologies[2].cache_manager.sdp.get("by_key") == {foo.key: foo} + + # So I can pop + manager.pop_ontology(context) + assert manager.get("by_id", foo.id) == foo + assert manager.get("by_key", foo.key) == foo + + # and pop again + manager.pop_ontology(context) + assert manager.get("by_id", foo.id) == foo + assert manager.get("by_key", foo.key) == foo + + def test_i_can_get_all(self, context, manager): + + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + manager.put("cache_name", "key_to_remove1", "value1") + manager.put("cache_name", "key_to_remove2", "value1") + manager.put("cache_name", "key1", "value1") + manager.put("cache_name", "key2", "value2_in_sdp") + manager.put("cache_name", "key3", "value3") + manager.commit(context) + manager.put("cache_name", "key2", "value2_in_cache") # in cache, but not in remote sdp + + # add ontology layers + manager.push_ontology("new ontology") + manager.put("cache_name", "key1", "value1_from_new_ontology") + manager.put("cache_name", "key2", "value2_from_new_ontology") + manager.put("cache_name", "key4", "value4_in_sdp") + manager.commit(context) + manager.put("cache_name", "key4", "value4_in_cache") + manager.put("cache_name", "key_to_remove1", Removed) + + manager.push_ontology("another ontology") + with manager.current_sdp().get_transaction(context.event) as transaction: + # so that value is only in sdp, not in cache + transaction.add("cache_name", "key5", "value5") + transaction.add("cache_name", "key_to_remove2", Removed) + + assert manager.get_all("cache_name") == { + "key1": "value1_from_new_ontology", + "key2": "value2_from_new_ontology", + "key3": "value3", + "key4": "value4_in_cache", + "key5": "value5" + } + + def test_i_can_keep_track_of_created_concepts_by_ontologies(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo").auto_init(next_id) + manager.add_concept(foo) + + def from_cache(entry): + return manager.internal_cache_manager.copy(entry) + + def from_db(entry): + return manager.internal_cache_manager.sdp.get(entry) + + # check that the new concept is tracked + assert from_cache(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == {'__default__': {'1001'}} + assert from_cache(SheerkaOntologyManager.ONTOLOGY_BY_CONCEPT_ENTRY) == {'1001': '__default__'} + + # add a new ontology and make sure the new concepts are tracked + manager.push_ontology("new ontology") + bar = get_metadata("bar").auto_init(next_id) + manager.add_concept(bar) + assert from_cache(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == {'__default__': {'1001'}, + 'new ontology': {'1002'}} + assert from_cache(SheerkaOntologyManager.ONTOLOGY_BY_CONCEPT_ENTRY) == {'1001': '__default__', + '1002': 'new ontology'} + + # commit the info and check the DB + manager.commit(context) + assert from_db(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == {'__default__': {'1001'}, + 'new ontology': {'1002'}} + assert from_db(SheerkaOntologyManager.ONTOLOGY_BY_CONCEPT_ENTRY) == {'1001': '__default__', + '1002': 'new ontology'} + + # remove a concept a check + manager.remove_concept(foo) + assert from_cache(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == {'new ontology': {'1002'}, } + assert from_cache(SheerkaOntologyManager.ONTOLOGY_BY_CONCEPT_ENTRY) == {'1002': 'new ontology', } + + manager.remove_concept(bar) + assert from_cache(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == {} + assert from_cache(SheerkaOntologyManager.ONTOLOGY_BY_CONCEPT_ENTRY) == {} + + # commit again and check + manager.commit(context) + assert from_db(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == {} + assert from_db(SheerkaOntologyManager.ONTOLOGY_BY_CONCEPT_ENTRY) == {} + + @pytest.mark.skip("No rule yet !") + def test_i_can_keep_track_of_created_rules_by_ontologies(self, manager): + sheerka, context, rule1 = self.init_format_rules(("rule1", "id.attr == 'value'", "True")) + + def rules_by_ontology_from_cache(): + res = sheerka.om.internal_cache_manager.copy(SheerkaOntologyManager.RULES_BY_ONTOLOGY_ENTRY) + del res[SheerkaOntologyManager.ROOT_ONTOLOGY_NAME] # discard builtin rules + return res + + def ontologies_from_cache(): + res = sheerka.om.internal_cache_manager.copy(SheerkaOntologyManager.ONTOLOGY_BY_RULE_ENTRY) + return {k: v for k, v in res.items() if v != SheerkaOntologyManager.ROOT_ONTOLOGY_NAME} + + def rules_by_ontology_from_db(): + res = sheerka.om.internal_cache_manager.sdp.get(SheerkaOntologyManager.RULES_BY_ONTOLOGY_ENTRY) + del res[SheerkaOntologyManager.ROOT_ONTOLOGY_NAME] # discard builtin rules + return res + + def ontologies_from_db(): + res = sheerka.om.internal_cache_manager.sdp.get(SheerkaOntologyManager.ONTOLOGY_BY_RULE_ENTRY) + return {k: v for k, v in res.items() if v != SheerkaOntologyManager.ROOT_ONTOLOGY_NAME} + + assert rules_by_ontology_from_cache() == {"#unit_test#": {rule1.id}} + assert ontologies_from_cache() == {rule1.id: "#unit_test#"} + + # add a new rule from a new ontology and check + sheerka.push_ontology(context, "new ontology") + rule2 = Rule(ACTION_TYPE_EXEC, "rule2", "id2.attr2 == 'value'", "True") + sheerka.create_new_rule(context, rule2) + + assert rules_by_ontology_from_cache() == {"#unit_test#": {rule1.id}, "new ontology": {rule2.id}} + assert ontologies_from_cache() == {rule1.id: "#unit_test#", rule2.id: "new ontology"} + + # commit and check the result + sheerka.om.commit(context) + assert rules_by_ontology_from_db() == {"#unit_test#": {rule1.id}, "new ontology": {rule2.id}} + assert ontologies_from_db() == {rule1.id: "#unit_test#", rule2.id: "new ontology"} + + sheerka.remove_rule(context, rule1) + assert rules_by_ontology_from_cache() == {"new ontology": {rule2.id}} + assert ontologies_from_cache() == {rule2.id: "new ontology"} + + # remove the last rule + sheerka.remove_rule(context, rule2) + assert rules_by_ontology_from_cache() == {} + assert ontologies_from_cache() == {} + + # commit and check the db + sheerka.om.commit(context) + assert rules_by_ontology_from_db() == {} + assert ontologies_from_db() == {} + + def test_i_can_keep_track_of_created_concept_on_ontology_pop(self, context, manager, next_id): + self.init_by_id_and_by_name(manager) + foo = get_metadata("foo").auto_init(next_id) + manager.add_concept(foo) + + def from_cache(entry): + return manager.internal_cache_manager.copy(entry) + + manager.push_ontology("new ontology") + bar, baz = get_metadatas("bar", "baz", next_id=next_id) + manager.add_concept(bar) + manager.add_concept(baz) + + assert from_cache(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == {'__default__': {'1001'}, + 'new ontology': {'1002', '1003'}} + assert from_cache(SheerkaOntologyManager.ONTOLOGY_BY_CONCEPT_ENTRY) == {'1001': '__default__', + '1002': 'new ontology', + '1003': 'new ontology'} + + manager.pop_ontology(context) + assert from_cache(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == {'__default__': {'1001'}} + assert from_cache(SheerkaOntologyManager.ONTOLOGY_BY_CONCEPT_ENTRY) == {'1001': '__default__'} + + @pytest.mark.skip("No rule yet !") + def test_i_can_keep_track_of_created_rules_on_ontology_pop(self, manager): + sheerka, context, rule1 = self.init_format_rules(("rule1", "id.attr == 'value'", "True")) + + events_raised = set() + sheerka.subscribe(EVENT_RULE_ID_DELETED, lambda ctx, r: events_raised.add(r)) + + def rules_by_ontology_from_cache(): + res = sheerka.om.internal_cache_manager.copy(SheerkaOntologyManager.RULES_BY_ONTOLOGY_ENTRY) + del res[SheerkaOntologyManager.ROOT_ONTOLOGY_NAME] # discard builtin rules + return res + + def ontologies_from_cache(): + res = sheerka.om.internal_cache_manager.copy(SheerkaOntologyManager.ONTOLOGY_BY_RULE_ENTRY) + return {k: v for k, v in res.items() if v != SheerkaOntologyManager.ROOT_ONTOLOGY_NAME} + + sheerka.push_ontology(context, "new ontology") + sheerka.create_new_rule(context, Rule(ACTION_TYPE_EXEC, "rule2", "id2.attr2 == 'value'", "True")) + sheerka.create_new_rule(context, Rule(ACTION_TYPE_EXEC, "rule3", "id3.attr3 == 'value'", "True")) + + sheerka.pop_ontology(context) + assert rules_by_ontology_from_cache() == {'#unit_test#': {'11'}} + assert ontologies_from_cache() == {'11': '#unit_test#'} + + # check that the 'rule is deleted' events are raised + assert events_raised == {'12', '13'} + + def test_i_can_get_call_when_a_cache_is_cleared(self, manager): + + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + manager.put("cache_name", "key1", "value1") + manager.put("cache_name", "key2", "value2") + manager.put("cache_name", "key3", "value3") + + # add ontology layers + manager.push_ontology("new ontology") + manager.clear("cache_name") + manager.put("cache_name", "key1", "new value1") + manager.put("cache_name", "key4", "value4") + + manager.push_ontology("another ontology") + manager.put("cache_name", "key5", "value5") + + assert manager.get_all("cache_name") == { + "key1": "new value1", + "key4": "value4", + "key5": "value5" + } + + def test_i_can_get_all_when_inc_cache(self, manager): + + manager.register_cache("cache_name", IncCache().auto_configure("cache_name")) + manager.freeze() + + assert manager.get("cache_name", "key1") == 1 + assert manager.get("cache_name", "key1") == 2 + + manager.push_ontology("new ontology") + assert manager.get("cache_name", "key1") == 3 + assert manager.get("cache_name", "key2") == 1 + assert manager.get("cache_name", "key2") == 2 + + assert manager.get_all("cache_name") == { + "key1": 3, + "key2": 2, + } + + # a second time, to make sure that nothing was incremented + assert manager.get_all("cache_name") == { + "key1": 3, + "key2": 2, + } + + @pytest.mark.parametrize("all_ontologies, expected_in_layer_1", [ + (False, {}), + (True, {'key1': DummyObj(key='key1', value='value1'), + 'key2': DummyObj(key='key2', value='value2')}), + ]) + def test_i_can_populate(self, context, manager, all_ontologies, expected_in_layer_1): + + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + manager.put("cache_name", "key1", DummyObj("key1", "value1")) + manager.put("cache_name", "key2", DummyObj("key2", "value2")) + manager.commit(context) + manager.clear("cache_name") + + manager.push_ontology("new ontology") + manager.put("cache_name", "key2", DummyObj("key2", "value22")) + manager.put("cache_name", "key3", DummyObj("key3", "value3")) + manager.commit(context) + manager.clear("cache_name") + + # sanity check + assert manager.ontologies[0].cache_manager.get_inner_cache("cache_name").copy() == {} + assert manager.ontologies[1].cache_manager.get_inner_cache("cache_name").copy() == {} + + manager.populate("cache_name", + lambda sdp: sdp.list("cache_name"), + lambda obj: obj.key, + all_ontologies=all_ontologies) + + assert manager.ontologies[0].cache_manager.get_inner_cache("cache_name").copy() == { + 'key2': DummyObj(key='key2', value='value22'), + 'key3': DummyObj(key='key3', value='value3')} + assert manager.ontologies[1].cache_manager.get_inner_cache("cache_name").copy() == expected_in_layer_1 + + def test_i_can_clear_when_multiple_ontology_layers(self, context, manager): + + manager.register_cache("cache_name", Cache().auto_configure("cache_name")) + manager.freeze() + + manager.put("cache_name", "key1", "value1") + manager.put("cache_name", "key2", "value2") + assert manager.ontologies[0].cache_manager.get_inner_cache("cache_name").copy() == {'key1': 'value1', + 'key2': 'value2'} + + # I can clear in another layer + manager.push_ontology("new ontology") + manager.clear("cache_name") + assert manager.get("cache_name", "key1") is NotFound + assert manager.get("cache_name", "key2") is NotFound + assert manager.ontologies[0].cache_manager.get_inner_cache("cache_name").copy() == {} + + manager.put("cache_name", "key1", "new value1") + assert manager.get("cache_name", "key1") == "new value1" + assert manager.get("cache_name", "key2") is NotFound + assert manager.ontologies[0].cache_manager.get_inner_cache("cache_name").copy() == {'key1': "new value1"} + + manager.push_ontology("another ontology") + manager.put("cache_name", "key2", "new value2") + assert manager.get("cache_name", "key1") == "new value1" + assert manager.get("cache_name", "key2") == "new value2" + assert manager.ontologies[0].cache_manager.get_inner_cache("cache_name").copy() == {'key1': "new value1", + 'key2': "new value2"} + + manager.clear("cache_name") + assert manager.get("cache_name", "key1") is NotFound + assert manager.get("cache_name", "key2") is NotFound + assert manager.ontologies[0].cache_manager.get_inner_cache("cache_name").copy() == {} + assert manager.ontologies[1].cache_manager.get_inner_cache("cache_name").copy() == {'key1': "new value1"} + assert manager.ontologies[2].cache_manager.get_inner_cache("cache_name").copy() == {'key1': 'value1', + 'key2': 'value2'} + + manager.pop_ontology(context) + assert manager.get("cache_name", "key1") == "new value1" + assert manager.get("cache_name", "key2") is NotFound + + manager.pop_ontology(context) + assert manager.get("cache_name", "key1") == "value1" + assert manager.get("cache_name", "key2") == "value2" + + def test_already_on_the_top(self, manager): + manager.freeze() + + manager.push_ontology("new ontology") + + assert manager.already_on_top("new ontology") + assert not manager.already_on_top("another ontology") + + def test_already_on_the_top_when_the_ontology_already_exists(self, manager): + manager.freeze() + + manager.push_ontology("new ontology") + manager.push_ontology("another ontology") + + with pytest.raises(OntologyAlreadyExists): + assert manager.already_on_top("new ontology") + +# class TestSheerkaOntologyWithFileBasedSheerka(UsingFileBasedSheerka): +# def test_i_can_put_back_ontology(self, manager): +# sheerka = self.get_sheerka() +# +# +# +# manager.register_cache("cache_name", Cache().auto_configure("cache_name")) +# manager.freeze() +# +# # default layer +# with manager.current_sdp().get_transaction(context.event) as transaction: +# transaction.add("cache_name", "key", "value1") +# +# # add a layer +# manager.push_ontology("new ontology") +# with manager.current_sdp().get_transaction(context.event) as transaction: +# transaction.add("cache_name", "key", "value2") +# +# assert manager.get("cache_name", "key") == "value2" +# +# manager.pop_ontology(context) +# assert manager.get("cache_name", "key") == "value1" +# +# # put back the previous ontology +# manager.push_ontology("new ontology") +# assert manager.get("cache_name", "key") == "value2" +# +# def test_i_can_remember_concept_and_rules_by_ontology(self, manager): +# sheerka, context, foo, r1 = self.init_test().with_concepts( +# "foo", +# create_new=True +# ).with_format_rules( +# ("rule1", "__ret", "True"), +# ).unpack() +# sheerka.om.commit(context) +# +# sheerka = self.new_sheerka_instance(False) +# +# +# sheerka.create_new_concept(context, Concept("bar")) +# r2 = sheerka.create_new_rule(context, Rule(ACTION_TYPE_EXEC, "rule2", "__ret.status", "True")).body.body +# sheerka.om.commit(context) +# +# sheerka.push_ontology(context, "new ontology") +# sheerka.create_new_concept(context, Concept("baz")) +# sheerka.create_new_rule(context, Rule(ACTION_TYPE_EXEC, "rule3", "id3.attr3 == 'value'", "True")) +# sheerka.om.commit(context) +# +# sheerka = self.new_sheerka_instance(False) +# +# +# sheerka.push_ontology(context, "another ontology") +# sheerka.create_new_concept(context, Concept("qux")) +# r4 = sheerka.create_new_rule(context, Rule(ACTION_TYPE_EXEC, "rule4", "id4.attr4", "True")).body.body +# sheerka.remove_concept(context, foo) +# sheerka.remove_rule(context, r2) +# sheerka.om.commit(context) +# +# assert sheerka.om.self_cache_manager.copy(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == { +# '#unit_test#': {'1002'}, +# 'another ontology': {'1004'}, +# } +# +# assert sheerka.om.self_cache_manager.copy(SheerkaOntologyManager.RULES_BY_ONTOLOGY_ENTRY) == { +# '#unit_test#': {r1.id}, +# 'another ontology': {r4.id}, +# } +# +# # in db +# assert sheerka.om.self_cache_manager.sdp.get(SheerkaOntologyManager.CONCEPTS_BY_ONTOLOGY_ENTRY) == { +# '#unit_test#': {'1002'}, +# 'another ontology': {'1004'}, +# 'new ontology': {'1003'}} +# +# rules_from_db = sheerka.om.self_cache_manager.sdp.get(SheerkaOntologyManager.RULES_BY_ONTOLOGY_ENTRY) +# del rules_from_db["__default__"] +# assert rules_from_db == { +# '#unit_test#': {'11'}, +# 'another ontology': {'14'}, +# 'new ontology': {'13'}} diff --git a/tests/parsers/__init__.py b/tests/parsers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/parsers/test_parser_input.py b/tests/parsers/test_parser_input.py new file mode 100644 index 0000000..142d758 --- /dev/null +++ b/tests/parsers/test_parser_input.py @@ -0,0 +1,14 @@ +from parsers.ParserInput import ParserInput +from parsers.tokenizer import LexerError + + +def test_i_can_parser_input(): + parser_input = ParserInput("def concept a") + assert parser_input.init() is True + assert parser_input.exception is None + + +def test_i_can_detect_errors(): + parser_input = ParserInput('def concept "a') + assert parser_input.init() is False + assert isinstance(parser_input.exception, LexerError) diff --git a/tests/parsers/test_tokenizer.py b/tests/parsers/test_tokenizer.py new file mode 100644 index 0000000..67694ca --- /dev/null +++ b/tests/parsers/test_tokenizer.py @@ -0,0 +1,211 @@ +import pytest + +from parsers.tokenizer import LexerError, Token, TokenKind, Tokenizer + + +def test_i_can_tokenize(): + source = "+*-/{}[]() ,;:.?\n\n\r\r\r\nidentifier_0\t \t10.15 10 'string\n' \"another string\"=|&<>c:name:" + source += "$£€!_identifier°~_^\\`==#__var__10r/regex\nregex/r:xxx#1:**//%that's" + tokens = list(Tokenizer(source)) + assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1) + assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2) + assert tokens[2] == Token(TokenKind.MINUS, "-", 2, 1, 3) + assert tokens[3] == Token(TokenKind.SLASH, "/", 3, 1, 4) + assert tokens[4] == Token(TokenKind.LBRACE, "{", 4, 1, 5) + assert tokens[5] == Token(TokenKind.RBRACE, "}", 5, 1, 6) + assert tokens[6] == Token(TokenKind.LBRACKET, "[", 6, 1, 7) + assert tokens[7] == Token(TokenKind.RBRACKET, "]", 7, 1, 8) + assert tokens[8] == Token(TokenKind.LPAR, "(", 8, 1, 9) + assert tokens[9] == Token(TokenKind.RPAR, ")", 9, 1, 10) + assert tokens[10] == Token(TokenKind.WHITESPACE, " ", 10, 1, 11) + assert tokens[11] == Token(TokenKind.COMMA, ",", 14, 1, 15) + assert tokens[12] == Token(TokenKind.SEMICOLON, ";", 15, 1, 16) + assert tokens[13] == Token(TokenKind.COLON, ":", 16, 1, 17) + assert tokens[14] == Token(TokenKind.DOT, ".", 17, 1, 18) + assert tokens[15] == Token(TokenKind.QMARK, "?", 18, 1, 19) + assert tokens[16] == Token(TokenKind.NEWLINE, "\n", 19, 1, 20) + assert tokens[17] == Token(TokenKind.NEWLINE, "\n\r", 20, 2, 1) + assert tokens[18] == Token(TokenKind.NEWLINE, "\r", 22, 3, 1) + assert tokens[19] == Token(TokenKind.NEWLINE, "\r\n", 23, 4, 1) + assert tokens[20] == Token(TokenKind.IDENTIFIER, "identifier_0", 25, 5, 1) + assert tokens[21] == Token(TokenKind.WHITESPACE, "\t \t", 37, 5, 13) + assert tokens[22] == Token(TokenKind.NUMBER, "10.15", 41, 5, 17) + assert tokens[23] == Token(TokenKind.WHITESPACE, " ", 46, 5, 22) + assert tokens[24] == Token(TokenKind.NUMBER, "10", 47, 5, 23) + assert tokens[25] == Token(TokenKind.WHITESPACE, " ", 49, 5, 25) + assert tokens[26] == Token(TokenKind.STRING, "'string\n'", 50, 5, 26) + assert tokens[27] == Token(TokenKind.WHITESPACE, " ", 59, 6, 2) + assert tokens[28] == Token(TokenKind.STRING, '"another string"', 60, 6, 3) + assert tokens[29] == Token(TokenKind.EQUALS, '=', 76, 6, 19) + assert tokens[30] == Token(TokenKind.VBAR, '|', 77, 6, 20) + assert tokens[31] == Token(TokenKind.AMPER, '&', 78, 6, 21) + assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 22) + assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 23) + assert tokens[34] == Token(TokenKind.CONCEPT, ('name', None), 81, 6, 24) + assert tokens[35] == Token(TokenKind.DOLLAR, '$', 88, 6, 31) + assert tokens[36] == Token(TokenKind.STERLING, '£', 89, 6, 32) + assert tokens[37] == Token(TokenKind.EURO, '€', 90, 6, 33) + assert tokens[38] == Token(TokenKind.EMARK, '!', 91, 6, 34) + assert tokens[39] == Token(TokenKind.IDENTIFIER, '_identifier', 92, 6, 35) + assert tokens[40] == Token(TokenKind.DEGREE, '°', 103, 6, 46) + assert tokens[41] == Token(TokenKind.TILDE, '~', 104, 6, 47) + assert tokens[42] == Token(TokenKind.UNDERSCORE, '_', 105, 6, 48) + assert tokens[43] == Token(TokenKind.CARAT, '^', 106, 6, 49) + assert tokens[44] == Token(TokenKind.BACK_SLASH, '\\', 107, 6, 50) + assert tokens[45] == Token(TokenKind.BACK_QUOTE, '`', 108, 6, 51) + assert tokens[46] == Token(TokenKind.EQUALSEQUALS, '==', 109, 6, 52) + assert tokens[47] == Token(TokenKind.HASH, '#', 111, 6, 54) + assert tokens[48] == Token(TokenKind.VAR_DEF, '__var__10', 112, 6, 55) + assert tokens[49] == Token(TokenKind.REGEX, '/regex\nregex/', 121, 6, 64) + assert tokens[50] == Token(TokenKind.RULE, ("xxx", "1"), 135, 7, 7) + assert tokens[51] == Token(TokenKind.STARSTAR, "**", 143, 7, 15) + assert tokens[52] == Token(TokenKind.SLASHSLASH, "//", 145, 7, 17) + assert tokens[53] == Token(TokenKind.PERCENT, "%", 147, 7, 19) + assert tokens[54] == Token(TokenKind.IDENTIFIER, "that", 148, 7, 20) + assert tokens[55] == Token(TokenKind.QUOTE, "'", 152, 7, 24) + assert tokens[56] == Token(TokenKind.IDENTIFIER, "s", 153, 7, 25) + + assert tokens[57] == Token(TokenKind.EOF, '', 154, 7, 26) + + +@pytest.mark.parametrize("text, expected", [ + ("_ident", True), + ("__ident", True), + ("___ident", True), + ("ident", True), + ("ident123", True), + ("ident_123", True), + ("ident-like-this", True), + ("àèùéû", True), + ("011254", False), + ("0abcd", False), + ("-abcd", False) +]) +def test_i_can_tokenize_identifiers(text, expected): + tokens = list(Tokenizer(text)) + comparison = tokens[0].type == TokenKind.IDENTIFIER + assert comparison == expected + + +@pytest.mark.parametrize("text", [ + "123abc", + "123", + "abc", + "abc123" +]) +def test_i_can_parse_word(text): + tokens = list(Tokenizer(text, parse_word=True)) + assert tokens[0].type == TokenKind.WORD + assert tokens[0].value == text + assert tokens[1].index == len(text) + + +@pytest.mark.parametrize("text", [ + "__var__0", + "__var__1", + "__var__10", + "__var__999", +]) +def test_i_can_parse_var_def(text): + tokens = list(Tokenizer(text)) + assert len(tokens) == 2 + assert tokens[0].type == TokenKind.VAR_DEF + assert tokens[0].value == text + + +@pytest.mark.parametrize("text, message, error_text, index, line, column", [ + ("'string", "Missing Trailing quote", "'string", 7, 1, 8), + ('"string', "Missing Trailing quote", '"string', 7, 1, 8), + ('"a" + "string', "Missing Trailing quote", '"string', 13, 1, 14), + ('"a"\n\n"string', "Missing Trailing quote", '"string', 12, 3, 8), + ('"', "Missing Trailing quote", '"', 1, 1, 2), + ("c::", "Concept identifiers not found", "", 2, 1, 3), + ("c:foo\nbar:", "New line in concept name", "foo", 5, 1, 6), + ("c:foo", "Missing ending colon", "foo", 5, 1, 6) +]) +def test_i_can_detect_tokenizer_errors(text, message, error_text, index, line, column): + with pytest.raises(LexerError) as e: + list(Tokenizer(text)) + assert e.value.message == message + assert e.value.text == error_text + assert e.value.index == index + assert e.value.line == line + assert e.value.column == column + + +@pytest.mark.parametrize("text, expected_text, expected_newlines, expected_column", [ + ("'foo'", "'foo'", 0, 6), + ('"foo"', '"foo"', 0, 6), + ("'foo\nbar'", "'foo\nbar'", 1, 5), + ("'foo\rbar'", "'foo\rbar'", 0, 10), + ("'foo\n\rbar'", "'foo\n\rbar'", 1, 6), + ("'foo\r\nbar'", "'foo\r\nbar'", 1, 5), + ("'foo\n\nbar'", "'foo\n\nbar'", 2, 5), + ("'foo\r\n\n\rbar'", "'foo\r\n\n\rbar'", 2, 6), + ("'\nfoo\nbar\n'", "'\nfoo\nbar\n'", 3, 2), + ("'\n\rfoo\r\n'", "'\n\rfoo\r\n'", 2, 2), + (r"'foo\'bar'", r"'foo\'bar'", 0, 11), + (r'"foo\"bar"', r'"foo\"bar"', 0, 11), + ('"foo"bar"', '"foo"', 0, 6), + ("'foo'bar'", "'foo'", 0, 6), +]) +def test_i_can_parse_strings(text, expected_text, expected_newlines, expected_column): + lexer = Tokenizer(text) + text_found, nb_of_newlines, column_index = lexer.eat_string(0, 1, 1) + + assert text_found == expected_text + assert nb_of_newlines == expected_newlines + assert column_index == expected_column + + +@pytest.mark.parametrize("text", [ + "1", "3.1415", "0.5", "01", "-5", "-5.10" +]) +def test_i_can_parse_numbers(text): + tokens = list(Tokenizer(text)) + assert tokens[0].type == TokenKind.NUMBER + assert tokens[0].value == text + + +@pytest.mark.parametrize("text, expected", [ + ("c:key:", ("key", None)), + ("c:key|id:", ("key", "id")), + ("c:key|:", ("key", None)), + ("c:|id:", (None, "id")), + ("c:125:", ("125", None)), +]) +def test_i_can_parse_concept_token(text, expected): + tokens = list(Tokenizer(text)) + + assert tokens[0].type == TokenKind.CONCEPT + assert tokens[0].value == expected + + +@pytest.mark.parametrize("text, expected", [ + ("r:key:", ("key", None)), + ("r:key#id:", ("key", "id")), + ("r:key#:", ("key", None)), + ("r:#id:", (None, "id")), + ("r:125:", ("125", None)), +]) +def test_i_can_parse_concept_token(text, expected): + tokens = list(Tokenizer(text)) + + assert tokens[0].type == TokenKind.RULE + assert tokens[0].value == expected + + +@pytest.mark.parametrize("text, expected", [ + ("r|regex|", "|regex|"), + ("r/regex/", "/regex/"), + ("r'regex'", "'regex'"), + ('r"regex"', '"regex"'), +]) +def test_i_can_parse_regex_token(text, expected): + tokens = list(Tokenizer(text)) + + assert tokens[0].type == TokenKind.REGEX + assert tokens[0].value == expected + assert tokens[0].str_value == "r" + expected + assert tokens[0].repr_value == "r" + expected + assert tokens[0].strip_quote == expected[1:-1] diff --git a/tests/sdp/__init__.py b/tests/sdp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sdp/test_sheerkaDataProvider.py b/tests/sdp/test_sheerkaDataProvider.py index c4154cf..79df46a 100644 --- a/tests/sdp/test_sheerkaDataProvider.py +++ b/tests/sdp/test_sheerkaDataProvider.py @@ -6,11 +6,11 @@ from os import path import pytest -from core.global_symbols import NotFound +from common.global_symbols import NotFound from sdp.sheerkaDataProvider import Event, SheerkaDataProvider from sdp.sheerkaSerializer import JsonSerializer, PickleSerializer -tests_root = path.abspath("../../build/tests") +tests_root = path.abspath("../build/tests") evt_digest = "3a571cb6034ef6fc8d7fe91948d0d29728eed74de02bac7968b0e9facca2c2d7" @@ -71,7 +71,7 @@ class ObjWithDigestWithKey: @pytest.fixture(autouse=True) -def init_test(): +def init_current_dir(): """ I test both SheerkaDataProviderFileIO and SheerkaDataProviderDictionaryIO So it's important to reset the folders between two tests @@ -407,11 +407,11 @@ def test_i_can_add_an_object_and_save_it_as_a_reference(root): state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile)) assert state.data == { - "entry": {'key1': '##REF##:fbc2b1c60ed753b49217cae851e342371ee39ebabc9778105f450812e615a513', - 'key2': ['##REF##:fbc2b1c60ed753b49217cae851e342371ee39ebabc9778105f450812e615a513', - '##REF##:448420dbc57d61401d10a98759fccdabbe50e2e825b6da3bd018c190926bcda4'], - 'key3': {'##REF##:448420dbc57d61401d10a98759fccdabbe50e2e825b6da3bd018c190926bcda4', - '##REF##:fbc2b1c60ed753b49217cae851e342371ee39ebabc9778105f450812e615a513'}} + "entry": {'key1': '##REF##:4d20621e3c45e8977504016caa2539c0d518850d3a8f92eb20f3e9e5192c41cf', + 'key2': ['##REF##:4d20621e3c45e8977504016caa2539c0d518850d3a8f92eb20f3e9e5192c41cf', + '##REF##:c142f0a14ae4b52afa7cdcbb88dc16563468ca3fa99584323083968099cbaf6b'], + 'key3': {'##REF##:4d20621e3c45e8977504016caa2539c0d518850d3a8f92eb20f3e9e5192c41cf', + '##REF##:c142f0a14ae4b52afa7cdcbb88dc16563468ca3fa99584323083968099cbaf6b'}} } diff --git a/tests/sdp/test_sheerkaSerializer.py b/tests/sdp/test_sheerkaSerializer.py index 4e994e9..3835460 100644 --- a/tests/sdp/test_sheerkaSerializer.py +++ b/tests/sdp/test_sheerkaSerializer.py @@ -26,7 +26,7 @@ class ObjNoKey: def test_i_can_json_serialize(): - json_serializer = JsonSerializer(lambda obj: True) + json_serializer = JsonSerializer(lambda o: True) obj = ObjNoKey("a", "b") stream = io.BytesIO() diff --git a/tests/server/__init__.py b/tests/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/server/test_server.py b/tests/server/test_server.py new file mode 100644 index 0000000..995f401 --- /dev/null +++ b/tests/server/test_server.py @@ -0,0 +1,19 @@ +from starlette.testclient import TestClient + +from server.main import app + +client = TestClient(app) + + +def test_i_can_authenticate(): + data = { + "username": "kodjo", + "password": "kodjo" + } + response = client.post("/token", data=data) + assert response.status_code == 200 + as_json = response.json() + assert 'access_token' in as_json + assert 'first_name' in as_json + assert 'last_name' in as_json + assert 'token_type' in as_json diff --git a/tests/services/__init__.py b/tests/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/test_ConceptManager.py b/tests/services/test_ConceptManager.py new file mode 100644 index 0000000..4724f06 --- /dev/null +++ b/tests/services/test_ConceptManager.py @@ -0,0 +1,186 @@ +import pytest + +from base import BaseTest +from common.global_symbols import NotFound, NotInit +from conftest import NewOntology +from core.BuiltinConcepts import BuiltinConcepts +from core.ErrorContext import ErrorContext +from core.concept import ConceptMetadata +from core.services.SheerkaConceptManager import ConceptAlreadyDefined, ConceptManager +from helpers import get_metadata + + +class TestConceptManager(BaseTest): + + @pytest.fixture() + def service(self, sheerka): + return sheerka.services[ConceptManager.NAME] + + def test_i_can_compute_concept_digest(self, service): + """ + Two concepts with the same definition share the same digest + :return: + :rtype: + """ + metadata = get_metadata("foo", "body") + digest = service.compute_metadata_digest(metadata) + assert digest == "21a1c2f420da62f4dc60f600c95b19dd9527b19dd28fd38e17f5c0e28963d176" + + another_metadata = get_metadata("foo", "body") + other_digest = service.compute_metadata_digest(another_metadata) + + assert digest == other_digest + + def test_id_is_not_part_of_the_digest(self, service): + metadata1 = get_metadata("foo", "body", id=1) + metadata2 = get_metadata("foo", "body", id=2) + + assert service.compute_metadata_digest(metadata1) == service.compute_metadata_digest(metadata2) + + def test_i_can_compute_concept_attributes_based_on_the_metadata(self, service): + compute_all_attrs = service.compute_all_attrs + + m1 = get_metadata("foo") + assert compute_all_attrs(m1.variables) == ('#where#', '#pre#', '#post#', '#body#', '#ret#') + + m2 = get_metadata("bar", variables=[("var1", None), ("var2", None)]) + assert compute_all_attrs(m2.variables) == ('#where#', '#pre#', '#post#', '#body#', '#ret#', 'var1', 'var2') + + @pytest.mark.parametrize("definition, variables, expected", [ + ("foo", [], "foo"), + ("foo(bar)", [], "foo ( bar )"), + ("foo a", ["a"], "foo __var__0"), + ("a foo b", ["a", "b"], "__var__0 foo __var__1"), + ("a foo b", ["b", "a"], "__var__1 foo __var__0"), + ("foo", ["foo"], "foo"), + ("foo a", ["foo"], "__var__0 a"), + ("foo a b", ["a"], "foo __var__0 b"), + ("'foo'", [], "'foo'"), + ("my name is a", ["a"], "my name is __var__0"), + ("a b c d", ["b", "c"], "a __var__0 __var__1 d"), + ("a 'b c' d", ["b", "c"], "a 'b c' d"), + ("a | b", ["a", "b"], "__var__0 | __var__1"), + ("a b a c", ["a", "b"], "__var__0 __var__1 __var__0 c"), + ("a b a c", ["b", "a"], "__var__1 __var__0 __var__1 c"), + ("def concept a", ["a"], "def concept __var__0"), + ]) + def test_i_can_create_concept_key(self, service, definition, variables, expected): + expanded_variables = tuple((v, NotInit) for v in variables) + key = service.create_concept_key(definition, None, expanded_variables) + + assert key == expected + + def test_the_key_is_created_from_the_definition_if_it_is_set(self, service): + assert service.create_concept_key("from name", "from definition", None) == "from definition" + assert service.create_concept_key("from name", None, None) == "from name" + + def test_i_can_define_a_new_concept(self, context, service): + with NewOntology(context, "test_i_can_define_a_new_concept"): + res = service.define_new_concept(context, "name", body="body") + + assert res.status is True + + metadata = res.value.metadata + assert isinstance(metadata, ConceptMetadata) + assert metadata.id == "1001" + assert metadata.name == "name" + assert metadata.key == "name" + assert metadata.body == "body" + assert metadata.digest == "eb0620bd4a317af8a403c0ae1e185a528f9b58f8b0878d990e62278f89cf10d5" + assert metadata.all_attrs == ('#where#', '#pre#', '#post#', '#body#', '#ret#') + + # is sorted in db + om = context.sheerka.om + assert om.get(ConceptManager.CONCEPTS_BY_ID_ENTRY, metadata.id) == metadata + assert om.get(ConceptManager.CONCEPTS_BY_NAME_ENTRY, metadata.name) == metadata + assert om.get(ConceptManager.CONCEPTS_BY_KEY_ENTRY, metadata.key) == metadata + assert om.get(ConceptManager.CONCEPTS_BY_HASH_ENTRY, metadata.digest) == metadata + + def test_i_cannot_create_the_same_concept_twice(self, context, service): + with NewOntology(context, "test_i_cannot_create_the_same_concept_twice"): + res = service.define_new_concept(context, "name", body="body") + assert res.status + + res = service.define_new_concept(context, "name", body="body") + assert not res.status + assert isinstance(res.value, ErrorContext) + assert isinstance(res.value.value, ConceptAlreadyDefined) + + def test_i_can_add_the_same_concept_on_different_ontologies(self, context, service): + with NewOntology(context, "test_i_can_add_the_same_concept_on_different_ontologies"): + res = service.define_new_concept(context, "name", body="body") + assert res.status + + sheerka = context.sheerka + om = sheerka.om + om.push_ontology("my_new_ontology") + res = service.define_new_concept(context, "name", body="body") + assert res.status is True + + def test_i_can_get_a_newly_created_concept(self, context, service): + with NewOntology(context, "test_i_can_get_a_newly_created_concept"): + res = service.define_new_concept(context, "name", body="body") + assert res.status + metadata = res.value.metadata + + assert service.get_by_id(metadata.id).id == metadata.id + assert service.get_by_name(metadata.name).name == metadata.name + assert service.get_by_key(metadata.key).key == metadata.key + + def test_i_can_instantiate_a_new_concept_by_its_name(self, context, service): + with NewOntology(context, "test_i_can_instantiate_a_new_concept_by_its_name"): + res = service.define_new_concept(context, "foo", variables=[("var1", None), ("var2", None)]) + assert res.status + + foo = service.newn("foo", var1="value1", var2="value2") + + assert foo.id == "1001" + assert foo.key == "foo" + assert foo.name == "foo" + assert foo.str_id == "c:#1001:" + assert foo.var1 == "value1" + assert foo.var2 == "value2" + + def test_i_can_instantiate_a_new_concept_by_its_id(self, context, service): + with NewOntology(context, "test_i_can_instantiate_a_new_concept_by_its_id"): + res = service.define_new_concept(context, "foo", variables=[("var1", None), ("var2", None)]) + assert res.status + + foo = service.newi("1001", var1="value1", var2="value2") + + assert foo.id == "1001" + assert foo.key == "foo" + assert foo.name == "foo" + assert foo.str_id == "c:#1001:" + assert foo.var1 == "value1" + assert foo.var2 == "value2" + + def test_i_cannot_instantiate_a_concept_which_does_not_exist(self, context, service): + foo = service.newn("foo", var1="value1", var2="value2") + assert foo.key == BuiltinConcepts.UNKNOWN_CONCEPT + assert foo.requested_name == "foo" + + foo = service.newi("1001", var1="value1", var2="value2") + assert foo.key == BuiltinConcepts.UNKNOWN_CONCEPT + assert foo.requested_id == "1001" + + def test_i_can_instantiate_by_name_when_multiple_results(self, context, service): + with NewOntology(context, "test_i_can_instantiate_by_name_when_multiple_results"): + service.define_new_concept(context, "foo", body="body1") + service.define_new_concept(context, "foo", body="body2") + + concepts = service.newn("foo") + + assert len(concepts) == 2 + assert concepts[0].id == "1001" + assert concepts[0].get_metadata().body == "body1" + assert concepts[1].id == "1002" + assert concepts[1].get_metadata().body == "body2" + + def test_concepts_are_removed_when_ontology_is_popped(self, context, service): + context.sheerka.om.push_ontology("new ontology") + res = service.define_new_concept(context, "foo", body="body") + assert service.get_by_id(res.value.metadata.id) is not NotFound + + context.sheerka.om.pop_ontology(context) + assert service.get_by_id(res.value.metadata.id) is NotFound diff --git a/tests/services/test_SheerkaEngine.py b/tests/services/test_SheerkaEngine.py new file mode 100644 index 0000000..7429039 --- /dev/null +++ b/tests/services/test_SheerkaEngine.py @@ -0,0 +1,379 @@ +from typing import Callable + +import pytest + +from base import BaseTest +from core.BuiltinConcepts import BuiltinConcepts +from core.ExecutionContext import ExecutionContext, ExecutionContextActions +from core.ReturnValue import ReturnValue +from core.services.SheerkaEngine import SheerkaEngine +from evaluators.CreateParserInput import CreateParserInput +from evaluators.base_evaluator import AllReturnValuesEvaluator, BaseEvaluator, EvaluatorEvalResult, \ + EvaluatorMatchResult, \ + OneReturnValueEvaluator +from helpers import _rvc + +ALL_STEPS = [ + ExecutionContextActions.BEFORE_PARSING, + ExecutionContextActions.PARSING, + ExecutionContextActions.AFTER_PARSING, + ExecutionContextActions.BEFORE_EVALUATION, + ExecutionContextActions.EVALUATION, + ExecutionContextActions.AFTER_EVALUATION +] + + +class OneReturnValueEvaluatorForTesting(OneReturnValueEvaluator): + def __init__(self, name, + step: ExecutionContextActions, + priority: int, + enabled=True, + match: bool | Callable = True, + match_context=None, + eval_result: list[ReturnValue] = None, + eval_eaten: list[ReturnValue] = None): + super().__init__(name, step, priority, enabled) + self.matches_delegate = match + self.matches_context = match_context + self.eval_result = eval_result + self.eval_eaten = eval_eaten + + def matches(self, context: ExecutionContext, return_value: ReturnValue) -> EvaluatorMatchResult: + # if status is a bool, use it + # otherwise, it's a delegate, so apply to return_value + status = self.matches_delegate if \ + isinstance(self.matches_delegate, bool) else \ + self.matches_delegate(return_value) + return EvaluatorMatchResult(status, self.matches_context) + + def eval(self, context: ExecutionContext, + evaluation_context: object, + return_value: ReturnValue) -> EvaluatorEvalResult: + + # make sure to correctly set up the parent when the return value is modified + if self.eval_result: + for ret_val in self.eval_result: + if ret_val != return_value: + ret_val.parents = [return_value] + + return EvaluatorEvalResult(self.eval_result, self.eval_eaten or [return_value]) + + +class AllReturnValuesEvaluatorForTesting(AllReturnValuesEvaluator): + def __init__(self, name, + step: ExecutionContextActions, + priority: int, + enabled=True, + match: bool | Callable = True, + match_context=None, + eval_result: list[ReturnValue] = None, + eval_eaten: list[ReturnValue] = None): + super().__init__(name, step, priority, enabled) + self.matches_delegate = match + self.matches_context = match_context + self.eval_result = eval_result + self.eval_eaten = eval_eaten + + def matches(self, context: ExecutionContext, return_values: list[ReturnValue]) -> EvaluatorMatchResult: + # if status is a bool, use it + # otherwise, it's a delegate, so apply to return_value + status = self.matches_delegate if \ + isinstance(self.matches_delegate, bool) else \ + self.matches_delegate(return_values) + return EvaluatorMatchResult(status, self.matches_context) + + def eval(self, context: ExecutionContext, + evaluation_context: object, + return_values: list[ReturnValue]) -> EvaluatorEvalResult: + + # make sure to correctly set up the parent when the return value is modified + if self.eval_result: + for ret_val in self.eval_result: + ret_val.parents = return_values + + return EvaluatorEvalResult(self.eval_result, self.eval_eaten or return_values) + + +class TestSheerkaEngine(BaseTest): + @pytest.fixture() + def service(self, sheerka): + return SheerkaEngine(sheerka) + + def test_i_can_compute_execution_plan(self, service): + assert service.compute_execution_plan([]) == {} + + e1 = BaseEvaluator("eval1", ExecutionContextActions.BEFORE_EVALUATION, 5) + e2 = BaseEvaluator("eval2", ExecutionContextActions.BEFORE_EVALUATION, 5) + e3 = BaseEvaluator("eval3", ExecutionContextActions.BEFORE_EVALUATION, 10) + e4 = BaseEvaluator("eval4", ExecutionContextActions.EVALUATION, 10) + e5 = BaseEvaluator("eval5", ExecutionContextActions.AFTER_EVALUATION, 10, enabled=False) + res = service.compute_execution_plan([e1, e2, e3, e4, e5]) + assert res == {ExecutionContextActions.BEFORE_EVALUATION: {5: [e1, e2], 10: [e3]}, + ExecutionContextActions.EVALUATION: {10: [e4]}} + + def test_i_can_call_execute(self, sheerka, context, service): + service.execution_plan = {ExecutionContextActions.BEFORE_EVALUATION: {50: [CreateParserInput()]}} + start = [ReturnValue("TestSheerkaEngine", True, sheerka.newn(BuiltinConcepts.USER_INPUT, command="1 + 1"))] + + ret = service.execute(context, start, [ExecutionContextActions.BEFORE_EVALUATION]) + assert len(ret) == 1 + ret = ret[0] + assert isinstance(ret, ReturnValue) + assert ret.who == CreateParserInput.NAME + assert ret.status is True + assert ret.parents == start + + def test_that_return_values_is_unchanged_when_no_evaluator(self, context, service): + service.execution_plan = {} + start = [_rvc("foo")] + + ret = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + + assert ret == start + + def test_steps_are_executed_in_correct_order(self, context, service): + # properly init the service + _ = OneReturnValueEvaluatorForTesting + evaluators = [ + _("eval1", ExecutionContextActions.AFTER_PARSING, 21, match=False), + _("eval2", ExecutionContextActions.BEFORE_EVALUATION, 5, match=False), + _("eval3", ExecutionContextActions.AFTER_EVALUATION, 12, match=False), + _("eval4", ExecutionContextActions.EVALUATION, 99, match=False), + _("eval5", ExecutionContextActions.BEFORE_PARSING, 5, match=False), + _("eval6", ExecutionContextActions.PARSING, 25, match=False), + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + # init test variables + start = [_rvc("foo")] + service.execute(context, start, ALL_STEPS) + # to check what happened, look at the execution context children + executed_steps = [ec.action_context["step"] for ec in context.get_children(level=1)] + assert executed_steps == ALL_STEPS + + def test_higher_priority_evaluators_are_executed_first(self, context, service): + # properly init the service + _ = OneReturnValueEvaluatorForTesting + evaluators = [ + _("eval1", ExecutionContextActions.EVALUATION, 20, match=False), + _("eval2", ExecutionContextActions.EVALUATION, 5, match=False), + _("eval3", ExecutionContextActions.EVALUATION, 20, match=False), + _("eval4", ExecutionContextActions.EVALUATION, 99, match=False), + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [_rvc("foo")] + service.execute(context, start, [ExecutionContextActions.EVALUATION]) + + # to check what happened, look at the execution context children + evaluators_executed = [ec.action_context["evaluator"] for ec in context.get_children() if + "evaluator" in ec.action_context] + assert evaluators_executed == ["eval4", "eval1", "eval3", "eval2"] + + def test_evaluation_loop_stops_when_no_modification(self, context, service): + rv_foo, rv_bar = _rvc("foo"), _rvc("bar") # rv => ReturnValue + # properly init the service + _ = OneReturnValueEvaluatorForTesting + evaluators = [ + _("eval1", + ExecutionContextActions.EVALUATION, + 20, + match=lambda r: context.sheerka.isinstance(r.value, "foo"), + eval_result=[rv_bar]) + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [rv_foo] + service.execute(context, start, [ExecutionContextActions.EVALUATION]) + children = [ec for ec in context.get_children() if ec.action == ExecutionContextActions.EVALUATING_ITERATION] + assert len(children) == 2 + + def test_eval_is_not_called_if_match_fails_for_one_return(self, context, service): + # properly init the service + _ = OneReturnValueEvaluatorForTesting + evaluators = [ + _("eval1", + ExecutionContextActions.EVALUATION, + 20, + match=lambda r: context.sheerka.isinstance(r.value, "foo"), + eval_result=[_rvc("bar")]) + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [_rvc("baz")] + res = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + assert res == start + + # check what happen in details + exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children())) + evaluation_trace = exec_context.values["evaluation"] + assert evaluation_trace == [{"item": start[0], "match": False}] + + def test_eval_is_called_if_match_succeed_for_one_return(self, context, service): + # properly init the service + _ = OneReturnValueEvaluatorForTesting + evaluators = [ + _("eval1", + ExecutionContextActions.EVALUATION, + 20, + match=lambda r: context.sheerka.isinstance(r.value, "foo"), + eval_result=[_rvc("bar")]) + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [_rvc("foo")] + res = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + assert res == [_rvc("bar")] + assert res[0].parents == start + + # check what happen in details + exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children())) + evaluation_trace = exec_context.values["evaluation"] + assert evaluation_trace == [{"item": start[0], "match": True, "new": res, "eaten": start}] + + def test_all_item_are_processed_during_one_return(self, context, service): + rv_foo, rv_bar, rv_baz, rv_qux = _rvc("foo"), _rvc("bar"), _rvc("baz"), _rvc("qux") # rv => ReturnValue + + # properly init the service + _ = OneReturnValueEvaluatorForTesting + evaluators = [ + _("eval1", + ExecutionContextActions.EVALUATION, + 20, + match=lambda r: context.sheerka.isinstance(r.value, "foo"), + eval_result=[rv_qux]) + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [rv_bar, rv_foo, rv_baz] + res = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + assert res == [rv_bar, rv_qux, rv_baz] # We must keep the order ! rv_qux replaces rv_foo + assert res[0].parents is None + assert res[1].parents == [rv_foo] + assert res[2].parents is None + + # check what happen in details + exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children())) + evaluation_trace = exec_context.values["evaluation"] + assert evaluation_trace == [{"item": rv_bar, "match": False}, + {"item": rv_foo, "match": True, "new": [rv_qux], "eaten": [rv_foo]}, + {"item": rv_baz, "match": False}] + + def test_evaluators_with_the_same_priority_do_not_compete_with_each_other_one_return(self, context, service): + rv_foo, rv_bar, rv_baz, rv_qux = _rvc("foo"), _rvc("bar"), _rvc("baz"), _rvc("qux") # rv => ReturnValue + + # properly init the service + # both evaluator want to eat 'foo' + _ = OneReturnValueEvaluatorForTesting + evaluators = [ + _("eval1", + ExecutionContextActions.EVALUATION, + 20, + match=lambda r: context.sheerka.isinstance(r.value, "foo"), + eval_result=[rv_bar]), + _("eval2", + ExecutionContextActions.EVALUATION, + 20, + match=lambda r: context.sheerka.isinstance(r.value, "foo"), + eval_result=[rv_baz]) + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [rv_qux, rv_foo, rv_qux] + res = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + assert res == [rv_qux, rv_bar, rv_baz, rv_qux] # they both eat it ! + assert res[1].parents == [rv_foo] + assert res[2].parents == [rv_foo] + + def test_evaluators_with_higher_priority_take_precedence_one_return(self, context, service): + rv_foo, rv_bar, rv_baz = _rvc("foo"), _rvc("bar"), _rvc("baz") # rv => ReturnValue + + # properly init the service + # both evaluator want to eat 'foo' + _ = OneReturnValueEvaluatorForTesting + evaluators = [ + _("eval1", + ExecutionContextActions.EVALUATION, + 20, + match=lambda r: context.sheerka.isinstance(r.value, "foo"), + eval_result=[rv_bar]), + _("eval2", + ExecutionContextActions.EVALUATION, + 30, + match=lambda r: context.sheerka.isinstance(r.value, "foo"), + eval_result=[rv_baz]) + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [rv_foo] + res = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + assert res == [rv_baz] + assert res[0].parents == start + + def test_evaluator_matches_is_called_before_eval_for_all_return(self, context, service): + # properly init the service + _ = AllReturnValuesEvaluatorForTesting + evaluators = [ + _("eval1", + ExecutionContextActions.EVALUATION, + 20, + match=lambda r: context.sheerka.isinstance(r[0].value, "foo"), + eval_result=[_rvc("bar")]) + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [_rvc("baz")] + res = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + assert res == start + + start = [_rvc("foo")] + res = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + assert res == [_rvc("bar")] + assert res[0].parents == start + + def test_eval_is_not_call_if_match_fails_for_all_return(self, context, service): + rv_foo, rv_bar, rv_baz = _rvc("foo"), _rvc("bar"), _rvc("baz") # rv => ReturnValue + + # properly init the service + _ = AllReturnValuesEvaluatorForTesting + evaluators = [ + _("eval1", + ExecutionContextActions.EVALUATION, + 20, + match=lambda lst: context.sheerka.isinstance(lst[0].value, "foo"), + eval_result=[rv_bar]) + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [rv_baz, rv_foo] # foo is not the first in the list + res = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + assert res == start + + # check what happen in details + exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children())) + evaluation_trace = exec_context.values["evaluation"] + assert evaluation_trace == {"match": False} + + def test_eval_is_called_if_match_succeed_for_all_return(self, context, service): + rv_foo, rv_bar, rv_baz = _rvc("foo"), _rvc("bar"), _rvc("baz") # rv => ReturnValue + # properly init the service + _ = AllReturnValuesEvaluatorForTesting + evaluators = [ + _("eval1", + ExecutionContextActions.EVALUATION, + 20, + match=lambda lst: context.sheerka.isinstance(lst[0].value, "foo"), + eval_result=[rv_bar]) + ] + service.execution_plan = service.compute_execution_plan(evaluators) + + start = [rv_foo, rv_baz] + res = service.execute(context, start, [ExecutionContextActions.EVALUATION]) + assert res == [rv_bar] + assert res[0].parents == start + + children = list(context.get_children()) + # check what happen in details + exec_context = next(filter(lambda ec: "evaluator" in ec.action_context, context.get_children())) + evaluation_trace = exec_context.values["evaluation"] + assert evaluation_trace == {"match": True, "new": res, "eaten": start} diff --git a/tests/sheerkapickle/__init__.py b/tests/sheerkapickle/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sheerkapickle/test_SheerkaPickler.py b/tests/sheerkapickle/test_SheerkaPickler.py new file mode 100644 index 0000000..da894f0 --- /dev/null +++ b/tests/sheerkapickle/test_SheerkaPickler.py @@ -0,0 +1,183 @@ +import logging + +import pytest + +from base import BaseTest +from common.global_symbols import NoFirstToken, NotFound, NotInit, Removed +from helpers import get_concept, get_concepts +from ontologies.SheerkaOntologyManager import SheerkaOntologyManager +from parsers.tokenizer import Keywords +from sheerkapickle import tags +from sheerkapickle.sheerkaplicker import SheerkaPickler +from sheerkapickle.sheerkaunpickler import SheerkaUnpickler + + +class Obj: + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + + def __eq__(self, other): + if id(self) == id(other): + return True + + if not isinstance(other, Obj): + return False + + return self.a == other.a and self.b == other.b and self.c == other.c + + def __hash__(self): + return hash((self.a, self.b, self.c)) + + +class TestSheerkaPickler(BaseTest): + + @pytest.mark.parametrize("obj, expected", [ + (1, 1), + (3.14, 3.14), + ("a string", "a string"), + (True, True), + (None, None), + ([1, 3.14, "a string"], [1, 3.14, "a string"]), + ((1, 3.14, "a string"), {tags.TUPLE: [1, 3.14, "a string"]}), + ({1}, {tags.SET: [1]}), + ({"a": "a", "b": 3.14, "c": True}, {"a": "a", "b": 3.14, "c": True}), + ({1: "a", 2: 3.14, 3: True}, {1: "a", 2: 3.14, 3: True}), + ([1, [3.14, "a string"]], [1, [3.14, "a string"]]), + ([1, (3.14, "a string")], [1, {tags.TUPLE: [3.14, "a string"]}]), + ([], []), + (Keywords.DEF, {tags.ENUM: 'parsers.tokenizer.Keywords.DEF'}), + ]) + def test_i_can_flatten_and_restore_primitives(self, sheerka, obj, expected): + flatten = SheerkaPickler(sheerka).flatten(obj) + assert flatten == expected + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj + + @pytest.mark.parametrize("obj, expected", [ + (NotInit, {tags.CUSTOM: NotInit.value}), + (NotFound, {tags.CUSTOM: NotFound.value}), + (Removed, {tags.CUSTOM: Removed.value}), + (NoFirstToken, {tags.CUSTOM: NoFirstToken.value}), + ]) + def test_i_can_flatten_and_restore_custom_types(self, sheerka, obj, expected): + flatten = SheerkaPickler(sheerka).flatten(obj) + assert flatten == expected + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj + + def test_i_can_flatten_and_restore_instances(self, sheerka): + obj1 = Obj(1, "b", True) + obj2 = Obj(3.14, ("a", "b"), obj1) + + flatten = SheerkaPickler(sheerka).flatten(obj2) + assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj', + 'a': 3.14, + 'b': {'_sheerka/tuple': ['a', 'b']}, + 'c': {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj', + 'a': 1, + 'b': 'b', + 'c': True}} + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj2 + + def test_i_can_manage_circular_reference(self, sheerka): + obj1 = Obj(1, "b", True) + obj1.c = obj1 + + flatten = SheerkaPickler(sheerka).flatten(obj1) + assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj', + 'a': 1, + 'b': 'b', + 'c': {'_sheerka/id': 0}} + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded.a == obj1.a + assert decoded.b == obj1.b + assert decoded.c == decoded + + def test_i_can_flatten_obj_with_new_props(self, sheerka): + # property 'z' is not part of the `Obj` definition + obj = Obj(1, "b", True) + obj.z = "new prop" + + flatten = SheerkaPickler(sheerka).flatten(obj) + assert flatten == {'_sheerka/obj': 'tests.sheerkapickle.test_SheerkaPickler.Obj', + 'a': 1, + 'b': 'b', + 'c': True, + 'z': "new prop"} + + decoded = SheerkaUnpickler(sheerka).restore(flatten) + assert decoded == obj + + def test_i_cannot_correctly_flatten_compiled_and_generator(self, sheerka): + obj = Obj((i for i in range(3)), compile("a + b", "", mode="eval"), None) + + flatten = SheerkaPickler(sheerka).flatten(obj) + + assert isinstance(flatten["a"], str) + assert flatten["a"].startswith("