SettingUpClientServer (#7)

Reviewed-on: #7
This commit is contained in:
2023-01-19 16:08:23 +00:00
parent 95459c5b02
commit 21a397861a
19 changed files with 2480 additions and 2 deletions
+62
View File
@@ -0,0 +1,62 @@
import logging
from multiprocessing import Process
from time import sleep
import uvicorn
from fastapi import FastAPI
class MockServer:
""" Core application to test. """
def __init__(self, endpoints: list[dict]):
"""
:param endpoints:
:type endpoints: list of {path: '', response:''}
"""
self.api = FastAPI()
def raise_exception(ex):
raise ex
# register endpoints
for endpoint in endpoints:
method = endpoint["method"] if "method" in endpoint else "get"
if method == "post":
if "exception" in endpoint:
self.api.post(endpoint["path"])(lambda: raise_exception(endpoint["exception"]))
else:
self.api.post(endpoint["path"])(lambda: endpoint["response"])
else:
self.api.get(endpoint["path"])(lambda: endpoint["response"])
# register shutdown
self.api.on_event("shutdown")(self.close)
# create the process
self.proc = Process(target=uvicorn.run,
args=(self.api,),
kwargs={
"host": "127.0.0.1",
"port": 5000,
"log_level": "info"},
daemon=True)
async def close(self):
""" Gracefull shutdown. """
logging.warning("Shutting down the app.")
def start_server(self):
self.proc.start()
sleep(0.1)
def stop_server(self):
self.proc.terminate()
def __enter__(self):
self.start_server()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop_server()
+539
View File
@@ -0,0 +1,539 @@
import json
import os
import shutil
from datetime import date, datetime
from os import path
import pytest
from core.global_symbols import NotFound
from sdp.sheerkaDataProvider import Event, SheerkaDataProvider
from sdp.sheerkaSerializer import JsonSerializer, PickleSerializer
tests_root = path.abspath("../../build/tests")
evt_digest = "3a571cb6034ef6fc8d7fe91948d0d29728eed74de02bac7968b0e9facca2c2d7"
def read_json_file(sdp, file_name):
with sdp.io.open(file_name, "r") as f:
return json.load(f)
class ObjNoKey:
"""
Object with no key, they won't be ordered
Not suitable for Json dump as there is no to_dict() method
"""
def __init__(self, a, b):
self.a = a
self.b = b
def __hash__(self):
return hash((self.a, self.b))
def __eq__(self, obj):
return isinstance(obj, ObjNoKey) and \
self.a == obj.a and \
self.b == obj.b
def __repr__(self):
return f"ObjNoKey({self.a}, {self.b})"
class ObjWithDigestWithKey:
"""
Object with a key that can compute its digest.
It can be used to test objects sharing the same key (but that are different)
Not suitable for Json dump as there is no to_dict() method
"""
def __init__(self, a, b):
self.a = a
self.b = b
def __hash__(self):
return hash((self.a, self.b))
def __eq__(self, obj):
return isinstance(obj, ObjWithDigestWithKey) and \
self.a == obj.a and \
self.b == obj.b
def __repr__(self):
return f"ObjWithDigestWithKey({self.a}, {self.b})"
def get_key(self):
return self.a
def get_digest(self):
return str(self.a) + str(self.b)
@pytest.fixture(autouse=True)
def init_test():
"""
I test both SheerkaDataProviderFileIO and SheerkaDataProviderDictionaryIO
So it's important to reset the folders between two tests
:return:
:rtype:
"""
if path.exists(tests_root):
shutil.rmtree(tests_root)
if not path.exists(tests_root):
os.makedirs(tests_root)
current_pwd = os.getcwd()
os.chdir(tests_root)
yield None
os.chdir(current_pwd)
@pytest.mark.parametrize("root, expected", [
(".sheerka", path.abspath(path.join(tests_root, ".sheerka"))),
("mem://", "")
])
def test_i_can_init_the_data_provider(root, expected):
sdp = SheerkaDataProvider(root)
assert sdp.io.root == expected
assert sdp.io.exists(sdp.io.root)
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_save_and_load_an_event(root):
sdp = SheerkaDataProvider(root)
event = Event("hello world", date=date(year=2007, month=9, day=10), user_id="kodjo")
evt_digest = sdp.save_event(event)
evt = sdp.load_event(evt_digest)
assert evt.date == datetime(year=2007, month=9, day=10)
assert evt.user_id == "kodjo"
assert evt.message == "hello world"
assert evt.parents is None
assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.EventFolder, evt_digest[0:24], evt_digest))
# I can get the last event
evt = sdp.load_event()
assert evt.message == "hello world"
# check that the last event is updated
last_event_file = path.join(sdp.io.root, SheerkaDataProvider.LastEventFile)
assert sdp.io.exists(last_event_file)
assert sdp.io.read_text(last_event_file) == evt_digest
def test_i_can_save_and_load_events_with_multiple_sdp():
root = ".sheerka"
sdp1 = SheerkaDataProvider(root)
sdp1.save_event(Event("event 1", date=date(year=2007, month=9, day=10), user_id="kodjo"))
sdp1.save_event(Event("event 2", date=date(year=2007, month=9, day=10), user_id="kodjo"))
sdp2 = SheerkaDataProvider(root, "Another sdp")
sdp2.save_event(Event("event 3", date=date(year=2007, month=9, day=10), user_id="kodjo"))
sdp2.save_event(Event("event 4", date=date(year=2007, month=9, day=10), user_id="kodjo"))
events_from_1 = list(sdp1.load_events(-1))
events_from_2 = list(sdp2.load_events(-1))
assert [e.message for e in events_from_1] == ['event 4', 'event 3', 'event 2', 'event 1']
assert [e.message for e in events_from_2] == ['event 4', 'event 3', 'event 2', 'event 1']
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_get_event_history(root):
sdp = SheerkaDataProvider(root)
event = Event("hello world", date=date(year=2007, month=9, day=10), user_id="kodjo")
event2 = Event("hello world 2", date=date(year=2007, month=9, day=10), user_id="kodjo")
evt_digest1 = sdp.save_event(event)
evt_digest2 = sdp.save_event(event2)
evt = sdp.load_event(evt_digest2)
assert evt.date == datetime(year=2007, month=9, day=10)
assert evt.user_id == "kodjo"
assert evt.message == "hello world 2"
assert evt.parents == [evt_digest1]
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_load_events(root):
sdp = SheerkaDataProvider(root)
for i in range(15):
sdp.save_event(Event(f"TEST::Hello {i}"))
events = list(sdp.load_events(10)) # first ten
assert len(events) == 10
assert events[0].message == "TEST::Hello 14"
assert events[9].message == "TEST::Hello 5"
events = list(sdp.load_events(10, 5)) # skip first 5, then take 10
assert len(events) == 10
assert events[0].message == "TEST::Hello 9"
assert events[9].message == "TEST::Hello 0"
events = list(sdp.load_events(20, 10)) # skip first 10, take 20,(but only 5 remaining)
assert len(events) == 5
assert events[0].message == "TEST::Hello 4"
assert events[4].message == "TEST::Hello 0"
events = list(sdp.load_events(1, 20)) # skip first 20, take one
assert len(events) == 0
events = list(sdp.load_events(0)) # all
assert len(events) == 15
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_load_events_when_no_event(root):
sdp = SheerkaDataProvider(root)
events = list(sdp.load_events(1))
assert len(events) == 0
events = list(sdp.load_events(1, 5))
assert len(events) == 0
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_add_and_reload_one_item(root):
sdp = SheerkaDataProvider(root)
event = Event("hello world", date=date(year=2007, month=9, day=10), user_id="kodjo")
with sdp.get_transaction(event) as transaction:
transaction.add("entry", "key", "foo => bar")
transaction.add("entry", "key2", ObjNoKey("a", "b"))
transaction.add("entry2", "key", "value2")
last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile)
state = sdp.load_state(last_commit)
loaded1 = sdp.get("entry", "key")
loaded2 = sdp.get("entry", "key2")
loaded3 = sdp.get("entry2", "key")
load_entry = sdp.get("entry")
# check that the event is saved
evt_digest = event.get_digest()
assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.EventFolder, evt_digest[0:24], evt_digest))
# check the values
assert loaded1 == "foo => bar"
assert loaded2 == ObjNoKey("a", "b")
assert loaded3 == "value2"
assert load_entry == {
"key": "foo => bar",
"key2": ObjNoKey("a", "b")
}
assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.StateFolder, last_commit[0:24], last_commit))
assert sdp.io.exists(path.join(sdp.io.root, SheerkaDataProvider.RefFolder, sdp.name, SheerkaDataProvider.HeadFile))
assert state.date is not None
assert state.parents == []
assert state.events == [evt_digest]
assert state.data == {"entry": {'key': 'foo => bar', 'key2': ObjNoKey("a", "b")},
'entry2': {'key': 'value2'}}
assert sdp.io.read_text(
path.join(sdp.io.root, SheerkaDataProvider.RefFolder, sdp.name, SheerkaDataProvider.HeadFile)) == last_commit
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_load_an_entry(root):
sdp = SheerkaDataProvider(root)
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key1", "foo")
transaction.add("entry", "key2", "bar")
transaction.add("entry", "key3", "baz")
item = sdp.get("entry", "key1")
assert item == "foo"
load_entry = sdp.get("entry")
assert load_entry == {
"key1": "foo",
"key2": "bar",
"key3": "baz",
}
# load entry was a copy
load_entry["key1"] = "another foo"
assert sdp.get("entry", "key1") == "foo"
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_add_and_reload_a_list_of_items(root):
sdp = SheerkaDataProvider(root)
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key", ["foo => bar", ObjNoKey("a", "b")])
last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile)
state = sdp.load_state(last_commit)
loaded = sdp.get("entry", "key")
# check the values
assert loaded == ["foo => bar", ObjNoKey("a", "b")]
assert state.date is not None
assert state.parents == []
assert state.events == [evt_digest]
assert state.data == {"entry": {'key': ['foo => bar', ObjNoKey('a', 'b')]}}
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_add_and_reload_a_set_of_items(root):
sdp = SheerkaDataProvider(root)
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key", {"foo => bar", ObjNoKey("a", "b")})
last_commit = sdp.get_snapshot(SheerkaDataProvider.HeadFile)
state = sdp.load_state(last_commit)
loaded = sdp.get("entry", "key")
# check the values
assert loaded == {"foo => bar", ObjNoKey("a", "b")}
assert state.date is not None
assert state.parents == []
assert state.events == [evt_digest]
assert state.data == {"entry": {'key': {'foo => bar', ObjNoKey('a', 'b')}}}
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_add_and_reload_an_entry(root):
sdp = SheerkaDataProvider(root)
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry1", None, "foo")
transaction.add("entry2", None, {"key": "foo", "key1": "bar"})
transaction.add("entry3", None, {"foo", "bar"})
transaction.add("entry4", None, ["foo", "bar"])
loaded_entry1 = sdp.get("entry1")
loaded_entry2 = sdp.get("entry2")
loaded_entry3 = sdp.get("entry3")
loaded_entry4 = sdp.get("entry4")
assert loaded_entry1 == "foo"
assert loaded_entry2 == {"key": "foo", "key1": "bar"}
assert loaded_entry3 == {"foo", "bar"}
assert loaded_entry4 == ["foo", "bar"]
# loaded values are copies
loaded_entry2["key"] = "foo2"
assert sdp.get("entry2", "key") == "foo"
loaded_entry3.remove("foo")
assert sdp.get("entry3") == {"foo", "bar"}
loaded_entry4[0] = "foo2"
assert sdp.get("entry4")[0] == "foo"
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_override_values(root):
sdp = SheerkaDataProvider(root)
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key", {"foo => bar", ObjNoKey("a", "b")})
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key", "new_value")
loaded = sdp.get("entry", "key")
assert loaded == "new_value"
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_add_an_object_and_save_it_as_a_reference(root):
sdp = SheerkaDataProvider(root)
sdp.serializer.register(JsonSerializer(lambda o: isinstance(o, ObjNoKey)))
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key1", ObjNoKey("a", "b"), use_ref=True)
transaction.add("entry", "key2", [ObjNoKey("a", "b"), ObjNoKey("c", "d")], use_ref=True)
transaction.add("entry", "key3", {ObjNoKey("a", "b"), ObjNoKey("c", "d")}, use_ref=True)
assert sdp.get("entry", "key1") == ObjNoKey("a", "b")
assert sdp.get("entry", "key2") == [ObjNoKey("a", "b"), ObjNoKey("c", "d")]
assert sdp.get("entry", "key3") == {ObjNoKey("a", "b"), ObjNoKey("c", "d")}
# I can ask for the whole entry
assert sdp.get("entry") == {"key1": ObjNoKey("a", "b"),
"key2": [ObjNoKey("a", "b"), ObjNoKey("c", "d")],
"key3": {ObjNoKey("a", "b"), ObjNoKey("c", "d")}}
state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
assert state.data == {
"entry": {'key1': '##REF##:fbc2b1c60ed753b49217cae851e342371ee39ebabc9778105f450812e615a513',
'key2': ['##REF##:fbc2b1c60ed753b49217cae851e342371ee39ebabc9778105f450812e615a513',
'##REF##:448420dbc57d61401d10a98759fccdabbe50e2e825b6da3bd018c190926bcda4'],
'key3': {'##REF##:448420dbc57d61401d10a98759fccdabbe50e2e825b6da3bd018c190926bcda4',
'##REF##:fbc2b1c60ed753b49217cae851e342371ee39ebabc9778105f450812e615a513'}}
}
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_add_an_object_as_a_reference_using_its_own_digest(root):
sdp = SheerkaDataProvider(root)
sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey)))
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key1", ObjWithDigestWithKey("a", "b"), use_ref=True)
assert sdp.get("entry", "key1") == ObjWithDigestWithKey("a", "b")
state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
assert state.data == {
"entry": {'key1': '##REF##:ab'}
}
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_remove_elements(root):
sdp = SheerkaDataProvider(root)
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key", "value")
transaction.add("entry", "key2", "value2")
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.remove("entry", "key")
assert sdp.get("entry", "key") is NotFound
state = sdp.load_state(sdp.get_snapshot(SheerkaDataProvider.HeadFile))
assert state.data == {
"entry": {'key2': 'value2'}
}
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_keep_state_history(root):
sdp = SheerkaDataProvider(root)
with sdp.get_transaction(Event("first event")) as transaction:
transaction.add("entry", "key", "value")
state_digest1 = transaction.snapshot
with sdp.get_transaction(Event("second event")) as transaction:
transaction.add("entry", "key2", "value2")
state_digest2 = transaction.snapshot
with sdp.get_transaction(Event("third event")) as transaction:
transaction.add("entry", "key2", "value2")
state_digest3 = transaction.snapshot
state = sdp.load_state(state_digest3)
assert state.parents == [state_digest2]
state = sdp.load_state(state_digest2)
assert state.parents == [state_digest1]
state = sdp.load_state(state_digest1)
assert state.parents == []
@pytest.mark.parametrize("root", [
".sheerka",
"mem://"
])
def test_i_can_save_and_load_ontologies_names(root):
sdp = SheerkaDataProvider(root)
ontologies = ['new ontology', '#unit_test#', '__default__']
sdp.save_ontologies(ontologies)
assert sdp.load_ontologies() == ontologies
# extra
ontologies_files = path.join(sdp.io.root, SheerkaDataProvider.OntologiesFiles)
assert sdp.io.exists(ontologies_files)
assert sdp.io.read_text(ontologies_files) == """new ontology
#unit_test#
__default__"""
def test_i_can_remove_even_if_not_exist():
sdp = SheerkaDataProvider("mem://")
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.remove("entry", None)
transaction.remove(None, "key")
transaction.remove("entry", "key")
def test_i_get_default_value_if_entry_is_missing():
sdp = SheerkaDataProvider("mem://")
assert sdp.get("fake_entry", "fake_key", "default_value") == "default_value"
def test_exists():
sdp = SheerkaDataProvider("mem://")
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key", "value")
assert not sdp.exists("entry2")
assert not sdp.exists("entry", "key2")
assert sdp.exists("entry", "key")
def test_not_found_is_returned_when_an_entry_is_not_found():
sdp = SheerkaDataProvider("mem://")
with sdp.get_transaction(Event(f"TEST::{evt_digest}")) as transaction:
transaction.add("entry", "key", "value")
assert sdp.get("entry", "key") == "value"
assert sdp.get("entry", "key2") == NotFound
assert sdp.get("entry2") == NotFound
+35
View File
@@ -0,0 +1,35 @@
import io
from sdp.sheerkaSerializer import JsonSerializer
class ObjNoKey:
"""
Object with no key, they won't be ordered
Not suitable for Json dump as there is no to_dict() method
"""
def __init__(self, a, b):
self.a = a
self.b = b
def __hash__(self):
return hash((self.a, self.b))
def __eq__(self, obj):
return isinstance(obj, ObjNoKey) and \
self.a == obj.a and \
self.b == obj.b
def __repr__(self):
return f"ObjNoKey({self.a}, {self.b})"
def test_i_can_json_serialize():
json_serializer = JsonSerializer(lambda obj: True)
obj = ObjNoKey("a", "b")
stream = io.BytesIO()
stream = json_serializer.dump(stream, obj, None)
res = json_serializer.load(stream, None)
assert res == obj
+98
View File
@@ -0,0 +1,98 @@
import json
from fastapi import HTTPException
from starlette import status
from client import SheerkaClient, parse_arguments
from mockserver import MockServer
def test_i_can_start_with_a_default_hostname():
parsed = parse_arguments([])
assert parsed.hostname == "http://localhost"
assert parsed.port == 56356
def test_i_can_override_hostname_and_port():
parsed = parse_arguments(["new_host", "--port", "1515"])
assert parsed.hostname == "new_host"
assert parsed.port == 1515
parsed = parse_arguments(["new_host", "-p", "1515"])
assert parsed.hostname == "new_host"
assert parsed.port == 1515
def test_i_can_provide_user_and_password():
parsed = parse_arguments(["--username", "my_user", "--password", "my_password"])
assert parsed.username == "my_user"
assert parsed.password == "my_password"
parsed = parse_arguments(["-u", "my_user", "-P", "my_password"])
assert parsed.username == "my_user"
assert parsed.password == "my_password"
def test_i_can_manage_when_no_server():
client = SheerkaClient("http://localhost", 80)
res = client.check_url()
assert res.status is False
assert res.message == "Connection refused."
def test_i_can_manage_when_resource_is_not_found():
with MockServer([]):
client = SheerkaClient("http://localhost", 5000)
res = client.check_url()
assert not res.status
assert res.message == '{"detail":"Not Found"}'
def test_i_can_connect_to_a_server():
with MockServer([{
"path": "/",
"response": "Hello world"
}]):
client = SheerkaClient("http://localhost", 5000)
res = client.check_url()
assert res.status
assert res.message == '"Hello world"'
def test_i_can_authenticate_with_valid_credentials():
with MockServer([{
"path": "/",
"response": "Hello world"
}, {
"method": "post",
"path": "/token",
"response": {"access_token": "xxxx", "token_type": "bearer"}
}]):
client = SheerkaClient("http://localhost", 5000)
res = client.connect("valid_username", "valid_password")
assert res.status
assert res.message == "Connected as valid_username"
def test_i_can_manage_when_authentication_fails():
with MockServer([{
"path": "/",
"response": "Hello world"
}, {
"method": "post",
"path": "/token",
"exception": HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect username or password",
headers={"WWW-Authenticate": "Bearer"},
)
}]):
client = SheerkaClient("http://localhost", 5000)
res = client.connect("username", "wrong_password")
assert not res.status
assert res.message == 'Incorrect username or password'