Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

This commit is contained in:
2020-05-12 17:21:10 +02:00
parent 7d3a490bc5
commit 6e343ba996
110 changed files with 13865 additions and 7540 deletions
+218 -103
View File
@@ -1,24 +1,33 @@
import pytest
import os
import pytest
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
from core.concept import Concept, PROPERTIES_TO_SERIALIZE
from core.sheerka.Sheerka import Sheerka
from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts
from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
class ConceptWithGetValue(Concept):
def get_value(self):
return self.get_prop("my_prop")
class ConceptWithGetObjValue(Concept):
def get_obj_value(self):
return self.get_value("my_prop")
class TestSheerka(TestUsingFileBasedSheerka):
class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
def test_root_folder_is_created_after_initialization(self):
return_value = Sheerka().initialize(self.root_folder)
assert return_value.status, "initialisation should be successful"
assert os.path.exists(self.root_folder), "init folder should be created"
def test_i_can_initialize_builtin_parsers(self):
sheerka = self.get_sheerka()
# test existence of some parser (not all)
assert "parsers.DefaultParser.DefaultParser" in sheerka.parsers
assert "parsers.BnfNodeParser.BnfNodeParser" in sheerka.parsers
assert "parsers.SyaNodeParser.SyaNodeParser" in sheerka.parsers
assert "parsers.AtomNodeParser.AtomNodeParser" in sheerka.parsers
# make sure BaseNodeParser is properly initialized
assert BASE_NODE_PARSER_CLASS not in sheerka.parsers
assert sheerka.bnp is not None
def test_i_can_list_builtin_concepts(self):
sheerka = self.get_sheerka()
@@ -27,27 +36,6 @@ class TestSheerka(TestUsingFileBasedSheerka):
assert str(BuiltinConcepts.ERROR) in builtins
assert str(BuiltinConcepts.RETURN_VALUE) in builtins
def test_builtin_concepts_are_initialized(self):
sheerka = self.get_sheerka(skip_builtins_in_db=False)
assert len(sheerka.cache_by_key) == len(BuiltinConcepts)
for concept_name in BuiltinConcepts:
assert str(concept_name) in sheerka.cache_by_key
assert sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, str(concept_name)) is not None
for key, concept_class in sheerka.get_builtins_classes_as_dict().items():
assert isinstance(sheerka.cache_by_key[key], concept_class)
def test_builtin_concepts_can_be_updated(self):
sheerka = self.get_sheerka(use_dict=False, skip_builtins_in_db=False)
loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
loaded_sheerka.metadata.desc = "I have a description"
sheerka.sdp.modify("Test", sheerka.CONCEPTS_ENTRY, loaded_sheerka.key, loaded_sheerka)
sheerka = self.get_sheerka(use_dict=False, skip_builtins_in_db=False)
loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
assert loaded_sheerka.metadata.desc == "I have a description"
def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string(self):
"""
Checks that a concept can be found its name
@@ -56,27 +44,27 @@ class TestSheerka(TestUsingFileBasedSheerka):
"""
sheerka = self.get_sheerka()
for key in sheerka.get_builtins_classes_as_dict():
assert sheerka.get(key) is not None
assert sheerka.get(str(key)) is not None
assert sheerka.get_by_key(key) is not None
assert sheerka.get_by_key(str(key)) is not None
def test_i_cannot_get_when_key_is_none(self):
sheerka = self.get_sheerka()
res = sheerka.get(None)
res = sheerka.get_by_key(None)
assert sheerka.isinstance(res, BuiltinConcepts.ERROR)
assert res.body == "Concept key is undefined."
assert res.body == "Concept 'None' is undefined."
def test_unknown_concept_is_return_when_the_concept_key_is_not_found(self):
def test_i_cannot_get_when_key_is_not_found(self):
sheerka = self.get_sheerka()
loaded = sheerka.get("key_that_does_not_exist")
loaded = sheerka.get_by_key("key_that_does_not_exist")
assert loaded is not None
assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
assert loaded.body == ("key", "key_that_does_not_exist")
assert loaded.metadata.is_evaluated
def test_unknown_concept_is_return_when_the_concept_id_is_not_found(self):
def test_i_cannot_get_when_id_is_not_found(self):
sheerka = self.get_sheerka()
loaded = sheerka.get_by_id("id_that_does_not_exist")
@@ -97,6 +85,10 @@ class TestSheerka(TestUsingFileBasedSheerka):
assert ret.value == "value"
assert ret.message == "message"
# check the others
for key, concept_class in sheerka.get_builtins_classes_as_dict().items():
assert isinstance(sheerka.get_by_key(key), concept_class)
def test_i_can_instantiate_a_builtin_concept_when_no_specific_class(self):
sheerka = self.get_sheerka()
ret = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body="fake_concept")
@@ -106,9 +98,7 @@ class TestSheerka(TestUsingFileBasedSheerka):
assert ret.body == "fake_concept"
def test_i_can_instantiate_a_concept(self):
sheerka = self.get_sheerka()
concept = self.get_default_concept()
sheerka.create_new_concept(self.get_context(sheerka), concept)
sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
new = sheerka.new(concept.key, a=10, b="value")
@@ -116,27 +106,33 @@ class TestSheerka(TestUsingFileBasedSheerka):
for prop in PROPERTIES_TO_SERIALIZE:
assert getattr(new.metadata, prop) == getattr(concept.metadata, prop)
assert new.props["a"].value == 10
assert new.props["b"].value == "value"
assert new.get_value("a") == 10
assert new.get_value("b") == "value"
def test_i_can_instantiate_with_the_name_and_the_id(self):
sheerka = self.get_sheerka()
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
def test_i_can_instantiate_multiple_when_same_key(self):
sheerka, context, *concepts = self.init_concepts(
Concept("foo", body="foo1"),
Concept("foo", body="foo2"),
create_new=True)
# when no id, i get two instances
concepts = sheerka.new("foo")
assert len(concepts) == 2
assert concepts[0].id == "1001"
assert concepts[0].metadata.body == "foo1"
assert concepts[1].id == "1002"
assert concepts[1].metadata.body == "foo2"
# only one instance if the id is given
foo1 = sheerka.new(("foo", "1001"))
assert foo1.metadata.body == "foo1"
# only one instance if the id is given
foo2 = sheerka.new(("foo", "1002"))
assert foo2.metadata.body == "foo2"
def test_instances_are_different_when_asking_for_new(self):
sheerka = self.get_sheerka()
concept = self.get_default_concept()
sheerka.create_new_concept(self.get_context(sheerka), concept)
sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
new1 = sheerka.new(concept.key, a=10, b="value")
new2 = sheerka.new(concept.key, a=10, b="value")
@@ -144,17 +140,50 @@ class TestSheerka(TestUsingFileBasedSheerka):
assert new1 == new2
assert id(new1) != id(new2)
def test_i_get_the_same_instance_when_is_unique_is_true(self):
sheerka = self.get_sheerka()
concept = Concept(name="unique", is_unique=True)
sheerka.create_new_concept(self.get_context(sheerka), concept)
def test_new_instance_does_not_impact_each_others(self):
sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
new1 = sheerka.new(concept.key, a=10, b="value")
new2 = sheerka.new(concept.key, a=10, b="value")
new_foo = sheerka.new("foo")
new_foo.metadata.body = "metadata value" # modify metadata
new_foo.def_var("var_name", "default value") # modify definition of variables
new_foo.add_prop(BuiltinConcepts.ISA, bar) # modify property
new_foo.compiled["var_name"] = "'var value'"
new_foo.set_value(ConceptParts.BODY, "body value") # modify value
new_foo.set_value("var_name", "var value") # modify value
assert new_foo.metadata.body != foo.metadata.body
assert new_foo.metadata.variables != foo.metadata.variables
assert new_foo.metadata.props != foo.metadata.props
assert new_foo.values != foo.values
assert new_foo.compiled != foo.compiled
def test_i_get_the_same_instance_when_is_unique_is_true(self):
sheerka, context, concept = self.init_concepts(Concept(name="unique", is_unique=True), create_new=True)
new1 = sheerka.new(concept.key)
new2 = sheerka.new(concept.key, a=10, b="value") # not that variables are simply discareded
assert new1 == new2
assert id(new1) == id(new2)
def test_values_are_reset_when_asking_for_a_new_instance(self):
sheerka, context, template = self.init_concepts(
Concept("foo", body="'foo body'"),
create_new=True,
eval_body=True)
sheerka.evaluate_concept(context, sheerka.get_by_id(template.id))
assert template.metadata.is_evaluated
assert template.body == "foo body"
new = sheerka.new(template.key)
assert not new.metadata.is_evaluated
assert not new.body
new = sheerka.new((None, template.id))
assert not new.metadata.is_evaluated
assert not new.body
def test_i_cannot_instantiate_an_unknown_concept(self):
sheerka = self.get_sheerka()
@@ -164,9 +193,10 @@ class TestSheerka(TestUsingFileBasedSheerka):
assert new.body == ('key', 'fake_concept')
def test_i_cannot_instantiate_with_invalid_id(self):
sheerka = self.get_sheerka()
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
sheerka, context, *concepts = self.init_concepts(
Concept("foo", body="foo1"),
Concept("foo", body="foo2"),
create_new=True)
new = sheerka.new(("foo", "invalid_id"))
@@ -174,9 +204,10 @@ class TestSheerka(TestUsingFileBasedSheerka):
assert new.body == [('key', 'foo'), ('id', 'invalid_id')]
def test_i_cannot_instantiate_with_invalid_key(self):
sheerka = self.get_sheerka()
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
sheerka, context, *concepts = self.init_concepts(
Concept("foo", body="foo1"),
Concept("foo", body="foo2"),
create_new=True)
new = sheerka.new(("invalid_key", "1001"))
@@ -184,8 +215,9 @@ class TestSheerka(TestUsingFileBasedSheerka):
assert new.body == [('key', 'invalid_key'), ('id', '1001')]
def test_concept_id_is_irrelevant_when_only_one_concept(self):
sheerka = self.get_sheerka()
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
sheerka, context, *concepts = self.init_concepts(
Concept("foo", body="foo1"),
create_new=True)
new = sheerka.new(("foo", "invalid_id"))
@@ -193,9 +225,7 @@ class TestSheerka(TestUsingFileBasedSheerka):
assert new.metadata.body == "foo1"
def test_i_cannot_instantiate_when_properties_are_not_recognized(self):
sheerka = self.get_sheerka()
concept = self.get_default_concept()
sheerka.create_new_concept(self.get_context(sheerka), concept)
sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
new = sheerka.new(concept.key, a=10, c="value")
@@ -210,7 +240,7 @@ class TestSheerka(TestUsingFileBasedSheerka):
(True, False, True),
(Concept("name", body="foo"), False, "foo"),
(Concept("name"), False, Concept("name")),
(ConceptWithGetValue("name").set_prop("my_prop", "my_value"), False, "my_value"),
(ConceptWithGetObjValue("name").set_value("my_prop", "my_value"), False, "my_value"),
(ReturnValueConcept(value="return_value"), False, "return_value"),
(ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
(ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
@@ -230,53 +260,138 @@ class TestSheerka(TestUsingFileBasedSheerka):
c.auto_init()
c = c.body
assert sheerka.value(concept, reduce_simple_list) == expected
def test_list_of_concept_is_sorted_by_id(self):
sheerka = self.get_sheerka(use_dict=False, skip_builtins_in_db=False)
concepts = sheerka.concepts()
assert concepts[0].id < concepts[-1].id
assert sheerka.objvalue(concept, reduce_simple_list) == expected
def test_builtin_error_concept_are_errors(self):
# only test a random one, it will be the same for the others
sheerka = self.get_sheerka()
assert not sheerka.is_success(sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS))
def test_cache_is_updated_after_get(self):
sheerka = self.get_sheerka(skip_builtins_in_db=False)
# updated when by_key returns one element
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="1"))
sheerka.reset_cache()
sheerka.get("foo")
assert "foo" in sheerka.cache_by_key
assert "1001" in sheerka.cache_by_id
class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
# updated when by_key returns two elements
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="2"))
sheerka.reset_cache()
sheerka.get("foo")
assert "foo" in sheerka.cache_by_key
assert "1001" in sheerka.cache_by_id
assert "1002" in sheerka.cache_by_id
def test_root_folder_is_created_after_initialization(self):
return_value = Sheerka().initialize(self.root_folder)
assert return_value.status, "initialisation should be successful"
assert os.path.exists(self.root_folder), "init folder should be created"
def test_builtin_concepts_are_initialized(self):
sheerka = self.get_sheerka()
for concept_name in BuiltinConcepts:
assert sheerka.has_key(str(concept_name))
assert sheerka.sdp.get(sheerka.CONCEPTS_BY_KEY_ENTRY, str(concept_name)) is not None
# I can get back data from the sdp when the cache is empty
sheerka.cache_manager.clear()
# caches are empty
assert not sheerka.has_id("1")
assert not sheerka.has_key(str(BuiltinConcepts.SHEERKA))
assert sheerka.get_by_id("1") == sheerka # use sdp
# assert sheerka.has_key(str(BuiltinConcepts.SHEERKA)) # auto update the other caches
def test_builtin_concepts_can_be_updated(self):
sheerka = self.get_sheerka()
before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
before_parsing.metadata.desc = "I have a description"
before_parsing.metadata.full_serialization = True
with sheerka.sdp.get_transaction("Test") as transac:
transac.add(sheerka.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True)
sheerka = self.get_sheerka() # another fresh new instance
before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
assert before_parsing.metadata.desc == "I have a description"
def test_i_first_look_in_local_cache(self):
sheerka, context, concept = self.init_concepts("foo", create_new=True)
sheerka.cache_manager.commit(context)
sheerka.get_by_key(concept.key).new_property = "I have modified the concept in cache"
from_cache = sheerka.get_by_key(concept.key)
assert from_cache is not None
assert from_cache.key == concept.key
assert from_cache.new_property == "I have modified the concept in cache"
# sdp instance is not modified
sheerka.cache_manager.clear()
from_sdp = sheerka.get_by_key(concept.key)
assert from_sdp is not None
assert from_sdp.key == concept.key
assert not hasattr(from_sdp, "new_property")
def test_i_can_retrieve_from_sdp_when_cache_is_reset(self):
sheerka, context, concept = self.init_concepts(Concept("foo", body="1"))
sheerka.cache_manager.commit(context)
sheerka.cache_manager.clear()
sheerka.get_by_key("foo")
assert sheerka.has_key("foo")
# It's also updated when sdp returns more than one element
concept2 = Concept("foo", body="2")
sheerka.create_new_concept(context, concept2)
sheerka.cache_manager.commit(context)
sheerka.cache_manager.clear()
assert len(sheerka.get_by_key("foo")) == 2
assert sheerka.has_key("foo")
# updated when by_id
sheerka.reset_cache()
sheerka.get_by_id("1001")
assert "1001" in sheerka.cache_by_id
assert "foo" not in sheerka.cache_by_key # cache_by_key not updated as "1001" is not the only one
sheerka.cache_manager.clear()
assert sheerka.get_by_id("1001") == concept
assert sheerka.has_id("1001")
def test_i_can_get_by_key_several_times(self):
sheerka = self.get_sheerka()
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="1"))
sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="2"))
sheerka.cache_manager.clear()
assert sheerka.get_by_name("foo") == [concept, concept2]
assert sheerka.has_name("foo")
sheerka.reset_cache()
sheerka.get("foo", "1001") # only one element requested. But the cache must be updated with two elements
sheerka.cache_manager.clear()
assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
assert sheerka.has_hash(concept.get_definition_hash())
# let's check it
concepts = sheerka.get("foo")
def test_get_by_key_retrieve_all_elements(self):
sheerka, context, *concepts = self.init_concepts(
Concept("foo", body="1"),
Concept("foo", body="2"),
create_new=True)
sheerka.cache_manager.commit(context)
sheerka.cache_manager.clear()
sheerka.get_by_key("foo", "1001") # I ask only for the one with id = "1001"
# but the two keys are returned
concepts = sheerka.get_by_key("foo")
assert len(concepts) == 2
assert concepts[0].id == "1001"
assert concepts[1].id == "1002"
def test_concept_node_parsing_is_initialized_at_startup(self):
sheerka, context, foo, bar, baz = self.init_concepts(
"foo",
"bar",
Concept("baz", definition="foo"),
create_new=True)
sheerka.cache_manager.commit(context)
assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'bar': ['1002'],
'c:|1001:': ['1003'],
'foo': ['1001']}
assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'bar': ['1002'],
'foo': ['1001', '1003']
}
sheerka = self.get_sheerka() # another instance
assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'bar': ['1002'],
'c:|1001:': ['1003'],
'foo': ['1001']}
assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
'bar': ['1002'],
'foo': ['1001', '1003']
}