Implemented SheerkaOntology

2021-01-11 15:36:03 +01:00
parent e3c2adb533
commit e26c83a825
119 changed files with 6876 additions and 2002 deletions
@@ -1,10 +1,14 @@
 import os

 import pytest
+
+from conftest import SHEERKA_TEST_FOLDER
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
 from core.builtin_concepts_ids import AllBuiltinConcepts
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, NotInit
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, get_concept_attrs
+from core.global_symbols import NotInit
 from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS
+from core.sheerka.SheerkaOntologyManager import OntologyAlreadyExists
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.tokenizer import Token, TokenKind

@@ -107,7 +111,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert ret.body == "fake_concept"

    def test_i_can_instantiate_a_concept(self):
-        sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
+        sheerka, context, concept = self.init_test().with_concepts(self.get_default_concept(), create_new=True).unpack()

        new = sheerka.new(concept.key, a=10, b="value")

@@ -119,10 +123,10 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert new.get_value("b") == "value"

    def test_i_can_instantiate_multiple_when_same_key(self):
-        sheerka, context, *concepts = self.init_concepts(
+        sheerka, context, *concepts = self.init_test().with_concepts(
            Concept("foo", body="foo1"),
            Concept("foo", body="foo2"),
-            create_new=True)
+            create_new=True).unpack()

        # when no id, i get two instances
        concepts = sheerka.new("foo")
@@ -141,7 +145,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert foo2.get_metadata().body == "foo2"

    def test_instances_are_different_when_asking_for_new(self):
-        sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
+        sheerka, context, concept = self.init_test().with_concepts(self.get_default_concept(), create_new=True).unpack()

        new1 = sheerka.new(concept.key, a=10, b="value")
        new2 = sheerka.new(concept.key, a=10, b="value")
@@ -150,7 +154,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert id(new1) != id(new2)

    def test_new_instance_does_not_impact_each_others(self):
-        sheerka, context, foo, bar = self.init_concepts("foo", "bar", create_new=True)
+        sheerka, context, foo, bar = self.init_test().with_concepts("foo", "bar", create_new=True).unpack()

        new_foo = sheerka.new("foo")
        new_foo.get_metadata().body = "metadata value"  # modify metadata
@@ -167,7 +171,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert new_foo.get_compiled() != foo.get_compiled()

    def test_i_get_the_same_instance_when_is_unique_is_true(self):
-        sheerka, context, concept = self.init_concepts(Concept(name="unique", is_unique=True), create_new=True)
+        sheerka, context, concept = self.init_test(). \
+            with_concepts(Concept(name="unique", is_unique=True), create_new=True).unpack()

        new1 = sheerka.new(concept.key)
        new2 = sheerka.new(concept.key, a=10, b="value")  # not that variables are simply discareded
@@ -176,10 +181,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert id(new1) == id(new2)

    def test_values_are_reset_when_asking_for_a_new_instance(self):
-        sheerka, context, template = self.init_concepts(
-            Concept("foo", body="'foo body'"),
-            create_new=True,
-            eval_body=True)
+        sheerka, context, template = self.init_test(eval_body=True).with_concepts(Concept("foo", body="'foo body'"),
+                                                                                  create_new=True).unpack()

        sheerka.evaluate_concept(context, sheerka.get_by_id(template.id))
        assert template.get_metadata().is_evaluated
@@ -202,10 +205,9 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert new.body == ('key', 'fake_concept')

    def test_i_cannot_instantiate_with_invalid_id(self):
-        sheerka, context, *concepts = self.init_concepts(
-            Concept("foo", body="foo1"),
-            Concept("foo", body="foo2"),
-            create_new=True)
+        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
+                                                                     Concept("foo", body="foo2"),
+                                                                     create_new=True).unpack()

        new = sheerka.new(("foo", "invalid_id"))

@@ -213,10 +215,9 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert new.body == [('key', 'foo'), ('id', 'invalid_id')]

    def test_i_cannot_instantiate_with_invalid_key(self):
-        sheerka, context, *concepts = self.init_concepts(
-            Concept("foo", body="foo1"),
-            Concept("foo", body="foo2"),
-            create_new=True)
+        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
+                                                                     Concept("foo", body="foo2"),
+                                                                     create_new=True).unpack()

        new = sheerka.new(("invalid_key", "1001"))

@@ -224,9 +225,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert new.body == [('key', 'invalid_key'), ('id', '1001')]

    def test_concept_id_is_irrelevant_when_only_one_concept(self):
-        sheerka, context, *concepts = self.init_concepts(
-            Concept("foo", body="foo1"),
-            create_new=True)
+        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
+                                                                     create_new=True).unpack()

        new = sheerka.new(("foo", "invalid_id"))

@@ -234,7 +234,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert new.get_metadata().body == "foo1"

    def test_i_cannot_instantiate_when_properties_are_not_recognized(self):
-        sheerka, context, concept = self.init_concepts(self.get_default_concept(), create_new=True)
+        sheerka, context, concept = self.init_test().with_concepts(self.get_default_concept(),
+                                                                   create_new=True).unpack()

        new = sheerka.new(concept.key, a=10, c="value")

@@ -326,46 +327,138 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
        assert sheerka.resolve(concept) == real_expected

    def test_i_can_resolve_when_searching_by_definition(self):
-        sheerka, context, plus = self.init_concepts(
-            self.from_def_concept("plus", "a plus b", ["a", "b"]),
-            create_new=True
-        )
+        sheerka, context, plus = self.init_test().with_concepts(self.from_def_concept("plus", "a plus b", ["a", "b"]),
+                                                                create_new=True).unpack()

        assert sheerka.resolve("a plus b") == plus

+    def test_i_can_reset_global_concept_definition_when_adding_and_removing_layers(self):
+        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b"))
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+        sheerka.push_ontology(context, "another ontology")
+        sheerka.modify_concept(context, foo, to_add={"variables": {"c": None}}, to_remove={"variables": ["b"]})
+        assert get_concept_attrs(foo) == ["a", "c"]
+
+        sheerka.pop_ontology()
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+    def test_i_can_manage_concepts_ids_on_multiple_ontology_layers(self):
+        sheerka, context = self.init_concepts()
+
+        res = sheerka.create_new_concept(context, Concept("foo"))
+        assert res.body.body.id == "1001"
+
+        sheerka.push_ontology(context, "new ontology")
+        res = sheerka.create_new_concept(context, Concept("bar"))
+        assert res.body.body.id == "1002"
+
+        res = sheerka.create_new_concept(context, Concept("baz"))
+        assert res.body.body.id == "1003"
+
+        sheerka.pop_ontology()
+        res = sheerka.create_new_concept(context, Concept("baz"))
+        assert res.body.body.id == "1002"
+
+    def test_i_can_add_ontology(self):
+        sheerka, context = self.init_test().unpack()
+
+        # Create an ontology an set some values
+        res = sheerka.push_ontology(context, "new ontology")
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        foo = sheerka.create_new_concept(context, Concept("foo").def_var("a").def_var("b")).body.body
+        sheerka.locals = {"key1": "value1"}
+
+        # sanity check
+        assert sheerka.get_by_name("foo") == foo
+        assert not sheerka.is_known(sheerka.get_by_name("bar"))
+        assert sheerka.locals == {"key1": "value1"}
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+        # record the ontology
+        ontology = sheerka.get_ontology(context)
+        sheerka.pop_ontology()
+
+        # Create another ontology with some other values
+        sheerka.push_ontology(context, "another ontology")
+        foo2 = sheerka.create_new_concept(context, Concept("foo").def_var("a").def_var("b").def_var("c")).body.body
+        bar = sheerka.create_new_concept(context, Concept("bar")).body.body
+        sheerka.locals = {"key2": "value2"}
+
+        # sanity check
+        assert sheerka.get_by_name("foo") == foo2
+        assert sheerka.get_by_name("bar") == bar
+        assert sheerka.locals == {"key2": "value2"}
+        assert get_concept_attrs(foo) == ["a", "b", "c"]
+
+        # put pack the previous ontology
+        sheerka.add_ontology(context, ontology)
+
+        assert sheerka.get_by_name("foo") == foo  # not [foo, foo2], foo2 is not seen !!!
+        assert sheerka.get_by_name("bar") == bar
+        assert sheerka.locals == {"key1": "value1"}
+        assert get_concept_attrs(foo) == ["a", "b"]
+
+        # sanity check
+        sheerka.pop_ontology()
+        assert sheerka.get_by_name("foo") == foo2
+        assert sheerka.get_by_name("bar") == bar
+        assert sheerka.locals == {"key2": "value2"}
+        assert get_concept_attrs(foo) == ["a", "b", "c"]
+
+    def test_adding_the_same_ontology_twice_has_no_effect(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka.push_ontology(context, "#unit_test#")
+
+        assert len(sheerka.om.ontologies) == 2
+
+    def test_i_cannot_add_an_ontology_that_already_exists(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka.push_ontology(context, "new ontology")
+
+        res = sheerka.push_ontology(context, "#unit_test#")
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED)
+

 class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):

    def test_root_folder_is_created_after_initialization(self):
-        return_value = Sheerka().initialize(self.root_folder)
+        return_value = Sheerka().initialize(SHEERKA_TEST_FOLDER)
        assert return_value.status, "initialisation should be successful"
-        assert os.path.exists(self.root_folder), "init folder should be created"
+        assert os.path.exists(SHEERKA_TEST_FOLDER), "init folder should be created"

    def test_builtin_concepts_are_initialized(self):
-        sheerka = self.get_sheerka()
+        sheerka, context = self.init_test().unpack()
        service = sheerka.services[SheerkaConceptManager.NAME]
        for concept_name in AllBuiltinConcepts:
-            assert service.has_key(str(concept_name))
-            assert sheerka.sdp.get(service.CONCEPTS_BY_KEY_ENTRY, str(concept_name)) is not None
+            # check that the concept is already in cache
+            assert sheerka.om.ontologies[-1].cache_manager.has(service.CONCEPTS_BY_KEY_ENTRY, concept_name)
+
+            # check that we can access it
+            assert sheerka.om.get(service.CONCEPTS_BY_KEY_ENTRY, concept_name) is not None

        # I can get back data from the sdp when the cache is empty
-        sheerka.cache_manager.clear()
+        sheerka.push_ontology(context, 'new ontology')

        # caches are empty
        assert not service.has_id("1")
-        assert not service.has_key(str(BuiltinConcepts.SHEERKA))
+        assert not service.has_key(BuiltinConcepts.SHEERKA)

        assert sheerka.get_by_id("1") == sheerka  # use sdp

-    #   assert sheerka.has_key(str(BuiltinConcepts.SHEERKA))  # auto update the other caches
-
    def test_builtin_concepts_can_be_updated(self):
        sheerka = self.get_sheerka()
        service = sheerka.services[SheerkaConceptManager.NAME]
        before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
        before_parsing.get_metadata().desc = "I have a description"
        before_parsing.get_metadata().full_serialization = True
-        with sheerka.sdp.get_transaction("Test") as transac:
+        with sheerka.om.current_sdp().get_transaction("Test") as transac:
            transac.add(service.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True)

        sheerka = self.get_sheerka()  # another fresh new instance
@@ -374,8 +467,8 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
        assert before_parsing.get_metadata().desc == "I have a description"

    def test_i_first_look_in_local_cache(self):
-        sheerka, context, concept = self.init_concepts("foo", create_new=True)
-        sheerka.cache_manager.commit(context)
+        sheerka, context, concept = self.init_test().with_concepts("foo", create_new=True).unpack()
+        sheerka.om.commit(context)

        sheerka.get_by_key(concept.key).new_property = "I have modified the concept in cache"

@@ -385,7 +478,7 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
        assert from_cache.new_property == "I have modified the concept in cache"

        # sdp instance is not modified
-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
        from_sdp = sheerka.get_by_key(concept.key)
        assert from_sdp is not None
        assert from_sdp.key == concept.key
@@ -394,42 +487,42 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
    def test_i_can_retrieve_from_sdp_when_cache_is_reset(self):
        sheerka, context, concept = self.init_concepts(Concept("foo", body="1"))
        service = sheerka.services[SheerkaConceptManager.NAME]
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)

-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
        sheerka.get_by_key("foo")
        assert service.has_key("foo")

        # It's also updated when sdp returns more than one element
        concept2 = Concept("foo", body="2")
        sheerka.create_new_concept(context, concept2)
-        sheerka.cache_manager.commit(context)
+        sheerka.om.commit(context)

-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
        assert len(sheerka.get_by_key("foo")) == 2
        assert service.has_key("foo")

        # updated when by_id
-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
        assert sheerka.get_by_id("1001") == concept
        assert service.has_id("1001")

-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
        assert sheerka.get_by_name("foo") == [concept, concept2]
        assert service.has_name("foo")

-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
        assert service.has_hash(concept.get_definition_hash())

    def test_get_by_key_retrieve_all_elements(self):
-        sheerka, context, *concepts = self.init_concepts(
+        sheerka, context, *concepts = self.init_test().with_concepts(
            Concept("foo", body="1"),
            Concept("foo", body="2"),
-            create_new=True)
-        sheerka.cache_manager.commit(context)
+            create_new=True).unpack()
+        sheerka.om.commit(context)

-        sheerka.cache_manager.clear()
+        sheerka.om.clear()
        sheerka.get_by_key("foo", "1001")  # I ask only for the one with id = "1001"

        # but the two keys are returned
@@ -439,28 +532,55 @@ class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):
        assert concepts[1].id == "1002"

    def test_concept_node_parsing_is_initialized_at_startup(self):
-        sheerka, context, foo, bar, baz = self.init_concepts(
+        sheerka, context, foo, bar, baz = self.init_test().with_concepts(
            "foo",
            "bar",
            Concept("baz", definition="foo"),
-            create_new=True)
-        sheerka.cache_manager.commit(context)
+            create_new=True).unpack()
+        sheerka.om.commit(context)

-        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002'],
            'c:|1001:': ['1003'],
            'foo': ['1001']}
-        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002'],
            'foo': ['1001', '1003']
        }

        sheerka = self.get_sheerka()  # another instance
-        assert sheerka.cache_manager.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002'],
            'c:|1001:': ['1003'],
            'foo': ['1001']}
-        assert sheerka.cache_manager.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+        assert sheerka.om.copy(sheerka.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002'],
            'foo': ['1001', '1003']
        }
+
+    def test_i_can_remember_ontologies_when_sheerka_is_recreated(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka.push_ontology(context, "to remove")
+        sheerka.pop_ontology()
+        sheerka.push_ontology(context, "new ontology")
+        sheerka.push_ontology(context, "another ontology")
+
+        # sanity check
+        ontologies_names = [o.name for o in sheerka.om.ontologies]
+        assert ontologies_names == ['another ontology', 'new ontology', '#unit_test#', '__default__']
+
+        # get new instance
+        sheerka = self.new_sheerka_instance(False)
+
+        # make sure that ontologies are recreated
+        ontologies_names = [o.name for o in sheerka.om.ontologies]
+        assert ontologies_names == ['another ontology', 'new ontology', '#unit_test#', '__default__']
+
+    def test_adding_the_same_ontology_twice_has_no_effect(self):
+        sheerka, context = self.init_test().unpack()
+
+        sheerka = self.new_sheerka_instance(False)  # new instance that remembers the top layer ontology
+        sheerka.push_ontology(context, "#unit_test#")
+
+        assert len(sheerka.om.ontologies) == 2