Fixed #30 : Add variable support in BNF concept definition

Fixed #31 : Add regex support in BNF Concept Fixed #33 : Do not memorize object during restore
2021-02-24 17:23:03 +01:00
parent cac2dad17f
commit 646c428edb
32 changed files with 2107 additions and 360 deletions
@@ -8,7 +8,8 @@ from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF,
 from core.global_symbols import NotInit, NotFound
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \
    UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced, NoFirstTokenError
-from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore
+from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore, \
+    RegExDef, RegExMatch
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka

@@ -60,6 +61,50 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")

+    def test_i_can_create_a_bnf_concept_that_starts_with_a_regex(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+        foo = self.bnf_concept("foo", RegExMatch("[a-z]+"))
+        bar = self.bnf_concept("bar", RegExMatch("[0-9]+"))
+
+        res = sheerka.create_new_concept(context, foo)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        # I can get by the first regex
+        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[a-z]+").serialize()) == [foo.id]
+        assert len(service.compiled_concepts_by_regex) == 1
+
+        # I can commit
+        sheerka.om.commit(context)
+
+        # I can load from DB
+        entry = sheerka.om.current_sdp().get(service.CONCEPTS_BY_REGEX_ENTRY)
+        assert entry == {RegExDef("[a-z]+").serialize(): [foo.id]}
+
+        # I can create another concept
+        res = sheerka.create_new_concept(context, bar)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        # I can get by the first regex
+        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[0-9]+").serialize()) == [bar.id]
+        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[a-z]+").serialize()) == [foo.id]
+        assert len(service.compiled_concepts_by_regex) == 2
+
+        # I can commit
+        sheerka.om.commit(context)
+
+        # I can load from DB
+        entry = sheerka.om.current_sdp().get(service.CONCEPTS_BY_REGEX_ENTRY)
+        assert entry == {
+            RegExDef("[a-z]+").serialize(): [foo.id],
+            RegExDef("[0-9]+").serialize(): [bar.id]
+        }
+
    def test_i_cannot_create_a_bnf_concept_that_references_a_concept_that_cannot_be_resolved(self):
        sheerka, context, one_1, one_1_0 = self.init_concepts(Concept("one", body="1"), Concept("one", body="1.0"))
        twenty_one = Concept("twenty one", definition="'twenty' one", definition_type=DEFINITION_TYPE_BNF)
@@ -361,17 +406,16 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):

        # sdp is updated
        sheerka.om.commit(context)
-        from_sdp = sheerka.om.current_sdp().get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
+        sdp = sheerka.om.current_sdp()
+        from_sdp = sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
        assert from_sdp.get_metadata().body == "metadata value"
        assert from_sdp.get_metadata().variables == [("var_name", "default value")]
        assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}

-        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_NAME_ENTRY,
-                                            new_concept.name).get_metadata().body == "metadata value"
-        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY,
-                                            new_concept.key).get_metadata().body == "metadata value"
-        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_HASH_ENTRY,
-                                            new_concept.get_definition_hash()).get_metadata().body == "metadata value"
+        assert sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).get_metadata().body == "metadata value"
+        assert sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).get_metadata().body == "metadata value"
+        assert sdp.get(service.CONCEPTS_BY_HASH_ENTRY,
+                       new_concept.get_definition_hash()).get_metadata().body == "metadata value"

    def test_caches_are_update_when_i_modify_the_name(self):
        sheerka, context, foo = self.init_concepts("foo", cache_only=False)
@@ -496,6 +540,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
            Concept("baz", definition="foo"),
            create_new=True).unpack()

+        # sanity check
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            "foo": ["1001"],
            "bar": ["1002"],
@@ -514,6 +559,71 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002', '1001', '1003']}

+    def test_i_can_modify_bnf_definition_from_first_token_to_first_regex(self):
+        sheerka, context, foo, = self.init_test().with_concepts(
+            Concept("foo", definition="'hello'|'hola'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
+        to_add = {"meta": {"definition": "r'[a-z]+'"}}
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert res.status
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
+            RegExDef("[a-z]+").serialize(): ["1001"]
+        }
+        assert len(service.compiled_concepts_by_regex) == 1
+
+    def test_i_can_modify_bnf_definition_from_first_regex_to_first_token(self):
+        sheerka, context, foo, = self.init_test().with_concepts(
+            Concept("foo", definition="r'[a-z]+'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
+            RegExDef("[a-z]+").serialize(): ["1001"]
+        }
+        assert len(service.compiled_concepts_by_regex) == 1
+
+        to_add = {"meta": {"definition": "'hello'|'hola'"}}
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert res.status
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
+    def test_i_can_modify_when_multiple_bnf_definitions_are_already_defined(self):
+        sheerka, context, foo, bar, baz = self.init_test().with_concepts(
+            Concept("foo", definition="r'[a-z]+'"),
+            Concept("bar", definition="r'[0-1]+'"),
+            Concept("baz", definition="'one'|'twox'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+        # it does not matter than baz is a bnf
+
+        to_add = {"meta": {"definition": "'one'|'two'"}}
+        res = sheerka.modify_concept(context, baz, to_add)
+
+        assert res.status
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "one": ["1003"],
+            "two": ["1003"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
+            RegExDef("[a-z]+").serialize(): ["1001"],
+            RegExDef("[0-1]+").serialize(): ["1002"],
+        }
+        assert len(service.compiled_concepts_by_regex) == 2
+
    def test_references_are_updated_after_concept_modification(self):
        sheerka, context, one, twenty_one = self.init_test().with_concepts(
            "onz",
@@ -602,7 +712,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == NoModificationFound(foo, {"name": "foo", "body": "a body"})

-    def test_i_cannot_remove_meta_attributes(self):
+    def test_i_cannot_modify_and_remove_meta_attributes(self):
        sheerka, context, foo = self.init_concepts(Concept("foo"))

        res = sheerka.modify_concept(context, foo, to_remove={"meta": {"any_value": "foo"}})
@@ -611,7 +721,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == CannotRemoveMeta({"any_value": "foo"})

-    def test_i_cannot_remove_props_that_does_not_exists(self):
+    def test_i_cannot_modify_and_remove_props_that_does_not_exists(self):
        sheerka, context, foo = self.init_concepts(Concept("foo"))

        res = sheerka.modify_concept(context, foo, to_remove={"props": {"any_value": "foo"}})
@@ -620,7 +730,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == UnknownAttribute("any_value")

-    def test_i_cannot_remove_props_value_that_does_not_exists(self):
+    def test_i_cannot_modify_and_remove_props_value_that_does_not_exists(self):
        # Need to returns an error, otherwise, we will save a concept that is not modified
        sheerka, context, foo = self.init_concepts(Concept("foo", props={"a": {"value"}}))

@@ -630,7 +740,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == ValueNotFound("a", "dummy")

-    def test_i_cannot_remove_variable_that_does_not_exists(self):
+    def test_i_cannot_modify_and_remove_variable_that_does_not_exists(self):
        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a"))

        res = sheerka.modify_concept(context, foo, to_remove={"variables": ["b"]})
@@ -649,6 +759,30 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT)

+    def test_i_cannot_modify_with_an_invalid_regex_expression(self):
+        sheerka, context, foo, = self.init_test().with_concepts(
+            Concept("foo", definition="'hello'|'hola'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
+        to_add = {"meta": {"definition": "r'[a-z+'"}}  # invalid regex definition
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body.msg == 'unterminated character set'
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
    def test_i_can_get_and_set_attribute(self):
        sheerka, context = self.init_concepts()
        foo = Concept("foo")
@@ -683,6 +817,8 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
        assert sheerka.get_by_name(one.name) == one
        assert sheerka.get_by_key(one.key) == one
        assert sheerka.get_by_hash(one.get_definition_hash()) == one
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) != {}
+        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) != {}

        res = sheerka.remove_concept(context, one)

@@ -694,6 +830,35 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
        assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
        assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)

+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+
+    def test_i_can_remove_a_first_regex_concept(self):
+        sheerka, context, one = self.init_test().with_concepts(
+            Concept("one", definition="r'[a-z]+'"),
+            create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity check
+        assert sheerka.get_by_id(one.id) == one
+        assert sheerka.get_by_name(one.name) == one
+        assert sheerka.get_by_key(one.key) == one
+        assert sheerka.get_by_hash(one.get_definition_hash()) == one
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) != {}
+        assert len(service.compiled_concepts_by_regex) != 0
+
+        res = sheerka.remove_concept(context, one)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        assert sheerka.isinstance(sheerka.get_by_id(one.id), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_name(one.name), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
    def test_i_cannot_remove_a_concept_that_does_not_exist(self):
        sheerka, context = self.init_concepts()
        one = Concept("one", id="1001")