Fixed #30 : Add variable support in BNF concept definition

Fixed #31 : Add regex support in BNF Concept
Fixed #33 : Do not memorize object during restore

tests/core/test_SheerkaConceptManager.py

@@ -8,7 +8,8 @@ from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF,
 from core.global_symbols import NotInit, NotFound
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \
     UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced, NoFirstTokenError
-from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore
+from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore, \
+    RegExDef, RegExMatch
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -60,6 +61,50 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
         assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
 
+    def test_i_can_create_a_bnf_concept_that_starts_with_a_regex(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+        foo = self.bnf_concept("foo", RegExMatch("[a-z]+"))
+        bar = self.bnf_concept("bar", RegExMatch("[0-9]+"))
+
+        res = sheerka.create_new_concept(context, foo)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        # I can get by the first regex
+        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[a-z]+").serialize()) == [foo.id]
+        assert len(service.compiled_concepts_by_regex) == 1
+
+        # I can commit
+        sheerka.om.commit(context)
+
+        # I can load from DB
+        entry = sheerka.om.current_sdp().get(service.CONCEPTS_BY_REGEX_ENTRY)
+        assert entry == {RegExDef("[a-z]+").serialize(): [foo.id]}
+
+        # I can create another concept
+        res = sheerka.create_new_concept(context, bar)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        # I can get by the first regex
+        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[0-9]+").serialize()) == [bar.id]
+        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[a-z]+").serialize()) == [foo.id]
+        assert len(service.compiled_concepts_by_regex) == 2
+
+        # I can commit
+        sheerka.om.commit(context)
+
+        # I can load from DB
+        entry = sheerka.om.current_sdp().get(service.CONCEPTS_BY_REGEX_ENTRY)
+        assert entry == {
+            RegExDef("[a-z]+").serialize(): [foo.id],
+            RegExDef("[0-9]+").serialize(): [bar.id]
+        }
+
     def test_i_cannot_create_a_bnf_concept_that_references_a_concept_that_cannot_be_resolved(self):
         sheerka, context, one_1, one_1_0 = self.init_concepts(Concept("one", body="1"), Concept("one", body="1.0"))
         twenty_one = Concept("twenty one", definition="'twenty' one", definition_type=DEFINITION_TYPE_BNF)
@@ -361,17 +406,16 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
 
         # sdp is updated
         sheerka.om.commit(context)
-        from_sdp = sheerka.om.current_sdp().get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
+        sdp = sheerka.om.current_sdp()
+        from_sdp = sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
         assert from_sdp.get_metadata().body == "metadata value"
         assert from_sdp.get_metadata().variables == [("var_name", "default value")]
         assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}
 
-        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_NAME_ENTRY,
-                                            new_concept.name).get_metadata().body == "metadata value"
-        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY,
-                                            new_concept.key).get_metadata().body == "metadata value"
-        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_HASH_ENTRY,
-                                            new_concept.get_definition_hash()).get_metadata().body == "metadata value"
+        assert sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).get_metadata().body == "metadata value"
+        assert sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).get_metadata().body == "metadata value"
+        assert sdp.get(service.CONCEPTS_BY_HASH_ENTRY,
+                       new_concept.get_definition_hash()).get_metadata().body == "metadata value"
 
     def test_caches_are_update_when_i_modify_the_name(self):
         sheerka, context, foo = self.init_concepts("foo", cache_only=False)
@@ -496,6 +540,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
             Concept("baz", definition="foo"),
             create_new=True).unpack()
 
+        # sanity check
         assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             "foo": ["1001"],
             "bar": ["1002"],
@@ -514,6 +559,71 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             'bar': ['1002', '1001', '1003']}
 
+    def test_i_can_modify_bnf_definition_from_first_token_to_first_regex(self):
+        sheerka, context, foo, = self.init_test().with_concepts(
+            Concept("foo", definition="'hello'|'hola'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
+        to_add = {"meta": {"definition": "r'[a-z]+'"}}
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert res.status
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
+            RegExDef("[a-z]+").serialize(): ["1001"]
+        }
+        assert len(service.compiled_concepts_by_regex) == 1
+
+    def test_i_can_modify_bnf_definition_from_first_regex_to_first_token(self):
+        sheerka, context, foo, = self.init_test().with_concepts(
+            Concept("foo", definition="r'[a-z]+'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
+            RegExDef("[a-z]+").serialize(): ["1001"]
+        }
+        assert len(service.compiled_concepts_by_regex) == 1
+
+        to_add = {"meta": {"definition": "'hello'|'hola'"}}
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert res.status
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
+    def test_i_can_modify_when_multiple_bnf_definitions_are_already_defined(self):
+        sheerka, context, foo, bar, baz = self.init_test().with_concepts(
+            Concept("foo", definition="r'[a-z]+'"),
+            Concept("bar", definition="r'[0-1]+'"),
+            Concept("baz", definition="'one'|'twox'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+        # it does not matter than baz is a bnf
+
+        to_add = {"meta": {"definition": "'one'|'two'"}}
+        res = sheerka.modify_concept(context, baz, to_add)
+
+        assert res.status
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "one": ["1003"],
+            "two": ["1003"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
+            RegExDef("[a-z]+").serialize(): ["1001"],
+            RegExDef("[0-1]+").serialize(): ["1002"],
+        }
+        assert len(service.compiled_concepts_by_regex) == 2
+
     def test_references_are_updated_after_concept_modification(self):
         sheerka, context, one, twenty_one = self.init_test().with_concepts(
             "onz",
@@ -602,7 +712,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == NoModificationFound(foo, {"name": "foo", "body": "a body"})
 
-    def test_i_cannot_remove_meta_attributes(self):
+    def test_i_cannot_modify_and_remove_meta_attributes(self):
         sheerka, context, foo = self.init_concepts(Concept("foo"))
 
         res = sheerka.modify_concept(context, foo, to_remove={"meta": {"any_value": "foo"}})
@@ -611,7 +721,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == CannotRemoveMeta({"any_value": "foo"})
 
-    def test_i_cannot_remove_props_that_does_not_exists(self):
+    def test_i_cannot_modify_and_remove_props_that_does_not_exists(self):
         sheerka, context, foo = self.init_concepts(Concept("foo"))
 
         res = sheerka.modify_concept(context, foo, to_remove={"props": {"any_value": "foo"}})
@@ -620,7 +730,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == UnknownAttribute("any_value")
 
-    def test_i_cannot_remove_props_value_that_does_not_exists(self):
+    def test_i_cannot_modify_and_remove_props_value_that_does_not_exists(self):
         # Need to returns an error, otherwise, we will save a concept that is not modified
         sheerka, context, foo = self.init_concepts(Concept("foo", props={"a": {"value"}}))
 
@@ -630,7 +740,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == ValueNotFound("a", "dummy")
 
-    def test_i_cannot_remove_variable_that_does_not_exists(self):
+    def test_i_cannot_modify_and_remove_variable_that_does_not_exists(self):
         sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a"))
 
         res = sheerka.modify_concept(context, foo, to_remove={"variables": ["b"]})
@@ -649,6 +759,30 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT)
 
+    def test_i_cannot_modify_with_an_invalid_regex_expression(self):
+        sheerka, context, foo, = self.init_test().with_concepts(
+            Concept("foo", definition="'hello'|'hola'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
+        to_add = {"meta": {"definition": "r'[a-z+'"}}  # invalid regex definition
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body.msg == 'unterminated character set'
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
     def test_i_can_get_and_set_attribute(self):
         sheerka, context = self.init_concepts()
         foo = Concept("foo")
@@ -683,6 +817,8 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.get_by_name(one.name) == one
         assert sheerka.get_by_key(one.key) == one
         assert sheerka.get_by_hash(one.get_definition_hash()) == one
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) != {}
+        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) != {}
 
         res = sheerka.remove_concept(context, one)
 
@@ -694,6 +830,35 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
         assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)
 
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+
+    def test_i_can_remove_a_first_regex_concept(self):
+        sheerka, context, one = self.init_test().with_concepts(
+            Concept("one", definition="r'[a-z]+'"),
+            create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity check
+        assert sheerka.get_by_id(one.id) == one
+        assert sheerka.get_by_name(one.name) == one
+        assert sheerka.get_by_key(one.key) == one
+        assert sheerka.get_by_hash(one.get_definition_hash()) == one
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) != {}
+        assert len(service.compiled_concepts_by_regex) != 0
+
+        res = sheerka.remove_concept(context, one)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        assert sheerka.isinstance(sheerka.get_by_id(one.id), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_name(one.name), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
     def test_i_cannot_remove_a_concept_that_does_not_exist(self):
         sheerka, context = self.init_concepts()
         one = Concept("one", id="1001")

tests/core/test_sheerka.py

@@ -73,7 +73,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
 
         assert loaded is not None
         assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert loaded.body == ("key", "key_that_does_not_exist")
+        assert loaded.body == {"key": "key_that_does_not_exist"}
         assert loaded.get_metadata().is_evaluated
 
     def test_i_cannot_get_when_id_is_not_found(self):
@@ -83,7 +83,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
 
         assert loaded is not None
         assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert loaded.body == ("id", "id_that_does_not_exist")
+        assert loaded.body == {"id": "id_that_does_not_exist"}
         assert loaded.get_metadata().is_evaluated
 
     def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self):
@@ -200,7 +200,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         new = sheerka.new("fake_concept")
 
         assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert new.body == ('key', 'fake_concept')
+        assert new.body == {'key': 'fake_concept'}
 
     def test_i_cannot_instantiate_with_invalid_id(self):
         sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
@@ -210,7 +210,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         new = sheerka.new(("foo", "invalid_id"))
 
         assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert new.body == [('key', 'foo'), ('id', 'invalid_id')]
+        assert new.body == {'key': 'foo', 'id': 'invalid_id'}
 
     def test_i_cannot_instantiate_with_invalid_key(self):
         sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
@@ -220,7 +220,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         new = sheerka.new(("invalid_key", "1001"))
 
         assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert new.body == [('key', 'invalid_key'), ('id', '1001')]
+        assert new.body == {'key': 'invalid_key', 'id': '1001'}
 
     def test_concept_id_is_irrelevant_when_only_one_concept(self):
         sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),

tests/core/test_tokenizer.py

@@ -1,4 +1,5 @@
 import pytest
+
 from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
 
 
@@ -172,6 +173,7 @@ def test_i_can_parse_concept_token(text, expected):
     assert tokens[0].type == TokenKind.CONCEPT
     assert tokens[0].value == expected
 
+
 @pytest.mark.parametrize("text, expected", [
     ("r:key:", ("key", None)),
     ("r:key|id:", ("key", "id")),
@@ -197,3 +199,6 @@ def test_i_can_parse_regex_token(text, expected):
 
     assert tokens[0].type == TokenKind.REGEX
     assert tokens[0].value == expected
+    assert tokens[0].str_value == "r" + expected
+    assert tokens[0].repr_value == "r" + expected
+    assert tokens[0].strip_quote == expected[1:-1]