Sheerka-Old/tests/core/test_SheerkaConceptManager.py

import pytest

from cache.CacheManager import ConceptNotFound
from core.builtin_concepts import BuiltinConcepts
from core.builtin_helpers import ensure_bnf
from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF, get_concept_attrs, \
    DEFINITION_TYPE_BNF
from core.global_symbols import NotInit, NotFound, SyaAssociativity, CONCEPT_COMPARISON_CONTEXT
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \
    UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced, NoFirstTokenError
from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore, \
    RegExDef, RegExMatch
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka

compute_concepts_by_first_token = SheerkaConceptManager.compute_concepts_by_first_token
resolve_concepts_by_first_keyword = SheerkaConceptManager.resolve_concepts_by_first_keyword


class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
    def test_i_can_create_a_concept(self):
        sheerka = self.get_sheerka(cache_only=False)
        context = self.get_context(sheerka)
        concept = self.get_default_concept()
        service = sheerka.services[SheerkaConceptManager.NAME]

        res = sheerka.create_new_concept(context, concept)
        sheerka.om.commit(context)

        assert res.status
        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)

        concept_found = res.value.body
        for prop in PROPERTIES_TO_SERIALIZE:
            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)

        assert concept_found.key == "__var__0 + __var__1"
        assert concept_found.id == "1001"
        assert get_concept_attrs(concept) == ['a', 'b']

        # saved in cache
        assert service.has_id(concept.id)
        assert service.has_key(concept.key)
        assert service.has_name(concept.name)
        assert service.has_hash(concept.get_definition_hash())

        # I can get the concept using various index
        assert sheerka.get_by_id(concept.id) == concept
        assert sheerka.get_by_key(concept.key) == concept
        assert sheerka.get_by_name(concept.name) == concept
        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept

        # I can get by the first entry
        assert sheerka.om.get(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]
        assert sheerka.om.get(service.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+") == [concept.id]

        # saved in sdp
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")

    def test_i_can_create_a_bnf_concept_that_starts_with_a_regex(self):
        sheerka = self.get_sheerka(cache_only=False)
        context = self.get_context(sheerka)
        service = sheerka.services[SheerkaConceptManager.NAME]
        foo = self.bnf_concept("foo", RegExMatch("[a-z]+"))
        bar = self.bnf_concept("bar", RegExMatch("[0-9]+"))

        res = sheerka.create_new_concept(context, foo)

        assert res.status
        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)

        # I can get by the first regex
        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[a-z]+").serialize()) == [foo.id]
        assert len(service.compiled_concepts_by_regex) == 1

        # I can commit
        sheerka.om.commit(context)

        # I can load from DB
        entry = sheerka.om.current_sdp().get(service.CONCEPTS_BY_REGEX_ENTRY)
        assert entry == {RegExDef("[a-z]+").serialize(): [foo.id]}

        # I can create another concept
        res = sheerka.create_new_concept(context, bar)

        assert res.status
        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)

        # I can get by the first regex
        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[0-9]+").serialize()) == [bar.id]
        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[a-z]+").serialize()) == [foo.id]
        assert len(service.compiled_concepts_by_regex) == 2

        # I can commit
        sheerka.om.commit(context)

        # I can load from DB
        entry = sheerka.om.current_sdp().get(service.CONCEPTS_BY_REGEX_ENTRY)
        assert entry == {
            RegExDef("[a-z]+").serialize(): [foo.id],
            RegExDef("[0-9]+").serialize(): [bar.id]
        }

    def test_i_cannot_create_a_bnf_concept_that_references_a_concept_that_cannot_be_resolved(self):
        sheerka, context, one_1, one_1_0 = self.init_concepts(Concept("one", body="1"), Concept("one", body="1.0"))
        twenty_one = Concept("twenty one", definition="'twenty' one", definition_type=DEFINITION_TYPE_BNF)

        res = sheerka.create_new_concept(context, twenty_one)

        assert not res.status
        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
        assert res.value.body == ("key", "one")

    def test_i_can_add_a_concept_when_name_differs_from_the_key(self):
        sheerka = self.get_sheerka(cache_only=False)
        context = self.get_context(sheerka)
        concept = Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a")
        service = sheerka.services[SheerkaConceptManager.NAME]

        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
        sheerka.om.commit(context)

        assert res.status
        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)

        concept_found = res.value.body
        for prop in PROPERTIES_TO_SERIALIZE:
            assert getattr(concept_found.get_metadata(), prop) == getattr(concept.get_metadata(), prop)

        assert concept_found.key == "hello __var__0"
        assert concept_found.id == "1001"

        # saved in cache
        assert service.has_id(concept.id)
        assert service.has_key(concept.key)
        assert service.has_name(concept.name)
        assert service.has_hash(concept.get_definition_hash())

        # I can get the concept using various index
        assert sheerka.get_by_id(concept.id) == concept
        assert sheerka.get_by_key(concept.key) == concept
        assert sheerka.get_by_name(concept.name) == concept
        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept

        # saved in sdp
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, concept.id)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, concept.key)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, concept.name)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "hello")

    def test_i_cannot_add_the_same_concept_twice(self):
        """
        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
        :return:
        """
        sheerka = self.get_sheerka()
        concept = self.get_default_concept()

        sheerka.create_new_concept(self.get_context(sheerka), concept)
        res = sheerka.create_new_concept(self.get_context(sheerka), concept)

        assert not res.status
        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
        assert res.value.body == concept

    def test_i_can_get_a_newly_created_concept(self):
        sheerka = self.get_sheerka()
        concept = self.get_default_concept()

        sheerka.create_new_concept(self.get_context(sheerka), concept)

        from_cache = sheerka.get_by_key(concept.key)
        assert from_cache is not None
        assert from_cache == concept

        from_cache = sheerka.get_by_id(concept.id)
        assert from_cache is not None
        assert from_cache == concept

    def test_i_can_get_list_of_concept_when_same_key_using_cache(self):
        sheerka = self.get_sheerka()
        concept1 = self.get_default_concept()
        concept2 = self.get_default_concept()
        concept2.get_metadata().body = "a+b"

        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)

        assert res1.value.body.key == res2.value.body.key  # same key

        result = sheerka.get_by_key(concept1.key)
        assert len(result) == 2
        assert result[0] == concept1
        assert result[1] == concept2

    def test_concept_that_references_itself_is_correctly_created(self):
        sheerka = self.get_sheerka()
        concept = Concept("foo", body="foo")

        res = sheerka.create_new_concept(self.get_context(sheerka), concept)

        assert res.status

    def test_i_can_get_by_name_when_created_with_def_definition(self):
        sheerka = self.get_sheerka(cache_only=False)
        context = self.get_context(sheerka)
        concept = self.from_def_concept("plus", "a plus b", ["a", "b"])

        res = sheerka.create_new_concept(context, concept)

        assert res.status
        assert sheerka.get_by_name(concept.name) == concept
        assert sheerka.get_by_name(concept.get_metadata().definition) == concept

        concept = Concept(name="foo", definition="foo", definition_type=DEFINITION_TYPE_DEF)
        res = sheerka.create_new_concept(context, concept)

        assert res.status
        assert sheerka.get_by_name(concept.name) == concept  # it's not a list, ie the entry is not duplicated

    def test_i_can_get_first_token_when_not_a_letter(self):
        sheerka = self.get_sheerka(cache_only=False)
        context = self.get_context(sheerka)
        concept = Concept("--filter a").def_var("a")

        res = sheerka.create_new_concept(context, concept)
        assert res.status

        # I can get by the first entry
        assert sheerka.om.get(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]
        assert sheerka.om.get(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [concept.id]

    @pytest.mark.parametrize("expression", [
        "--'filter' ('one' | 'two') ",
        "'--filter' ('one' | 'two') ",
    ])
    def test_i_can_get_first_token_when_bnf_concept_and_not_a_letter(self, expression):
        sheerka, context, bnf_concept = self.init_test().with_concepts(
            Concept("foo", definition=expression),
            create_new=True).unpack()

        # I can get by the first entry
        assert sheerka.om.get(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]
        assert sheerka.om.get(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "-") == [bnf_concept.id]

    def test_concept_references_are_updated_1(self):
        sheerka, context, one, two, number, twenty, twenties = self.init_test().with_concepts(
            "one",
            "two",
            "number",
            "twenty",
            Concept("twenties", definition="twenty one | two 'hundred'"),
            create_new=True
        ).unpack()
        service = sheerka.services[SheerkaConceptManager.NAME]

        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) == {twenties.id}
        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) == {twenties.id}
        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) is NotFound
        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is NotFound

    def test_concept_references_are_updated_2(self):
        sheerka, context, one, two, number, twenty, twenties = self.init_test().with_concepts(
            "one",
            "two",
            "number",
            "twenty",
            Concept("twenties", definition="twenty number"),
            create_new=True
        ).unpack()
        service = sheerka.services[SheerkaConceptManager.NAME]

        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, one.id) is NotFound
        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, two.id) is NotFound
        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, number.id) == {twenties.id}
        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, twenty.id) == {twenties.id}
        assert sheerka.om.get(service.CONCEPTS_REFERENCES_ENTRY, twenties.id) is NotFound

    @pytest.mark.parametrize("attr", [
        "name",
        "is_unique",
        "body",
        "where",
        "pre",
        "post",
        "ret",
        "definition",
        "definition_type",
        "desc",
        "full_serialization",
    ])
    def test_i_can_modify_a_metadata_attribute(self, attr):
        sheerka, context, foo = self.init_concepts("foo")

        res = sheerka.modify_concept(context, foo, to_add={"meta": {attr: "new value"}})

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert getattr(res.body.body.get_metadata(), attr) == "new value"

    def test_i_can_modify_a_concept_when_at_least_one_attr_is_different(self):
        sheerka, context, foo = self.init_concepts(Concept("foo", body="a body"))

        res = sheerka.modify_concept(context, foo, to_add={"meta": {"name": "foo", "body": "a body", "pre": "new pre"}})

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert getattr(res.body.body.get_metadata(), "name") == "foo"
        assert getattr(res.body.body.get_metadata(), "body") == "a body"
        assert getattr(res.body.body.get_metadata(), "pre") == "new pre"

    def test_i_can_modify_add_a_property(self):
        sheerka, context, one, foo = self.init_concepts("one", Concept("foo", props={BuiltinConcepts.ISA: {"value"}}))

        res = sheerka.modify_concept(context, foo, to_add={"props": {BuiltinConcepts.ISA: {"value2"},
                                                                     BuiltinConcepts.HASA: one}})

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert res.body.body.get_prop(BuiltinConcepts.ISA) == {"value2"}
        assert res.body.body.get_prop(BuiltinConcepts.HASA) == sheerka.new("one")

    def test_i_can_modify_remove_a_property(self):
        sheerka, context, foo = self.init_concepts(
            Concept("foo", props={"a": {"value1", "value2", "value3"},
                                  "b": {"value4"}}))

        res = sheerka.modify_concept(context, foo, to_remove={"props": {"a": {"value2", "value3"},
                                                                        "b": "value4"}})

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert res.body.body.get_prop("a") == {"value1"}
        assert res.body.body.get_prop("b") is None

    def test_i_can_modify_add_variables(self):
        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a", "value"))

        res = sheerka.modify_concept(context, foo, to_add={"variables": {"b": "some_value",
                                                                         "a": "new_value",
                                                                         "c": None}})

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert res.body.body.get_metadata().variables == [("a", "new_value"), ("b", "some_value"), ("c", None)]
        assert res.body.body.values() == {"a": NotInit, "b": NotInit, "c": NotInit}

    def test_i_can_modify_remove_variables(self):
        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b", "value").def_var("c"))

        res = sheerka.modify_concept(context, foo, to_remove={"variables": ["a", "c"]})

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert res.body.body.get_metadata().variables == [("b", "value")]
        assert res.body.body.values() == {"b": NotInit}

    def test_i_can_modify_the_concept_source(self):
        sheerka, context, foo, bar = self.init_concepts("foo", "bar")

        res = sheerka.modify_concept(context, foo, to_add={"meta": {"body": "new value"}})
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert getattr(foo.get_metadata(), "body") is None

        res = sheerka.modify_concept(context, bar, to_add={"meta": {"body": "new value"}}, modify_source=True)
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert getattr(bar.get_metadata(), "body") == "new value"

    def test_caches_are_updated_when_i_modify_the_properties_and_the_variables(self):
        sheerka, context, foo, bar = self.init_concepts("foo", "bar", cache_only=False)
        service = sheerka.services[SheerkaConceptManager.NAME]

        to_add = {"meta": {"body": "metadata value"},
                  "variables": {"var_name": "default value"},
                  "props": {BuiltinConcepts.ISA: {bar}}}

        res = sheerka.modify_concept(context, foo, to_add)
        new_concept = res.body.body

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert new_concept.get_metadata().body == "metadata value"
        assert new_concept.get_metadata().variables == [("var_name", "default value")]
        assert new_concept.get_prop(BuiltinConcepts.ISA) == {bar}

        # test that object
        foo_from_sheerka = sheerka.get_by_key(new_concept.key)
        assert foo_from_sheerka.get_metadata().body == "metadata value"
        assert foo_from_sheerka.get_metadata().variables == [("var_name", "default value")]
        assert foo_from_sheerka.get_prop(BuiltinConcepts.ISA) == {bar}

        # other caches are also updated
        assert sheerka.get_by_id(new_concept.id).get_metadata().body == "metadata value"
        assert sheerka.get_by_name(new_concept.name).get_metadata().body == "metadata value"
        assert sheerka.get_by_hash(new_concept.get_definition_hash()).get_metadata().body == "metadata value"

        # sdp is updated
        sheerka.om.commit(context)
        sdp = sheerka.om.current_sdp()
        from_sdp = sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
        assert from_sdp.get_metadata().body == "metadata value"
        assert from_sdp.get_metadata().variables == [("var_name", "default value")]
        assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}

        assert sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).get_metadata().body == "metadata value"
        assert sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).get_metadata().body == "metadata value"
        assert sdp.get(service.CONCEPTS_BY_HASH_ENTRY,
                       new_concept.get_definition_hash()).get_metadata().body == "metadata value"

    def test_caches_are_update_when_i_modify_the_name(self):
        sheerka, context, foo = self.init_concepts("foo", cache_only=False)
        service = sheerka.services[SheerkaConceptManager.NAME]

        sheerka.is_known(sheerka.get_by_name(foo.name))
        sheerka.is_known(sheerka.get_by_key(foo.key))
        sheerka.get_by_hash(foo.get_definition_hash())

        to_add = {"meta": {"name": "bar"}}

        res = sheerka.modify_concept(context, foo, to_add)
        new_concept = res.body.body

        assert new_concept.name == "bar"
        assert sheerka.get_by_id(new_concept.id).name == "bar"
        assert sheerka.get_by_key(new_concept.key).name == "bar"
        assert sheerka.get_by_name(new_concept.name).name == "bar"
        assert sheerka.get_by_hash(new_concept.get_definition_hash()).name == "bar"

        assert not sheerka.is_known(sheerka.get_by_name(foo.name))
        assert not sheerka.is_known(sheerka.get_by_key(foo.key))
        assert not sheerka.is_known(sheerka.get_by_hash(foo.get_definition_hash()))

        sheerka.om.commit(context)
        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id).name == "bar"
        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).name == "bar"
        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).name == "bar"
        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_HASH_ENTRY,
                                            new_concept.get_definition_hash()).name == "bar"
        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY, foo.key) is NotFound
        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_NAME_ENTRY, foo.name) is NotFound
        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_HASH_ENTRY, foo.get_definition_hash()) is NotFound

    def test_i_can_modify_a_concept_from_a_list_of_concepts(self):
        sheerka, context, foo1, foo2 = self.init_concepts(
            Concept("foo", body="1"),
            Concept("foo", body="2"))

        to_add = {"meta": {"body": "new_value"}}

        res = sheerka.modify_concept(context, foo1, to_add)

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)

        new_concept = res.body.body

        assert new_concept.id == foo1.id
        assert res.body.body.get_metadata().body == "new_value"

        assert sheerka.get_by_id(foo1.id).get_metadata().body == "new_value"
        assert sheerka.get_by_id(foo2.id).get_metadata().body == "2"

    def test_values_are_modified_when_variables_are_added_or_removed(self):
        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b"))

        to_add = {"meta": {"body": "metadata value"},
                  "variables": {"c": "default value"}}

        to_remove = {"variables": ["a"]}

        assert get_concept_attrs(foo) == ["a", "b"]

        res = sheerka.modify_concept(context, foo, to_add, to_remove)
        new_concept = res.body.body

        assert res.status
        assert get_concept_attrs(foo) == ["b", "c"]
        assert get_concept_attrs(new_concept) == ["b", "c"]

        new_foo = sheerka.new(foo)
        assert get_concept_attrs(new_foo) == ["b", "c"]

    def test_key_is_modified_when_modifying_name_or_variables(self):
        sheerka, context, foo = self.init_concepts(Concept("foo a b").def_var("a").def_var("b"))

        to_add = {"meta": {"name": "b bar c d"},
                  "variables": {"c": None, "d": None}}

        to_remove = {"variables": ["a"]}

        res = sheerka.modify_concept(context, foo, to_add, to_remove)
        new_concept = res.body.body

        assert res.status
        assert new_concept.key == "__var__0 bar __var__1 __var__2"

    def test_key_is_modified_when_modifying_the_definition(self):
        sheerka, context, foo = self.init_concepts(
            Concept(name="foo", definition="foo a b", definition_type=DEFINITION_TYPE_DEF).def_var("a").def_var("b"))

        to_add = {"meta": {"definition": "b bar c d"},
                  "variables": {"c": None, "d": None}}

        to_remove = {"variables": ["a"]}

        res = sheerka.modify_concept(context, foo, to_add, to_remove)
        new_concept = res.body.body

        assert res.status
        assert new_concept.key == "__var__0 bar __var__1 __var__2"

    def test_bnf_is_modified_when_modifying_the_definition(self):
        sheerka, context, one, two, foo = self.init_test().with_concepts(
            "one",
            "two",
            Concept(name="foo", definition="'twenty' one"),
            create_new=True
        ).unpack()

        to_add = {"meta": {"definition": "'twenty' two"}}

        res = sheerka.modify_concept(context, foo, to_add)
        new_concept = res.body.body

        assert res.status
        assert new_concept.get_metadata().definition == "'twenty' two"
        assert new_concept.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(two, rule_name='two'))

    def test_concept_by_first_keyword_is_updated_after_concept_modification(self):
        sheerka, context, foo, bar, baz = self.init_test().with_concepts(
            Concept("foo"),
            Concept("bar"),
            Concept("baz", definition="foo"),
            create_new=True).unpack()

        # sanity check
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            "foo": ["1001"],
            "bar": ["1002"],
            'c:|1001:': ['1003']}
        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'foo': ['1001', '1003'],
            'bar': ['1002']}

        to_add = {"meta": {"name": "bar"}}
        res = sheerka.modify_concept(context, foo, to_add)

        assert res.status
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            "bar": ["1002", "1001"],
            'c:|1001:': ['1003']}
        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002', '1001', '1003']}

    def test_i_can_modify_bnf_definition_from_first_token_to_first_regex(self):
        sheerka, context, foo, = self.init_test().with_concepts(
            Concept("foo", definition="'hello'|'hola'"), create_new=True).unpack()
        service = sheerka.services[SheerkaConceptManager.NAME]

        # sanity
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            "hello": ["1001"],
            "hola": ["1001"]}
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
        assert len(service.compiled_concepts_by_regex) == 0

        to_add = {"meta": {"definition": "r'[a-z]+'"}}
        res = sheerka.modify_concept(context, foo, to_add)

        assert res.status
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
            RegExDef("[a-z]+").serialize(): ["1001"]
        }
        assert len(service.compiled_concepts_by_regex) == 1

    def test_i_can_modify_bnf_definition_from_first_regex_to_first_token(self):
        sheerka, context, foo, = self.init_test().with_concepts(
            Concept("foo", definition="r'[a-z]+'"), create_new=True).unpack()
        service = sheerka.services[SheerkaConceptManager.NAME]

        # sanity
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
            RegExDef("[a-z]+").serialize(): ["1001"]
        }
        assert len(service.compiled_concepts_by_regex) == 1

        to_add = {"meta": {"definition": "'hello'|'hola'"}}
        res = sheerka.modify_concept(context, foo, to_add)

        assert res.status
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            "hello": ["1001"],
            "hola": ["1001"]}
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
        assert len(service.compiled_concepts_by_regex) == 0

    def test_i_can_modify_when_multiple_bnf_definitions_are_already_defined(self):
        sheerka, context, foo, bar, baz = self.init_test().with_concepts(
            Concept("foo", definition="r'[a-z]+'"),
            Concept("bar", definition="r'[0-1]+'"),
            Concept("baz", definition="'one'|'twox'"), create_new=True).unpack()
        service = sheerka.services[SheerkaConceptManager.NAME]
        # it does not matter than baz is a bnf

        to_add = {"meta": {"definition": "'one'|'two'"}}
        res = sheerka.modify_concept(context, baz, to_add)

        assert res.status
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            "one": ["1003"],
            "two": ["1003"]}
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
            RegExDef("[a-z]+").serialize(): ["1001"],
            RegExDef("[0-1]+").serialize(): ["1002"],
        }
        assert len(service.compiled_concepts_by_regex) == 2

    def test_references_are_updated_after_concept_modification(self):
        sheerka, context, one, twenty_one = self.init_test().with_concepts(
            "onz",
            Concept("twenty one", definition="'twenty' onz"),
            create_new=True
        ).unpack()

        assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(one, rule_name='onz'))

        to_add = {"meta": {"name": "one"}}
        res = sheerka.modify_concept(context, one, to_add)
        modified = res.body.body

        assert res.status

        twenty_one = sheerka.get_by_name("twenty one")
        assert twenty_one.get_metadata().definition == "'twenty' one"
        assert twenty_one.get_bnf() is None

        ensure_bnf(context, twenty_one)
        assert twenty_one.get_bnf() == Sequence(StrMatch('twenty'), ConceptExpression(modified, rule_name='one'))

    def test_i_can_modify_on_top_of_a_new_ontology_layer(self):
        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b"), cache_only=False)

        sheerka.push_ontology(context, "new ontology")

        to_add = {
            "meta": {"body": "a body"},
            "props": {BuiltinConcepts.ISA: {"bar"}},
            "variables": {"c": "value"}
        }
        to_remove = {
            "variables": ["b"]
        }

        res = sheerka.modify_concept(context, foo, to_add=to_add, to_remove=to_remove)
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)
        assert res.body.body.get_metadata().body == "a body"
        assert res.body.body.get_metadata().variables == [("a", None), ("c", "value")]
        assert res.body.body.get_metadata().props == {BuiltinConcepts.ISA: {"bar"}}

        # and correctly set in cache
        updated = sheerka.get_by_id(foo.id)
        assert updated.get_metadata().body == "a body"
        assert updated.get_metadata().variables == [("a", None), ("c", "value")]
        assert updated.get_metadata().props == {BuiltinConcepts.ISA: {"bar"}}

        sheerka.pop_ontology(context)

    def test_i_cannot_modify_without_any_modification(self):
        sheerka, context, foo = self.init_concepts("foo")
        service = sheerka.services[SheerkaConceptManager.NAME]

        res = service.modify_concept(context, foo)

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == NoModificationFound(foo)

    def test_i_cannot_modify_forbidden_attributes(self):
        sheerka, context, foo = self.init_concepts("foo")
        service = sheerka.services[SheerkaConceptManager.NAME]

        for attr in service.forbidden_meta:
            res = service.modify_concept(context, foo, to_add={"meta": {attr: "new value"}})
            assert not res.status
            assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
            assert res.body.body == ForbiddenAttribute(attr)

    def test_i_cannot_modify_unknown_attributes(self):
        sheerka, context, foo = self.init_concepts("foo")

        res = sheerka.modify_concept(context, foo, to_add={"meta": {"dummy": "new value"}})
        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == UnknownAttribute("dummy")

    def test_i_cannot_modify_if_all_new_values_are_the_same(self):
        sheerka, context, foo = self.init_concepts(Concept("foo", body="a body"))

        res = sheerka.modify_concept(context, foo, to_add={"meta": {"name": "foo", "body": "a body"}})

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == NoModificationFound(foo, {"name": "foo", "body": "a body"})

    def test_i_cannot_modify_and_remove_meta_attributes(self):
        sheerka, context, foo = self.init_concepts(Concept("foo"))

        res = sheerka.modify_concept(context, foo, to_remove={"meta": {"any_value": "foo"}})

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == CannotRemoveMeta({"any_value": "foo"})

    def test_i_cannot_modify_and_remove_props_that_does_not_exists(self):
        sheerka, context, foo = self.init_concepts(Concept("foo"))

        res = sheerka.modify_concept(context, foo, to_remove={"props": {"any_value": "foo"}})

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == UnknownAttribute("any_value")

    def test_i_cannot_modify_and_remove_props_value_that_does_not_exists(self):
        # Need to returns an error, otherwise, we will save a concept that is not modified
        sheerka, context, foo = self.init_concepts(Concept("foo", props={"a": {"value"}}))

        res = sheerka.modify_concept(context, foo, to_remove={"props": {"a": "dummy"}})

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == ValueNotFound("a", "dummy")

    def test_i_cannot_modify_and_remove_variable_that_does_not_exists(self):
        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a"))

        res = sheerka.modify_concept(context, foo, to_remove={"variables": ["b"]})

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == UnknownAttribute("b")

    def test_i_cannot_modify_a_concept_that_is_not_known(self):
        sheerka, context = self.init_concepts()
        foo = Concept("foo")
        sheerka.set_id_if_needed(foo, False)

        res = sheerka.modify_concept(context, foo, to_add={"meta": {"body": "new value"}})

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT)

    def test_i_cannot_modify_with_an_invalid_regex_expression(self):
        sheerka, context, foo, = self.init_test().with_concepts(
            Concept("foo", definition="'hello'|'hola'"), create_new=True).unpack()
        service = sheerka.services[SheerkaConceptManager.NAME]

        # sanity
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            "hello": ["1001"],
            "hola": ["1001"]}
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
        assert len(service.compiled_concepts_by_regex) == 0

        to_add = {"meta": {"definition": "r'[a-z+'"}}  # invalid regex definition
        res = sheerka.modify_concept(context, foo, to_add)

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body.msg == 'unterminated character set'
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            "hello": ["1001"],
            "hola": ["1001"]}
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
        assert len(service.compiled_concepts_by_regex) == 0

    def test_i_can_get_and_set_attribute(self):
        sheerka, context = self.init_concepts()
        foo = Concept("foo")
        prop = Concept("property")
        bar = Concept("bar")

        res = sheerka.set_attr(foo, prop, bar)
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)

        assert sheerka.get_attr(foo, prop) == bar

    def test_i_setting_twice_the_same_property_creates_a_list(self):
        sheerka, context = self.init_concepts()
        foo = Concept("foo")
        prop = Concept("property")
        bar = Concept("bar")
        baz = Concept("baz")
        qux = Concept("qux")

        res = sheerka.set_attr(foo, prop, bar)
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
        assert sheerka.get_attr(foo, prop) == bar

        res = sheerka.set_attr(foo, prop, bar)  # again, same value as no effect
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
        assert sheerka.get_attr(foo, prop) == bar

        res = sheerka.set_attr(foo, prop, baz)
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
        assert sheerka.get_attr(foo, prop) == [bar, baz]

        res = sheerka.set_attr(foo, prop, qux)
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
        assert sheerka.get_attr(foo, prop) == [bar, baz, qux]

    def test_i_can_get_and_set_property(self):
        sheerka, context, foo, prop, bar = self.init_concepts("foo", "property", "bar")

        foo_instance = sheerka.new(foo)
        res = sheerka.set_property(context, foo_instance, prop, bar)
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
        assert sheerka.get_property(foo_instance, prop) == bar

        res = sheerka.set_property(context, foo_instance, BuiltinConcepts.ASSOCIATIVITY, SyaAssociativity.Left)
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
        assert sheerka.get_property(foo_instance, BuiltinConcepts.ASSOCIATIVITY) == SyaAssociativity.Left

        res = sheerka.set_property(context, foo_instance, sheerka.new(BuiltinConcepts.ASSOCIATIVITY),
                                   SyaAssociativity.Right)
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
        assert sheerka.get_property(foo_instance, BuiltinConcepts.ASSOCIATIVITY) == SyaAssociativity.Right

        # by default, the concept itself is not modified
        new_foo = sheerka.new(foo)
        not_found = sheerka.get_property(new_foo, prop)
        assert sheerka.isinstance(not_found, BuiltinConcepts.NOT_FOUND)
        assert not_found.body == {"#concept": new_foo, "#prop": prop}

        # I can modify the concept itself
        another_foo_instance = sheerka.new(foo)
        res = sheerka.set_property(context, another_foo_instance, prop, bar, all_concepts=True)
        assert res.status
        assert sheerka.get_property(another_foo_instance, prop) == bar

        new_foo = sheerka.new(foo)
        assert sheerka.get_property(new_foo, prop) == bar

    def test_i_cannot_remove_a_concept_which_has_reference(self):
        sheerka, context, one, twenty_one = self.init_test().with_concepts(
            Concept("one"),
            Concept("twenty one", definition="'twenty' one"),
            create_new=True).unpack()

        res = sheerka.remove_concept(context, one)

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == ConceptIsReferenced([twenty_one])

    def test_i_can_remove_a_concept(self):
        sheerka, context, one = self.init_test().with_concepts(
            Concept("one"),
            create_new=True).unpack()

        # sanity check
        assert sheerka.get_by_id(one.id) == one
        assert sheerka.get_by_name(one.name) == one
        assert sheerka.get_by_key(one.key) == one
        assert sheerka.get_by_hash(one.get_definition_hash()) == one
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) != {}
        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) != {}

        res = sheerka.remove_concept(context, one)

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)

        assert sheerka.isinstance(sheerka.get_by_id(one.id), BuiltinConcepts.UNKNOWN_CONCEPT)
        assert sheerka.isinstance(sheerka.get_by_name(one.name), BuiltinConcepts.UNKNOWN_CONCEPT)
        assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
        assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)

        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}

    def test_i_can_remove_a_first_regex_concept(self):
        sheerka, context, one = self.init_test().with_concepts(
            Concept("one", definition="r'[a-z]+'"),
            create_new=True).unpack()
        service = sheerka.services[SheerkaConceptManager.NAME]

        # sanity check
        assert sheerka.get_by_id(one.id) == one
        assert sheerka.get_by_name(one.name) == one
        assert sheerka.get_by_key(one.key) == one
        assert sheerka.get_by_hash(one.get_definition_hash()) == one
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) != {}
        assert len(service.compiled_concepts_by_regex) != 0

        res = sheerka.remove_concept(context, one)

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)

        assert sheerka.isinstance(sheerka.get_by_id(one.id), BuiltinConcepts.UNKNOWN_CONCEPT)
        assert sheerka.isinstance(sheerka.get_by_name(one.name), BuiltinConcepts.UNKNOWN_CONCEPT)
        assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
        assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
        assert len(service.compiled_concepts_by_regex) == 0

    def test_i_cannot_remove_a_concept_that_does_not_exist(self):
        sheerka, context = self.init_concepts()
        one = Concept("one", id="1001")

        res = sheerka.remove_concept(context, one)

        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
        assert res.body.body == ConceptNotFound(one)

    def test_i_can_create_concepts_in_multiple_ontology_layers(self):
        sheerka, context = self.init_concepts(cache_only=False)

        res = sheerka.create_new_concept(context, Concept("foo"))
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)

        sheerka.push_ontology(context, "new ontology")
        res = sheerka.create_new_concept(context, Concept("bar"))
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)

        # I cannot defined foo again, even if it's not the same layer
        res = sheerka.create_new_concept(context, Concept("foo"))
        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)

        # I cannot define bar again in this layer
        res = sheerka.create_new_concept(context, Concept("bar"))
        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)

        sheerka.pop_ontology(context)
        # But I can if I remove the layer
        res = sheerka.create_new_concept(context, Concept("bar"))
        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.NEW_CONCEPT)

    @pytest.mark.parametrize("concept, expected", [
        (Concept("foo"), {"foo": ["1001"]}),
        (Concept("foo a").def_var("a"), {"foo": ["1001"]}),
        (Concept("a b foo").def_var("a").def_var("b"), {"foo": ["1001"]}),
    ])
    def test_i_can_get_concepts_by_first_keyword(self, concept, expected):
        """
        Given a concept, i can find the first know token
        example:
            Concept("a foo b").def_var("a").def_var("b")
            'a' and 'b' are properties
            the first 'real' token is foo
        :return:
        """

        sheerka, context, *updated = self.init_concepts(concept)

        res = SheerkaConceptManager.compute_concepts_by_first_token(context, updated)

        assert res.status
        assert res.body == expected

    @pytest.mark.parametrize("bnf, expected", [
        (StrMatch("foo"), {"foo": ["1002"]}),
        (StrMatch("bar"), {"bar": ["1002"]}),
        (ConceptExpression("bar"), {"c:|1001:": ["1002"]}),
        (Sequence(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"]}),
        (Sequence(StrMatch("foo"), ConceptExpression("bar")), {"foo": ["1002"]}),
        (Sequence(ConceptExpression("bar"), StrMatch("foo")), {"c:|1001:": ["1002"]}),
        (OrderedChoice(StrMatch("foo"), StrMatch("bar")), {"foo": ["1002"], "bar": ["1002"]}),
        (Optional(StrMatch("foo")), {"foo": ["1002"]}),
        (ZeroOrMore(StrMatch("foo")), {"foo": ["1002"]}),
        (OneOrMore(StrMatch("foo")), {"foo": ["1002"]}),
        (StrMatch("--filter"), {"--filter": ["1002"]}),  # add both entries
    ])
    def test_i_can_get_concepts_by_first_keyword_with_bnf(self, bnf, expected):
        sheerka, context = self.init_test().unpack()

        bar = Concept("bar").init_key()
        sheerka.set_id_if_needed(bar, False)
        sheerka.test_only_add_in_cache(bar)

        concept = Concept("foo").init_key()
        concept.set_bnf(bnf)
        sheerka.set_id_if_needed(concept, False)

        res = compute_concepts_by_first_token(context, [concept])

        assert res.status
        assert res.body == expected

    def test_i_can_get_concepts_by_first_keyword_when_multiple_concepts(self):
        sheerka = self.get_sheerka()
        context = self.get_context(sheerka)

        bar = Concept("bar").init_key()
        sheerka.set_id_if_needed(bar, False)
        sheerka.test_only_add_in_cache(bar)

        baz = Concept("baz").init_key()
        sheerka.set_id_if_needed(baz, False)
        sheerka.test_only_add_in_cache(baz)

        foo = Concept("foo").init_key()
        foo.set_bnf(OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux")))
        sheerka.set_id_if_needed(foo, False)

        res = compute_concepts_by_first_token(context, [bar, baz, foo])

        assert res.status
        assert res.body == {
            "bar": ["1001"],
            "baz": ["1002"],
            "c:|1001:": ["1003"],
            "c:|1002:": ["1003"],
            "qux": ["1003"],
        }

    def test_i_can_get_concepts_by_first_keyword_using_sheerka(self):
        sheerka, context, *updated = self.init_test().with_concepts(
            "one",
            "two",
            Concept("twenty", definition="'twenty' (one|two)"),
            create_new=True
        ).unpack()

        bar = Concept("bar").init_key()
        sheerka.set_id_if_needed(bar, False)
        sheerka.test_only_add_in_cache(bar)

        foo = Concept("foo").init_key()
        foo.set_bnf(OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux")))
        sheerka.set_id_if_needed(foo, False)

        res = compute_concepts_by_first_token(context, [bar, foo], use_sheerka=True)

        assert res.status
        assert res.body == {
            "one": ["1001"],
            "two": ["1002"],
            "twenty": ["1003"],
            "bar": ["1004"],
            "c:|1001:": ["1005"],
            "c:|1004:": ["1005"],
            "qux": ["1005"],
        }

    def test_i_cannot_get_concept_by_first_keyword_when_no_first_keyword(self):
        sheerka, context, foo = self.init_concepts(Concept("x y", body="x y").def_var("x").def_var("y"))
        res = compute_concepts_by_first_token(context, [foo])

        assert not res.status
        assert res.body == NoFirstTokenError(foo, foo.key)

    def test_i_can_resolve_concepts_by_first_keyword(self):
        sheerka, context, *updated = self.init_concepts(
            "one",
            Concept("two", definition="one"),
            Concept("three", definition="two"))

        concepts_by_first_keywords = {
            "one": ["1001"],
            "c:|1001:": ["1002"],
            "c:|1002:": ["1003"],
        }

        resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)

        assert resolved_ret_val.status
        assert resolved_ret_val.body == {
            "one": ["1001", "1002", "1003"],
        }

    def test_i_can_resolve_when_concepts_are_sets(self):
        sheerka, context, number, *concepts = self.init_concepts(
            "number",
            "one",
            "two",
            "twenty",
            "hundred",
            Concept("twenties", definition="twenty number"),
            Concept("hundreds", definition="number hundred"),
        )

        global_truth_context = self.get_context(sheerka, global_truth=True)
        sheerka.set_isa(global_truth_context, sheerka.new("one"), number)
        sheerka.set_isa(global_truth_context, sheerka.new("two"), number)
        sheerka.set_isa(global_truth_context, sheerka.new("twenty"), number)
        sheerka.set_isa(global_truth_context, sheerka.new("thirty"), number)
        sheerka.set_isa(global_truth_context, sheerka.new("hundred"), number)
        sheerka.set_isa(global_truth_context, sheerka.new("twenties"), number)
        sheerka.set_isa(global_truth_context, sheerka.new("hundreds"), number)

        sheerka.clear_bnf_definition()  # reset all the grammar to simulate Sheerka restart

        # cbft : concept_by_first_token (I usually don't  use abbreviation)
        cbft = compute_concepts_by_first_token(context, [number] + concepts).body
        resolved_ret_val = resolve_concepts_by_first_keyword(context, cbft)

        assert resolved_ret_val.status
        assert resolved_ret_val.body == {
            'number': ['1001'],
            'one': ['1002', '1007'],
            'two': ['1003', '1007'],
            'twenty': ['1004', '1006', '1007'],
            'hundred': ['1005', '1007'],
        }

    def test_i_can_resolve_when_concepts_have_multiple_levels_of_sets(self):
        sheerka, context, adjective, color, red, qualified_table = self.init_test(global_truth=True).with_concepts(
            "adjective",
            "color",
            "red",
            Concept("qualified table", definition="adjective 'table'"),
        ).unpack()

        sheerka.set_isa(context, color, adjective)
        sheerka.set_isa(context, red, color)

        by_first_token = compute_concepts_by_first_token(context, [adjective, color, red, qualified_table]).body
        resolved_ret_val = resolve_concepts_by_first_keyword(context, by_first_token)

        assert resolved_ret_val.status
        assert resolved_ret_val.body == {
            'adjective': ['1001'],
            'color': ['1002'],
            'red': ['1003', '1004'],
        }

    def test_concepts_are_defined_once(self):
        sheerka = self.get_sheerka()
        context = self.get_context(sheerka)
        good = self.create_and_add_in_cache_concept(sheerka, "good")
        foo = self.create_and_add_in_cache_concept(sheerka, "foo", bnf=ConceptExpression("good"))
        bar = self.create_and_add_in_cache_concept(sheerka, "bar", bnf=ConceptExpression("good"))
        baz = self.create_and_add_in_cache_concept(sheerka, "baz", bnf=OrderedChoice(
            ConceptExpression("foo"),
            ConceptExpression("bar")))

        concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, foo, bar, baz]).body

        resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
        assert resolved_ret_val.status
        assert resolved_ret_val.body == {
            "good": ["1001", "1002", "1003", "1004"],
        }

    def test_i_can_resolve_more_complex(self):
        sheerka = self.get_sheerka()
        context = self.get_context(sheerka)

        a = self.create_and_add_in_cache_concept(sheerka, "a", bnf=Sequence("one", "two"))
        b = self.create_and_add_in_cache_concept(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two"))

        concepts_by_first_keywords = compute_concepts_by_first_token(context, [a, b]).body

        resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
        assert resolved_ret_val.status
        assert resolved_ret_val.body == {
            "one": ["1001", "1002"],
        }

    def tests_i_can_detect_direct_recursion(self):
        sheerka, context, good, foo, bar = self.init_concepts(
            "good",
            self.bnf_concept("foo", ConceptExpression("bar")),
            self.bnf_concept("bar", ConceptExpression("foo")),
        )

        concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, foo, bar]).body
        resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
        assert resolved_ret_val.status
        assert resolved_ret_val.body == {
            "good": ["1001"],
        }
        assert sheerka.chicken_and_eggs.get(foo.id) == {foo.id, bar.id}
        assert sheerka.chicken_and_eggs.get(bar.id) == {foo.id, bar.id}

    def test_i_can_detect_indirect_infinite_recursion(self):
        sheerka, context, good, one, two, three = self.init_concepts(
            "good",
            self.bnf_concept("one", ConceptExpression("two")),
            self.bnf_concept("two", ConceptExpression("three")),
            self.bnf_concept("three", ConceptExpression("two")),
        )

        concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, one, two, three]).body
        resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
        assert resolved_ret_val.status
        assert resolved_ret_val.body == {
            "good": ["1001"],
        }
        assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id}
        assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id}
        assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id}

    def test_i_can_detect_the_longest_infinite_recursion_chain(self):
        sheerka, context, good, one, two, three = self.init_concepts(
            "good",
            self.bnf_concept("two", ConceptExpression("three")),
            self.bnf_concept("three", ConceptExpression("two")),
            self.bnf_concept("one", ConceptExpression("three")),
        )

        concepts_by_first_keywords = compute_concepts_by_first_token(context, [good, one, two, three]).body
        resolved_ret_val = resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
        assert resolved_ret_val.status
        assert resolved_ret_val.body == {
            "good": ["1001"],
        }
        assert sheerka.chicken_and_eggs.get(one.id) == {one.id, two.id, three.id}
        assert sheerka.chicken_and_eggs.get(two.id) == {one.id, two.id, three.id}
        assert sheerka.chicken_and_eggs.get(three.id) == {one.id, two.id, three.id}

    def test_i_can_smart_get_attr_when_the_value_is_known(self):
        sheerka, context = self.init_concepts()
        foo = Concept("foo")
        prop = Concept("property")
        bar = Concept("bar")

        sheerka.set_attr(foo, prop, bar)
        assert sheerka.smart_get_attr(foo, prop) == bar

    def test_i_can_smart_get_attr_when_simple_isa(self):
        sheerka, context, adjective, color, red, table = self.init_concepts("adjective",
                                                                            "color",
                                                                            "red",
                                                                            "table",
                                                                            create_new=True)
        sheerka.set_isa(context, color, adjective)
        sheerka.set_isa(context, red, color)

        color_instance = sheerka.new(color, body=red)
        adjective_instance = sheerka.new(adjective, body=color_instance)
        table_instance = sheerka.new(table)
        sheerka.set_attr(table_instance, adjective, adjective_instance)

        assert sheerka.smart_get_attr(table_instance, color) == color_instance
        assert sheerka.objvalue(sheerka.smart_get_attr(table_instance, color)) == red

    def test_i_can_smart_get_when_multiple_levels_of_isa(self):
        sheerka, context, adjective, color, reddish, red, table = self.init_concepts("adjective",
                                                                                     "color",
                                                                                     "reddish",
                                                                                     "red",
                                                                                     "table",
                                                                                     create_new=True)
        sheerka.set_isa(context, color, adjective)
        sheerka.set_isa(context, reddish, color)
        sheerka.set_isa(context, red, reddish)

        reddish_instance = sheerka.new(reddish, body=red)
        color_instance = sheerka.new(color, body=reddish_instance)
        adjective_instance = sheerka.new(adjective, body=color_instance)
        table_instance = sheerka.new(table)
        sheerka.set_attr(table_instance, adjective, adjective_instance)

        assert sheerka.smart_get_attr(table_instance, reddish_instance) == reddish_instance
        assert sheerka.objvalue(sheerka.smart_get_attr(table_instance, color)) == red

    def test_i_can_smart_get_when_multiple_values(self):
        sheerka, context, adjective, color, red, blue, table = self.init_concepts("adjective",
                                                                                  "color",
                                                                                  "red",
                                                                                  "blue",
                                                                                  "table",
                                                                                  create_new=True)
        sheerka.set_isa(context, color, adjective)
        sheerka.set_isa(context, red, color)
        sheerka.set_isa(context, blue, color)
        table_instance = sheerka.new(table)

        # add red color
        red_color_instance = sheerka.new(color, body=red)
        red_adjective_instance = sheerka.new(adjective, body=red_color_instance)
        sheerka.set_attr(table_instance, adjective, red_adjective_instance)

        # add blue color
        blue_color_instance = sheerka.new(color, body=blue)
        blue_adjective_instance = sheerka.new(adjective, body=blue_color_instance)
        sheerka.set_attr(table_instance, adjective, blue_adjective_instance)

        res = sheerka.smart_get_attr(table_instance, color)
        assert res == [red_color_instance, blue_color_instance]

    def test_i_can_smart_get_when_ancestor_is_asked(self):
        sheerka, context, adjective, color, red, table = self.init_concepts("adjective",
                                                                            "color",
                                                                            "red",
                                                                            "table",
                                                                            create_new=True)
        sheerka.set_isa(context, color, adjective)
        sheerka.set_isa(context, red, color)

        color_instance = sheerka.new(color, body=red)
        table_instance = sheerka.new(table)
        sheerka.set_attr(table_instance, color, color_instance)

        assert sheerka.smart_get_attr(table_instance, adjective) == color_instance

    def test_i_can_smart_get_when_ancestor_is_asked_and_multiple_values(self):
        sheerka, context, adjective, color, red, size, large, table = self.init_concepts("adjective",
                                                                                         "color",
                                                                                         "red",
                                                                                         "size",
                                                                                         "large",
                                                                                         "table",
                                                                                         create_new=True)
        sheerka.set_isa(context, color, adjective)
        sheerka.set_isa(context, red, color)
        sheerka.set_isa(context, size, adjective)
        sheerka.set_isa(context, large, size)

        table_instance = sheerka.new(table)
        color_instance = sheerka.new(color, body=red)
        sheerka.set_attr(table_instance, color, color_instance)
        size_instance = sheerka.new(size, body=large)
        sheerka.set_attr(table_instance, size, size_instance)

        assert sheerka.smart_get_attr(table_instance, adjective) == [color_instance, size_instance]

    def test_i_can_smart_get_when_direct_value_takes_precedence_over_child_value(self):
        sheerka, context, adjective, color, red, blue, table = self.init_concepts("adjective",
                                                                                  "color",
                                                                                  "red",
                                                                                  "blue",
                                                                                  "table",
                                                                                  create_new=True)
        sheerka.set_isa(context, color, adjective)
        sheerka.set_isa(context, red, color)

        color_instance = sheerka.new(color, body=red)
        adjective_instance = sheerka.new(adjective, body=color_instance)
        table_instance = sheerka.new(table)
        sheerka.set_attr(table_instance, adjective, adjective_instance)
        sheerka.set_attr(table_instance, color, blue)  # set direct color value

        assert sheerka.smart_get_attr(table_instance, color) == blue

    def test_i_can_smart_get_when_direct_value_takes_precedence_over_ancestor_value(self):
        sheerka, context, adjective, color, red, blue, table = self.init_concepts("adjective",
                                                                                  "color",
                                                                                  "red",
                                                                                  "blue",
                                                                                  "table",
                                                                                  create_new=True)
        sheerka.set_isa(context, color, adjective)
        sheerka.set_isa(context, red, color)

        color_instance = sheerka.new(color, body=red)
        table_instance = sheerka.new(table)
        sheerka.set_attr(table_instance, color, color_instance)
        sheerka.set_attr(table_instance, adjective, blue)  # set direct color value

        assert sheerka.smart_get_attr(table_instance, adjective) == blue

    def test_i_cannot_smart_get_attr_if_value_is_unknown_and_attribute_not_a_concept(self):
        sheerka, context = self.init_concepts()
        foo = Concept("foo")

        res = sheerka.smart_get_attr(foo, "attribute")
        assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)

    def test_i_cannot_smart_get_attr_if_value_is_unknown_and_no_isa(self):
        sheerka, context, adjective, color, red, table = self.init_concepts("adjective",
                                                                            "color",
                                                                            "red",
                                                                            "table",
                                                                            create_new=True)

        # sheerka.set_isa(context, color, adjective) #  color is not defined as an adjective
        sheerka.set_isa(context, red, color)

        color_instance = sheerka.new(color, body=red)
        adjective_instance = sheerka.new(adjective, body=color_instance)
        table_instance = sheerka.new(table)
        sheerka.set_attr(table_instance, adjective, adjective_instance)

        res = sheerka.smart_get_attr(table_instance, color)
        assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)

    def test_i_cannot_smart_get_attr_when_attribute_does_not_exist(self):
        sheerka, context, adjective, color, red, size, table = self.init_concepts("adjective",
                                                                                  "color",
                                                                                  Concept("red",
                                                                                          body="red").auto_init(),
                                                                                  "size",
                                                                                  "table",
                                                                                  create_new=True)
        sheerka.set_isa(context, color, adjective)
        sheerka.set_isa(context, red, color)
        sheerka.set_isa(context, size, adjective)

        table_instance = sheerka.new(table)
        color_instance = sheerka.new(color, body=red)
        adjective_instance = sheerka.new(adjective, body=color_instance)
        sheerka.set_attr(table_instance, adjective, adjective_instance)

        res = sheerka.smart_get_attr(table_instance, size)
        assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)

    def test_i_can_set_concept_precedence(self):
        sheerka, context, one, two, three = self.init_concepts("one", "two", "three")

        res = sheerka.set_precedence(context, one, two, three)
        assert sheerka.isinstance(res, BuiltinConcepts.SUCCESS)

        weights = sheerka.get_weights(BuiltinConcepts.PRECEDENCE, comparison_context=CONCEPT_COMPARISON_CONTEXT)
        assert weights == {'c:one|1001:': 3, 'c:two|1002:': 2, 'c:three|1003:': 1}

    def test_i_cannot_set_precedence_when_too_few_argument(self):
        sheerka, context, one = self.init_concepts("one")

        res = sheerka.set_precedence(context)
        assert res == sheerka.err("Not enough elements")

        res = sheerka.set_precedence(context, one)
        assert res == sheerka.err("Not enough elements")

    def test_i_cannot_set_precedence_when_error(self):
        sheerka, context, one, two = self.init_concepts("one", "two")

        ret = sheerka.set_precedence(context, one, two, one)
        assert not ret.status

        weights = sheerka.get_weights(BuiltinConcepts.PRECEDENCE, comparison_context=CONCEPT_COMPARISON_CONTEXT)
        assert weights == {'c:one|1001:': 2, 'c:two|1002:': 1}


class TestSheerkaConceptManagerUsingFileBasedSheerka(TestUsingFileBasedSheerka):
    def test_i_can_add_several_concepts(self):
        sheerka = self.get_sheerka()
        context = self.get_context(sheerka)
        service = sheerka.services[SheerkaConceptManager.NAME]

        hello = Concept("Hello world a").def_var("a")
        res = sheerka.create_new_concept(context, hello)
        sheerka.om.commit(context)
        assert res.status

        sheerka = self.get_sheerka()  # another instance
        context = self.get_context(sheerka)
        greeting = Concept("Greeting a").def_var("a")
        res = sheerka.create_new_concept(context, greeting)
        sheerka.om.commit(context)
        assert res.status

        sheerka = self.get_sheerka()  # another instance again
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, hello.key)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_KEY_ENTRY, greeting.key)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, hello.id)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_ID_ENTRY, greeting.id)
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, "Hello world a")
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_NAME_ENTRY, "Greeting a")
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, hello.get_definition_hash())
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, greeting.get_definition_hash())

        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Hello")
        assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "Greeting")

    def test_i_cannot_add_the_same_concept_twice_using_sdp(self):
        """
        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
        :return:
        """
        sheerka, context, concept = self.init_concepts("foo")

        sheerka.create_new_concept(context, concept)
        sheerka.om.commit(context)

        sheerka.om.current_cache_manager().clear(set_is_cleared=False)
        res = sheerka.create_new_concept(context, concept)

        assert not res.status
        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
        assert res.value.body == concept

    def test_new_entry_does_not_override_the_previous_ones(self):
        sheerka = self.get_sheerka()
        context = self.get_context(sheerka)
        service = sheerka.services[SheerkaConceptManager.NAME]

        sheerka.create_new_concept(context, Concept("foo", body="1"))
        sheerka.create_new_concept(context, Concept("foo", body="2"))
        sheerka.om.commit(context)

        assert len(sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 2

        sheerka = self.get_sheerka()  # new instance
        context = self.get_context(sheerka)
        sheerka.create_new_concept(context, Concept("foo", body="3"))
        sheerka.om.commit(context)

        assert len(sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY, "foo")) == 3