Sheerka-Old/tests/core/test_sheerka.py

import os

import pytest

from conftest import SHEERKA_TEST_FOLDER
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
from core.builtin_concepts_ids import AllBuiltinConcepts
from core.concept import Concept, PROPERTIES_TO_SERIALIZE, ConceptParts, get_concept_attrs
from core.global_symbols import NotInit
from core.sheerka.Sheerka import Sheerka, BASE_NODE_PARSER_CLASS
from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
from core.tokenizer import Token, TokenKind
from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka


class ConceptWithGetObjValue(Concept):
    def get_obj_value(self):
        return self.get_value("my_prop")


class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):

    def test_i_can_initialize_services(self):
        sheerka = self.get_sheerka()

        assert len(sheerka.services) > 0
        assert None not in sheerka.services
        assert "VariableManager" in sheerka.services  # test at least one service

    def test_i_can_initialize_builtin_parsers(self):
        sheerka = self.get_sheerka()

        # test existence of some parser (not all)
        assert "parsers.DefConceptParser.DefConceptParser" in sheerka.parsers
        assert "parsers.BnfNodeParser.BnfNodeParser" in sheerka.parsers
        assert "parsers.SyaNodeParser.SyaNodeParser" in sheerka.parsers
        assert "parsers.SequenceNodeParser.SequenceNodeParser" in sheerka.parsers

        # make sure BaseNodeParser is properly initialized
        assert BASE_NODE_PARSER_CLASS not in sheerka.parsers
        assert sheerka.bnp is not None

    def test_i_can_list_builtin_concepts(self):
        sheerka = self.get_sheerka()
        builtins = list(sheerka.get_builtins_classes_as_dict())

        assert str(BuiltinConcepts.ERROR) in builtins
        assert str(BuiltinConcepts.RETURN_VALUE) in builtins

    def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string(self):
        """
        Checks that a concept can be found its name
        even when there are variables in the name (ex 'hello + a' or 'a + b' )
        :return:
        """
        sheerka = self.get_sheerka()
        for key in sheerka.get_builtins_classes_as_dict():
            assert sheerka.get_by_key(key) is not None
            assert sheerka.get_by_key(str(key)) is not None

    def test_i_cannot_get_when_key_is_none(self):
        sheerka = self.get_sheerka()

        res = sheerka.get_by_key(None)
        assert sheerka.isinstance(res, BuiltinConcepts.ERROR)
        assert res.body == "Concept 'None' is undefined."

    def test_i_cannot_get_when_key_is_not_found(self):
        sheerka = self.get_sheerka()

        loaded = sheerka.get_by_key("key_that_does_not_exist")

        assert loaded is not None
        assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
        assert loaded.body == {"key": "key_that_does_not_exist"}
        assert loaded.get_metadata().is_evaluated

    def test_i_cannot_get_when_id_is_not_found(self):
        sheerka = self.get_sheerka()

        loaded = sheerka.get_by_id("id_that_does_not_exist")

        assert loaded is not None
        assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
        assert loaded.body == {"id": "id_that_does_not_exist"}
        assert loaded.get_metadata().is_evaluated

    def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self):
        sheerka = self.get_sheerka()
        ret = sheerka.new(BuiltinConcepts.RETURN_VALUE, who="who", status="status", value="value")

        assert isinstance(ret, ReturnValueConcept)
        assert ret.key == str(BuiltinConcepts.RETURN_VALUE)
        assert ret.who == "who"
        assert ret.status == "status"
        assert ret.value == "value"

        # check the others
        for key, concept_class in sheerka.get_builtins_classes_as_dict().items():
            assert isinstance(sheerka.get_by_key(key), concept_class)

    def test_i_can_instantiate_a_builtin_concept_when_no_specific_class(self):
        sheerka = self.get_sheerka()
        ret = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body="fake_concept")

        assert isinstance(ret, Concept)
        assert ret.key == str(BuiltinConcepts.UNKNOWN_CONCEPT)
        assert ret.body == "fake_concept"

    def test_i_can_instantiate_a_concept(self):
        sheerka, context, concept = self.init_test().with_concepts(self.get_default_concept(), create_new=True).unpack()

        new = sheerka.new(concept.key, a=10, b="value")

        assert sheerka.isinstance(new, concept)
        for prop in PROPERTIES_TO_SERIALIZE:
            assert getattr(new.get_metadata(), prop) == getattr(concept.get_metadata(), prop)

        assert new.get_value("a") == 10
        assert new.get_value("b") == "value"

    def test_i_can_instantiate_concepts_when_variables_are_defined_in_constructor(self):
        sheerka, context, foo = self.init_test().with_concepts(
            Concept(name="foo", variables=[("x", "default value for x"), ("y", None)]),
            create_new=True).unpack()

        new = sheerka.new(foo.key)
        assert new.values() == {"x": NotInit, "y": NotInit}  # default values are not used ?

    def test_i_can_instantiate_multiple_when_same_key(self):
        sheerka, context, *concepts = self.init_test().with_concepts(
            Concept("foo", body="foo1"),
            Concept("foo", body="foo2"),
            create_new=True).unpack()

        # when no id, i get two instances
        concepts = sheerka.new("foo")
        assert len(concepts) == 2
        assert concepts[0].id == "1001"
        assert concepts[0].get_metadata().body == "foo1"
        assert concepts[1].id == "1002"
        assert concepts[1].get_metadata().body == "foo2"

        # only one instance if the id is given
        foo1 = sheerka.new(("foo", "1001"))
        assert foo1.get_metadata().body == "foo1"

        # only one instance if the id is given
        foo2 = sheerka.new(("foo", "1002"))
        assert foo2.get_metadata().body == "foo2"

    def test_instances_are_different_when_asking_for_new(self):
        sheerka, context, concept = self.init_test().with_concepts(self.get_default_concept(), create_new=True).unpack()

        new1 = sheerka.new(concept.key, a=10, b="value")
        new2 = sheerka.new(concept.key, a=10, b="value")

        assert new1 == new2
        assert id(new1) != id(new2)

    def test_new_instance_does_not_impact_each_others(self):
        sheerka, context, foo, bar = self.init_test().with_concepts("foo", "bar", create_new=True).unpack()

        new_foo = sheerka.new("foo")
        new_foo.get_metadata().body = "metadata value"  # modify metadata
        new_foo.def_var("var_name", "default value")  # modify definition of variables
        new_foo.add_prop(BuiltinConcepts.ISA, bar)  # modify property
        new_foo.get_compiled()["var_name"] = "'var value'"
        new_foo.set_value(ConceptParts.BODY, "body value")  # modify value
        new_foo.set_value("var_name", "var value")  # modify value

        assert new_foo.get_metadata().body != foo.get_metadata().body
        assert new_foo.get_metadata().variables != foo.get_metadata().variables
        assert new_foo.get_metadata().props != foo.get_metadata().props
        assert new_foo.values != foo.values
        assert new_foo.get_compiled() != foo.get_compiled()

    def test_i_get_the_same_instance_when_is_unique_is_true(self):
        sheerka, context, concept = self.init_test(). \
            with_concepts(Concept(name="unique", is_unique=True), create_new=True).unpack()

        new1 = sheerka.new(concept.key)
        new2 = sheerka.new(concept.key, a=10, b="value")  # not that variables are simply discareded

        assert new1 == new2
        assert id(new1) == id(new2)

    def test_values_are_reset_when_asking_for_a_new_instance(self):
        sheerka, context, template = self.init_test(eval_body=True).with_concepts(Concept("foo", body="'foo body'"),
                                                                                  create_new=True).unpack()

        sheerka.evaluate_concept(context, sheerka.get_by_id(template.id))
        assert template.get_metadata().is_evaluated
        assert template.body == "foo body"

        new = sheerka.new(template.key)
        assert not new.get_metadata().is_evaluated
        assert new.body == NotInit

        new = sheerka.new((None, template.id))
        assert not new.get_metadata().is_evaluated
        assert new.body == NotInit

    def test_i_cannot_instantiate_an_unknown_concept(self):
        sheerka = self.get_sheerka()

        new = sheerka.new("fake_concept")

        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
        assert new.body == {'key': 'fake_concept'}

    def test_i_cannot_instantiate_with_invalid_id(self):
        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
                                                                     Concept("foo", body="foo2"),
                                                                     create_new=True).unpack()

        new = sheerka.new(("foo", "invalid_id"))

        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
        assert new.body == {'key': 'foo', 'id': 'invalid_id'}

    def test_i_cannot_instantiate_with_invalid_key(self):
        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
                                                                     Concept("foo", body="foo2"),
                                                                     create_new=True).unpack()

        new = sheerka.new(("invalid_key", "1001"))

        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
        assert new.body == {'key': 'invalid_key', 'id': '1001'}

    def test_concept_id_is_irrelevant_when_only_one_concept(self):
        sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
                                                                     create_new=True).unpack()

        new = sheerka.new(("foo", "invalid_id"))

        assert sheerka.isinstance(new, "foo")
        assert new.get_metadata().body == "foo1"

    def test_i_cannot_instantiate_when_properties_are_not_recognized(self):
        sheerka, context, concept = self.init_test().with_concepts(self.get_default_concept(),
                                                                   create_new=True).unpack()

        new = sheerka.new(concept.key, a=10, c="value")

        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_PROPERTY)
        assert new.property_name == "c"
        assert sheerka.isinstance(new.concept, concept)

    @pytest.mark.parametrize("concept, reduce_simple_list, expected", [
        (None, False, None),
        (3.14, False, 3.14),
        ("foo", False, "foo"),
        (True, False, True),
        (Concept("name", body="foo"), False, "foo"),
        (Concept("name"), False, Concept("name")),
        (ConceptWithGetObjValue("name").set_value("my_prop", "my_value"), False, "my_value"),
        (ReturnValueConcept(value="return_value"), False, "return_value"),
        (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
        (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
        (ReturnValueConcept(value=Concept("foo", body=False).auto_init(), status=True), False, False),
        (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]),
        (Concept("name", body=["foo"]), True, "foo"),
        (Concept("name", body=Concept("foo")), False, Concept("foo")),
        (Concept("name", body=Concept("foo", body="value")), False, "value"),
        (Concept("name", body=Concept("foo", body=Concept("bar", body="value"))), False, "value"),
        (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"),
    ])
    def test_i_can_get_value(self, concept, reduce_simple_list, expected):
        sheerka = self.get_sheerka()

        # I use auto_init() instead of evaluate_concept() to be quicker
        c = concept
        while isinstance(c, Concept):
            c.auto_init()
            c = c.body

        assert sheerka.objvalue(concept, reduce_simple_list) == expected

    def test_builtin_error_concept_are_errors(self):
        # only test a random one, it will be the same for the others
        sheerka = self.get_sheerka()
        assert not sheerka.is_success(sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS))

    @pytest.mark.parametrize("concept, expected", [
        # by name
        ("foo", ["foo", "foo2"]),
        ("bar", "bar"),
        ("plus", "plus"),
        ("a mult b", "mult"),

        # by tuple
        (("foo", None), ["foo", "foo2"]),
        (("foo", "1002"), "foo2"),
        ((None, "1001"), "foo"),
        (("plus", None), "plus"),
        (("1001", None), "1001"),

        # by token
        (Token(TokenKind.CONCEPT, ("foo", None), 0, 0, 0), ["foo", "foo2"]),

        # by concept token str
        ("c:foo:", ["foo", "foo2"]),

        # I cannot resolve
        (None, None),
        ("unknown", None),
        ((None, None), None),
        (("unknown", None), None),
        ((None, "unknown"), None),
        (Token(TokenKind.CONCEPT, (None, None), 0, 0, 0), None),
        ("c:unknown:", None),
        ("c:foo:.id", None),
        ("c:foo: + 1", None),

    ])
    def test_i_can_resolve_concept(self, concept, expected):
        sheerka, context, *concepts = self.init_concepts(
            "foo",
            Concept("foo", body="another one"),
            "bar",
            self.from_def_concept("plus", "a plus b", ["a", "b"]),
            Concept("a mult b").def_var("a").def_var("b"),
            Concept("1001"),
        )

        cmap = {k: concepts[i] for i, k in enumerate(["foo", "foo2", "bar", "plus", "mult", "1001"])}
        cmap[None] = None

        real_expected = [cmap[e] for e in expected] if isinstance(expected, list) else cmap[expected]
        assert sheerka.resolve(concept) == real_expected

    def test_i_can_resolve_when_searching_by_definition(self):
        sheerka, context, plus = self.init_test().with_concepts(self.from_def_concept("plus", "a plus b", ["a", "b"]),
                                                                create_new=True).unpack()

        assert sheerka.resolve("a plus b") == plus

    def test_i_can_reset_global_concept_definition_when_adding_and_removing_layers(self):
        sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a").def_var("b"))
        assert get_concept_attrs(foo) == ["a", "b"]

        sheerka.push_ontology(context, "another ontology")
        sheerka.modify_concept(context, foo, to_add={"variables": {"c": None}}, to_remove={"variables": ["b"]})
        assert get_concept_attrs(foo) == ["a", "c"]

        sheerka.pop_ontology(context)
        assert get_concept_attrs(foo) == ["a", "b"]

    def test_i_can_manage_concepts_ids_on_multiple_ontology_layers(self):
        sheerka, context = self.init_concepts()

        res = sheerka.create_new_concept(context, Concept("foo"))
        assert res.body.body.id == "1001"

        sheerka.push_ontology(context, "new ontology")
        res = sheerka.create_new_concept(context, Concept("bar"))
        assert res.body.body.id == "1002"

        res = sheerka.create_new_concept(context, Concept("baz"))
        assert res.body.body.id == "1003"

        sheerka.pop_ontology(context)
        res = sheerka.create_new_concept(context, Concept("baz"))
        assert res.body.body.id == "1002"

    def test_i_can_add_ontology(self):
        sheerka, context = self.init_test().unpack()

        # Create an ontology an set some values
        res = sheerka.push_ontology(context, "new ontology")

        assert res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)

        foo = sheerka.create_new_concept(context, Concept("foo").def_var("a").def_var("b")).body.body

        # sanity check
        assert sheerka.get_by_name("foo") == foo
        assert not sheerka.is_known(sheerka.get_by_name("bar"))
        assert get_concept_attrs(foo) == ["a", "b"]

        # record the ontology
        ontology = sheerka.get_ontology(context)
        sheerka.pop_ontology(context)

        # Create another ontology with some other values
        sheerka.push_ontology(context, "another ontology")
        foo2 = sheerka.create_new_concept(context, Concept("foo").def_var("a").def_var("b").def_var("c")).body.body
        bar = sheerka.create_new_concept(context, Concept("bar")).body.body

        # sanity check
        assert sheerka.get_by_name("foo") == foo2
        assert sheerka.get_by_name("bar") == bar
        assert get_concept_attrs(foo) == ["a", "b", "c"]

        # put pack the previous ontology
        sheerka.add_ontology(context, ontology)

        assert sheerka.get_by_name("foo") == foo  # not [foo, foo2], foo2 is not seen !!!
        assert sheerka.get_by_name("bar") == bar
        assert get_concept_attrs(foo) == ["a", "b"]

        # sanity check
        sheerka.pop_ontology(context)
        assert sheerka.get_by_name("foo") == foo2
        assert sheerka.get_by_name("bar") == bar
        assert get_concept_attrs(foo) == ["a", "b", "c"]

    def test_adding_the_same_ontology_twice_has_no_effect(self):
        sheerka, context = self.init_test().unpack()

        sheerka.push_ontology(context, "#unit_test#")

        assert len(sheerka.om.ontologies) == 2

    def test_i_cannot_add_an_ontology_that_already_exists(self):
        sheerka, context = self.init_test().unpack()

        sheerka.push_ontology(context, "new ontology")

        res = sheerka.push_ontology(context, "#unit_test#")
        assert not res.status
        assert sheerka.isinstance(res.body, BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED)


class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):

    def test_root_folder_is_created_after_initialization(self):
        return_value = Sheerka().initialize(SHEERKA_TEST_FOLDER)
        assert return_value.status, "initialisation should be successful"
        assert os.path.exists(SHEERKA_TEST_FOLDER), "init folder should be created"

    def test_builtin_concepts_are_initialized(self):
        sheerka, context = self.init_test().unpack()
        service = sheerka.services[SheerkaConceptManager.NAME]
        for concept_name in AllBuiltinConcepts:
            # check that the concept is already in cache
            assert sheerka.om.ontologies[-1].cache_manager.has(service.CONCEPTS_BY_KEY_ENTRY, concept_name)

            # check that we can access it
            assert sheerka.om.get(service.CONCEPTS_BY_KEY_ENTRY, concept_name) is not None

        # I can get back data from the sdp when the cache is empty
        sheerka.push_ontology(context, 'new ontology')

        # caches are empty
        assert not service.has_id("1")
        assert not service.has_key(BuiltinConcepts.SHEERKA)

        assert sheerka.get_by_id("1") == sheerka  # use sdp

    def test_builtin_concepts_can_be_updated(self):
        sheerka = self.get_sheerka()
        service = sheerka.services[SheerkaConceptManager.NAME]
        before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)
        before_parsing.get_metadata().desc = "I have a description"
        before_parsing.get_metadata().full_serialization = True
        with sheerka.om.current_sdp().get_transaction("Test") as transac:
            transac.add(service.CONCEPTS_BY_KEY_ENTRY, before_parsing.key, before_parsing, use_ref=True)

        sheerka = self.get_sheerka()  # another fresh new instance
        before_parsing = sheerka.get_by_key(BuiltinConcepts.BEFORE_PARSING)

        assert before_parsing.get_metadata().desc == "I have a description"

    def test_i_first_look_in_local_cache(self):
        sheerka, context, concept = self.init_test().with_concepts("foo", create_new=True).unpack()
        sheerka.om.commit(context)

        sheerka.get_by_key(concept.key).new_property = "I have modified the concept in cache"

        from_cache = sheerka.get_by_key(concept.key)
        assert from_cache is not None
        assert from_cache.key == concept.key
        assert from_cache.new_property == "I have modified the concept in cache"

        # sdp instance is not modified
        sheerka.om.clear()
        from_sdp = sheerka.get_by_key(concept.key)
        assert from_sdp is not None
        assert from_sdp.key == concept.key
        assert not hasattr(from_sdp, "new_property")

    def test_i_can_retrieve_from_sdp_when_cache_is_reset(self):
        sheerka, context, concept = self.init_concepts(Concept("foo", body="1"))
        service = sheerka.services[SheerkaConceptManager.NAME]
        sheerka.om.commit(context)

        sheerka.om.clear()
        sheerka.get_by_key("foo")
        assert service.has_key("foo")

        # It's also updated when sdp returns more than one element
        concept2 = Concept("foo", body="2")
        sheerka.create_new_concept(context, concept2)
        sheerka.om.commit(context)

        sheerka.om.clear()
        assert len(sheerka.get_by_key("foo")) == 2
        assert service.has_key("foo")

        # updated when by_id
        sheerka.om.clear()
        assert sheerka.get_by_id("1001") == concept
        assert service.has_id("1001")

        sheerka.om.clear()
        assert sheerka.get_by_name("foo") == [concept, concept2]
        assert service.has_name("foo")

        sheerka.om.clear()
        assert sheerka.get_by_hash(concept.get_definition_hash()) == concept
        assert service.has_hash(concept.get_definition_hash())

    def test_get_by_key_retrieve_all_elements(self):
        sheerka, context, *concepts = self.init_test().with_concepts(
            Concept("foo", body="1"),
            Concept("foo", body="2"),
            create_new=True).unpack()
        sheerka.om.commit(context)

        sheerka.om.clear()
        sheerka.get_by_key("foo", "1001")  # I ask only for the one with id = "1001"

        # but the two keys are returned
        concepts = sheerka.get_by_key("foo")
        assert len(concepts) == 2
        assert concepts[0].id == "1001"
        assert concepts[1].id == "1002"

    def test_concept_node_parsing_is_initialized_at_startup(self):
        sheerka, context, foo, bar, baz = self.init_test().with_concepts(
            "foo",
            "bar",
            Concept("baz", definition="foo"),
            create_new=True).unpack()
        sheerka.om.commit(context)

        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002'],
            'c:|1001:': ['1003'],
            'foo': ['1001']}
        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002'],
            'foo': ['1001', '1003']
        }

        sheerka = self.get_sheerka()  # another instance
        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002'],
            'c:|1001:': ['1003'],
            'foo': ['1001']}
        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
            'bar': ['1002'],
            'foo': ['1001', '1003']
        }

    def test_i_can_remember_ontologies_when_sheerka_is_recreated(self):
        sheerka, context = self.init_test().unpack()

        sheerka.push_ontology(context, "to remove")
        sheerka.pop_ontology(context)
        sheerka.push_ontology(context, "new ontology")
        sheerka.push_ontology(context, "another ontology")

        # sanity check
        ontologies_names = [o.name for o in sheerka.om.ontologies]
        assert ontologies_names == ['another ontology', 'new ontology', '#unit_test#', '__default__']

        # get new instance
        sheerka = self.new_sheerka_instance(False)

        # make sure that ontologies are recreated
        ontologies_names = [o.name for o in sheerka.om.ontologies]
        assert ontologies_names == ['another ontology', 'new ontology', '#unit_test#', '__default__']

    def test_adding_the_same_ontology_twice_has_no_effect(self):
        sheerka, context = self.init_test().unpack()

        sheerka = self.new_sheerka_instance(False)  # new instance that remembers the top layer ontology
        sheerka.push_ontology(context, "#unit_test#")

        assert len(sheerka.om.ontologies) == 2