We can now use concept sets in BNF definitions

core/builtin_concepts.py

@@ -29,6 +29,7 @@ class BuiltinConcepts(Enum):
     SUCCESS = "success"
     ERROR = "error"
     UNKNOWN_CONCEPT = "unknown concept"  # the request concept is not recognized
+    CANNOT_RESOLVE_CONCEPT = "cannot resolve concept"  # when too many concepts with the same name
     RETURN_VALUE = "return value"  # a value is returned
     CONCEPT_TOO_LONG = "concept too long"  # concept cannot be processed by exactConcept parser
     NEW_CONCEPT = "new concept"  # when a new concept is added
@@ -48,6 +49,7 @@ class BuiltinConcepts(Enum):
     EVALUATOR_PRE_PROCESS = "evaluator pre process"  # used modify / tweak behaviour of evaluators
     CONCEPT_EVAL_REQUESTED = "concept eval requested"
     REDUCE_REQUESTED = "reduce requested"  # remove meaningless error when possible
+    NOT_A_SET = "not a set"  # the concept has no entry in sets
 
     NODE = "node"
     GENERIC_NODE = "generic node"
@@ -79,6 +81,7 @@ BuiltinUnique = [
 BuiltinErrors = [str(e) for e in {
     BuiltinConcepts.ERROR,
     BuiltinConcepts.UNKNOWN_CONCEPT,
+    BuiltinConcepts.CANNOT_RESOLVE_CONCEPT,
     BuiltinConcepts.CONCEPT_TOO_LONG,
     BuiltinConcepts.UNKNOWN_PROPERTY,
     BuiltinConcepts.TOO_MANY_SUCCESS,
@@ -87,6 +90,7 @@ BuiltinErrors = [str(e) for e in {
     BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
     BuiltinConcepts.CONCEPT_EVAL_ERROR,
     BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
+    BuiltinConcepts.NOT_A_SET,
 }]
 
 """
@@ -124,6 +128,16 @@ class ErrorConcept(Concept):
         return f"({self.id}){self.name}: {self.body}"
 
 
+class UnknownConcept(Concept):
+    def __init__(self, metadata=None):
+        super().__init__(BuiltinConcepts.UNKNOWN_CONCEPT, True, False, BuiltinConcepts.UNKNOWN_CONCEPT)
+        self.set_metadata_value(ConceptParts.BODY, metadata)
+        self.metadata.is_evaluated = True
+
+    def __repr__(self):
+        return f"({self.id}){self.name}: {self.body}"
+
+
 class ReturnValueConcept(Concept):
     """
     This class represents the result of a data flow processing

core/sheerka.py

@@ -1,4 +1,5 @@
-from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique
+from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
+    UnknownConcept
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW, DoNotResolve
 from parsers.BaseParser import BaseParser
 from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderDuplicateKeyError
@@ -17,6 +18,7 @@ CONCEPT_EVALUATION_STEPS = [
 
 CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
 DEBUG_TAB_SIZE = 4
+GROUP_PREFIX = 'All_'
 
 
 class Sheerka(Concept):
@@ -25,6 +27,7 @@ class Sheerka(Concept):
     """
 
     CONCEPTS_ENTRY = "All_Concepts"  # to store all the concepts
+    CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"
     CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions"  # to store definitions (bnf) of concepts
     BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts"  # sequential key for builtin concepts
     USER_CONCEPTS_KEYS = "User_Concepts"  # sequential key for user defined concepts
@@ -40,7 +43,8 @@ class Sheerka(Concept):
         # They are used as a footprint for instantiation
         # Except of source when the concept is supposed to be unique
         # key is the key of the concept (not the name or the id)
-        self.concepts_cache = {}
+        self.cache_by_key = {}
+        self.cache_by_id = {}
 
         # cache for concept definitions,
         # Primarily used for unit test that does not have access to sdp
@@ -176,6 +180,20 @@ class Sheerka(Concept):
 
         self.concepts_grammars = lexer_parser.concepts_grammars
 
+    def reset_cache(self, filter_to_use=None):
+        """
+        reset the different cache that exists
+        :param filter_to_use:
+        :return:
+        """
+        if filter_to_use is None:
+            self.cache_by_key = {}
+            self.cache_by_id = {}
+        else:
+            raise NotImplementedError()
+
+        return self
+
     def evaluate_user_input(self, text: str, user_name="kodjo"):
         """
         Note to KSI: If you try to add execution context to this function,
@@ -455,6 +473,7 @@ class Sheerka(Concept):
     def set_id_if_needed(self, obj: Concept, is_builtin: bool):
         """
         Set the key for the concept if needed
+        For test purpose only !!!!!
         :param obj:
         :param is_builtin:
         :return:
@@ -509,7 +528,12 @@ class Sheerka(Concept):
 
         # save the new concept in sdp
         try:
+            # TODO : needs to make these calls atomic (or at least one single call)
             self.sdp.add(context.event.get_digest(), self.CONCEPTS_ENTRY, concept, use_ref=True)
+            self.sdp.add(context.event.get_digest(),
+                         self.CONCEPTS_BY_ID_ENTRY,
+                         {concept.id: concept.get_digest()},
+                         is_ref=True)
             if concepts_definitions is not None:
                 self.sdp.set(context.event.get_digest(),
                              self.CONCEPTS_DEFINITIONS_ENTRY,
@@ -523,7 +547,8 @@ class Sheerka(Concept):
                 error.args[0])
 
         # Updates the caches
-        self.concepts_cache[concept.key] = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.key)
+        self.cache_by_key[concept.key] = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.key)  # reset from sdp
+        self.cache_by_id[concept.id] = concept  # no need to reset
         if init_ret_value is not None and init_ret_value.status:
             self.concepts_grammars = init_ret_value.body
 
@@ -548,7 +573,7 @@ class Sheerka(Concept):
         assert concept_set.id
 
         try:
-            ret = self.sdp.add_unique(context.event.get_digest(), "All_" + str(concept_set.id), concept.id)
+            ret = self.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id)
             if ret == (None, None):  # concept already in set
                 return self.ret(
                     self.add_concept_to_set.__name__,
@@ -560,6 +585,23 @@ class Sheerka(Concept):
             context.log_error(logger, "Failed to add to set.", who=self.add_concept_to_set.__name__)
             return self.ret(self.create_new_concept.__name__, False, ErrorConcept(error), error.args[0])
 
+    def get_set_elements(self, concept):
+        """
+        Concept is supposed to be a set
+        Returns all elements if the set
+        :param concept:
+        :return:
+        """
+
+        assert concept.id
+
+        ids = self.sdp.get_safe(GROUP_PREFIX + concept.id)
+        if ids is None:
+            return self.new(BuiltinConcepts.NOT_A_SET, body=concept)
+
+        elements = [self.get_by_id(element_id) for element_id in ids]
+        return elements
+
     def initialize_concept_asts(self, context, concept: Concept, logger=None):
         """
         Updates the codes of the newly created concept
@@ -608,13 +650,13 @@ class Sheerka(Concept):
                     sub_context.add_values(return_values=res)
 
         # Updates the cache of concepts when possible
-        if concept.key in self.concepts_cache:
+        if concept.key in self.cache_by_key:
-            entry = self.concepts_cache[concept.key]
+            entry = self.cache_by_key[concept.key]
             if isinstance(entry, list):
                 # TODO : manage when there are multiple entries
                 pass
             else:
-                self.concepts_cache[concept.key].compiled = concept.compiled
+                self.cache_by_key[concept.key].compiled = concept.compiled
 
     def evaluate_concept(self, context, concept: Concept, logger=None):
         """
@@ -751,7 +793,11 @@ class Sheerka(Concept):
         if concept.key is None:
             raise KeyError()
 
-        self.concepts_cache[concept.key] = concept
+        self.cache_by_key[concept.key] = concept
+
+        if concept.id:
+            self.cache_by_id[concept.id] = concept
+
         return concept
 
     def get(self, concept_key, concept_id=None):
@@ -771,7 +817,7 @@ class Sheerka(Concept):
             concept_key = str(concept_key)
 
         # first search in cache
-        result = self.concepts_cache[concept_key] if concept_key in self.concepts_cache else \
+        result = self.cache_by_key[concept_key] if concept_key in self.cache_by_key else \
             self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key)
 
         if result and (concept_id is None or not isinstance(result, list)):
@@ -785,14 +831,18 @@ class Sheerka(Concept):
             else:
                 return result
 
-        # else return new Unknown concept
+        metadata = [("key", concept_key), ("id", concept_id)] if concept_id else ("key", concept_key)
-        # Note that I don't call the new() method to prevent cyclic call
+        return self._get_unknown(metadata)
-        unknown_concept = Concept()
+
-        template = self.concepts_cache[str(BuiltinConcepts.UNKNOWN_CONCEPT)]
+    def get_by_id(self, concept_id):
-        unknown_concept.update_from(template)
+        if concept_id is None:
-        unknown_concept.set_metadata_value(ConceptParts.BODY, concept_key)
+            return ErrorConcept("Concept id is undefined.")
-        unknown_concept.metadata.is_evaluated = True
+
-        return unknown_concept
+        # first search in cache
+        result = self.cache_by_id[concept_id] if concept_id in self.cache_by_id else \
+            self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id)
+
+        return result or self._get_unknown(('id', concept_id))
 
     def new(self, concept_key, **kwargs):
         """
@@ -945,14 +995,20 @@ class Sheerka(Concept):
         if isinstance(a, BuiltinConcepts):  # common KSI error ;-)
             raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept")
 
-        if not isinstance(a, Concept):
+        assert isinstance(a, Concept)
-            return False
+        assert isinstance(b, Concept)
 
-        b_key = b.key if isinstance(b, Concept) else str(b)
+        # TODO, first check the 'isa' property of a
 
-        # TODO : manage when a is the list of all possible b
+        return self.sdp.exists(GROUP_PREFIX + b.id, a.id)
-        # for example, if a is a color, it will be found the entry 'All_Colors'
+
-        return a.key == b_key
+    def isagroup(self, concept):
+        """True if exists All_<concept_id> in sdp"""
+        if not concept.id:
+            return None
+
+        res = self.sdp.get_safe(GROUP_PREFIX + concept.id)
+        return res is not None
 
     def get_evaluator_name(self, name):
         if self.evaluators_prefix is None:
@@ -1029,6 +1085,30 @@ class Sheerka(Concept):
                 self.log.info(f"digest : {c.get_digest()}")
                 first = False
 
+    @staticmethod
+    def _get_unknown(metadata):
+        """
+        Returns the concept 'UnknownConcept' for a requested id or key
+        Note that I don't call the new() method to prevent cyclic call
+        :param metadata:
+        :return:
+        """
+
+        # metadata is a list of tuple that contains the known metadata for this concept
+        # ex : (key, 'not_found)
+        # or
+        # (id, invalid_id)
+        #
+        # the metadata can be a list, if several attributes where given
+        # (key, 'not_found), (id, invalid_id)
+
+        unknown_concept = UnknownConcept()
+        unknown_concept.set_metadata_value(ConceptParts.BODY, metadata)
+        for meta in (metadata if isinstance(metadata, list) else [metadata]):
+            unknown_concept.set_prop(meta[0], meta[1])
+        unknown_concept.metadata.is_evaluated = True
+        return unknown_concept
+
     @staticmethod
     def get_builtins_classes_as_dict():
         res = {}
@@ -1145,6 +1225,25 @@ class ExecutionContext:
             self.values[k] = v
         return self
 
+    def get_concept(self, key):
+        # search in obj
+        if isinstance(self.obj, Concept):
+            if self.obj.key == key:
+                return self.obj
+            for prop in self.obj.props:
+                if prop == key:
+                    value = self.obj.props[prop].value
+                    if isinstance(value, Concept):
+                        return value
+
+        # search in concepts
+        if self.concepts:
+            for k, c in self.concepts.items():
+                if k == key:
+                    return c
+
+        return self.sheerka.get(key)
+
     def new_concept(self, key, **kwargs):
         # search in obj
         if self.obj:

parsers/BaseParser.py

@@ -1,4 +1,7 @@
 from dataclasses import dataclass
+
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
 from core.tokenizer import TokenKind, Keywords
 from core.sheerka_logger import get_logger
 import logging
@@ -83,6 +86,17 @@ class BaseParser:
             value = context.return_value_to_str(r)
             context.log(self.log, f"  Recognized '{value}'", self.name)
 
+    def get_return_value_body(self, sheerka, source, tree, try_parse):
+        if len(self.error_sink) == 1 and isinstance(self.error_sink[0], Concept):
+            return self.error_sink[0]
+
+        return sheerka.new(
+            BuiltinConcepts.PARSER_RESULT,
+            parser=self,
+            source=source,
+            body=self.error_sink if self.has_error else tree,
+            try_parsed=try_parse)
+
     @staticmethod
     def get_text_from_tokens(tokens, custom_switcher=None):
         if tokens is None:

parsers/BnfParser.py

@@ -127,15 +127,12 @@ class BnfParser(BaseParser):
         except LexerError as e:
             self.add_error(e, False)
 
+        value = self.get_return_value_body(context.sheerka, self.source, tree, tree)
+
         ret = self.sheerka.ret(
             self.name,
             not self.has_error,
-            self.sheerka.new(
+            value)
-                BuiltinConcepts.PARSER_RESULT,
-                parser=self,
-                source=self.source,
-                body=self.error_sink if self.has_error else tree,
-                try_parsed=tree))
 
         return ret
 
@@ -231,15 +228,26 @@ class BnfParser(BaseParser):
 
         if token.type == TokenKind.IDENTIFIER:
             self.next_token()
-            return ConceptExpression(token.value)
+
-            # concept = self.sheerka.get(str(token.value))
+            concept_name = str(token.value)
-            # if hasattr(concept, "__iter__") or self.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
+
-            #     self.add_error(CannotResolveConceptNode(str(token.value)))
+            # we are trying to match against a concept which is still under construction !
-            #     self.next_token()
+            # (for example of recursive bnf definition)
-            #     return None
+            if self.context.obj and hasattr(self.context.obj, "name"):
-            # else:
+                if concept_name == str(self.context.obj.name):
-            #     self.next_token()
+                    return ConceptExpression(concept_name)
-            #     return concept
+
+            concept = self.context.get_concept(concept_name)
+            if not self.sheerka.is_known(concept):
+                self.add_error(concept)
+                return None
+            elif hasattr(concept, "__iter__"):
+                self.add_error(
+                    self.sheerka.new(BuiltinConcepts.CANNOT_RESOLVE_CONCEPT,
+                                     body=("key", concept_name)))
+                return None
+            else:
+                return concept
 
         ret = StrMatch(core.utils.strip_quotes(token.value))
         self.next_token()

parsers/ConceptLexerParser.py

@@ -269,8 +269,12 @@ class ConceptExpression(ParsingExpression):
         if isinstance(self.concept, Concept):
             return self.concept.name == other.concept.name
 
+        # when it's only the name of the concept
         return self.concept == other.concept
 
+    def __hash__(self):
+        return hash((self.concept, self.rule_name))
+
     @staticmethod
     def get_parsing_expression_from_name(name):
         tokens = Tokenizer(name)
@@ -302,6 +306,29 @@ class ConceptExpression(ParsingExpression):
         return NonTerminalNode(self, node.start, node.end, parser.tokens[node.start: node.end + 1], [node])
 
 
+class ConceptGroupExpression(ConceptExpression):
+    def _parse(self, parser):
+        to_match = parser.get_concept(self.concept) if isinstance(self.concept, str) else self.concept
+        if parser.sheerka.isinstance(to_match, BuiltinConcepts.UNKNOWN_CONCEPT):
+            return None
+
+        self.concept = to_match  # Memoize
+
+        if to_match not in parser.concepts_grammars:
+            concepts_in_group = parser.sheerka.get_set_elements(self.concept)
+            nodes = [ConceptExpression(c, rule_name=c.name) for c in concepts_in_group]
+            expr = OrderedChoice(nodes)
+            expr.nodes = nodes
+            node = expr.parse(parser)
+        else:
+            node = parser.concepts_grammars[to_match].parse(parser)
+
+        if node is None:
+            return None
+
+        return NonTerminalNode(self, node.start, node.end, parser.tokens[node.start: node.end + 1], [node])
+
+
 class Sequence(ParsingExpression):
     """
     Will match sequence of parser expressions in exact order they are defined.
@@ -667,7 +694,10 @@ class ConceptLexerParser(BaseParser):
         # A copy must be created
         def inner_get_model(expression):
             if isinstance(expression, Concept):
-                ret = ConceptExpression(expression, rule_name=expression.name)
+                if self.sheerka.isagroup(expression):
+                    ret = ConceptGroupExpression(expression, rule_name=expression.name)
+                else:
+                    ret = ConceptExpression(expression, rule_name=expression.name)
                 concepts_to_resolve.add(expression)
             elif isinstance(expression, ConceptExpression):
                 if expression.rule_name is None or expression.rule_name == "":

parsers/DefaultParser.py

@@ -210,12 +210,13 @@ class DefaultParser(BaseParser):
         if self.has_error and isinstance(self.error_sink[0], CannotHandleErrorNode):
             body = self.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=self.error_sink)
         else:
-            body = self.sheerka.new(
+            body = self.get_return_value_body(context.sheerka, text, tree, tree)
-                BuiltinConcepts.PARSER_RESULT,
+            # body = self.sheerka.new(
-                parser=self,
+            #     BuiltinConcepts.PARSER_RESULT,
-                source=text,
+            #     parser=self,
-                body=self.error_sink if self.has_error else tree,
+            #     source=text,
-                try_parsed=tree)
+            #     body=self.error_sink if self.has_error else tree,
+            #     try_parsed=tree)
 
         ret = self.sheerka.ret(
             self.name,
@@ -261,7 +262,7 @@ class DefaultParser(BaseParser):
         concept_found.name = self.get_concept_name(first_token, tokens_found_by_parts)
 
         # get the definition
-        concept_found.definition = self.get_concept_definition(tokens_found_by_parts)
+        concept_found.definition = self.get_concept_definition(concept_found, tokens_found_by_parts)
 
         # get the ASTs for the remaining parts
         asts_found_by_parts = self.get_concept_parts(tokens_found_by_parts)
@@ -359,7 +360,7 @@ class DefaultParser(BaseParser):
         name_node = NameNode(name_tokens[name_first_token_index:])  # skip the first token
         return name_node
 
-    def get_concept_definition(self, tokens_found_by_parts):
+    def get_concept_definition(self, current_concept_def, tokens_found_by_parts):
         if tokens_found_by_parts[Keywords.FROM] is None:
             return NotInitializedNode()
 
@@ -373,7 +374,7 @@ class DefaultParser(BaseParser):
             return NotInitializedNode()
 
         regex_parser = BnfParser()
-        with self.context.push(self.name) as sub_context:
+        with self.context.push(self.name, obj=current_concept_def) as sub_context:
             parsing_result = regex_parser.parse(sub_context, tokens)
             sub_context.add_values(return_values=parsing_result)
 

sdp/sheerkaDataProvider.py

@@ -346,7 +346,7 @@ class SheerkaDataProvider:
     def is_reference(obj):
         return isinstance(obj, str) and obj.startswith(SheerkaDataProvider.REF_PREFIX)
 
-    def add(self, event_digest: str, entry, obj, allow_multiple=True, use_ref=False):
+    def add(self, event_digest: str, entry, obj, allow_multiple=True, use_ref=False, is_ref=False):
         """
         Adds obj to the entry 'entry'
         :param event_digest: digest of the event that triggers the modification of the state
@@ -359,6 +359,12 @@ class SheerkaDataProvider:
         :return: (entry, key) to retrieve the object
         """
 
+        if use_ref and is_ref:
+            raise SheerkaDataProviderError("Cannot use use_ref and is_ref at the same time", None)
+
+        if is_ref and not isinstance(obj, dict):
+            raise SheerkaDataProviderError("is_ref can only be used with dictionaries", obj)
+
         snapshot = self.get_snapshot()
         state = self.load_state(snapshot)
 
@@ -387,6 +393,10 @@ class SheerkaDataProvider:
             obj.set_digest(self.save_obj(obj.obj))
             obj.obj = self.REF_PREFIX + obj.get_digest()
 
+        if is_ref:
+            for k, v in obj.obj.items():
+                obj.obj[k] = self.REF_PREFIX + v
+
         state.update(entry, obj)
 
         new_snapshot = self.save_state(state)
@@ -427,16 +437,24 @@ class SheerkaDataProvider:
         self.set_snapshot(new_snapshot)
         return (None if already_exist else entry), None
 
-    def set(self, event_digest, entry, obj, use_ref=False):
+    def set(self, event_digest, entry, obj, use_ref=False, is_ref=False):
         """
         Add or replace an entry. The entry is reinitialized.
         If the previous value was dict, all keys are lost
         :param event_digest:
         :param entry:
         :param obj:
-        :param use_ref:
+        :param use_ref: Do not save obj in State (save it under objects), use_ref in State
+        :param is_ref: obj is supposed to be a reference
         :return:
         """
+
+        if use_ref and is_ref:
+            raise SheerkaDataProviderError("Cannot use use_ref and is_ref at the same time", None)
+
+        if is_ref and not isinstance(obj, dict):
+            raise SheerkaDataProviderError("is_ref can only be used with dictionaries", obj)
+
         snapshot = self.get_snapshot()
         state = self.load_state(snapshot)
 
@@ -447,6 +465,10 @@ class SheerkaDataProvider:
         key = self.get_obj_key(obj)
         obj = self.save_ref_if_needed(use_ref, obj)
 
+        if is_ref:
+            for k, v in obj.items():
+                obj[k] = self.REF_PREFIX + v
+
         state.data[entry] = obj if key is None else {key: obj}
 
         new_snapshot = self.save_state(state)

tests/test_BnfParser.py

@@ -1,5 +1,6 @@
 import pytest
 
+from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
 from core.tokenizer import Tokenizer, TokenKind, LexerError
@@ -17,6 +18,11 @@ def get_context():
     return ExecutionContext("sheerka", Event(), sheerka)
 
 
+class ClassWithName():
+    def __init__(self, name):
+        self.name = name
+
+
 @pytest.mark.parametrize("expression, expected", [
     ("'str'", StrMatch("str")),
     ("1", StrMatch("1")),
@@ -41,12 +47,6 @@ def get_context():
     ("(1|*) +", Sequence(OrderedChoice(StrMatch("1"), StrMatch("*")), StrMatch("+"))),
     ("1, :&", Sequence(StrMatch("1"), StrMatch(","), StrMatch(":"), StrMatch("&"))),
     ("(1 )", StrMatch("1")),
-    ("foo", ConceptExpression("foo")),
-    ("foo*", ZeroOrMore(ConceptExpression("foo"))),
-    ("foo 'and' bar+", Sequence(ConceptExpression("foo"), StrMatch("and"), OneOrMore(ConceptExpression("bar")))),
-    ("foo | bar?", OrderedChoice(ConceptExpression("foo"), Optional(ConceptExpression("bar")))),
-    ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), ConceptExpression("var"))),
-    ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), ConceptExpression("var"))),
     ("'str'=var", StrMatch("str", rule_name="var")),
     ("'foo'?=var", Optional(StrMatch("foo"), rule_name="var")),
     ("('foo'?)=var", Optional(StrMatch("foo"), rule_name="var")),
@@ -75,6 +75,47 @@ def test_i_can_parse_regex(expression, expected):
     assert res.value.source == expression
 
 
+@pytest.mark.parametrize("expression, expected", [
+    ("foo", Concept("foo").init_key()),
+    ("foo*", ZeroOrMore(Concept("foo").init_key())),
+    ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))),
+    ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))),
+    ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+    ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+])
+def test_i_can_parse_regex_with_concept(expression, expected):
+    foo = Concept("foo")
+    bar = Concept("bar")
+    var = Concept("var")
+    context = get_context()
+
+    for c in (foo, bar, var):
+        context.sheerka.add_in_cache(c)
+    parser = BnfParser()
+    res = parser.parse(context, Tokenizer(expression))
+
+    assert not parser.has_error
+    assert res.status
+    assert res.value.value == expected
+    assert res.value.source == expression
+
+
+def test_i_can_parse_regex_with_concept_when_the_concept_is_still_under_definition():
+    expression = "foo"
+    expected = ConceptExpression("foo")
+
+    context = get_context()
+    context.obj = ClassWithName("foo")
+
+    parser = BnfParser()
+    res = parser.parse(context, Tokenizer(expression))
+
+    assert not parser.has_error
+    assert res.status
+    assert res.value.value == expected
+    assert res.value.source == expression
+
+
 @pytest.mark.parametrize("expression, error", [
     ("1 ", UnexpectedEndOfFileError()),
     ("1|", UnexpectedEndOfFileError()),
@@ -117,3 +158,28 @@ def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text():
     res = concept_parser.parse(context, "twenty")
     assert res.status
     assert res.value.body == [cnode("foo", 0, 0, "twenty")]
+
+
+def test_i_cannot_parse_when_too_many_concepts():
+    foo1 = Concept(name="foo", body="1")
+    foo2 = Concept(name="foo", body="2")
+    context = get_context()
+    context.sheerka.cache_by_key["foo"] = [foo1, foo2]
+
+    regex_parser = BnfParser()
+    res = regex_parser.parse(context, "foo")
+
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
+    assert res.value.body == ('key', 'foo')
+
+
+def test_i_cannot_parse_when_unknown_concept():
+    context = get_context()
+
+    regex_parser = BnfParser()
+    res = regex_parser.parse(get_context(), "foo")
+
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body == ('key', 'foo')

tests/test_ConceptLexerParser.py

@@ -61,6 +61,7 @@ def get_expected(concept, text=None):
     c = Concept(name=concept.name)
     c.compiled[ConceptParts.BODY] = DoNotResolve(text or concept.name)
     c.init_key()
+    c.metadata.id = concept.id
     return c
 
 
@@ -606,9 +607,6 @@ def test_i_can_parse_concept_reference_that_is_not_in_grammar():
     grammar = {foo: Sequence("twenty", OrderedChoice(one, two))}
     context, parser = init([one, two, foo], grammar)
 
-    parser = ConceptLexerParser()
-    parser.initialize(context, grammar)
-
     res = parser.parse(context, "twenty two")
     assert res.status
     assert res.value.body == [cnode("foo", 0, 2, "twenty two")]
@@ -621,6 +619,46 @@ def test_i_can_parse_concept_reference_that_is_not_in_grammar():
     assert res.value.body == [cnode("foo", 0, 2, "twenty one")]
 
 
+def test_i_can_parse_concept_reference_that_is_group():
+    """
+    if one is  number, then number is a 'group'
+    a group can be found under the sdp entry 'all_<group_name>'
+    """
+
+    context = get_context()
+    one = Concept(name="one")
+    two = Concept(name="two")
+    number = Concept(name="number")
+    foo = Concept(name="foo")
+    for c in [one, two, number, foo]:
+        context.sheerka.set_id_if_needed(c, False)
+        context.sheerka.add_in_cache(c)
+
+    context.sheerka.add_concept_to_set(context, one, number)
+    context.sheerka.add_concept_to_set(context, two, number)
+
+    grammar = {foo: Sequence("twenty", number)}
+
+    parser = ConceptLexerParser()
+    parser.initialize(context, grammar)
+
+    res = parser.parse(context, "twenty two")
+    assert res.status
+    assert res.value.body == [cnode("foo", 0, 2, "twenty two")]
+    concept_found = res.value.body[0].concept
+    assert cbody(concept_found) == DoNotResolve("twenty two")
+    assert cprop(concept_found, "two") == get_expected(two, "two")
+    assert cprop(concept_found, "number") == get_expected(number, get_expected(two, "two"))
+
+    res = parser.parse(context, "twenty one")
+    assert res.status
+    assert res.value.body == [cnode("foo", 0, 2, "twenty one")]
+    concept_found = res.value.body[0].concept
+    assert cbody(concept_found) == DoNotResolve("twenty one")
+    assert cprop(concept_found, "one") == get_expected(one, "one")
+    assert cprop(concept_found, "number") == get_expected(number, get_expected(one, "one"))
+
+
 def test_i_can_parse_zero_or_more():
     foo = Concept(name="foo")
     grammar = {foo: ZeroOrMore("one")}

tests/test_DefaultParser.py

@@ -2,6 +2,7 @@ import pytest
 import ast
 
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
+from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
 from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptExpression
 from parsers.PythonParser import PythonParser, PythonNode
@@ -191,8 +192,7 @@ def concept add one to a as
     return_value = res.value
 
     assert not res.status
-    assert isinstance(return_value, ParserResultConcept)
+    assert context.sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
-    assert sheerka.isinstance(return_value.value[0], BuiltinConcepts.TOO_MANY_ERRORS)
 
 
 def test_name_is_mandatory():
@@ -239,8 +239,7 @@ def test_i_can_detect_error_in_declaration(text):
     return_value = res.value
 
     assert not res.status
-    assert isinstance(return_value, ParserResultConcept)
+    assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
-    assert sheerka.isinstance(return_value.value[0], BuiltinConcepts.TOO_MANY_ERRORS)
 
 
 def test_new_line_is_not_allowed_in_the_name():
@@ -255,11 +254,15 @@ def test_new_line_is_not_allowed_in_the_name():
 
 
 def test_i_can_parse_def_concept_from_regex():
+    context = get_context()
+    a_concept = Concept("a_concept")
+    context.sheerka.add_in_cache(a_concept)
+
     text = "def concept name from bnf a_concept | 'a_string' as __definition[0]"
     parser = DefaultParser()
-    res = parser.parse(get_context(), text)
+    res = parser.parse(context, text)
     node = res.value.value
-    definition = OrderedChoice(ConceptExpression("a_concept"), StrMatch("a_string"))
+    definition = OrderedChoice(a_concept, StrMatch("a_string"))
     parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition)
     expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result)
 
@@ -270,6 +273,18 @@ def test_i_can_parse_def_concept_from_regex():
     assert node == expected
 
 
+def test_i_can_parse_def_concept_where_bnf_references_itself():
+    context = get_context()
+    a_concept = Concept("a_concept")
+    context.sheerka.add_in_cache(a_concept)
+
+    text = "def concept name from bnf 'a' + name?"
+    parser = DefaultParser()
+    parser.parse(context, text)
+
+    assert not parser.has_error
+
+
 def test_i_can_detect_empty_bnf_declaration():
     text = "def concept name from bnf as __definition[0]"
 
@@ -339,3 +354,15 @@ def test_i_cannot_parse_when_tokenizer_fails(text, error_msg, error_text):
     assert isinstance(res.body.body[0], LexerError)
     assert res.body.body[0].message == error_msg
     assert res.body.body[0].text == error_text
+
+
+def test_i_cannot_parse_bnf_definition_referencing_unknown_concept():
+    context = get_context()
+    text = "def concept name from bnf unknown"
+
+    parser = DefaultParser()
+    res = parser.parse(context, text)
+
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body == ("key", "unknown")

tests/test_ExactConceptParser.py

@@ -90,7 +90,7 @@ def test_i_can_recognize_a_concept_with_variables():
 def test_i_can_recognize_a_concept_with_duplicate_variables():
     context = get_context()
     concept = get_concept("a + b + a", ["a", "b"])
-    context.sheerka.concepts_cache[concept.key] = concept
+    context.sheerka.cache_by_key[concept.key] = concept
     source = "10 + 5 + 10"
     results = ExactConceptParser().parse(context, source)
 

tests/test_PythonParser.py

@@ -50,8 +50,6 @@ def test_i_can_parse_from_tokens(text, expected):
     "foo = 'name"
 ])
 def test_i_can_detect_error(text):
-    text = "1+"
-
     parser = PythonParser()
     res = parser.parse(get_context(), text)
 

tests/test_sheerka.py

@@ -76,13 +76,13 @@ def test_i_can_list_builtin_concepts():
 
 def test_builtin_concepts_are_initialized():
     sheerka = get_sheerka(skip_builtins_in_db=False)
-    assert len(sheerka.concepts_cache) == len(BuiltinConcepts)
+    assert len(sheerka.cache_by_key) == len(BuiltinConcepts)
     for concept_name in BuiltinConcepts:
-        assert str(concept_name) in sheerka.concepts_cache
+        assert str(concept_name) in sheerka.cache_by_key
         assert sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, str(concept_name)) is not None
 
     for key, concept_class in sheerka.get_builtins_classes_as_dict().items():
-        assert isinstance(sheerka.concepts_cache[key], concept_class)
+        assert isinstance(sheerka.cache_by_key[key], concept_class)
 
 
 def test_builtin_concepts_can_be_updated():
@@ -113,7 +113,8 @@ def test_i_can_add_a_concept():
     assert concept_found.key == "__var__0 + __var__1"
     assert concept_found.id == "1001"
 
-    assert concept.key in sheerka.concepts_cache
+    assert concept.key in sheerka.cache_by_key
+    assert concept.id in sheerka.cache_by_id
     assert sheerka.sdp.io.exists(
         sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest()))
 
@@ -154,7 +155,10 @@ def test_i_can_get_a_newly_created_concept():
 
     from_cache = sheerka.get(concept.key)
     assert from_cache is not None
-    assert from_cache.key == concept.key
+    assert from_cache == concept
+
+    from_cache = sheerka.get_by_id(concept.id)
+    assert from_cache is not None
     assert from_cache == concept
 
 
@@ -163,7 +167,7 @@ def test_i_first_look_in_local_cache():
     concept = get_default_concept()
 
     sheerka.create_new_concept(get_context(sheerka), concept)
-    sheerka.concepts_cache[concept.key].pre = "I have modified the concept in cache"
+    sheerka.cache_by_key[concept.key].pre = "I have modified the concept in cache"
 
     from_cache = sheerka.get(concept.key)
     assert from_cache is not None
@@ -180,12 +184,29 @@ def test_i_can_get_a_known_concept_when_not_in_cache():
     concept = get_default_concept()
     sheerka.create_new_concept(get_context(sheerka), concept)
 
-    sheerka.concepts_cache = {}  # reset the cache
+    sheerka.cache_by_key = {}  # reset the cache
     loaded = sheerka.get(concept.key)
 
     assert loaded is not None
     assert loaded == concept
 
+    # I can also get it by its id
+    loaded = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
+    assert loaded is not None
+    assert loaded == concept
+
+
+def test_i_can_get_a_concept_by_its_id():
+    sheerka = get_sheerka()
+    concept = get_default_concept()
+    sheerka.create_new_concept(get_context(sheerka), concept)
+
+    sheerka.cache_by_key = {}  # reset the cache
+    loaded = sheerka.get_by_id(concept.id)
+
+    assert loaded is not None
+    assert loaded == concept
+
 
 def test_i_can_get_list_of_concept_when_same_key_when_no_cache():
     sheerka = get_sheerka()
@@ -198,7 +219,7 @@ def test_i_can_get_list_of_concept_when_same_key_when_no_cache():
 
     assert res1.value.body.key == res2.value.body.key  # same key
 
-    sheerka.concepts_cache = {}  # reset the cache
+    sheerka.cache_by_key = {}  # reset the cache
 
     result = sheerka.get(concept1.key)
     assert len(result) == 2
@@ -217,7 +238,7 @@ def test_i_can_get_list_of_concept_when_same_key_when_cache():
 
     assert res1.value.body.key == res2.value.body.key  # same key
 
-    # sheerka.concepts_cache = {}  # Do not reset the cache
+    # sheerka.cache_by_key = {}  # Do not reset the cache
 
     result = sheerka.get(concept1.key)
     assert len(result) == 2
@@ -280,14 +301,25 @@ def test_i_cannot_get_when_key_is_none():
     assert res.body == "Concept key is undefined."
 
 
-def test_unknown_concept_is_return_when_the_concept_is_not_found():
+def test_unknown_concept_is_return_when_the_concept_key_is_not_found():
     sheerka = get_sheerka()
 
-    loaded = sheerka.get("concept_that_does_not_exist")
+    loaded = sheerka.get("key_that_does_not_exist")
 
     assert loaded is not None
     assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert loaded.body == "concept_that_does_not_exist"
+    assert loaded.body == ("key", "key_that_does_not_exist")
+    assert loaded.metadata.is_evaluated
+
+
+def test_unknown_concept_is_return_when_the_concept_id_is_not_found():
+    sheerka = get_sheerka()
+
+    loaded = sheerka.get_by_id("id_that_does_not_exist")
+
+    assert loaded is not None
+    assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert loaded.body == ("id", "id_that_does_not_exist")
     assert loaded.metadata.is_evaluated
 
 
@@ -372,7 +404,7 @@ def test_i_cannot_instantiate_an_unknown_concept():
     new = sheerka.new("fake_concept")
 
     assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert new.body == "fake_concept"
+    assert new.body == ('key', 'fake_concept')
 
 
 def test_i_cannot_instantiate_with_invalid_id():
@@ -383,7 +415,7 @@ def test_i_cannot_instantiate_with_invalid_id():
     new = sheerka.new(("foo", "invalid_id"))
 
     assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert new.body == "foo"
+    assert new.body == [('key', 'foo'), ('id', 'invalid_id')]
 
 
 def test_i_cannot_instantiate_with_invalid_key():
@@ -394,7 +426,7 @@ def test_i_cannot_instantiate_with_invalid_key():
     new = sheerka.new(("invalid_key", "1001"))
 
     assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert new.body == "invalid_key"
+    assert new.body == [('key', 'invalid_key'), ('id', '1001')]
 
 
 def test_concept_id_is_irrelevant_when_only_one_concept():
@@ -457,8 +489,8 @@ def test_list_of_concept_is_sorted_by_id():
 
 
 @pytest.mark.parametrize("body, expected", [
-    (None, None),
+    # (None, None),
-    ("", ""),
+    # ("", ""),
     ("1", 1),
     ("1+1", 2),
     ("'one'", "one"),
@@ -871,3 +903,53 @@ def test_i_cannot_add_the_same_concept_twice_in_a_set():
     all_entries = sheerka.sdp.get("All_" + all_foos.id, None, False)
     assert len(all_entries) == 1
     assert foo.id in all_entries
+
+
+def test_i_get_elements_from_a_set():
+    sheerka = get_sheerka(False, False)
+
+    one = Concept("one")
+    two = Concept("two")
+    three = Concept("three")
+    number = Concept("number")
+
+    for c in [one, two, three, number]:
+        sheerka.set_id_if_needed(c, False)
+        sheerka.add_in_cache(c)
+
+    context = get_context(sheerka)
+    for c in [one, two, three]:
+        sheerka.add_concept_to_set(context, c, number)
+
+    elements = sheerka.get_set_elements(number)
+
+    assert set(elements) == set([one, two, three])
+
+
+def test_i_cannot_get_elements_if_not_a_set():
+    sheerka = get_sheerka(False, False)
+    one = Concept("one")
+    sheerka.set_id_if_needed(one, False)
+    sheerka.add_in_cache(one)
+
+    error = sheerka.get_set_elements(one)
+
+    assert sheerka.isinstance(error, BuiltinConcepts.NOT_A_SET)
+    assert error.body == one
+
+
+def test_isa_and_isa_group():
+    sheerka = get_sheerka()
+
+    group = Concept("group").init_key()
+    group.metadata.id = "1001"
+    assert not sheerka.isagroup(group)
+
+    foo = Concept("foo").init_key()
+    foo.metadata.id = "1002"
+    assert not sheerka.isa(foo, group)
+
+    context = get_context(sheerka)
+    sheerka.add_concept_to_set(context, foo, group)
+    assert sheerka.isagroup(group)
+    assert sheerka.isa(foo, group)

tests/test_sheerkaDataProvider.py

@@ -164,7 +164,7 @@ class ObjWithDigestWithKey:
         return hash((self.a, self.b))
 
     def __eq__(self, obj):
-        return isinstance(obj, ObjNoKey) and \
+        return isinstance(obj, ObjWithDigestWithKey) and \
                self.a == obj.a and \
                self.b == obj.b
 
@@ -529,6 +529,44 @@ def test_i_can_add_obj_with_key_to_a_list(root):
     assert loaded == ["foo", "bar", ObjWithKey("a", "b")]
 
 
+@pytest.mark.parametrize("root", [
+    ".sheerka",
+    "mem://"
+])
+def test_i_can_add_a_reference(root):
+    sdp = SheerkaDataProvider(root)
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey)))
+    obj1 = ObjWithDigestWithKey(1, "foo")
+    sdp.add(evt_digest, "entry", obj1, use_ref=True)
+    sdp.add(evt_digest, "entry_by_value", {obj1.b: obj1.get_digest()}, is_ref=True)
+
+    # another object
+    obj2 = ObjWithDigestWithKey(2, "bar")
+    sdp.add(evt_digest, "entry", obj2, use_ref=True)
+    sdp.add(evt_digest, "entry_by_value", {obj2.b: obj2.get_digest()}, is_ref=True)
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {
+        "entry": {
+            "1": '##REF##:' + obj1.get_digest(),
+            "2": '##REF##:' + obj2.get_digest(),
+        },
+        "entry_by_value": {
+            "foo": '##REF##:' + obj1.get_digest(),
+            "bar": '##REF##:' + obj2.get_digest()
+        },
+    }
+
+    # sanity check, make sure that I can load back
+    loaded1 = sdp.get("entry_by_value", "foo")
+    assert loaded1 == ObjWithDigestWithKey(1, "foo")
+    assert getattr(loaded1, Serializer.ORIGIN) == obj1.get_digest()
+
+    loaded2 = sdp.get("entry_by_value", "bar")
+    assert loaded2 == ObjWithDigestWithKey(2, "bar")
+    assert getattr(loaded2, Serializer.ORIGIN) == obj2.get_digest()
+
+
 @pytest.mark.parametrize("root", [
     ".sheerka",
     "mem://"
@@ -650,6 +688,20 @@ def test_i_cannot_add_the_same_digest_twice_in_the_same_entry4(root):
     assert error.value.args[0] == "Duplicate object."
 
 
+def test_i_cannot_add_using_use_ref_and_is_ref():
+    sdp = SheerkaDataProvider("mem://")
+
+    with pytest.raises(SheerkaDataProviderError) as error:
+        sdp.add(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), use_ref=True, is_ref=True)
+
+
+def test_i_cannot_add_using_is_ref_if_obj_is_not_a_dictionary():
+    sdp = SheerkaDataProvider("mem://")
+
+    with pytest.raises(SheerkaDataProviderError) as error:
+        sdp.add(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), is_ref=True)
+
+
 @pytest.mark.parametrize("root", [
     ".sheerka",
     "mem://"
@@ -782,6 +834,43 @@ def test_i_can_set_using_reference(root):
     assert getattr(loaded, Serializer.ORIGIN) == "95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268"
 
 
+@pytest.mark.parametrize("root", [
+    ".sheerka",
+    "mem://"
+])
+def test_i_can_set_a_reference(root):
+    sdp = SheerkaDataProvider(root)
+    sdp.serializer.register(PickleSerializer(lambda o: isinstance(o, ObjWithDigestWithKey)))
+    obj = ObjWithDigestWithKey(1, "foo")
+    sdp.add(evt_digest, "entry", obj, use_ref=True)
+    sdp.set(evt_digest, "entry_by_value", {obj.b: obj.get_digest()}, is_ref=True)
+
+    state = sdp.load_state(sdp.get_snapshot())
+    assert state.data == {
+        "entry": {"1": '##REF##:' + obj.get_digest()},
+        "entry_by_value": {"foo": '##REF##:' + obj.get_digest()},
+    }
+
+    # sanity check, make sure that I can load back
+    loaded = sdp.get("entry_by_value", "foo")
+    assert loaded == ObjWithDigestWithKey(1, "foo")
+    assert getattr(loaded, Serializer.ORIGIN) == obj.get_digest()
+
+
+def test_i_cannot_set_using_use_ref_and_is_ref():
+    sdp = SheerkaDataProvider("mem://")
+
+    with pytest.raises(SheerkaDataProviderError) as error:
+        sdp.set(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), use_ref=True, is_ref=True)
+
+
+def test_i_cannot_set_using_is_ref_if_obj_is_not_a_dictionary():
+    sdp = SheerkaDataProvider("mem://")
+
+    with pytest.raises(SheerkaDataProviderError) as error:
+        sdp.set(evt_digest, "entry", ObjWithDigestWithKey("a", "b"), is_ref=True)
+
+
 @pytest.mark.parametrize("root", [
     ".sheerka",
     "mem://"

tests/test_sheerka_call_evaluators.py

@@ -18,8 +18,8 @@ def get_context(sheerka):
 
 def get_ret_val(sheerka, concept, who="who"):
     concept.init_key()
-    if concept.key not in sheerka.concepts_cache:
+    if concept.key not in sheerka.cache_by_key:
-        sheerka.concepts_cache[concept.key] = concept
+        sheerka.cache_by_key[concept.key] = concept
     return sheerka.ret(who, True, sheerka.new(concept.key))
 
 

tests/test_sheerka_non_reg.py

@@ -152,7 +152,8 @@ as:
     for prop in PROPERTIES_TO_SERIALIZE:
         assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
 
-    assert concept_saved.key in sheerka.concepts_cache
+    assert concept_saved.key in sheerka.cache_by_key
+    assert concept_saved.id in sheerka.cache_by_id
     assert sheerka.sdp.io.exists(
         sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
 
@@ -182,7 +183,8 @@ def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept()
     for prop in PROPERTIES_TO_SERIALIZE:
         assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
 
-    assert concept_saved.key in sheerka.concepts_cache
+    assert concept_saved.key in sheerka.cache_by_key
+    assert concept_saved.id in sheerka.cache_by_id
     assert sheerka.sdp.io.exists(
         sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
 
@@ -331,8 +333,7 @@ def test_i_can_create_concept_with_bnf_definition():
 
     saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY)
     expected_bnf = Sequence(
-        ConceptExpression("a", rule_name="a"),
+        a, Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus"))))
-        Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus"))))
     assert saved_definitions[saved_concept] == expected_bnf
 
     new_concept = res[0].value.body
@@ -421,6 +422,23 @@ def test_i_can_eval_bnf_definitions_from_separate_instances():
         "def concept digit from bnf one|two",
         "def concept twenties from bnf twenty digit as twenty + digit"
     ]),
+    ("When using isa and concept twenty", [
+        "def concept one as 1",
+        "def concept two as 2",
+        "def concept number",
+        "one isa number",
+        "two isa number",
+        "def concept twenties from bnf 'twenty' number as 20 + number"
+    ]),
+    ("When using isa and concept twenty", [
+        "def concept one as 1",
+        "def concept two as 2",
+        "def concept twenty as 20",
+        "def concept number",
+        "one isa number",
+        "two isa number",
+        "def concept twenties from bnf twenty number as 20 + number"
+    ]),
 ])
 def test_i_can_mix_concept_with_python_to_define_numbers(desc, definitions):
     sheerka = get_sheerka()
@@ -460,6 +478,50 @@ def test_i_can_mix_concept_with_python_to_define_numbers(desc, definitions):
     assert res[0].body == 23
 
 
+def test_i_can_mix_bnf_and_isa():
+    """
+    if 'one' isa 'number, twenty number should be recognized
+    :return:
+    """
+    sheerka = get_sheerka()
+    sheerka.evaluate_user_input("def concept one as 1")
+    sheerka.evaluate_user_input("def concept two as 2")
+    sheerka.evaluate_user_input("def concept number")
+    sheerka.evaluate_user_input("one isa number")
+    sheerka.evaluate_user_input("two isa number")
+    sheerka.evaluate_user_input("def concept twenties from bnf 'twenty' number as 20 + number")
+
+    res = sheerka.evaluate_user_input("twenty one")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == simplec("twenties", 21)
+
+    res = sheerka.evaluate_user_input("twenty one + 1")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == 22
+
+    res = sheerka.evaluate_user_input("twenty one + one")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == 22
+
+    res = sheerka.evaluate_user_input("twenty one + twenty two")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == 43
+
+    res = sheerka.evaluate_user_input("1 + twenty one")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == 22
+
+    res = sheerka.evaluate_user_input("1 + 1 + twenty one")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == 23
+
+
 def test_i_can_mix_concept_of_concept():
     sheerka = get_sheerka()