Added basic implentation for where

.gitignore

@@ -4,3 +4,4 @@ venv
 __pycache__
 build
 prof
+tests/_concepts.txt

_concepts.txt

@@ -40,4 +40,3 @@ eighteen isa number
 nineteen isa number
 twenty isa number
 def concept twenties from bnf twenty number where number < 10 as twenty + number
-

src/core/builtin_concepts.py

@@ -50,6 +50,7 @@ class BuiltinConcepts(Enum):
     CONCEPT_EVAL_REQUESTED = "concept eval requested"
     REDUCE_REQUESTED = "reduce requested"  # remove meaningless error when possible
     NOT_A_SET = "not a set"  # the concept has no entry in sets
+    WHERE_CLAUSE_FAILED = "where clause failed"  # failed to validate where clause during evaluation
 
     NODE = "node"
     GENERIC_NODE = "generic node"
@@ -91,6 +92,7 @@ BuiltinErrors = [str(e) for e in {
     BuiltinConcepts.CONCEPT_EVAL_ERROR,
     BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
     BuiltinConcepts.NOT_A_SET,
+    BuiltinConcepts.WHERE_CLAUSE_FAILED
 }]
 
 """
@@ -385,3 +387,20 @@ class ConceptAlreadyInSet(Concept):
     @property
     def concept_set(self):
         return self.props["concept_set"].value
+
+
+class WhereClauseFailed(Concept):
+    def __init__(self, concept=None):
+        super().__init__(BuiltinConcepts.WHERE_CLAUSE_FAILED,
+                         True,
+                         False,
+                         BuiltinConcepts.WHERE_CLAUSE_FAILED)
+        self.set_metadata_value(ConceptParts.BODY, concept)
+        self.metadata.is_evaluated = True
+
+    def __repr__(self):
+        return f"WhereClauseFailed(concept={self.concept})"
+
+    @property
+    def concept(self):
+        return self.body

src/core/concept.py

@@ -231,7 +231,8 @@ class Concept:
             if token.value in variables:
                 key += VARIABLE_PREFIX + str(variables.index(token.value))
             else:
-                key += token.value[1:-1] if token.type == TokenKind.STRING else token.value
+                value = token.value[1:-1] if token.type == TokenKind.STRING else token.value
+                key += value
             first = False
 
         self.metadata.key = key

src/core/sheerka/Services/SheerkaCreateNewConcept.py

@@ -45,7 +45,7 @@ class SheerkaCreateNewConcept:
 
         # add the BNF if known
         if concept.bnf:
-            concepts_definitions = self.sheerka.get_concept_definition()
+            concepts_definitions = self.sheerka.get_concepts_definitions(context)
             concepts_definitions[concept] = concept.bnf
 
             # check if it's a valid BNF or whether it breaks the known rules
@@ -61,22 +61,26 @@ class SheerkaCreateNewConcept:
         # save the new concept in sdp
         try:
             # TODO : needs to make these calls atomic (or at least one single call)
+            # save the new concept
             self.sheerka.sdp.add(
                 context.event.get_digest(),
                 self.sheerka.CONCEPTS_ENTRY,
                 concept,
                 use_ref=True)
+            # save it by id
             self.sheerka.sdp.add(
                 context.event.get_digest(),
                 self.sheerka.CONCEPTS_BY_ID_ENTRY,
                 {concept.id: concept.get_digest()},
                 is_ref=True)
+            # update the definition table
             if concepts_definitions is not None:
                 self.sheerka.sdp.set(
                     context.event.get_digest(),
                     self.sheerka.CONCEPTS_DEFINITIONS_ENTRY,
-                    concepts_definitions,
+                    concept_lexer_parser.encode_grammar(init_ret_value.body),
                     use_ref=True)
+                self.sheerka.concepts_definitions_cache = None  # invalidate cache
         except SheerkaDataProviderDuplicateKeyError as error:
             context.log_error(logger, "Failed to create a new concept.", who=self.logger_name)
             return self.sheerka.ret(
@@ -94,6 +98,3 @@ class SheerkaCreateNewConcept:
         # process the return in needed
         ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
         return ret
-
-
-

src/core/sheerka/Services/SheerkaDump.py

@@ -1,5 +1,14 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+import pprint
+import os
+
+
+def get_pp():
+    rows, columns = os.popen('stty size', 'r').read().split()
+    pp = pprint.PrettyPrinter(width=columns, compact=True)
+    return pp
 
 
 class SheerkaDump:
@@ -71,3 +80,8 @@ class SheerkaDump:
                     break
 
             page_count += 1
+
+    def dump_state(self):
+        snapshot = self.sheerka.sdp.get_snapshot(SheerkaDataProvider.HeadFile)
+        state = self.sheerka.sdp.load_state(snapshot)
+        self.sheerka.log.info(get_pp().pformat(state.data))

src/core/sheerka/Services/SheerkaEvaluateConcept.py

@@ -147,9 +147,6 @@ class SheerkaEvaluateConcept:
         if concept.metadata.is_evaluated:
             return concept
 
-        # WHERE condition should already be validated by the parser.
-        # It's a mandatory condition for the concept before it can be recognized
-
         #
         # TODO : Validate the PRE condition
         #
@@ -157,8 +154,8 @@ class SheerkaEvaluateConcept:
         self.initialize_concept_asts(context, concept, logger)
 
         # to make sure of the order, it don't use ConceptParts.get_parts()
-        # props must be evaluated first
+        # props must be evaluated first, body must be evaluated before where
-        all_metadata_to_eval = ["props", "where", "pre", "post", "body"]
+        all_metadata_to_eval = ["pre", "post", "props", "body", "where"]
 
         for metadata_to_eval in all_metadata_to_eval:
             if metadata_to_eval == "props":
@@ -186,6 +183,12 @@ class SheerkaEvaluateConcept:
                     else:
                         concept.values[part_key] = resolved
 
+        # validate where clause
+        if concept.metadata.where is not None:
+            where_value = concept.values[ConceptParts.WHERE]
+            if not (where_value is None or self.sheerka.value(where_value) is True):
+                return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept)
+
         #
         # TODO : Validate the POST condition
         #

src/core/sheerka/Services/SheerkaHistoryManager.py

@@ -19,7 +19,7 @@ class History:
         return msg
 
     def __repr__(self):
-        return f"event={self.event!r}, status={self.status}, result={self.result}"
+        return f"History(event={self.event!r}, status={self.status}, result={self.result})"
 
     def __eq__(self, other):
         if id(self) == id(other):

src/core/sheerka/Services/SheerkaSetsManager.py

@@ -38,6 +38,21 @@ class SheerkaSetsManager:
             context.log_error(logger, "Failed to add to set.", who=self.logger_name)
             return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
 
+    def add_concepts_to_set(self, context, concepts, concept_set, logger=None):
+        """Adding multiple concepts at the same time"""
+        logger = logger or self.sheerka.log
+
+        context.log(logger, f"Adding concepts {concepts} to set {concept_set}", who=self.logger_name)
+        previous = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept_set.id)
+
+        new_ids = [c.id for c in concepts] if previous is None else previous + [c.id for c in concepts]
+        try:
+            self.sheerka.sdp.set(context.event.get_digest(), GROUP_PREFIX + concept_set.id, new_ids)
+            return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+        except Exception as error:
+            context.log_error(logger, "Failed to add to set.", who=self.logger_name)
+            return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
+
     def get_set_elements(self, concept):
         """
         Concept is supposed to be a set

src/core/sheerka/Sheerka.py

@@ -22,6 +22,7 @@ import logging
 #     BuiltinConcepts.AFTER_EVALUATION]
 
 CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
+BNF_PARSER_CLASS = "parsers.BnfParser.BnfParser"
 CONCEPTS_FILE = "_concepts.txt"
 
 
@@ -52,7 +53,7 @@ class Sheerka(Concept):
 
         # cache for concept definitions,
         # Primarily used for unit test that does not have access to sdp
-        self.concepts_definition_cache = {}
+        self.concepts_definitions_cache = {}
 
         #
         # cache for concepts grammars
@@ -187,7 +188,8 @@ class Sheerka(Concept):
 
     def initialize_concepts_definitions(self, execution_context):
         self.init_log.debug("Initializing concepts definitions")
-        definitions = self.sdp.get_safe(self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False)
+        # definitions = self.sdp.get_safe(self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False)
+        definitions = self.get_concepts_definitions(execution_context)
 
         if definitions is None:
             self.init_log.debug("No BNF defined")
@@ -389,14 +391,26 @@ class Sheerka(Concept):
 
         return result or self._get_unknown(('id', concept_id))
 
-    def get_concept_definition(self):
+    def get_concepts_definitions(self, context):
-        if self.concepts_definition_cache:
+        if self.concepts_definitions_cache:
-            return self.concepts_definition_cache
+            return self.concepts_definitions_cache
 
-        self.concepts_definition_cache = self.sdp.get_safe(
+        encoded = self.sdp.get_safe(
             self.CONCEPTS_DEFINITIONS_ENTRY,
             load_origin=False) or {}
-        return self.concepts_definition_cache
+
+        self.concepts_definitions_cache = {}
+        bnf_parser = self.parsers[BNF_PARSER_CLASS]()
+        for k, v in encoded.items():
+            key, id_ = core.utils.unstr_concept(k)
+            concept = self.new((key, id_))
+            rule_result = bnf_parser.parse(context, v)
+            if rule_result.status:
+                self.concepts_definitions_cache[concept] = rule_result.value.value
+            else:
+                self.log.error(f"Failed to load bnf rule for concept {key}")
+
+        return self.concepts_definitions_cache
 
     def new(self, concept_key, **kwargs):
         """
@@ -411,7 +425,7 @@ class Sheerka(Concept):
         else:
             concept_id = None
 
-        template = self.get(concept_key, concept_id)
+        template = self.get_by_id(concept_id) if not concept_key else self.get(concept_key, concept_id)
 
         # manage concept not found
         if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \
@@ -579,7 +593,10 @@ class Sheerka(Concept):
             self.during_restore = True
             with open(CONCEPTS_FILE, "r") as f:
                 for line in f.readlines():
-                    self.log.info(line.strip())
+                    line = line.strip()
+                    if line == "" or line.startswith("#"):
+                        continue
+                    self.log.info(line)
                     self.evaluate_user_input(line)
             self.during_restore = False
         except IOError:

src/core/tokenizer.py

@@ -266,10 +266,10 @@ class Tokenizer:
                 self.column = 1
                 self.line += 1
             elif c == "c" and self.i + 1 < self.text_len and self.text[self.i + 1] == ":":
-                concept_name = self.eat_concept_name(self.i + 2, self.line, self.column)
+                name, id, length = self.eat_concept(self.i + 2, self.line, self.column + 2)
-                yield Token(TokenKind.CONCEPT, concept_name, self.i, self.line, self.column)
+                yield Token(TokenKind.CONCEPT, (name, id), self.i, self.line, self.column)
-                self.i += len(concept_name) + 3
+                self.i += length + 2
-                self.column += len(concept_name) + 3
+                self.column += length + 2
             elif c.isalpha() or c == "_":
                 identifier = self.eat_identifier(self.i)
                 token_type = TokenKind.KEYWORD if identifier in self.KEYWORDS else TokenKind.IDENTIFIER
@@ -297,31 +297,41 @@ class Tokenizer:
 
         yield Token(TokenKind.EOF, "", self.i, self.line, self.column)
 
-    def eat_concept_name(self, start, line, column):
+    def eat_concept(self, start, line, column):
-        result = ""
+        key, id, buffer = None, None, ""
         i = start
-        end_colon_found = False
+        processing_key = True
 
         while i < self.text_len:
-            c = self.text[i]
 
+            c = self.text[i]
             if c == "\n":
-                raise LexerError(f"New line is forbidden in concept name", result, i, line, column + 2 + len(result))
+                raise LexerError(f"New line in concept name", self.text[start:i], i, line, column + i - start)
 
             if c == ":":
-                end_colon_found = True
+                if processing_key:
+                    key = buffer if buffer else None
+                else:
+                    id = buffer if buffer else None
+                i += 1  # eat the colon
                 break
 
-            result += c
+            if c == "|":
+                key = buffer if buffer else None
+                buffer = ""
+                processing_key = False
                 i += 1
+                continue
 
-        if not end_colon_found:
+            buffer += c
-            raise LexerError(f"Missing ending colon", result, i, line, column + 2 + len(result))
+            i += 1
+        else:
+            raise LexerError(f"Missing ending colon", self.text[start:i], i, line, column + i - start)
 
-        if result == "":
+        if (key, id) == (None, None):
-            raise LexerError(f"Concept name not found", result, start, line, column + 2 + len(result))
+            raise LexerError(f"Concept identifiers not found", "", start, line, column)
 
-        return result
+        return key, id, i - start
 
     def eat_whitespace(self, start):
         result = self.text[start]

src/core/utils.py

@@ -1,6 +1,7 @@
 import importlib
 import inspect
 import pkgutil
+import re
 
 from core.tokenizer import TokenKind
 
@@ -239,43 +240,6 @@ def pp(items):
     return "    \n" + "    \n".join(str(item) for item in items)
 
 
-def decode_concept(concept_repr):
-    """
-    if concept_repr is like :c:key:id:
-    return the key and the id
-    :param concept_repr:
-    :return:
-    """
-    if not (concept_repr and isinstance(concept_repr, str) and concept_repr.startswith(":c:")):
-        return None, None
-
-    i = 3
-    length = len(concept_repr)
-    key = ""
-    while i < length:
-        if concept_repr[i] == ":":
-            break
-        key += concept_repr[i]
-        i += 1
-    else:
-        return None, None
-
-    i += 1
-    if i >= length:
-        return key, None
-
-    id = ""
-    while i < length:
-        if concept_repr[i] == ":":
-            break
-        id += concept_repr[i]
-        i += 1
-    else:
-        return None, None
-
-    return key, id
-
-
 def decode_enum(enum_repr: str):
     """
     Tries to transform ClassName.Name into an enum
@@ -300,3 +264,110 @@ def decode_enum(enum_repr: str):
 
     except TypeError:
         return None
+
+
+def str_concept(t):
+    """
+    The key,id identifiers of a concept are stored in a tuple
+    we want to return the key and the id, separated by a pipe
+    None value must be replaced by an empty string
+
+    >>> assert str_concept(("key", "id")) == "c:key|id:"
+    >>> assert str_concept((None, "id")) == "c:|id:"
+    >>> assert str_concept(("key", None)) == "c:key:"
+    >>> assert str_concept((None, None)) == ""
+    :param t:
+    :return:
+    """
+    if isinstance(t, tuple):
+        key, id_ = t[0], t[1]
+    else:
+        key, id_ = t.key, t.id
+
+    if key is None and id_ is None:
+        return ""
+
+    result = 'c:' if key is None else "c:" + key
+    if id_:
+        result += "|" + id_
+    return result + ":"
+
+
+def unstr_concept(concept_repr):
+    """
+    if concept_repr is like :c:key:id:
+    return the key and the id
+    :param concept_repr:
+    :return:
+    """
+    if not (concept_repr and isinstance(concept_repr, str) and concept_repr.startswith("c:")):
+        return None, None
+
+    i = 2
+    length = len(concept_repr)
+    key = ""
+    while i < length:
+        c = concept_repr[i]
+        if c in (":", "|"):
+            break
+        key += c
+        i += 1
+    else:
+        return None, None
+
+    if c == ":":
+        return key if key != "" else None, None
+
+    i += 1
+    id = ""
+    while i < length:
+        c = concept_repr[i]
+        if c == ":":
+            break
+        id += c
+        i += 1
+    else:
+        return None, None
+
+    return key if key != "" else None, id if id != "" else None
+
+
+def encode_concept(t, use_concept=False):
+    """
+    Given a tuple of concept id, concept id
+    Create a valid Python identifier that can be parsed back
+
+    >>> assert encode_concept(("key", "id")) == "__C__KEY_key__ID_id__C__"
+    >>> assert encode_concept((None, "id")) == "__C__KEY_00None00__ID_id__C__"
+    >>> assert encode_concept(("key", None)) == "__C__KEY_key__ID_00None00__C__"
+    >>> assert encode_concept(("key", "id"), True) == "__C__USE_CONCEPT__KEY_key__ID_id__C__"
+
+    :param t:
+    :param use_concept:
+    :return:
+    """
+
+    key, id_ = (t[0], t[1]) if isinstance(t, tuple) else (t.key, t.id)
+    prefix = "__C__USE_CONCEPT" if use_concept else "__C"
+    sanitized_key = "".join(c if c.isalnum() else "0" for c in key) if key else "00None00"
+    return prefix + f"__KEY_{sanitized_key}__ID_{id_ or '00None00'}__C__"
+
+
+decode_regex = re.compile(r"__KEY_(\w+)__ID_(\w+)__C__")
+
+
+def decode_concept(text):
+    """
+    Decode what was encoded by encode_concept_key_id
+    :param text:
+    :return:
+    """
+    use_concept = text.startswith("__C__USE_CONCEPT")
+    m = decode_regex.search(text)
+    lookup = {"00None00": None}
+    if m:
+        key = lookup.get(m.group(1), m.group(1))
+        id_ = lookup.get(m.group(2), m.group(2))
+        return key, id_, use_concept
+
+    return None, None, None

src/evaluators/PythonEvaluator.py

@@ -8,6 +8,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.PythonParser import PythonNode
 import ast
 import core.ast.nodes
+import core.utils
 
 
 class PythonEvaluator(OneReturnValueEvaluator):
@@ -40,9 +41,11 @@ class PythonEvaluator(OneReturnValueEvaluator):
                     not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node)
                     return sheerka.ret(self.name, False, not_for_me, parents=[return_value])
 
+            # get locals
             my_locals = self.get_locals(context, node)
             context.log(self.verbose_log, f"locals={my_locals}", self.name)
 
+            # eval
             if isinstance(node.ast_, ast.Expression):
                 context.log(self.verbose_log, "Evaluating using 'eval'.", self.name)
                 compiled = compile(node.ast_, "<string>", "eval")
@@ -53,6 +56,7 @@ class PythonEvaluator(OneReturnValueEvaluator):
 
             context.log(self.verbose_log, f"{evaluated=}", self.name)
             return sheerka.ret(self.name, True, evaluated, parents=[return_value])
+
         except Exception as error:
             context.log_error(self.verbose_log, error, self.name)
             error = sheerka.new(BuiltinConcepts.ERROR, body=error)
@@ -65,16 +69,19 @@ class PythonEvaluator(OneReturnValueEvaluator):
             "concepts": context.sheerka.dump_handler.dump_concepts,
             "definitions": context.sheerka.dump_handler.dump_definitions,
             "history": context.sheerka.dump_handler.dump_history,
+            "state": context.sheerka.dump_handler.dump_state,
         }
         if context.obj:
             context.log(self.verbose_log,
-                        f"Concept '{context.obj}' is in context. Adding its properties to locals if any.", self.name)
+                        f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name)
 
             for prop_name, prop_value in context.obj.props.items():
-                if not isinstance(prop_value.value, Concept):
+                if isinstance(prop_value.value, Concept):
-                    my_locals[prop_name] = prop_value.value
-                else:
                     my_locals[prop_name] = context.sheerka.value(prop_value.value)
+                else:
+                    my_locals[prop_name] = prop_value.value
+
+            my_locals["self"] = context.obj.body
 
         node_concept = core.ast.nodes.python_to_concept(node.ast_)
         unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka)
@@ -89,16 +96,17 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 return_concept = False
 
             else:
-                concept_key, concept_id, return_concept = self.resolve_name(context, name)
+                c_key, c_id, return_concept = self.resolve_name(name)
 
-                if concept_key in my_locals:
+                if c_key in my_locals:
                     context.log(self.verbose_log, f"Using value from property.", self.name)
                     continue
 
-                context.log(self.verbose_log, f"Instantiating new concept.", self.name)
+                context.log(self.verbose_log, f"Instantiating new concept with {c_key=}, {c_id=}.", self.name)
-                concept = context.sheerka.new((concept_key, concept_id))
+                new = context.sheerka.new
+                concept = new((None, c_id)) if c_id else new(c_key)
                 if context.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT):
-                    context.log(self.verbose_log, f"'{concept_key}' is not a concept. Skipping.", self.name)
+                    context.log(self.verbose_log, f"({c_key=}, {c_id=}) is not a concept. Skipping.", self.name)
                     continue
 
             context.log(self.verbose_log, f"Evaluating '{concept}'", self.name)
@@ -110,62 +118,68 @@ class PythonEvaluator(OneReturnValueEvaluator):
                 if evaluated.key == concept.key:
                     my_locals[name] = evaluated if return_concept else context.sheerka.value(evaluated)
 
-        if self.locals:
+        if self.locals:  # when exta values are given. Add them
             my_locals.update(self.locals)
 
         return my_locals
 
-    def resolve_name(self, context, to_resolve):
+    @staticmethod
+    def resolve_name(to_resolve):
         """
         Try to match
         __C__concept_key__C__
         or
         __C__concept_key__concept_id__C__
 
-        :param context:
         :param to_resolve:
         :return:
         """
-        if not to_resolve.startswith("__C__"):
+        key, id_, use_concept = core.utils.decode_concept(to_resolve)
-            return to_resolve, None, False
+        if key or id_:
-
+            return key, id_, use_concept
-        context.log(self.verbose_log, f"Resolving name '{to_resolve}'.", self.name)
-
-        if len(to_resolve) >= 18 and to_resolve[:18] == "__C__USE_CONCEPT__":
-            use_concept = True
-            index = 18
         else:
-            use_concept = False
+            return to_resolve, None, False
-            index = 5
+        #
-
+        # if not to_resolve.startswith("__C__"):
-        try:
+        #     return to_resolve, None, False
-            next_index = to_resolve.index("__", index)
+        #
-            if next_index == index:
+        # context.log(self.verbose_log, f"Resolving name '{to_resolve}'.", self.name)
-                context.log(self.verbose_log, f"Error: no key between '__'.", self.name)
+        #
-                return None
+        # if len(to_resolve) >= 18 and to_resolve[:18] == "__C__USE_CONCEPT__":
-            concept_key = to_resolve[index: next_index]
+        #     use_concept = True
-        except ValueError:
+        #     index = 18
-            context.log(self.verbose_log, f"Error: Missing trailing '__'.", self.name)
+        # else:
-            return None
+        #     use_concept = False
-
+        #     index = 5
-        if next_index == len(to_resolve) - 5:
+        #
-            context.log(self.verbose_log, f"Recognized concept '{concept_key}'", self.name)
+        # try:
-            return concept_key, None, use_concept
+        #     next_index = to_resolve.index("__", index)
-
+        #     if next_index == index:
-        index = next_index + 2
+        #         context.log(self.verbose_log, f"Error: no key between '__'.", self.name)
-        try:
+        #         return None
-            next_index = to_resolve.index("__", index)
+        #     concept_key = to_resolve[index: next_index]
-            if next_index == index:
+        # except ValueError:
-                context.log(self.verbose_log, f"Error: no id between '__'.", self.name)
+        #     context.log(self.verbose_log, f"Error: Missing trailing '__'.", self.name)
-                return None
+        #     return None
-
+        #
-            concept_id = to_resolve[index: next_index]
+        # if next_index == len(to_resolve) - 5:
-        except ValueError:
+        #     context.log(self.verbose_log, f"Recognized concept '{concept_key}'", self.name)
-            context.log(self.verbose_log, f"Recognized concept '{concept_key}'.", self.name)
+        #     return concept_key, None, use_concept
-            return concept_key, None, use_concept
+        #
-
+        # index = next_index + 2
-        context.log(self.verbose_log, f"Recognized concept '{concept_key}' (id='{concept_id}').", self.name)
+        # try:
-        return concept_key, concept_id, use_concept
+        #     next_index = to_resolve.index("__", index)
+        #     if next_index == index:
+        #         context.log(self.verbose_log, f"Error: no id between '__'.", self.name)
+        #         return None
+        #
+        #     concept_id = to_resolve[index: next_index]
+        # except ValueError:
+        #     context.log(self.verbose_log, f"Recognized concept '{concept_key}'.", self.name)
+        #     return concept_key, None, use_concept
+        #
+        # context.log(self.verbose_log, f"Recognized concept '{concept_key}' (id='{concept_id}').", self.name)
+        # return concept_key, concept_id, use_concept
 
     @staticmethod
     def expr_to_expression(expr):

src/parsers/BaseParser.py

@@ -2,8 +2,9 @@ from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.tokenizer import TokenKind, Keywords
+from core.tokenizer import TokenKind, Keywords, Token
 from core.sheerka_logger import get_logger
+import core.utils
 import logging
 
 
@@ -35,8 +36,34 @@ class ErrorNode(Node):
 @dataclass()
 class UnexpectedTokenErrorNode(ErrorNode):
     message: str
+    token: Token
     expected_tokens: list
 
+    def __eq__(self, other):
+        if id(other) == id(self):
+            return True
+
+        if not isinstance(other, UnexpectedTokenErrorNode):
+            return False
+
+        if self.message != other.message:
+            return False
+
+        if self.token.type != other.token.type or self.token.value != other.token.value:
+            return False
+
+        if len(self.expected_tokens) != len(other.expected_tokens):
+            return False
+
+        for i, t in enumerate(self.expected_tokens):
+            if t != other.expected_tokens[i]:
+                return False
+
+        return True
+
+    def __hash__(self):
+        return hash((self.message, self.token, self.expected_tokens))
+
 
 class BaseParser:
     PREFIX = "parsers."
@@ -108,7 +135,7 @@ class BaseParser:
 
         switcher = {
             TokenKind.KEYWORD: lambda t: Keywords(t.value).value,
-            TokenKind.CONCEPT: lambda t: "c:" + t.value + ":",
+            TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value),
         }
 
         if custom_switcher:

src/parsers/BnfParser.py

@@ -5,7 +5,8 @@ from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.Sheerka import ExecutionContext
 from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
 from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode
-from parsers.ConceptLexerParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, ConceptExpression, StrMatch
+from parsers.ConceptLexerParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, ConceptExpression, \
+    StrMatch, ConceptGroupExpression
 
 
 @dataclass()
@@ -119,11 +120,11 @@ class BnfParser(BaseParser):
         tree = None
         try:
             self.reset_parser(context, text)
-            tree = self.parser_outer_rule_name()
+            tree = self.parse_choice()
 
             token = self.get_token()
             if token and token.type != TokenKind.EOF:
-                self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", []))
+                self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, []))
         except LexerError as e:
             self.add_error(e, False)
 
@@ -136,10 +137,11 @@ class BnfParser(BaseParser):
 
         return ret
 
-    def parser_outer_rule_name(self):
-        return self.parser_rule_name(self.parse_choice)
-
     def parse_choice(self):
+        """
+        a | b | c
+        :return:
+        """
         sequence = self.parse_sequence()
 
         self.eat_white_space()
@@ -159,9 +161,13 @@ class BnfParser(BaseParser):
             sequence = self.parse_sequence()
             elements.append(sequence)
 
-        return OrderedChoice(*elements)
+        return self.eat_rule_name_if_needed(OrderedChoice(*elements))
 
     def parse_sequence(self):
+        """
+        a b c
+        :return:
+        """
         expr_and_modifier = self.parse_modifier()
         token = self.get_token()
         if token is None or \
@@ -185,30 +191,31 @@ class BnfParser(BaseParser):
             sequence = self.parse_modifier()
             elements.append(sequence)
 
-        return Sequence(*elements)
+        return self.eat_rule_name_if_needed(Sequence(*elements))
 
     def parse_modifier(self):
-        expression = self.parser_inner_rule_name()
+        """
+        a? | a* | a+
+        :return:
+        """
+        expression = self.parse_expression()
 
         token = self.get_token()
 
         if token.type == TokenKind.QMARK:
             self.next_token()
-            return Optional(expression)
+            return self.eat_rule_name_if_needed(Optional(expression))
 
         if token.type == TokenKind.STAR:
             self.next_token()
-            return ZeroOrMore(expression)
+            return self.eat_rule_name_if_needed(ZeroOrMore(expression))
 
         if token.type == TokenKind.PLUS:
             self.next_token()
-            return OneOrMore(expression)
+            return self.eat_rule_name_if_needed(OneOrMore(expression))
 
         return expression
 
-    def parser_inner_rule_name(self):
-        return self.parser_rule_name(self.parse_expression)
-
     def parse_expression(self):
         token = self.get_token()
         if token.type == TokenKind.EOF:
@@ -216,15 +223,21 @@ class BnfParser(BaseParser):
         if token.type == TokenKind.LPAR:
             self.nb_open_par += 1
             self.next_token()
-            expression = self.parse_choice()
+            expr = self.parse_choice()
             token = self.get_token()
             if token.type == TokenKind.RPAR:
                 self.nb_open_par -= 1
                 self.next_token()
-                return expression
+                return self.eat_rule_name_if_needed(expr)
             else:
-                self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", [TokenKind.RPAR]))
+                self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, [TokenKind.RPAR]))
-                return expression
+                return expr
+
+        if token.type == TokenKind.CONCEPT:
+            self.next_token()
+            concept = self.sheerka.new((token.value[0], token.value[1]))
+            expr = ConceptGroupExpression(concept) if self.sheerka.isaset(concept) else ConceptExpression(concept)
+            return self.eat_rule_name_if_needed(expr)
 
         if token.type == TokenKind.IDENTIFIER:
             self.next_token()
@@ -247,14 +260,15 @@ class BnfParser(BaseParser):
                                      body=("key", concept_name)))
                 return None
             else:
-                return concept
+                expr = ConceptGroupExpression(concept) if self.sheerka.isaset(concept) else ConceptExpression(concept)
+                expr.rule_name = concept.name
+                return expr
 
         ret = StrMatch(core.utils.strip_quotes(token.value))
         self.next_token()
-        return ret
+        return self.eat_rule_name_if_needed(ret)
 
-    def parser_rule_name(self, next_to_parse):
+    def eat_rule_name_if_needed(self, expression):
-        expression = next_to_parse()
         token = self.get_token()
         if token is None or token.type != TokenKind.EQUALS:
             return expression
@@ -263,7 +277,8 @@ class BnfParser(BaseParser):
         token = self.get_token()
 
         if token is None or token.type != TokenKind.IDENTIFIER:
-            return self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", [TokenKind.IDENTIFIER]))
+            return self.add_error(
+                UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, [TokenKind.IDENTIFIER]))
 
         expression.rule_name = token.value
         self.next_token()

src/parsers/ConceptLexerParser.py

@@ -243,6 +243,9 @@ class ParsingExpression:
     def parse(self, parser):
         return self._parse(parser)
 
+    def add_rule_name_if_needed(self, text):
+        return text + "=" + self.rule_name if self.rule_name else text
+
 
 class ConceptExpression(ParsingExpression):
     """
@@ -257,7 +260,7 @@ class ConceptExpression(ParsingExpression):
         self.concept = concept
 
     def __repr__(self):
-        return f"{self.concept}"
+        return  self.add_rule_name_if_needed(f"{self.concept}")
 
     def __eq__(self, other):
         if not super().__eq__(other):
@@ -352,7 +355,7 @@ class Sequence(ParsingExpression):
 
     def __repr__(self):
         to_str = ", ".join(repr(n) for n in self.elements)
-        return f"({to_str})"
+        return self.add_rule_name_if_needed(f"({to_str})")
 
 
 class OrderedChoice(ParsingExpression):
@@ -375,7 +378,7 @@ class OrderedChoice(ParsingExpression):
 
     def __repr__(self):
         to_str = "| ".join(repr(n) for n in self.elements)
-        return f"({to_str})"
+        return self.add_rule_name_if_needed(f"({to_str})")
 
 
 class Optional(ParsingExpression):
@@ -413,7 +416,7 @@ class Optional(ParsingExpression):
             return f"{self.elements[0]}?"
         else:
             to_str = ", ".join(repr(n) for n in self.elements)
-        return f"({to_str})?"
+        return self.add_rule_name_if_needed(f"({to_str})?")
 
 
 class Repetition(ParsingExpression):
@@ -467,7 +470,7 @@ class ZeroOrMore(Repetition):
 
     def __repr__(self):
         to_str = ", ".join(repr(n) for n in self.elements)
-        return f"({to_str})*"
+        return self.add_rule_name_if_needed(f"({to_str})*")
 
 
 class OneOrMore(Repetition):
@@ -507,7 +510,7 @@ class OneOrMore(Repetition):
 
     def __repr__(self):
         to_str = ", ".join(repr(n) for n in self.elements)
-        return f"({to_str})+"
+        return self.add_rule_name_if_needed(f"({to_str})+")
 
 
 class UnorderedGroup(Repetition):
@@ -541,13 +544,13 @@ class StrMatch(Match):
     Matches a literal
     """
 
-    def __init__(self, to_match, rule_name="", root=False, ignore_case=True):
+    def __init__(self, to_match, rule_name="", ignore_case=True):
-        super(Match, self).__init__(rule_name=rule_name, root=root)
+        super(Match, self).__init__(rule_name=rule_name)
         self.to_match = to_match
         self.ignore_case = ignore_case
 
     def __repr__(self):
-        return f"'{self.to_match}'"
+        return self.add_rule_name_if_needed(f"'{self.to_match}'")
 
     def __eq__(self, other):
         if not super().__eq__(other):
@@ -699,10 +702,14 @@ class ConceptLexerParser(BaseParser):
                 else:
                     ret = ConceptExpression(expression, rule_name=expression.name)
                 concepts_to_resolve.add(expression)
-            elif isinstance(expression, ConceptExpression):
+            elif isinstance(expression, ConceptExpression): # it includes ConceptGroupExpression
                 if expression.rule_name is None or expression.rule_name == "":
                     expression.rule_name = expression.concept.name if isinstance(expression.concept, Concept) \
                         else expression.concept
+                if isinstance(expression.concept, str):
+                    concept = self.get_concept(expression.concept)
+                    if self.sheerka.is_known(concept):
+                        expression.concept = concept
                 concepts_to_resolve.add(expression.concept)
                 ret = expression
             elif isinstance(expression, str):
@@ -955,6 +962,47 @@ class ConceptLexerParser(BaseParser):
 
         return concept
 
+    def encode_grammar(self, grammar):
+        """
+        Transform the grammar into something that can easily can be serialized
+        :param grammar:
+        :return:
+        """
+
+        def _encode(expression):
+            if isinstance(expression, StrMatch):
+                res = f"'{expression.to_match}'"
+
+            elif isinstance(expression, ConceptExpression):
+                res = core.utils.str_concept(expression.concept)
+
+            elif isinstance(expression, Sequence):
+                res = "(" + " ".join(_encode(c) for c in expression.nodes) + ")"
+                
+            elif isinstance(expression, OrderedChoice):
+                res = "(" + "|".join(_encode(c) for c in expression.nodes) + ")"
+
+            elif isinstance(expression, Optional):
+                res = _encode(expression.nodes[0]) + "?"
+                
+            elif isinstance(expression, ZeroOrMore):
+                res = _encode(expression.nodes[0]) + "*"
+                
+            elif isinstance(expression, OneOrMore):
+                res = _encode(expression.nodes[0]) + "+"
+
+            if expression.rule_name:
+                res += "=" + expression.rule_name
+
+            return res
+
+        result = {}
+        for k, v in grammar.items():
+            key = core.utils.str_concept(k)
+            value = _encode(v)
+            result[key] = value
+        return result
+
     @staticmethod
     def get_bests(results):
         """

src/parsers/PythonParser.py

@@ -1,9 +1,10 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.tokenizer import Tokenizer, LexerError, TokenKind
 from parsers.BaseParser import BaseParser, Node, ErrorNode
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 import ast
 import logging
+import core.utils
 
 from parsers.ConceptLexerParser import ConceptNode
 
@@ -71,7 +72,7 @@ class PythonParser(BaseParser):
         tree = None
 
         python_switcher = {
-            TokenKind.CONCEPT: lambda t: f"__C__USE_CONCEPT__{t.value}__C__"
+            TokenKind.CONCEPT: lambda t: core.utils.encode_concept(t.value, True)
         }
 
         try:
@@ -136,6 +137,7 @@ class PythonGetNamesVisitor(ast.NodeVisitor):
     def visit_Name(self, node):
         self.names.add(node.id)
 
+
 class LexerNodeParserHelperForPython:
     """Helper class to parse mix of concepts and Python"""
 

src/sdp/sheerkaSerializer.py

@@ -223,7 +223,7 @@ class PickleSerializer(BaseSerializer):
 
 
 class StateSerializer(PickleSerializer):
-    def __init__(self, ):
+    def __init__(self):
         PickleSerializer.__init__(
             self,
             lambda obj: core.utils.get_full_qualified_name(obj) == "sdp.sheerkaDataProvider.State",
@@ -239,13 +239,23 @@ class ConceptSerializer(JsonSerializer):
         return isinstance(obj, Concept)
 
 
-class DictionarySerializer(PickleSerializer):
+class DictionarySerializer(BaseSerializer):
-    def __init__(self, ):
+    def __init__(self):
-        PickleSerializer.__init__(
+        super().__init__("D", 1)
-            self,
+
-            lambda obj: isinstance(obj, dict),
+    def matches(self, obj):
-            "D",
+        return isinstance(obj, dict)
-            1)
+
+    def dump(self, stream, obj, context):
+        stream.write(json.dumps(obj, default=json_default_converter).encode("utf-8"))
+        stream.seek(0)
+        return stream
+
+    def load(self, stream, context):
+        json_stream = stream.read().decode("utf-8")
+        obj = json.loads(json_stream)
+
+        return obj
 
 
 class ExecutionContextSerializer(BaseSerializer):

src/sheerkapickle/SheerkaPickler.py

@@ -70,7 +70,7 @@ class SheerkaPickler:
             elif utils.is_enum(k):
                 k_str = core.utils.get_full_qualified_name(k) + "." + k.name
             elif isinstance(k, Concept):
-                k_str = f":c:{k.key}:{k.id}:"
+                k_str = core.utils.str_concept(k)
             else:
                 k_str = k
 

src/sheerkapickle/SheerkaUnpickler.py

@@ -90,7 +90,7 @@ class SheerkaUnpickler:
         if key == "null":
             return None
 
-        concept_key, concept_id = core.utils.decode_concept(key)
+        concept_key, concept_id = core.utils.unstr_concept(key)
         if concept_key is not None:
             return self.sheerka.new((concept_key, concept_id)) if concept_id else self.sheerka.new(concept_key)
 

tests/_concepts.txt

@@ -1,15 +0,0 @@
-def concept one as 1
-def concept two as 2
-def concept three as 3
-def concept four as 4
-def concept five as 5
-def concept one as 1
-def concept two as 2
-def concept three as 3
-def concept four as 4
-def concept five as 5
-def concept one as 1
-def concept two as 2
-def concept three as 3
-def concept four as 4
-def concept five as 5

tests/core/_concepts.txt

@@ -8,3 +8,13 @@ def concept two as 2
 def concept three as 3
 def concept four as 4
 def concept five as 5
+def concept one as 1
+def concept two as 2
+def concept three as 3
+def concept four as 4
+def concept five as 5
+def concept one as 1
+def concept two as 2
+def concept three as 3
+def concept four as 4
+def concept five as 5

tests/core/test_SheerkaEvaluateConcept.py

@@ -45,7 +45,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
     ])
     def test_i_can_evaluate_the_other_metadata(self, expr, expected):
         """
-        I only test WHERE, it's the same for the others
+        I only test PRE, it's the same for the others
         :param expr:
         :param expected:
         :return:
@@ -53,15 +53,15 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         sheerka = self.get_sheerka()
 
-        concept = Concept("foo", where=expr)
+        concept = Concept("foo", pre=expr)
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
 
         assert evaluated.key == concept.key
         assert evaluated.metadata.body is None
-        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.pre == expr
         assert evaluated.metadata.post is None
-        assert evaluated.metadata.where == expr
+        assert evaluated.metadata.where is None
-        assert evaluated.get_metadata_value(ConceptParts.WHERE) == expected
+        assert evaluated.get_metadata_value(ConceptParts.PRE) == expected
         assert evaluated.props == {}
         assert evaluated.metadata.is_evaluated
         assert len(evaluated.values) == 0 if expr is None else 1
@@ -330,3 +330,42 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         assert evaluated.key == concept.init_key().key
 
+    @pytest.mark.parametrize("where_clause, expected", [
+        ("True", True),
+        ("False", False),
+        ("self < 10", False),
+        ("self < 11", True),
+        ("a < 20", False),
+        ("a > 19", True),
+        ("a + self > 20", True),
+    ])
+    def test_i_can_evaluate_simple_where(self, where_clause, expected):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body="10", where=where_clause).def_prop("a", "20")
+        sheerka.add_in_cache(concept)
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        if expected:
+            assert evaluated.key == concept.key
+        else:
+            assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED)
+            assert evaluated.body == concept
+
+    def test_i_can_evaluate_where_when_using_other_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo_true = Concept("foo_true", body="True").init_key()
+        foo_false = Concept("foo_false", body="False").init_key()
+        sheerka.add_in_cache(foo_false)
+        sheerka.add_in_cache(foo_true)
+
+        concept = Concept("foo", where="foo_true").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+
+        concept = Concept("foo", where="foo_false")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED)
+        assert evaluated.body == concept

tests/core/test_SheerkaSetsManager.py

@@ -1,8 +1,3 @@
-import os
-import shutil
-from os import path
-
-import pytest
 from core.builtin_concepts import ConceptAlreadyInSet, BuiltinConcepts
 from core.concept import Concept
 
@@ -11,42 +6,36 @@ from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 
 class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
 
-
+    def init(self, use_dict, *concepts):
-
+        sheerka = self.get_sheerka(use_dict, True)
-    def test_i_can_add_concept_to_set(self):
+        for c in concepts:
-        sheerka = self.get_sheerka(False, False)
+            sheerka.set_id_if_needed(c, False)
-
+            sheerka.add_in_cache(c)
-        foo = Concept("foo")
-        sheerka.set_id_if_needed(foo, False)
-
-        all_foos = Concept("all_foos")
-        sheerka.set_id_if_needed(all_foos, False)
 
         context = self.get_context(sheerka)
+        return sheerka, context
+
+    def test_i_can_add_concept_to_set(self):
+        foo = Concept("foo")
+        all_foos = Concept("all_foos")
+        sheerka, context = self.init(False, foo, all_foos)
+
         res = sheerka.add_concept_to_set(context, foo, all_foos)
 
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
-        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        all_entries = self.get_sheerka(False, True).sdp.get("All_" + all_foos.id, None, False)
         assert len(all_entries) == 1
         assert foo.id in all_entries
 
     def test_i_can_add_several_concepts_to_set(self):
-        sheerka = self.get_sheerka(False, False)
-
         foo1 = Concept("foo1")
-        sheerka.set_id_if_needed(foo1, False)
+        foo2 = Concept("foo2")
-
-        foo2 = Concept("foo1")
-        sheerka.set_id_if_needed(foo2, False)
-
         all_foos = Concept("all_foos")
-        sheerka.set_id_if_needed(all_foos, False)
+        sheerka, context = self.init(False, foo1, foo2, all_foos)
 
-        context = self.get_context(sheerka)
+        res = sheerka.sets_handler.add_concepts_to_set(context, (foo1, foo2), all_foos)
-        sheerka.add_concept_to_set(context, foo1, all_foos)
-        res = sheerka.add_concept_to_set(context, foo2, all_foos)
 
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
@@ -56,16 +45,30 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
         assert foo1.id in all_entries
         assert foo2.id in all_entries
 
+        # I can add another elements
+        foo3 = Concept("foo3")
+        foo4 = Concept("foo4")
+        for c in [foo3, foo4]:
+            sheerka.set_id_if_needed(c, False)
+            sheerka.add_in_cache(c)
+
+        res = sheerka.sets_handler.add_concepts_to_set(context, (foo3, foo4), all_foos)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 4
+        assert foo1.id in all_entries
+        assert foo2.id in all_entries
+        assert foo3.id in all_entries
+        assert foo4.id in all_entries
+
     def test_i_cannot_add_the_same_concept_twice_in_a_set(self):
-        sheerka = self.get_sheerka()
-
         foo = Concept("foo")
-        sheerka.set_id_if_needed(foo, False)
-
         all_foos = Concept("all_foos")
-        sheerka.set_id_if_needed(all_foos, False)
+        sheerka, context = self.init(True, foo, all_foos)
 
-        context = self.get_context(sheerka)
         sheerka.add_concept_to_set(context, foo, all_foos)
         res = sheerka.add_concept_to_set(context, foo, all_foos)
 
@@ -77,18 +80,12 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
         assert foo.id in all_entries
 
     def test_i_get_elements_from_a_set(self):
-        sheerka = self.get_sheerka()
-
         one = Concept("one")
         two = Concept("two")
         three = Concept("three")
         number = Concept("number")
+        sheerka, context = self.init(True, one, two, three, number)
 
-        for c in [one, two, three, number]:
-            sheerka.set_id_if_needed(c, False)
-            sheerka.add_in_cache(c)
-
-        context = self.get_context(sheerka)
         for c in [one, two, three]:
             sheerka.add_concept_to_set(context, c, number)
 
@@ -97,10 +94,8 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
         assert set(elements) == {one, two, three}
 
     def test_i_cannot_get_elements_if_not_a_set(self):
-        sheerka = self.get_sheerka()
         one = Concept("one")
-        sheerka.set_id_if_needed(one, False)
+        sheerka, context = self.init(True, one)
-        sheerka.add_in_cache(one)
 
         error = sheerka.get_set_elements(one)
 
@@ -108,17 +103,17 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
         assert error.body == one
 
     def test_isa_and_isa_group(self):
-        sheerka = self.get_sheerka()
+        group = Concept("group")
+        foo = Concept("foo")
+        sheerka, context = self.init(True, group, foo)
 
-        group = Concept("group").init_key()
-        group.metadata.id = "1001"
         assert not sheerka.isaset(group)
-
-        foo = Concept("foo").init_key()
-        foo.metadata.id = "1002"
         assert not sheerka.isa(foo, group)
 
         context = self.get_context(sheerka)
         sheerka.add_concept_to_set(context, foo, group)
         assert sheerka.isaset(group)
         assert sheerka.isa(foo, group)
+
+    def test_i_can_a_multiples_concepts(self):
+        pass

tests/core/test_tokenizer.py

@@ -40,7 +40,7 @@ def test_i_can_tokenize():
     assert tokens[31] == Token(TokenKind.AMPER, '&', 78, 6, 20)
     assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 21)
     assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 22)
-    assert tokens[34] == Token(TokenKind.CONCEPT, 'name', 81, 6, 23)
+    assert tokens[34] == Token(TokenKind.CONCEPT, ('name', None), 81, 6, 23)
     assert tokens[35] == Token(TokenKind.DOLLAR, '$', 88, 6, 30)
     assert tokens[36] == Token(TokenKind.STERLING, '£', 89, 6, 31)
     assert tokens[37] == Token(TokenKind.EURO, '€', 90, 6, 32)
@@ -79,8 +79,8 @@ def test_i_can_tokenize_identifiers(text, expected):
     ('"string', "Missing Trailing quote", '"string', 7, 1, 8),
     ('"a" + "string', "Missing Trailing quote", '"string', 13, 1, 14),
     ('"a"\n\n"string', "Missing Trailing quote", '"string', 12, 3, 8),
-    ("c::", "Concept name not found", "", 2, 1, 3),
+    ("c::", "Concept identifiers not found", "", 2, 1, 3),
-    ("c:foo\nbar:", "New line is forbidden in concept name", "foo", 5, 1, 6),
+    ("c:foo\nbar:", "New line in concept name", "foo", 5, 1, 6),
     ("c:foo", "Missing ending colon", "foo", 5, 1, 6)
 ])
 def test_i_can_detect_tokenizer_errors(text, message, error_text, index, line, column):
@@ -139,3 +139,17 @@ def test_i_can_recognize_keywords(text, expected):
     tokens = list(Tokenizer(text))
     assert tokens[0].type == TokenKind.KEYWORD
     assert tokens[0].value == expected
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("c:key:", ("key", None)),
+    ("c:key|id:", ("key", "id")),
+    ("c:key|:", ("key", None)),
+    ("c:|id:", (None, "id")),
+    ("c:125:", ("125", None)),
+])
+def test_i_can_parse_concept_token(text, expected):
+    tokens = list(Tokenizer(text))
+
+    assert tokens[0].type == TokenKind.CONCEPT
+    assert tokens[0].value == expected

tests/core/test_utils.py

@@ -1,10 +1,25 @@
 import core.utils
 import pytest
-from core.concept import ConceptParts
+from core.concept import ConceptParts, Concept
 
 from core.tokenizer import Token, TokenKind
 
 
+def get_tokens(lst):
+    res = []
+    for e in lst:
+        if e == " ":
+            res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0))
+        elif e == "\n":
+            res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0))
+        elif e == "<EOF>":
+            res.append(Token(TokenKind.EOF, "\n", 0, 0, 0))
+        else:
+            res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0))
+
+    return res
+
+
 @pytest.mark.parametrize("lst, as_string", [
     (None, "",),
     ([], ""),
@@ -136,18 +151,33 @@ def test_i_can_escape():
     (10, None, None),
     ("", None, None),
     ("xxx", None, None),
-    (":c:", None, None),
+    ("c:", None, None),
-    (":c:key", None, None),
+    ("c:key", None, None),
-    (":c:key:", "key", None),
+    ("c:key:", "key", None),
-    (":c:key:id", None, None),
+    ("c:key|id", None, None),
-    (":c:key:id:", "key", "id"),
+    ("c:key|id:", "key", "id"),
+    ("c:|id:", None, "id"),
+    ("c:key|:", "key", None),
 ])
-def test_i_can_decode_concept_repr(text, expected_key, expected_id):
+def test_i_can_unstr_concept(text, expected_key, expected_id):
-    k, i = core.utils.decode_concept(text)
+    k, i = core.utils.unstr_concept(text)
     assert k == expected_key
     assert i == expected_id
 
 
+def test_i_can_str_concept():
+    assert core.utils.str_concept(("key", "id")) == "c:key|id:"
+    assert core.utils.str_concept((None, "id")) == "c:|id:"
+    assert core.utils.str_concept(("key", None)) == "c:key:"
+    assert core.utils.str_concept((None, None)) == ""
+
+    concept = Concept("foo").init_key()
+    assert core.utils.str_concept(concept) == "c:foo:"
+
+    concept.metadata.id = "1001"
+    assert core.utils.str_concept(concept) == "c:foo|1001:"
+
+
 @pytest.mark.parametrize("text, expected", [
     (None, None),
     (10, None),
@@ -162,16 +192,22 @@ def test_i_can_decode_enum(text, expected):
     assert actual == expected
 
 
-def get_tokens(lst):
+def test_encode_concept_key_id():
-    res = []
+    assert core.utils.encode_concept(("key", "id")) == "__C__KEY_key__ID_id__C__"
-    for e in lst:
+    assert core.utils.encode_concept((None, "id")) == "__C__KEY_00None00__ID_id__C__"
-        if e == " ":
+    assert core.utils.encode_concept(("key", None)) == "__C__KEY_key__ID_00None00__C__"
-            res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0))
+    assert core.utils.encode_concept(("key", "id"), True) == "__C__USE_CONCEPT__KEY_key__ID_id__C__"
-        elif e == "\n":
+    assert core.utils.encode_concept(("k + y", "id")) == "__C__KEY_k000y__ID_id__C__"
-            res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0))
-        elif e == "<EOF>":
-            res.append(Token(TokenKind.EOF, "\n", 0, 0, 0))
-        else:
-            res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0))
 
-    return res
+    concept = Concept("foo").init_key()
+    assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_00None00__C__"
+
+    concept.metadata.id = "1001"
+    assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_1001__C__"
+
+
+def test_decode_concept_key_id():
+    assert core.utils.decode_concept("__C__KEY_key__ID_id__C__") == ("key", "id", False)
+    assert core.utils.decode_concept("__C__KEY_00None00__ID_id__C__") == (None, "id", False)
+    assert core.utils.decode_concept("__C__KEY_key__ID_00None00__C__") == ("key", None, False)
+    assert core.utils.decode_concept("__C__USE_CONCEPT__KEY_key__ID_id__C__") == ("key", "id", True)

tests/evaluators/test_ConceptEvaluator.py

@@ -21,7 +21,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
     def test_i_can_evaluate_concept(self):
         context = self.get_context()
         concept = Concept(name="foo",
-                          where="1",
+                          where="True",
                           pre="2",
                           post="3").def_prop("a", "4").def_prop("b", "5")
 
@@ -32,7 +32,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert result.who == evaluator.name
         assert result.status
         assert result.value.name == "foo"
-        assert result.value.get_metadata_value(ConceptParts.WHERE) == 1
+        assert result.value.get_metadata_value(ConceptParts.WHERE) == True
         assert result.value.get_metadata_value(ConceptParts.PRE) == 2
         assert result.value.get_metadata_value(ConceptParts.POST) == 3
         assert result.value.get_prop("a") == 4
@@ -44,7 +44,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         concept = Concept(name="foo",
                           body="'I have a value'",
-                          where="1",
+                          where="True",
                           pre="2",
                           post="3").set_prop("a", "4").set_prop("b", "5")
 
@@ -61,7 +61,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         concept = Concept(name="foo",
                           body="'I have a value'",
-                          where="1",
+                          where="True",
                           pre="2",
                           post="3").set_prop("a", "4").set_prop("b", "5")
 

tests/evaluators/test_PythonEvaluator.py

@@ -116,23 +116,23 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert not evaluated.status
         assert evaluated.body.body.args[0] == "'int' object has no attribute 'name'"
 
-    @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [
+    # @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [
-        ("__C__key__C__", "key", None, False),
+    #     ("__C__key__C__", "key", None, False),
-        ("__C__key__id__C__", "key", "id", False),
+    #     ("__C__key__id__C__", "key", "id", False),
-        ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+    #     ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
-        ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+    #     ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
-    ])
+    # ])
-    def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept):
+    # def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept):
-        context = self.get_context()
+    #     context = self.get_context()
-        assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept)
+    #     assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept)
-
+    #
-    @pytest.mark.parametrize("text", [
+    # @pytest.mark.parametrize("text", [
-        "__C__",
+    #     "__C__",
-        "__C__key",
+    #     "__C__key",
-        "__C__key____",
+    #     "__C__key____",
-        "__C____",
+    #     "__C____",
-        "__C__USE_CONCEPT__",
+    #     "__C__USE_CONCEPT__",
-    ])
+    # ])
-    def test_i_cannot_resolve_name(self, text):
+    # def test_i_cannot_resolve_name(self, text):
-        context = self.get_context()
+    #     context = self.get_context()
-        assert PythonEvaluator().resolve_name(context, text) is None
+    #     assert PythonEvaluator().resolve_name(context, text) is None

tests/non_reg/test_sheerka_non_reg.py

@@ -273,8 +273,9 @@ as:
 
         saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY)
         expected_bnf = Sequence(
-            a, Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus"))))
+            ConceptExpression(a, rule_name="a"),
-        assert saved_definitions[saved_concept] == expected_bnf
+            Optional(Sequence(StrMatch("plus"), ConceptExpression(saved_concept, rule_name="plus"))))
+        assert saved_definitions["c:plus|1001:"] == "(c:a:=a ('plus' c:plus|1001:=plus)?)"
 
         new_concept = res[0].value.body
         assert new_concept.metadata.name == "plus"
@@ -456,6 +457,26 @@ as:
         assert res[0].status
         assert res[0].body == 23
 
+    def test_i_can_mix_bnf_and_isa_when_concept_other_case(self):
+        sheerka = self.get_sheerka()
+
+        init = [
+            "def concept one as 1",
+            "def concept twenty as 20",
+            "def concept number",
+            "one isa number",
+            "twenty isa number",
+            "def concept twenties from bnf twenty number as twenty + number"
+        ]
+
+        for exp in init:
+            sheerka.evaluate_user_input(exp)
+
+        res = sheerka.evaluate_user_input("twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == simplec("twenties", 21)
+
     def test_i_can_mix_concept_of_concept(self):
         sheerka = self.get_sheerka()
 
@@ -623,3 +644,30 @@ as:
         assert len(res) == 1
         assert res[0].status
         assert res[0].body == 21
+
+    def test_i_can_use_where_in_bnf(self):
+        sheerka = self.get_sheerka()
+
+        init = [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept three as 3",
+            "def concept twenty as 20",
+            "def concept number",
+            "one isa number",
+            "two isa number",
+            "three isa number",
+            "def concept twenties from bnf twenty number where number <= 2 as twenty + number"
+        ]
+
+        for exp in init:
+            sheerka.evaluate_user_input(exp)
+
+        res = sheerka.evaluate_user_input("eval twenty one")
+        assert len(res) == 1 and res[0].status and res[0].body == 21
+
+        res = sheerka.evaluate_user_input("eval twenty two")
+        assert len(res) == 1 and res[0].status and res[0].body == 22
+
+        res = sheerka.evaluate_user_input("eval twenty three")
+        assert len(res) > 1

tests/parsers/test_BaseParser.py

@@ -18,7 +18,7 @@ def test_i_can_get_text_from_tokens(text, expected_text):
 
 
 @pytest.mark.parametrize("text, custom, expected_text", [
-    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value}"}, "execute(__C__concept_name)")
+    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)")
 ])
 def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
     tokens = list(Tokenizer(text))

tests/parsers/test_BnfParser.py

@@ -2,7 +2,7 @@ import pytest
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.tokenizer import Tokenizer, TokenKind, LexerError
+from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
 from parsers.BaseParser import UnexpectedTokenErrorNode
 from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
 from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
@@ -16,6 +16,14 @@ class ClassWithName:
         self.name = name
 
 
+def c(name):
+    concept = Concept(name).init_key()
+    return ConceptExpression(concept, rule_name=name)
+
+
+eof_token = Token(TokenKind.EOF, "", 0, 0, 0)
+
+
 class TestBnfParser(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("expression, expected", [
@@ -33,9 +41,10 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("1 2 | 3 4+", OrderedChoice(
             Sequence(StrMatch("1"), StrMatch("2")),
             Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))),
-        (
+        ("1 (2 | 3) 4+", Sequence(
-            "1 (2 | 3) 4+",
+            StrMatch("1"),
-            Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))),
+            OrderedChoice(StrMatch("2"), StrMatch("3")),
+            OneOrMore(StrMatch("4")))),
         ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))),
         ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))),
         ("1 *", Sequence(StrMatch("1"), StrMatch("*"))),
@@ -61,6 +70,13 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")),
         ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")),
         ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))),
+        ("(1=a 2=b)=c", Sequence(StrMatch("1", rule_name="a"), StrMatch("2", rule_name="b"), rule_name="c")),
+        ("(1*=a)", ZeroOrMore(StrMatch("1"), rule_name="a")),
+        ("'a'* 'b'+", Sequence(ZeroOrMore(StrMatch("a")), OneOrMore(StrMatch("b")))),
+        ("('a'* 'b'+)", Sequence(ZeroOrMore(StrMatch("a")), OneOrMore(StrMatch("b")))),
+        ("('a'*=x 'b'+=y)=z", Sequence(
+            ZeroOrMore(StrMatch("a"), rule_name="x"),
+            OneOrMore(StrMatch("b"), rule_name="y"), rule_name="z")),
     ])
     def test_i_can_parse_regex(self, expression, expected):
         parser = BnfParser()
@@ -72,12 +88,12 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         assert res.value.source == expression
 
     @pytest.mark.parametrize("expression, expected", [
-        ("foo", Concept("foo").init_key()),
+        ("foo", c("foo")),
-        ("foo*", ZeroOrMore(Concept("foo").init_key())),
+        ("foo*", ZeroOrMore(c("foo"))),
-        ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))),
+        ("foo 'and' bar+", Sequence(c("foo"), StrMatch("and"), OneOrMore(c("bar")))),
-        ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))),
+        ("foo | bar?", OrderedChoice(c("foo"), Optional(c("bar")))),
-        ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+        ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), c("var"))),
-        ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+        ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), c("var"))),
     ])
     def test_i_can_parse_regex_with_concept(self, expression, expected):
         foo = Concept("foo")
@@ -113,8 +129,8 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
     @pytest.mark.parametrize("expression, error", [
         ("1 ", UnexpectedEndOfFileError()),
         ("1|", UnexpectedEndOfFileError()),
-        ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.RPAR])),
+        ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.RPAR])),
-        ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.IDENTIFIER])),
+        ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.IDENTIFIER])),
         ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
     ])
     def test_i_can_detect_errors(self, expression, error):

tests/parsers/test_ConceptLexerParser.py

@@ -1,10 +1,13 @@
+from ast import Str
+
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DoNotResolve
 from core.tokenizer import Tokenizer, TokenKind, Token
+from parsers.BnfParser import BnfParser
 from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \
     ParsingExpressionVisitor, TerminalNode, NonTerminalNode, ZeroOrMore, OneOrMore, \
-    UnrecognizedTokensNode, cnode, short_cnode
+    UnrecognizedTokensNode, cnode, short_cnode, ConceptExpression, ConceptGroupExpression
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -75,6 +78,7 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         for c in concepts:
             context.sheerka.add_in_cache(c)
+            context.sheerka.set_id_if_needed(c, False)
 
         parser = ConceptLexerParser()
         parser.initialize(context, grammar)
@@ -586,6 +590,32 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka):
         assert res.status
         assert res.value.body == [cnode("foo", 0, 2, "twenty one")]
 
+    def test_i_can_initialize_when_cyclic_reference(self):
+        foo = Concept(name="foo")
+        grammar = {foo: Optional("one", ConceptExpression("foo"))}
+        context, parser = self.init([foo], grammar)
+
+        assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression(foo, rule_name="foo"))
+
+    def test_i_cannot_initialize_when_cyclic_reference_when_concept_is_under_construction_and_not_known(self):
+        foo = Concept(name="foo").init_key()
+        grammar = {foo: Optional("one", ConceptExpression("foo"))}
+
+        context = self.get_context()
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+        assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression("foo", rule_name="foo"))
+
+    def test_i_can_initialize_when_cyclic_reference_when_concept_is_under_construction_and_known(self):
+        foo = Concept(name="foo").init_key()
+        grammar = {foo: Optional("one", ConceptExpression("foo"))}
+
+        context = self.get_context()
+        context.concepts["foo"] = foo
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+        assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression(foo, rule_name="foo"))
+
     def test_i_can_parse_concept_reference_that_is_group(self):
         """
         if one is  number, then number is a 'group'
@@ -1092,6 +1122,109 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka):
         assert cprop(concept_found, "seq")[1] == DoNotResolve("un ok")
         assert cprop(concept_found, "seq")[2] == DoNotResolve("uno ok")
 
+    @pytest.mark.parametrize("rule, expected", [
+        (StrMatch("string"), "'string'"),
+        (StrMatch("string", rule_name="rule_name"), "'string'=rule_name"),
+        (Sequence(StrMatch("foo"), StrMatch("bar")), "('foo' 'bar')"),
+        (Sequence(StrMatch("foo"), StrMatch("bar"), rule_name="rule_name"), "('foo' 'bar')=rule_name"),
+        (OrderedChoice(StrMatch("foo"), StrMatch("bar")), "('foo'|'bar')"),
+        (OrderedChoice(StrMatch("foo"), StrMatch("bar"), rule_name="rule_name"), "('foo'|'bar')=rule_name"),
+        (Optional(StrMatch("foo")), "'foo'?"),
+        (Optional(StrMatch("foo"), rule_name="rule_name"), "'foo'?=rule_name"),
+        (ZeroOrMore(StrMatch("foo")), "'foo'*"),
+        (ZeroOrMore(StrMatch("foo"), rule_name="rule_name"), "'foo'*=rule_name"),
+        (OneOrMore(StrMatch("foo")), "'foo'+"),
+        (OneOrMore(StrMatch("foo"), rule_name="rule_name"), "'foo'+=rule_name"),
+        (Sequence(
+            Optional(StrMatch("foo"), rule_name="a"),
+            ZeroOrMore(StrMatch("bar"), rule_name="b"),
+            OneOrMore(StrMatch("baz"), rule_name="c"),
+            rule_name="d"), "('foo'?=a 'bar'*=b 'baz'+=c)=d"),
+        (OrderedChoice(
+            Optional(StrMatch("foo"), rule_name="a"),
+            ZeroOrMore(StrMatch("bar"), rule_name="b"),
+            OneOrMore(StrMatch("baz"), rule_name="c"),
+            rule_name="d"), "('foo'?=a|'bar'*=b|'baz'+=c)=d"),
+        (Sequence(
+            OrderedChoice(StrMatch("foo"), StrMatch("bar"), rule_name="a"),
+            OrderedChoice(StrMatch("x"), StrMatch("y"), rule_name="b"),
+            rule_name="c"), "(('foo'|'bar')=a ('x'|'y')=b)=c")
+    ])
+    def test_i_can_encode_grammar(self, rule, expected):
+        foo = Concept(name="foo")
+        grammar = {foo: rule}
+        context, parser = self.init([foo], grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == expected
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+        assert parse_res.value.value == rule
+
+    def test_i_can_encode_grammar_when_concept_simple(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        grammar = {foo: ConceptExpression(bar)}
+        context, parser = self.init([foo, bar], grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == "c:bar|1002:=bar"
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+        assert parse_res.value.value == grammar[foo]
+
+    def test_i_can_encode_grammar_when_concepts(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        baz = Concept(name="baz")
+        grammar = {foo: Sequence(
+            StrMatch("a"),
+            OrderedChoice(ConceptExpression(bar),
+                          OneOrMore(ConceptExpression(baz)), rule_name="oc"), rule_name="s")}
+        context, parser = self.init([foo, bar, baz], grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == "('a' (c:bar|1002:=bar|c:baz|1003:=baz+)=oc)=s"
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+        assert parse_res.value.value == grammar[foo]
+
+    def test_i_can_encode_grammar_when_set_concepts(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        baz = Concept(name="baz")
+        grammar = {foo: Sequence(
+            StrMatch("a"),
+            OrderedChoice(bar,
+                          OneOrMore(ConceptExpression(baz)), rule_name="oc"), rule_name="s")}
+        context = self.get_context()
+        for c in [foo, bar, baz]:
+            context.sheerka.add_in_cache(c)
+            context.sheerka.set_id_if_needed(c, False)
+        context.sheerka.add_concept_to_set(context, baz, bar)
+
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == "('a' (c:bar|1002:=bar|c:baz|1003:=baz+)=oc)=s"
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+
+        expected = Sequence(
+            StrMatch("a"),
+            OrderedChoice(ConceptGroupExpression(bar, rule_name="bar"),
+                          OneOrMore(ConceptExpression(baz, rule_name="baz")), rule_name="oc"), rule_name="s")
+        assert parse_res.value.value == expected
+
     #
     # def test_i_can_parse_basic_arithmetic_operations_and_resolve_properties(self):
     #     context = self.get_context()

tests/parsers/test_DefaultParser.py

@@ -3,7 +3,7 @@ import ast
 
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept
-from parsers.ConceptLexerParser import OrderedChoice, StrMatch
+from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptExpression
 from parsers.PythonParser import PythonParser, PythonNode
 from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode
@@ -246,7 +246,7 @@ def concept add one to a as
         parser = DefaultParser()
         res = parser.parse(context, text)
         node = res.value.value
-        definition = OrderedChoice(a_concept, StrMatch("a_string"))
+        definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
         parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition)
         expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result)
 
@@ -321,7 +321,7 @@ def concept add one to a as
         ("def concept 'name", "Missing Trailing quote", "'name"),
         ("def concept name as 'body", "Missing Trailing quote", "'body"),
         ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"),
-        ("def concept c::", "Concept name not found", ""),
+        ("def concept c::", "Concept identifiers not found", ""),
     ])
     def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text):
         parser = DefaultParser()

tests/parsers/test_PythonParser.py

@@ -3,6 +3,7 @@ import pytest
 from core.builtin_concepts import ParserResultConcept
 from core.tokenizer import Tokenizer, LexerError
 from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
+import core.utils
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -52,8 +53,8 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
         assert isinstance(res.value.value[0].exception, SyntaxError)
 
     @pytest.mark.parametrize("text, error_msg, error_text", [
-        ("c::", "Concept name not found", ""),
+        ("c::", "Concept identifiers not found", ""),
-        ("c:: + 1", "Concept name not found", ""),
+        ("c:: + 1", "Concept identifiers not found", ""),
     ])
     def test_i_can_detect_lexer_errors(self, text, error_msg, error_text):
         parser = PythonParser()
@@ -66,12 +67,12 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
         assert res.body.body[0].text == error_text
 
     def test_i_can_parse_a_concept(self):
-        text = "c:concept_name: + 1"
+        text = "c:name|key: + 1"
 
         parser = PythonParser()
         res = parser.parse(self.get_context(), text)
 
         assert res
         assert res.value.value == PythonNode(
-            "c:concept_name: + 1",
+            "c:name|key: + 1",
-            ast.parse("__C__USE_CONCEPT__concept_name__C__+1", mode="eval"))
+            ast.parse(core.utils.encode_concept(("name", "key"), True) + "+1", mode="eval"))

tests/sheerkapickle/test_SheerkaPickler.py

@@ -127,7 +127,7 @@ class TestSheerkaPickler(TestUsingFileBasedSheerka):
         sheerka.add_in_cache(concept)
         obj = {concept: "a"}
         flatten = SheerkaPickler(sheerka).flatten(obj)
-        assert flatten == {':c:foo:1001:': 'a'}
+        assert flatten == {'c:foo|1001:': 'a'}
 
         decoded = SheerkaUnpickler(sheerka).restore(flatten)
         assert decoded == obj