Fixed infinite recursion when parsing complex BNF node

.gitignore

@@ -6,5 +6,6 @@ __pycache__
 build
 _build
 prof
+log.txt
 tests/_concepts.txt
 tests/**/*result_test

_concepts.txt

@@ -69,10 +69,10 @@ def concept ninety as 90
 ninety isa number
 def concept nineties from bnf ninety number where number < 10 as ninety + number
 nineties isa number
-def concept hundreds1 from number hundred where number1 < 10 as number1 * 100
+# def concept hundreds1 from number 'hundred' where number < 10 as number * 100
-def concept hundreds2 from number1 hundred and number2 where number1 < 10 and number2 < 100 as number1 * 100 + number2
+# def concept hundreds2 from number=number1 'hundred' 'and' number=number2 where number1 < 10 and number2 < 100 as number1 * 100 + number2
-def concept one hundred as 100
+# def concept one hundred as 100
-one hundred isa number
+# c:one hundred: isa number
-hundreds1 isa number
+# hundreds1 isa number
-hundreds2 isa number
+# hundreds2 isa number
 def concept history as history()

src/cache/CacheManager.py

@@ -262,6 +262,7 @@ class CacheManager:
 
     def reset(self, cache_only):
         """For unit test speed enhancement"""
+        self.clear()
         self.cache_only = cache_only
         self.caches.clear()
         self.concept_caches.clear()

src/core/builtin_concepts.py

@@ -37,6 +37,7 @@ class BuiltinConcepts(Enum):
     MANAGE_INFINITE_RECURSION = "manage infinite recursion"
     PARSE_CODE = "execute source code"
     EXEC_CODE = "execute source code"
+    TESTING = "testing"
 
     USER_INPUT = "user input"  # represent an input from an user
     SUCCESS = "success"

src/core/concept.py

@@ -332,6 +332,9 @@ class Concept:
             for k in other.values:
                 self.set_value(k, other.get_value(k))
 
+        # update bnf definition
+        self.bnf = other.bnf
+
         # origin
         from sdp.sheerkaSerializer import Serializer
         if hasattr(other, Serializer.ORIGIN):
@@ -533,6 +536,10 @@ class CC:
         self.end = None  # for debug purpose, indicate where the concept ends
         self.exclude_body = exclude_body
 
+        if "body" in self.compiled:
+            self.compiled[ConceptParts.BODY] = self.compiled["body"]
+            del self.compiled["body"]
+
     def __eq__(self, other):
         if id(self) == id(other):
             return True
@@ -544,7 +551,10 @@ class CC:
                 to_compare = {k: v for k, v in other.compiled.items() if k != ConceptParts.BODY}
             else:
                 to_compare = other.compiled
-            return self.compiled == to_compare
+            if self.compiled == to_compare:
+                return True
+            else:
+                return False
 
         if not isinstance(other, CC):
             return False

src/core/sheerka/ExecutionContext.py

@@ -334,7 +334,10 @@ class ExecutionContext:
         for k, v in self._bag.items():
             bag[k] = v
             bag["bag." + k] = v
-        for prop in ("id", "who", "desc", "obj", "inputs", "values", "concepts"):
+        for prop in ("id", "who", "action", "desc", "obj", "inputs", "values", "concepts"):
+            bag[prop] = getattr(self, prop)
+        bag["action"] = self.action_context
+        for prop in ("desc", "obj", "inputs", "values", "concepts"):
             bag[prop] = getattr(self, prop)
         bag["status"] = self.get_status()
         bag["elapsed"] = self.elapsed
@@ -362,15 +365,23 @@ class ExecutionContext:
         :param predicate:
         :return:
         """
-        res = []
+        return list(self.search(predicate, None, False))
-        current = self
+
-        while True:
+    def search(self, predicate=None, get_obj=None, start_with_self=False, stop=None):
-            parent = current._parent
+        """
-            if parent:
+        Iter thru execution context parent and return the list of obj
-                if predicate is None or predicate(parent):
+        :param predicate: what execution context to keep
-                    res.append(parent)
+        :param get_obj: lambda to compute what to return
-                current = parent
+        :param start_with_self: include the current execution context in the search
-            else:
+        :param stop: stop the search if matched
+        :return:
+        """
+        current = self if start_with_self else self._parent
+        while current:
+            if stop and stop(current):
                 break
 
-        return res
+            if predicate is None or predicate(current):
+                yield current if get_obj is None else get_obj(current)
+
+            current = current._parent

src/core/sheerka/Sheerka.py

@@ -328,7 +328,7 @@ class Sheerka(Concept):
             self.evaluators.append(evaluator)
 
     def initialize_concept_node_parsing(self, context):
-        self.init_log.debug("Initializing concept node parsing.")
+        self.init_log.debug("siInitializing concepts by first keyword.")
 
         concepts_by_first_keyword = self.cache_manager.copy(self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY)
         res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)

src/core/sheerka/services/SheerkaAdmin.py

@@ -1,3 +1,5 @@
+import time
+
 from core.builtin_concepts import BuiltinConcepts
 from core.sheerka.services.sheerka_service import BaseService
 
@@ -39,6 +41,7 @@ class SheerkaAdmin(BaseService):
         :return:
         """
         try:
+            start = time.time_ns()
             self.sheerka.during_restore = True
             with open(CONCEPTS_FILE, "r") as f:
                 for line in f.readlines():
@@ -50,5 +53,11 @@ class SheerkaAdmin(BaseService):
                     if len(res) > 1 or not res[0].status:
                         self.sheerka.log.error("Error detected !")
             self.sheerka.during_restore = False
+            stop = time.time_ns()
+
+            nano_sec = stop - start
+            dt = nano_sec / 1e6
+            elapsed = f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
+            print(f"Execution time: {elapsed}")
         except IOError:
             pass

src/core/sheerka/services/SheerkaComparisonManager.py

@@ -90,12 +90,12 @@ class SheerkaComparisonManager(BaseService):
         cache = Cache()
         self.sheerka.cache_manager.register_cache(self.RESOLVED_COMPARISON_ENTRY, cache, persist=False)
 
-        self.sheerka.bind_service_method(self.is_greater_than)
+        self.sheerka.bind_service_method(self.set_is_greater_than)
-        self.sheerka.bind_service_method(self.is_less_than)
+        self.sheerka.bind_service_method(self.set_is_less_than)
         self.sheerka.bind_service_method(self.get_partition)
         self.sheerka.bind_service_method(self.get_concepts_weights)
 
-    def is_greater_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
+    def set_is_greater_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
         """
         Records that the property of concept a is greater than concept b's one
         :param context:
@@ -112,7 +112,7 @@ class SheerkaComparisonManager(BaseService):
         comparison_obj = ComparisonObj(event_digest, prop_name, concept_a.id, concept_b.id, ">", comparison_context)
         return self._inner_add_comparison(comparison_obj)
 
-    def is_less_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
+    def set_is_less_than(self, context, prop_name, concept_a, concept_b, comparison_context="#"):
         """
         Records that the property of concept a is lesser than concept b's one
         :param context:

src/core/sheerka/services/SheerkaCreateNewConcept.py

@@ -52,7 +52,7 @@ class SheerkaCreateNewConcept(BaseService):
         sheerka.set_id_if_needed(concept, False)
 
         # compute new concepts_by_first_keyword
-        init_ret_value = self.bnp.get_concepts_by_first_keyword(context, [concept], True)
+        init_ret_value = self.bnp.get_concepts_by_first_token(context, [concept], True)
         if not init_ret_value.status:
             return sheerka.ret(self.NAME, False, ErrorConcept(init_ret_value.value))
         concepts_by_first_keyword = init_ret_value.body

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -296,6 +296,11 @@ class SheerkaEvaluateConcept(BaseService):
 
         concept.init_key()  # only does it if needed
         concept.metadata.is_evaluated = "body" in all_metadata_to_eval
+
+        # # update the cache for concepts with no variable
+        # if len(concept.metadata.variables) == 0:
+        #     self.sheerka.cache_manager.put(self.sheerka.CONCEPTS_BY_ID_ENTRY, concept.id, concept)
+
         return concept
 
     def choose_metadata_to_eval(self, context, concept):

src/core/sheerka/services/SheerkaExecute.py

@@ -173,6 +173,17 @@ class SheerkaExecute(BaseService):
         return pi
 
     def call_parsers(self, context, return_values):
+        """
+        Call all the parsers, ordered by priority
+        Possible return value for a parser:
+            None : indicate that you do no need to care about the result
+            ParserResult with status False : Success
+            ParserResult with status False : failed to parse, but the result will be reused by other parsers
+            NotForMe (status is False) : Failed to parse. Do no reuse the result
+        :param context:
+        :param return_values:
+        :return:
+        """
 
         # return_values must be a list
         if not isinstance(return_values, list):

src/core/sheerka/services/SheerkaSetsManager.py

@@ -1,4 +1,5 @@
 import core.builtin_helpers
+from cache.Cache import Cache
 from cache.SetCache import SetCache
 from core.ast.nodes import python_to_concept
 from core.builtin_concepts import BuiltinConcepts
@@ -11,9 +12,12 @@ GROUP_PREFIX = 'All_'
 class SheerkaSetsManager(BaseService):
     NAME = "SetsManager"
     CONCEPTS_GROUPS_ENTRY = "SetsManager:Concepts_Groups"
+    CONCEPTS_IN_GROUPS_ENTRY = "SetsManager:Concepts_In_Groups" # cache for get_set_elements()
 
     def __init__(self, sheerka):
         super().__init__(sheerka)
+        self.sets = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k))
+        self.concepts_in_set = Cache()
 
     def initialize(self):
         self.sheerka.bind_service_method(self.set_isa)
@@ -23,8 +27,8 @@ class SheerkaSetsManager(BaseService):
         self.sheerka.bind_service_method(self.isa)
         self.sheerka.bind_service_method(self.isaset)
 
-        cache = SetCache(default=lambda k: self.sheerka.sdp.get(self.CONCEPTS_GROUPS_ENTRY, k))
+        self.sheerka.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, self.sets)
-        self.sheerka.cache_manager.register_cache(self.CONCEPTS_GROUPS_ENTRY, cache)
+        self.sheerka.cache_manager.register_cache(self.CONCEPTS_IN_GROUPS_ENTRY, self.concepts_in_set, persist=False)
 
     def set_isa(self, context, concept, concept_set):
         """
@@ -64,14 +68,14 @@ class SheerkaSetsManager(BaseService):
         context.log(f"Adding concept {concept} to set {concept_set}", who=self.NAME)
         ensure_concept(concept, concept_set)
 
-        set_elements = self.sheerka.cache_manager.get(self.CONCEPTS_GROUPS_ENTRY, concept_set.id)
+        set_elements = self.sets.get(concept_set.id)
         if set_elements and concept.id in set_elements:
             return self.sheerka.ret(
                 self.NAME,
                 False,
                 self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
 
-        self.sheerka.cache_manager.put(self.CONCEPTS_GROUPS_ENTRY, concept_set.id, concept.id)
+        self.sets.put(concept_set.id, concept.id)
         return self.sheerka.ret(self.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
     def add_concepts_to_set(self, context, concepts, concept_set):
@@ -109,8 +113,8 @@ class SheerkaSetsManager(BaseService):
             if not self.isaset(context, sub_concept):
                 return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
 
-            # first, try to see if sub_context has it's own group entry
+            # first, try to see if sub_concept has it's own group entry
-            ids = self.sheerka.cache_manager.get(self.CONCEPTS_GROUPS_ENTRY, sub_concept.id)
+            ids = self.sets.get(sub_concept.id)
             concepts = self._get_concepts(context, ids, True)
 
             # aggregate with en entries from its body
@@ -139,7 +143,12 @@ class SheerkaSetsManager(BaseService):
 
             return concepts
 
-        return _get_set_elements(concept)
+        if res := self.concepts_in_set.get(concept.id):
+            return res
+
+        res = _get_set_elements(concept)
+        self.concepts_in_set.put(concept.id, res)
+        return res
 
     def isinset(self, a, b):
         """
@@ -156,7 +165,7 @@ class SheerkaSetsManager(BaseService):
         if not (a.id and b.id):
             return False
 
-        group_elements = self.sheerka.cache_manager.get(self.CONCEPTS_GROUPS_ENTRY, b.id)
+        group_elements = self.sets.get(b.id)
         return group_elements and a.id in group_elements
 
     def isa(self, a, b):
@@ -187,7 +196,7 @@ class SheerkaSetsManager(BaseService):
 
         # check if it has a group
         # TODO: use cache instead of directly requesting sdp
-        if self.sheerka.cache_manager.get(self.CONCEPTS_GROUPS_ENTRY, concept.id):
+        if self.sets.get(concept.id):
             return True
 
         # it may be a concept that references a set
@@ -240,9 +249,18 @@ for x in xx__concepts__xx:
                           desc=f"Evaluating concepts of a set") as sub_context:
             sub_context.add_inputs(ids=ids)
             sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+            errors = []
             for element_id in ids:
                 concept = self.sheerka.get_by_id(element_id)
+                if len(concept.metadata.variables) == 0:
+                    # only evaluate
                     evaluated = self.sheerka.evaluate_concept(sub_context, concept)
+                    if context.sheerka.is_success(evaluated):
                         result.append(evaluated)
-            sub_context.add_inputs(return_value=result)
+                    else:
+                        errors.append(evaluated)
+                else:
+                    result.append(concept)
+            sub_context.add_values(return_value=result)
+            sub_context.add_values(errors=errors)
         return result

src/core/sheerka/services/SheerkaVariableManager.py

@@ -30,6 +30,8 @@ class SheerkaVariableManager(BaseService):
         self.sheerka.bind_service_method(self.record)
         self.sheerka.bind_service_method(self.load)
         self.sheerka.bind_service_method(self.delete)
+        self.sheerka.bind_service_method(self.set)
+        self.sheerka.bind_service_method(self.get)
 
         cache = Cache(default=lambda k: self.sheerka.sdp.get(self.VARIABLES_ENTRY, k))
         self.sheerka.cache_manager.register_cache(self.VARIABLES_ENTRY, cache, True, True)
@@ -56,3 +58,9 @@ class SheerkaVariableManager(BaseService):
 
     def delete(self, context, who, key):
         self.sheerka.cache_manager.delete(self.VARIABLES_ENTRY, who + "|" + key)
+
+    def set(self, context, key, value):
+        return self.record(context, context.event.user_id, key, value)
+
+    def get(self, context, key):
+        return self.load(context.event.user_id, key)

src/core/utils.py

@@ -187,6 +187,32 @@ def remove_list_from_list(lst, to_remove):
     return lst
 
 
+def make_unique(lst, get_id=None):
+    """
+    All items in the list are now uniq and the order is kept
+    >>> assert make_unique(["a", "a", "b", "c", "c"]) == ["a", "b",  "c"]
+    :param lst:
+    :param get_id: define your own way to recognize the items
+    :return:
+    """
+
+    def _make_unique(seq, get_id=None):
+        seen = set()
+        if get_id is None:
+            for x in seq:
+                if x not in seen:
+                    seen.add(x)
+                    yield x
+        else:
+            for x in seq:
+                x = get_id(x)
+                if x not in seen:
+                    seen.add(x)
+                    yield x
+
+    return list(_make_unique(lst, get_id))
+
+
 def product(a, b):
     """
     Kind of cartesian product between lists a and b
@@ -241,7 +267,6 @@ def dict_product(a, b):
     return res
 
 
-
 def strip_quotes(text):
     if not isinstance(text, str):
         return text

src/parsers/BaseNodeParser.py

@@ -1,6 +1,7 @@
 from collections import namedtuple
 from dataclasses import dataclass
 from enum import Enum
+from typing import Set
 
 import core.utils
 from core.builtin_concepts import BuiltinConcepts
@@ -12,6 +13,11 @@ from parsers.BaseParser import Node, BaseParser, ErrorNode
 DEBUG_COMPILED = True
 
 
+@dataclass
+class ChickenAndEggError(Exception):
+    concepts: Set[str]
+
+
 @dataclass()
 class LexerNode(Node):
     start: int  # starting index in the tokens list
@@ -422,7 +428,7 @@ class CN(HelperWithPos):
     ConceptNode tester class
     It matches with ConceptNode but with less constraints
 
-    CNC == ConceptNode if concept key, start, end and source are the same
+    CN == ConceptNode if concept key, start, end and source are the same
     """
 
     def __init__(self, concept, start=None, end=None, source=None):
@@ -496,6 +502,9 @@ class CNC(CN):
         super().__init__(concept_key, start, end, source)
         self.compiled = kwargs
         self.exclude_body = exclude_body
+        if "body" in self.compiled:
+            self.compiled[ConceptParts.BODY] = self.compiled["body"]
+            del self.compiled["body"]
 
     def __eq__(self, other):
         if id(self) == id(other):
@@ -516,7 +525,10 @@ class CNC(CN):
                 to_compare = {k: v for k, v in other.concept.compiled.items() if k != ConceptParts.BODY}
             else:
                 to_compare = other.concept.compiled
-            return self.compiled == to_compare
+            if self.compiled == to_compare:
+                return True
+            else:
+                return False
 
         if not isinstance(other, CNC):
             return False
@@ -613,7 +625,7 @@ class BaseNodeParser(BaseParser):
         :param concepts
         :return:
         """
-        concepts_by_first_keyword = self.get_concepts_by_first_keyword(context, concepts).body
+        concepts_by_first_keyword = self.get_concepts_by_first_token(context, concepts).body
         self.concepts_by_first_keyword = self.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword).body
 
     def reset_parser(self, context, parser_input: ParserInput):
@@ -626,38 +638,6 @@ class BaseNodeParser(BaseParser):
             self.add_error(self.sheerka.new(BuiltinConcepts.ERROR, body=e), False)
             return False
         return True
-        # self.text = text
-        #
-        # try:
-        #     self.tokens = list(self.get_input_as_tokens(text))
-        #
-        #
-        # self.token = None
-        # self.pos = -1
-        # return True
-
-    # def add_error(self, error, next_token=True):
-    #     self.error_sink.append(error)
-    #     if next_token:
-    #         self.parser_input.next_token()
-    #     return error
-
-    # def get_token(self) -> Token:
-    #     return self.token
-    #
-    # def next_token(self, skip_whitespace=True):
-    #     if self.token and self.token.type == TokenKind.EOF:
-    #         return False
-    #
-    #     self.pos += 1
-    #     self.token = self.tokens[self.pos]
-    #
-    #     if skip_whitespace:
-    #         while self.token.type == TokenKind.WHITESPACE or self.token.type == TokenKind.NEWLINE:
-    #             self.pos += 1
-    #             self.token = self.tokens[self.pos]
-    #
-    #     return self.token.type != TokenKind.EOF
 
     def get_concepts(self, token, to_keep, custom=None, to_map=None, strip_quotes=False):
         """
@@ -698,7 +678,7 @@ class BaseNodeParser(BaseParser):
         return custom_concepts if custom else None
 
     @staticmethod
-    def get_concepts_by_first_keyword(context, concepts, use_sheerka=False):
+    def get_concepts_by_first_token(context, concepts, use_sheerka=False):
         """
         Create the map describing the first token expected by a concept
         :param context:
@@ -718,22 +698,26 @@ class BaseNodeParser(BaseParser):
             for keyword in keywords:
                 res.setdefault(keyword, []).append(concept.id)
 
+        # 'uniquify' the lists
+        for k, v in res.items():
+            res[k] = core.utils.make_unique(v)
+
         return sheerka.ret("BaseNodeParser", True, res)
 
     @staticmethod
     def resolve_concepts_by_first_keyword(context, concepts_by_first_keyword):
         sheerka = context.sheerka
 
-        def _make_unique(elements):
+        def resolve_concepts(concept_str):
-            keys = {}
+            resolved = set()
-            for e in elements:
+            to_resolve = set()
-                keys[e] = 1
-            return list(keys.keys())
-
-        def _resolve_concepts(concept_str):
-            resolved = []
-            to_resolve = []
             concept = sheerka.get_by_id(core.utils.unstr_concept(concept_str)[1])
+
+            if concept.id in already_seen:
+                raise ChickenAndEggError(already_seen)
+            else:
+                already_seen.add(concept.id)
+
             if sheerka.isaset(context, concept):
                 concepts = sheerka.get_set_elements(context, concept)
             else:
@@ -743,25 +727,31 @@ class BaseNodeParser(BaseParser):
                 BaseNodeParser.ensure_bnf(context, concept)  # need to make sure that it cannot fail
                 keywords = BaseNodeParser.get_first_tokens(sheerka, concept)
                 for keyword in keywords:
-                    (to_resolve if keyword.startswith("c:|") else resolved).append(keyword)
+                    (to_resolve if keyword.startswith("c:|") else resolved).add(keyword)
 
                 for concept_to_resolve_str in to_resolve:
-                    resolved += _resolve_concepts(concept_to_resolve_str)
+                    resolved |= resolve_concepts(concept_to_resolve_str)
 
             return resolved
 
         res = {}
         for k, v in concepts_by_first_keyword.items():
             if k.startswith("c:|"):
-                resolved_keywords = _resolve_concepts(k)
+                try:
+                    already_seen = set()
+                    resolved_keywords = resolve_concepts(k)
                     for resolved in resolved_keywords:
                         res.setdefault(resolved, []).extend(v)
+                except ChickenAndEggError as ex:
+                    context.log(f"Chicken and egg detected for {k}, concepts={ex.concepts}")
+                    # res[k] = sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG,
+                    #                      body=[sheerka.get_by_id(c) for c in ex.concepts])
             else:
                 res.setdefault(k, []).extend(v)
 
         # 'uniquify' the lists
         for k, v in res.items():
-            res[k] = _make_unique(v)
+            res[k] = core.utils.make_unique(v)
 
         return sheerka.ret("BaseNodeParser", True, res)
 
@@ -797,7 +787,7 @@ class BaseNodeParser(BaseParser):
         if concept.metadata.definition_type == DEFINITION_TYPE_BNF and not concept.bnf:
             from parsers.BnfParser import BnfParser
             regex_parser = BnfParser()
-            desc = f"Resolving BNF {concept.metadata.definition}"
+            desc = f"Resolving BNF '{concept.metadata.definition}'"
             with context.push(BuiltinConcepts.INIT_BNF,
                               concept,
                               who=parser_name,

src/parsers/BnfNodeParser.py

@@ -154,18 +154,6 @@ class ConceptExpression(ParsingExpression):
                                [node])
 
 
-# class ConceptGroupExpression(ConceptExpression):
-#     def _parse(self, parser_helper):
-#         node = self.nodes[0].parse(parser_helper)
-#         if node is None:
-#             return None
-#         return NonTerminalNode(self,
-#                                node.start,
-#                                node.end,
-#                                node.tokens,  # node is an OrderedChoice
-#                                [node])
-
-
 class Sequence(ParsingExpression):
     """
     Will match sequence of parser expressions in exact order they are defined.
@@ -422,6 +410,69 @@ class StrMatch(Match):
         return None
 
 
+# class RegExMatch(Match):
+#     '''
+#     This Match class will perform input matching based on Regular Expressions.
+#
+#     Args:
+#         to_match (regex string): A regular expression string to match.
+#             It will be used to create regular expression using re.compile.
+#         ignore_case(bool): If case insensitive match is needed.
+#             Default is None to support propagation from global parser setting.
+#         multiline(bool): allow regex to works on multiple lines
+#             (re.DOTALL flag). Default is None to support propagation from
+#             global parser setting.
+#         str_repr(str): A string that is used to represent this regex.
+#         re_flags: flags parameter for re.compile if neither ignore_case
+#             or multiple are set.
+#
+#     '''
+#     def __init__(self, to_match, rule_name='', root=False, ignore_case=None,
+#                  multiline=None, str_repr=None, re_flags=re.MULTILINE):
+#         super(RegExMatch, self).__init__(rule_name, root)
+#         self.to_match_regex = to_match
+#         self.ignore_case = ignore_case
+#         self.multiline = multiline
+#         self.explicit_flags = re_flags
+#
+#         self.to_match = str_repr if str_repr is not None else to_match
+#
+#     def compile(self):
+#         flags = self.explicit_flags
+#         if self.multiline is True:
+#             flags |= re.DOTALL
+#         if self.multiline is False and flags & re.DOTALL:
+#             flags -= re.DOTALL
+#         if self.ignore_case is True:
+#             flags |= re.IGNORECASE
+#         if self.ignore_case is False and flags & re.IGNORECASE:
+#             flags -= re.IGNORECASE
+#         self.regex = re.compile(self.to_match_regex, flags)
+#
+#     def __str__(self):
+#         return self.to_match
+#
+#     def __unicode__(self):
+#         return self.__str__()
+#
+#     def _parse(self, parser):
+#         c_pos = parser.position
+#         m = self.regex.match(parser.input, c_pos)
+#         if m:
+#             matched = m.group()
+#             if parser.debug:
+#                 parser.dprint(
+#                     "++ Match '%s' at %d => '%s'" %
+#                     (matched, c_pos, parser.context(len(matched))))
+#             parser.position += len(matched)
+#             if matched:
+#                 return Terminal(self, c_pos, matched, extra_info=m)
+#         else:
+#             if parser.debug:
+#                 parser.dprint("-- NoMatch at {}".format(c_pos))
+#             parser._nm_raise(self, c_pos, parser)
+
+
 class ParsingExpressionVisitor:
     """
     visit ParsingExpression
@@ -550,7 +601,7 @@ class BnfConceptParserHelper:
             forked.eat_concept(concept, token)
 
         # init
-        parsing_expression = self.parser.get_parsing_expression(concept)
+        parsing_expression = self.parser.get_parsing_expression(self.parser.context, concept)
         if not isinstance(parsing_expression, ParsingExpression):
             self.debug.append(concept)
             error_msg = f"Failed to parse concept '{concept}'"
@@ -733,6 +784,11 @@ class BnfConceptParserHelper:
         return concept
 
 
+@dataclass
+class UnderConstruction:
+    concept_id: str
+
+
 class BnfNodeParser(BaseNodeParser):
     def __init__(self, **kwargs):
         super().__init__("BnfNode", 50, **kwargs)
@@ -769,6 +825,11 @@ class BnfNodeParser(BaseNodeParser):
         return valid_parser_helpers
 
     def get_concepts_sequences(self):
+        """
+        Main method that parses the tokens and extract the concepts
+        :return:
+        """
+
         def _add_forked_to_concept_parser_helpers():
             # check that if some new InfixToPostfix are created
             for parser in concept_parser_helpers:
@@ -836,36 +897,146 @@ class BnfNodeParser(BaseNodeParser):
 
         return concept_parser_helpers
 
-    def get_parsing_expression(self, concept, already_seen=None):
+    def check_for_infinite_recursion(self, parsing_expression, already_found, only_first=False):
+
+        if isinstance(parsing_expression, ConceptExpression):
+            if parsing_expression.concept in already_found:
+                return True
+            already_found.add(parsing_expression.concept)
+            return self.check_for_infinite_recursion(parsing_expression.nodes[0], already_found, False)
+
+        if isinstance(parsing_expression, Sequence):
+            # for sequence, we need to check all nodes
+            if only_first:
+                nodes = [] if len(parsing_expression.nodes) == 0 else [parsing_expression.nodes[0]]
+            else:
+                nodes = parsing_expression.nodes
+            for node in nodes:
+                already_found_for_current_node = already_found.copy()
+                if self.check_for_infinite_recursion(node, already_found_for_current_node, False):
+                    already_found.update(already_found_for_current_node)
+                    return True
+            return False
+
+        if isinstance(parsing_expression, OrderedChoice):
+            # for ordered choice, if there is at least one node that does not resolved to a recursion
+            # we are safe
+            for node in parsing_expression.nodes:
+                already_found_for_current_node = already_found.copy()
+                if self.check_for_infinite_recursion(node, already_found, True):
+                    already_found.update(already_found_for_current_node)
+                    return True
+                else:
+                    return False
+            return False
+
+        return False
+
+    def get_parsing_expression(self, context, concept):
         if concept.id in self.concepts_grammars:
             return self.concepts_grammars.get(concept.id)
 
-        if not concept.bnf:
+        grammar = self.concepts_grammars.copy()
-            BaseNodeParser.ensure_bnf(self.context, concept, self.name)
+        to_resolve = {}  # the key is the instance id of the parsing expression
+        isa_concepts = set()
+        self.resolve_concept_parsing_expression(context, concept, grammar, to_resolve, isa_concepts)
 
-        expression = concept.bnf
+        for _id, pe in to_resolve.items():
-        desc = f"Resolving parsing expression {expression}"
+            for i, node in enumerate(pe.nodes):
-        with self.context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context:
+                if isinstance(node, UnderConstruction):
-            sub_context.add_inputs(expression=expression)
+                    pe.nodes[i] = grammar.get(node.concept_id)
-            resolved = self.resolve_parsing_expression(expression, already_seen or set())
-            sub_context.add_values(return_values=resolved)
 
-            self.concepts_grammars.put(concept.id, resolved)
+            concepts_in_recursion = set()
+            if self.check_for_infinite_recursion(pe, concepts_in_recursion):
+                cycle = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c.id for c in concepts_in_recursion})
+                for concept in concepts_in_recursion:
+                    grammar[concept.id] = cycle
 
-            if self.has_error:
+        # Make sure you do not put isa concepts in cache
-                return None
+        # why :
+        #   twenties = 'twenty' number where number < 10
+        #   hundreds = number 'hundred' where number < 99
+        # the concept of number depends on its utilisation
+        for concept_id in [c for c in grammar if c not in isa_concepts]:
+            self.concepts_grammars.put(concept_id, grammar[concept_id])
 
         return self.concepts_grammars.get(concept.id)
 
-    def resolve_parsing_expression(self, parsing_expression, already_seen):
+    def resolve_concept_parsing_expression(self, context, concept, grammar, to_resolve, isa_concepts):
+        if concept.id in grammar:
+            return grammar.get(concept.id)
+
+        desc = f"Get parsing expression for '{concept}'"
+        with context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context:
+            if not concept.bnf:  # to save a function call. Not sure it worth it.
+                BaseNodeParser.ensure_bnf(sub_context, concept, self.name)
+
+            grammar[concept.id] = UnderConstruction(concept.id)
+            sheerka = context.sheerka
+
+            if concept.metadata.definition_type == DEFINITION_TYPE_BNF:
+                expression = concept.bnf
+                desc = f"Bnf concept detected. Resolving parsing expression '{expression}'"
+                with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc:
+                    ssc.add_inputs(expression=expression)
+                    resolved = self.resolve_parsing_expression(ssc, expression, grammar, to_resolve, isa_concepts)
+                    ssc.add_values(return_values=resolved)
+
+            elif sheerka.isaset(context, concept):
+                desc = f"Concept is a group. Resolving parsing expression using 'isa'"
+                with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc:
+                    ssc.add_inputs(concept=concept)
+                    isa_concepts.add(concept.id)
+                    concepts_in_group = self.sheerka.get_set_elements(ssc, concept)
+
+                    # concepts_in_group comes from a set, so the order of its elements is not guaranteed
+                    # to avoid random failure (ie random CHICKEN_AND_EGG), we need to rearrange
+                    # We also remove the root concept (the one from get_parsing_expression())
+
+                    root_concept_as_set = set(context.search(
+                        predicate=lambda ec: ec.action == BuiltinConcepts.INIT_BNF,
+                        get_obj=lambda ec: ec.obj,
+                        stop=lambda ec: ec.action != BuiltinConcepts.INIT_BNF))  # there only one item in the set
+                    root_concept = list(root_concept_as_set)[0]
+                    reordered = []
+                    for c in concepts_in_group:
+                        if c.id == root_concept.id:
+                            continue
+
+                        # I do not guaranty the same order every time, but I minimize the ChickenAndEgg random issue
+                        if c.metadata.definition_type == DEFINITION_TYPE_BNF or sheerka.isaset(ssc, c):
+                            reordered.append(c)
+                        else:
+                            reordered.insert(0, c)
+
+                    nodes = [ConceptExpression(c, rule_name=c.name) for c in reordered]
+                    resolved = self.resolve_parsing_expression(ssc,
+                                                               OrderedChoice(*nodes),
+                                                               grammar,
+                                                               to_resolve,
+                                                               isa_concepts)
+                    ssc.add_values(concepts_in_group=concepts_in_group)
+                    ssc.add_values(return_values=resolved)
+
+            else:
+                desc = f"Concept is a simple concept."
+                with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc:
+                    tokens = Tokenizer(concept.name, yield_eof=False)
+                    nodes = [StrMatch(token.strip_quote) for token in tokens]
+                    expression = nodes[0] if len(nodes) == 1 else Sequence(nodes)
+                    resolved = self.resolve_parsing_expression(ssc, expression, grammar, to_resolve, isa_concepts)
+
+            grammar[concept.id] = resolved
+
+            if self.has_error:
+                sub_context.add_values(errors=self.error_sink)
+                return None
+
+            sub_context.add_values(return_values=resolved)
+            return resolved
+
+    def resolve_parsing_expression(self, context, expression, grammar, to_resolve, isa_concepts):
 
-        def inner_resolve(expression, inner_already_seen):
-            # if isinstance(expression, Concept):
-            #     if self.sheerka.isaset(self.context, expression):
-            #         ret = ConceptGroupExpression(expression, rule_name=expression.name)
-            #     else:
-            #         ret = ConceptExpression(expression, rule_name=expression.name)
-            #     possible_recursion.add(expression)
         if isinstance(expression, str):
             ret = StrMatch(expression, ignore_case=self.ignore_case)
 
@@ -873,33 +1044,24 @@ class BnfNodeParser(BaseNodeParser):
             return expression  # escalate the error
 
         elif isinstance(expression, ConceptExpression):
-                concept = self.get_concept(expression.concept)
+            concept = self.get_concept(context, expression.concept)
-                if concept in inner_already_seen:
-                    return self.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concept)
             expression.concept = concept
-                inner_already_seen.add(concept)
 
             if not self.sheerka.is_known(concept):
                 unknown_concept = self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept)
                 return self.add_error(unknown_concept)
 
-                # bnf concept
+            pe = self.resolve_concept_parsing_expression(context, concept, grammar, to_resolve, isa_concepts)
-                elif concept.metadata.definition_type == DEFINITION_TYPE_BNF:
-                    pe = self.get_parsing_expression(concept, inner_already_seen)
 
-                elif self.sheerka.isaset(self.context, concept):
+            if not isinstance(pe, (ParsingExpression, UnderConstruction)):
-                    concepts_in_group = self.sheerka.get_set_elements(self.context, concept)
+                return pe  # an error is detected, escalate it
-                    nodes = [ConceptExpression(c, rule_name=c.name) for c in concepts_in_group]
+            #
-                    pe = inner_resolve(OrderedChoice(*nodes), inner_already_seen)
+            # if isinstance(pe, UnderConstruction) and expression.concept.id == pe.concept_id:
+            #     return pe  # we are looking for ourself, just return it
 
-                else:
+            if isinstance(pe, UnderConstruction):
-                    # regular concepts
+                to_resolve[id(expression)] = expression
-                    tokens = Tokenizer(concept.name)
-                    nodes = [StrMatch(token.strip_quote) for token in list(tokens)[:-1]]
-                    pe = inner_resolve(nodes[0] if len(nodes) == 1 else Sequence(nodes), inner_already_seen)
 
-                if not isinstance(pe, ParsingExpression):
-                    return pe
             expression.nodes = [pe]
             expression.rule_name = expression.rule_name or concept.name
             ret = expression
@@ -917,29 +1079,32 @@ class BnfNodeParser(BaseNodeParser):
             ret = expression
             ret.nodes = []
             for e in ret.elements:
-                    pe = inner_resolve(e, already_seen.copy())
+                pe = self.resolve_parsing_expression(context, e, grammar, to_resolve, isa_concepts)
-                    if not isinstance(pe, ParsingExpression):
+                if not isinstance(pe, (ParsingExpression, UnderConstruction)):
-                        return pe
+                    return pe  # an error is detected, escalate it
+                if isinstance(pe, UnderConstruction):
+                    to_resolve[id(ret)] = ret  # remember that there is an unresolved parsing expression
                 ret.nodes.append(pe)
 
         else:
             ret = self.add_error(GrammarErrorNode(f"Unrecognized grammar element '{expression}'."), False)
 
         # Translate separator expression.
-            if isinstance(expression, Repetition) and expression.sep:
+        if isinstance(ret, Repetition) and expression.sep:
-                expression.sep = inner_resolve(expression.sep, already_seen)
+            expression.sep = self.resolve_parsing_expression(context,
+                                                             expression.sep,
+                                                             grammar,
+                                                             to_resolve,
+                                                             isa_concepts)
 
         return ret
 
-        parsing_expression = inner_resolve(parsing_expression, already_seen)
+    def get_concept(self, context, concept):
-        return parsing_expression
-
-    def get_concept(self, concept):
         if isinstance(concept, Concept):
             return concept
 
-        if concept in self.context.concepts:
+        if concept in context.concepts:
-            return self.context.concepts[concept]
+            return context.concepts[concept]
         return self.sheerka.get_by_key(concept)
 
     def parse(self, context, parser_input: ParserInput):

tests/BaseTest.py

@@ -13,7 +13,7 @@ class BaseTest:
         pass
 
     def get_context(self, sheerka=None, eval_body=False, eval_where=False):
-        context = ExecutionContext("test", Event(), sheerka or self.get_sheerka(), BuiltinConcepts.NOP, None)
+        context = ExecutionContext("test", Event(), sheerka or self.get_sheerka(), BuiltinConcepts.TESTING, None)
         if eval_body:
             context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
         if eval_where:
@@ -145,3 +145,13 @@ class BaseTest:
                 else:
                     concept.metadata.variables[k] = v
         return concept
+
+    def init_scenario(self, init_expressions):
+        sheerka = self.get_sheerka()
+
+        for expression in init_expressions:
+            res = sheerka.evaluate_user_input(expression)
+            assert len(res) == 1, f"Failed to execute '{expression}'"
+            assert res[0].status, f"Error while executing '{expression}'"
+
+        return sheerka

tests/core/test_ExecutionContext.py

@@ -103,3 +103,21 @@ def test_global_hits_are_global_even_when_empty():
 
     assert a.global_hints == {"global hint 2"}
     assert b.global_hints == {"global hint 2"}
+
+
+def test_i_can_search():
+    a = ExecutionContext("foo", Event("event_1"), "fake_sheerka", BuiltinConcepts.TESTING, "a")
+    ab = a.push(BuiltinConcepts.TESTING, "ab", obj="obj_ab")
+    ac = a.push(BuiltinConcepts.TESTING, "ac", obj="obj_ac")
+    abb = ab.push(BuiltinConcepts.TESTING, "abb", obj="skip")
+    abbb = abb.push(BuiltinConcepts.TESTING, "abbb", obj="obj_abbb")
+
+    assert list(abbb.search()) == [abb, ab, a]
+    assert list(abbb.search(start_with_self=True)) == [abbb, abb, ab, a]
+    assert list(abbb.search(lambda ec: ec.obj != "skip")) == [ab, a]
+    assert list(abbb.search(lambda ec: ec.obj != "skip", lambda ec: ec.action_context)) == ["ab", "a"]
+    assert list(abbb.search(stop=lambda ec: ec.obj == "skip")) == []
+    assert list(abbb.search(
+        stop=lambda ec: ec.obj == "skip",
+        start_with_self=True,
+        get_obj=lambda ec: ec.obj)) == ["obj_abbb"]

tests/core/test_SheerkaComparisonManager.py

@@ -10,7 +10,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         sheerka, context, one, two = self.init_concepts("one", "two", cache_only=False)
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
-        res = service.is_greater_than(context, "prop_name", two, one)
+        res = service.set_is_greater_than(context, "prop_name", two, one)
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
@@ -29,7 +29,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         sheerka, context, one, two = self.init_concepts("one", "two", cache_only=False)
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
-        res = service.is_less_than(context, "prop_name", one, two)
+        res = service.set_is_less_than(context, "prop_name", one, two)
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
@@ -48,8 +48,8 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         sheerka, context, one, two, three, four = self.init_concepts("one", "two", "three", "four", cache_only=False)
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
-        service.is_greater_than(context, "prop_name", two, one)
+        service.set_is_greater_than(context, "prop_name", two, one)
-        service.is_greater_than(context, "prop_name", three, two)
+        service.set_is_greater_than(context, "prop_name", three, two)
 
         in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [
@@ -67,7 +67,7 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
 
         sheerka.cache_manager.clear(SheerkaComparisonManager.COMPARISON_ENTRY)  # reset the cache
 
-        service.is_greater_than(context, "prop_name", four, three)
+        service.set_is_greater_than(context, "prop_name", four, three)
         in_cache = sheerka.cache_manager.get(SheerkaComparisonManager.COMPARISON_ENTRY, "prop_name|#")
         assert in_cache == [
             ComparisonObj(context.event.get_digest(), "prop_name", two.id, one.id, ">", "#"),
@@ -92,10 +92,10 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         for entry in entries:
             if ">" in entry:
                 a, b = [concepts_map[e.strip()] for e in entry.split(">")]
-                service.is_greater_than(context, "prop_name", a, b)
+                service.set_is_greater_than(context, "prop_name", a, b)
             else:
                 a, b = [concepts_map[e.strip()] for e in entry.split("<")]
-                service.is_less_than(context, "prop_name", a, b)
+                service.set_is_less_than(context, "prop_name", a, b)
 
         assert service.get_concepts_weights("prop_name") == expected
 
@@ -103,8 +103,8 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         sheerka, context, one, two, three = self.init_concepts("one", "two", "three")
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
-        service.is_greater_than(context, "prop_name", two, one)
+        service.set_is_greater_than(context, "prop_name", two, one)
-        service.is_less_than(context, "prop_name", two, three)
+        service.set_is_less_than(context, "prop_name", two, three)
 
         res = service.get_partition("prop_name")
 
@@ -118,8 +118,8 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         sheerka, context, one, two = self.init_concepts("one", "two")
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
-        service.is_greater_than(context, "prop_name", two, one)
+        service.set_is_greater_than(context, "prop_name", two, one)
-        res = service.is_greater_than(context, "prop_name", one, two)
+        res = service.set_is_greater_than(context, "prop_name", one, two)
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG)
@@ -129,15 +129,15 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         sheerka, context, one, two, three, four, five = self.init_concepts("one", "two", "three", "four", "five")
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
-        service.is_greater_than(context, "prop_name", two, one)
+        service.set_is_greater_than(context, "prop_name", two, one)
-        service.is_greater_than(context, "prop_name", five, four)
+        service.set_is_greater_than(context, "prop_name", five, four)
-        service.is_greater_than(context, "prop_name", four, three)
+        service.set_is_greater_than(context, "prop_name", four, three)
-        service.is_greater_than(context, "prop_name", five, two)
+        service.set_is_greater_than(context, "prop_name", five, two)
 
-        res = service.is_greater_than(context, "prop_name", two, one)
+        res = service.set_is_greater_than(context, "prop_name", two, one)
         assert res.status
 
-        res = service.is_greater_than(context, "prop_name", one, five)
+        res = service.set_is_greater_than(context, "prop_name", one, five)
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.CHICKEN_AND_EGG)
@@ -147,13 +147,13 @@ class TestSheerkaGreaterThanManager(TestUsingMemoryBasedSheerka):
         sheerka, context, one, two = self.init_concepts("one", "two")
         service = sheerka.services[SheerkaComparisonManager.NAME]
 
-        service.is_greater_than(context, "prop_name", two, one)
+        service.set_is_greater_than(context, "prop_name", two, one)
-        service.is_less_than(context, "prop_name", one, two)
+        service.set_is_less_than(context, "prop_name", one, two)
 
         weighted = sheerka.cache_manager.get(SheerkaComparisonManager.RESOLVED_COMPARISON_ENTRY, "prop_name|#")
         assert weighted == {"1001": 1, "1002": 2}
 
     def test_methods_are_correctly_bound(self):
         sheerka, context, one, two = self.init_concepts("one", "two")
-        res = sheerka.is_greater_than(context, "prop_name", two, one)
+        res = sheerka.set_is_greater_than(context, "prop_name", two, one)
         assert res.status

tests/core/test_SheerkaVariableManager.py

@@ -1,5 +1,4 @@
 from core.concept import Concept, ConceptParts
-from core.sheerka.Sheerka import Sheerka
 from core.sheerka.services.SheerkaVariableManager import SheerkaVariableManager
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -50,6 +49,26 @@ class TestSheerkaVariable(TestUsingMemoryBasedSheerka):
         sheerka.delete(context, "TestSheerkaVariable", "my_variable")
         assert sheerka.load("TestSheerkaVariable", "my_variable") is None
 
+    def test_i_can_set_and_get_a_value(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        context.event.user_id = "Test_user"
+
+        sheerka.set(context, "my_variable", "my value")
+        res = sheerka.get(context, "my_variable")
+        assert res == "my value"
+
+        # I can persist in db
+        sheerka.cache_manager.commit(context)
+
+        assert sheerka.sdp.exists(SheerkaVariableManager.VARIABLES_ENTRY, "Test_user|my_variable")
+        loaded = sheerka.sdp.get(SheerkaVariableManager.VARIABLES_ENTRY, "Test_user|my_variable")
+        assert loaded.event_id == context.event.get_digest()
+        assert loaded.key == "my_variable"
+        assert loaded.value == "my value"
+        assert loaded.who == "Test_user"
+        assert loaded.parents is None
+
     # def test_i_can_get_the_parent_when_modified(self):
     #     sheerka = self.get_sheerka()
     #     context = self.get_context(sheerka)

tests/evaluators/test_PythonEvaluator.py

@@ -132,7 +132,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
             self.from_def_concept("mult", "a mult b", ["a", "b"]),
         )
 
-        parsed = PythonParser().parse(context, ParserInput("is_greater_than(BuiltinConcepts.PRECEDENCE, mult, plus)"))
+        parsed = PythonParser().parse(context, ParserInput("set_is_greater_than(BuiltinConcepts.PRECEDENCE, mult, plus)"))
         python_evaluator = PythonEvaluator()
 
         evaluated = python_evaluator.eval(context, parsed)

tests/non_reg/test_sheerka_non_reg.py

@@ -1,6 +1,6 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV, CB, CC, CV
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, simplec, CMV
 from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from evaluators.PythonEvaluator import PythonEvalError
 from parsers.BaseNodeParser import SyaAssociativity
@@ -12,16 +12,6 @@ from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
 
-    def init_scenario(self, init_expressions):
-        sheerka = self.get_sheerka()
-
-        for expression in init_expressions:
-            res = sheerka.evaluate_user_input(expression)
-            assert len(res) == 1, f"Failed to execute '{expression}'"
-            assert res[0].status, f"Error while executing '{expression}'"
-
-        return sheerka
-
     @pytest.mark.parametrize("text, expected", [
         ("1 + 1", 2),
         ("sheerka.test()", 'I have access to Sheerka !')
@@ -880,10 +870,10 @@ as:
 
         sheerka = self.init_scenario(definitions)
 
-        res = sheerka.evaluate_user_input("is_greater_than('some_prop', two, one)")
+        res = sheerka.evaluate_user_input("set_is_greater_than('some_prop', two, one)")
         assert res[0].status
 
-        res = sheerka.evaluate_user_input("is_less_than('some_prop', two, three)")
+        res = sheerka.evaluate_user_input("set_is_less_than('some_prop', two, three)")
         assert res[0].status
 
         res = sheerka.evaluate_user_input("get_concepts_weights('some_prop')")
@@ -891,7 +881,7 @@ as:
         assert res[0].body == {'1001': 1, '1002': 2, '1003': 3}
 
         # test i use a concept to define relation
-        sheerka.evaluate_user_input("def concept a > b as is_greater_than('some_prop', a, b)")
+        sheerka.evaluate_user_input("def concept a > b as set_is_greater_than('some_prop', a, b)")
         res = sheerka.evaluate_user_input("eval four > three")
         assert res[0].status
 
@@ -901,7 +891,7 @@ as:
         sheerka, context, one, two, plus = self.init_concepts(
             Concept("one", body="1"),
             Concept("two", body="2"),
-            self.from_def_concept("<", "a < b", ["a", "b"], body="is_less_than('some_prop', a, b)")
+            self.from_def_concept("<", "a < b", ["a", "b"], body="set_is_less_than('some_prop', a, b)")
         )
 
         expression = "c:one: < c:two:"
@@ -946,6 +936,33 @@ as:
         assert isinstance(error1.error, TypeError)
         assert error1.error.args[0] == "unsupported operand type(s) for +: 'Concept' and 'int'"
 
+    def test_i_can_evaluate_bnf_concept_defined_with_group_after_restart(self):
+        """
+        BNF Concepts defined with group and being themselves part a s group get messed up after restart
+        :return:
+        """
+        init = [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenty as 20",
+            "def concept number",
+            "one isa number",
+            "two isa number",
+            "twenty isa number",
+            "def concept twenties from bnf twenty number where number < 10 as twenty + number",
+            "twenties isa number",
+        ]
+
+        sheerka = self.init_scenario(init)
+
+        # simulate that sheerka was stopped and restarted
+        sheerka.cache_manager.clear(sheerka.CONCEPTS_GRAMMARS_ENTRY)
+        sheerka.cache_manager.get(sheerka.CONCEPTS_BY_KEY_ENTRY, "twenties").compiled = {}
+
+        res = sheerka.evaluate_user_input("eval twenty one")
+        assert res[0].status
+        assert res[0].body == 21
+
 
 class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
     def test_i_can_def_several_concepts(self):
@@ -1013,3 +1030,23 @@ class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
         evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), res[0].value)
         assert evaluated.body == "one two three"
         assert evaluated.get_value("a") == sheerka.new(concept_a.key, body="one two").init_key()
+
+    def test_i_can_eval_sophisticated_bnf_concepts_after_restart(self):
+        self.init_scenario([
+            "def concept one as 1",
+            "def concept number",
+            "one isa number",
+            "def concept twenty as 20",
+            "twenty isa number",
+            "def concept twenties from bnf twenty number where number < 10 as twenty + number",
+            "twenties isa number",
+            "def concept thirty as 30",
+            "thirty isa number",
+            "def concept thirties from bnf thirty number where number < 10 as thirty + number",
+            "thirties isa number",
+        ])
+
+        sheerka = self.get_sheerka()  # another instance
+
+        assert sheerka.evaluate_user_input("eval twenty one")[0].body == 21
+        assert sheerka.evaluate_user_input("eval thirty one")[0].body == 31

tests/parsers/parsers_utils.py

@@ -71,7 +71,7 @@ def get_node(
     if sub_expr == "')'":
         return ")"
 
-    if isinstance(sub_expr, (scnode, utnode)):
+    if isinstance(sub_expr, (scnode, utnode, DoNotResolve)):
         return sub_expr
 
     if isinstance(sub_expr, cnode):

tests/parsers/test_BaseNodeParser.py

@@ -24,7 +24,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
 
         sheerka, context, *updated = self.init_concepts(concept)
 
-        res = BaseNodeParser.get_concepts_by_first_keyword(context, updated)
+        res = BaseNodeParser.get_concepts_by_first_token(context, updated)
 
         assert res.status
         assert res.body == expected
@@ -54,7 +54,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         concept.bnf = bnf
         sheerka.set_id_if_needed(concept, False)
 
-        res = BaseNodeParser.get_concepts_by_first_keyword(context, [concept])
+        res = BaseNodeParser.get_concepts_by_first_token(context, [concept])
 
         assert res.status
         assert res.body == expected
@@ -75,7 +75,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         foo.bnf = OrderedChoice(ConceptExpression("bar"), ConceptExpression("baz"), StrMatch("qux"))
         sheerka.set_id_if_needed(foo, False)
 
-        res = BaseNodeParser.get_concepts_by_first_keyword(context, [bar, baz, foo])
+        res = BaseNodeParser.get_concepts_by_first_token(context, [bar, baz, foo])
 
         assert res.status
         assert res.body == {
@@ -102,7 +102,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         foo.bnf = OrderedChoice(ConceptExpression("one"), ConceptExpression("bar"), StrMatch("qux"))
         sheerka.set_id_if_needed(foo, False)
 
-        res = BaseNodeParser.get_concepts_by_first_keyword(context, [bar, foo], use_sheerka=True)
+        res = BaseNodeParser.get_concepts_by_first_token(context, [bar, foo], use_sheerka=True)
 
         assert res.status
         assert res.body == {
@@ -149,7 +149,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         sheerka.set_isa(context, sheerka.new("one"), number)
         sheerka.set_isa(context, sheerka.new("two"), number)
 
-        cbfk = BaseNodeParser.get_concepts_by_first_keyword(context, [one, two, three, number, foo]).body
+        cbfk = BaseNodeParser.get_concepts_by_first_token(context, [one, two, three, number, foo]).body
 
         resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, cbfk)
 
@@ -171,7 +171,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
             ConceptExpression("foo"),
             ConceptExpression("bar")))
 
-        concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(
+        concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(
             context, [good, foo, bar, baz]).body
 
         resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
@@ -187,7 +187,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
         a = self.create_and_add_in_cache_concept(sheerka, "a", bnf=Sequence("one", "two"))
         b = self.create_and_add_in_cache_concept(sheerka, "b", bnf=Sequence(ConceptExpression("a"), "two"))
 
-        concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(
+        concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(
             context, [a, b]).body
 
         resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(context, concepts_by_first_keywords)
@@ -202,7 +202,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
     #     foo = self.get_concept(sheerka, "foo", ConceptExpression("bar"))
     #     bar = self.get_concept(sheerka, "bar", ConceptExpression("foo"))
     #
-    #     concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, foo, bar]).body
+    #     concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, foo, bar]).body
     #
     #     resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
     #     assert resolved_ret_val.status
@@ -218,7 +218,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
     #     two = self.get_concept(sheerka, "two", ConceptExpression("three"))
     #     three = self.get_concept(sheerka, "three", ConceptExpression("two"))
     #
-    #     concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, one, two, three]).body
+    #     concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two, three]).body
     #
     #     resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
     #     assert resolved_ret_val.status
@@ -233,7 +233,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
     #     one = self.get_concept(sheerka, "one", ConceptExpression("two"))
     #     two = self.get_concept(sheerka, "two", OrderedChoice(ConceptExpression("one"), ConceptExpression("two")))
     #
-    #     concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, one, two]).body
+    #     concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two]).body
     #
     #     resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
     #     assert resolved_ret_val.status
@@ -248,7 +248,7 @@ class TestBaseNodeParser(TestUsingMemoryBasedSheerka):
     #     one = self.get_concept(sheerka, "one", ConceptExpression("two"))
     #     two = self.get_concept(sheerka, "two", Sequence(StrMatch("yes"), ConceptExpression("one")))
     #
-    #     concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_keyword(sheerka, [good, one, two]).body
+    #     concepts_by_first_keywords = BaseNodeParser.get_concepts_by_first_token(sheerka, [good, one, two]).body
     #
     #     resolved_ret_val = BaseNodeParser.resolve_concepts_by_first_keyword(sheerka, concepts_by_first_keywords)
     #     assert resolved_ret_val.status

tests/parsers/test_BnfNodeParser.py

@@ -1,25 +1,40 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts, DoNotResolve
+from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYPE_BNF
 from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.BaseNodeParser import CNC, UTN, CN
 from parsers.BnfNodeParser import BnfNodeParser, StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
     Optional, ZeroOrMore, OneOrMore, ConceptExpression
+from parsers.BnfParser import BnfParser
 
 import tests.parsers.parsers_utils
+from tests.BaseTest import BaseTest
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 cmap = {
     "one": Concept("one"),
     "two": Concept("two"),
     "three": Concept("three"),
-    "plus": Concept(name="a plus b").def_var("a").def_var("b"),
+    "four": Concept("four"),
-    "bnf one": Concept("bnf_one", definition="'one'"),
+    "thirty": Concept("thirty", body=30),
-    'one and two': Concept("one and two", definition="one two"),
+    "forty": Concept("forty", body=40),
-    'one or more three': Concept("one or more three", definition="three+"),
+    "fifty": Concept("fifty", body=50),
-    'two or four': Concept("two or four", definition="two | 'four'"),
+    "number": Concept("number"),
-    "twenties": Concept("twenties", definition="'twenty' c:two or four:=unit"),
+    "foo": Concept("foo"),
-    "one or more plus": Concept("one or more plus", definition="c:a plus b:+"),  # TODO
+    "bar": Concept("bar"),
+    "baz": Concept("baz"),
+
+    "bnf baz": Concept("bnf baz", definition="'baz'"),  # this one should be chosen
+
+    "plus": Concept("plus", definition="one 'plus' two").def_var("a").def_var("b"),
+
+    'foo then bar': Concept("foo then bar", definition="foo bar").def_var("foo").def_var("bar"),
+    'foo or bar': Concept("foo or bar", definition="foo | bar").def_var("foo").def_var("bar"),
+    'one or more foo': Concept("one or more foo", definition="foo+").def_var("foo"),
+
+    "t1": Concept("t1", definition="'twenty' (one|two)=unit").def_var("unit").def_var("one").def_var("two"),
+    "three_four": Concept("three_four", definition="three | four").def_var("three").def_var("four"),
+    "t2": Concept("t2", definition="'twenty' three_four=unit").def_var("unit").def_var("three").def_var("four"),
 
     # testing keywords
     "def_only": Concept("def"),
@@ -65,15 +80,57 @@ def compute_expected_array(my_concepts_map, expression, expected, exclude_body=F
 class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
     sheerka = None
 
+    @staticmethod
+    def update_bnf(context, concept):
+        bnf_parser = BnfParser()
+        res = bnf_parser.parse(context, concept.metadata.definition)
+        if res.status:
+            concept.bnf = res.value.value
+            concept.metadata.definition_type = DEFINITION_TYPE_BNF
+        else:
+            raise Exception(res)
+        return concept
+
     @classmethod
     def setup_class(cls):
-        t = TestBnfNodeParser()
+        t = cls()
         TestBnfNodeParser.sheerka, context, _ = t.init_parser(
             cmap,
             singleton=False,
             create_new=True,
             init_from_sheerka=True)
 
+        # end of initialisation
+        sheerka = TestBnfNodeParser.sheerka
+        sheerka.set_isa(context, sheerka.new("one"), sheerka.new("number"))
+        sheerka.set_isa(context, sheerka.new("two"), sheerka.new("number"))
+        sheerka.set_isa(context, sheerka.new("three"), sheerka.new("number"))
+        sheerka.set_isa(context, sheerka.new("four"), sheerka.new("number"))
+        sheerka.set_isa(context, sheerka.new("thirty"), sheerka.new("number"))
+        sheerka.set_isa(context, sheerka.new("forty"), sheerka.new("number"))
+        sheerka.set_isa(context, sheerka.new("fifty"), sheerka.new("number"))
+
+        thirties = cls.update_bnf(context, Concept("thirties",
+                                                   definition="thirty number",
+                                                   where="number < 10",
+                                                   body="thirty + number").def_var("thirty").def_var("number"))
+        cmap["thirties"] = sheerka.create_new_concept(context, thirties).body.body
+        sheerka.set_isa(context, sheerka.new("thirties"), sheerka.new("number"))
+
+        forties = cls.update_bnf(context, Concept("forties",
+                                                  definition="forty number",
+                                                  where="number < 10",
+                                                  body="forty + number").def_var("forty").def_var("number"))
+        cmap["forties"] = sheerka.create_new_concept(context, forties).body.body
+        sheerka.set_isa(context, sheerka.new("forties"), sheerka.new("number"))
+
+        fifties = cls.update_bnf(context, Concept("fifties",
+                                                  definition="fifty number",
+                                                  where="number < 10",
+                                                  body="fifty + number").def_var("fifty").def_var("number"))
+        cmap["fifties"] = sheerka.create_new_concept(context, fifties).body.body
+        sheerka.set_isa(context, sheerka.new("fifties"), sheerka.new("number"))
+
     def init_parser(self, my_concepts_map=None, init_from_sheerka=False, **kwargs):
         if my_concepts_map is not None:
             sheerka, context, *updated = self.init_concepts(*my_concepts_map.values(), **kwargs)
@@ -174,6 +231,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         self.validate_get_concepts_sequences(my_map, text, expected)
 
     def test_i_can_use_skip_whitespace_when_mixing_sequence_and_strmatch(self):
+        # to match '--filter' in one word
         my_map = {
             "filter": self.bnf_concept("filter",
                                        Sequence(StrMatch("-", skip_whitespace=False),
@@ -236,20 +294,50 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         self.validate_get_concepts_sequences(my_map, text, expected)
 
+    @pytest.mark.parametrize("concept_three, expected", [
+        (Concept("three"), []),
+        (BaseTest.bnf_concept("three", StrMatch("three")), [UTN('twenty '), "three"])
+    ])
+    def test_i_can_manage_sequence_with_wrong_order_choice(self, concept_three, expected):
+        my_map = {
+            "foo": self.bnf_concept("foo",
+                                    Sequence(
+                                        StrMatch("twenty"),
+                                        OrderedChoice(StrMatch("one"), StrMatch("two")))),
+            "three": concept_three}
+
+        text = "twenty three"
+        self.validate_get_concepts_sequences(my_map, text, expected)
+
     @pytest.mark.parametrize("text, expected", [
-        ("thirty one ok", [CNC("foo", source="thirty one ok")]),
+        ("ok thirty one", [CNC("foo", source="ok thirty one")]),
-        ("twenty one ok", [CNC("foo", source="twenty one ok")]),
+        ("ok twenty one", [CNC("foo", source="ok twenty one")]),
+        ("ok one", []),
     ])
     def test_i_can_mix_sequence_and_ordered(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo",
                                     Sequence(
+                                        StrMatch("ok"),
                                         OrderedChoice(StrMatch("twenty"), StrMatch("thirty")),
-                                        StrMatch("one"),
+                                        StrMatch("one"))
-                                        StrMatch("ok"))
                                     )}
         self.validate_get_concepts_sequences(my_map, text, expected)
 
+    @pytest.mark.parametrize("text, expected", [
+        # ("twenty one", [CNC("foo", source="twenty one")]),
+        # ("twenty three", []),  # three does not exist
+        ("twenty four", []),  # four exists but should not be seen
+    ])
+    def test_i_can_mix_sequence_and_ordered_2(self, text, expected):
+        my_map = {
+            "foo": self.bnf_concept("foo",
+                                    Sequence(
+                                        StrMatch("twenty"),
+                                        OrderedChoice(StrMatch("one"), StrMatch("two")))),
+            "four": Concept("four")}
+        self.validate_get_concepts_sequences(my_map, text, expected)
+
     @pytest.mark.parametrize("text, expected", [
         ("twenty thirty", [CNC("foo", source="twenty thirty")]),
         ("one", [CNC("foo", source="one")]),
@@ -531,6 +619,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             "bar": self.bnf_concept("bar", Sequence(
                 ConceptExpression("foo"),
                 OrderedChoice(StrMatch("one"), StrMatch("two")))),
+            "three": Concept("three")
         }
 
         text = "twenty two"
@@ -553,6 +642,33 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         }
         assert concept_bar.compiled["foo"].compiled == {ConceptParts.BODY: DoNotResolve("thirty")}
 
+        text = "thirty three"
+        expected = [[CN("foo", source="thirty"), CN("three")], []]
+        self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True)
+
+    def test_i_can_mix_reference_to_other_concepts_2(self):
+        # this time, we use concept expression
+        my_map = {
+            "twenty": self.bnf_concept("twenty", StrMatch("twenty")),
+            "number": self.bnf_concept("number", OrderedChoice(StrMatch("one"), StrMatch("two"))),
+            "twenties": self.bnf_concept("twenties",
+                                         Sequence(ConceptExpression("twenty"), ConceptExpression("number"))),
+            "three": Concept("three")
+        }
+
+        text = "twenty two"
+
+        expected = [CNC("twenties",
+                        source="twenty two",
+                        twenty=CC("twenty", body=DoNotResolve("twenty")),
+                        number=CC("number", source="two", body=DoNotResolve("two"))
+                        )]
+        self.validate_get_concepts_sequences(my_map, text, expected)
+
+        text = "twenty three"
+        expected = [[CN("twenty"), CN("three")], []]
+        self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True)
+
     def test_i_can_mix_reference_to_other_concepts_when_body(self):
         my_map = {
             "foo": self.bnf_concept(Concept("foo", body="'foo'"),
@@ -654,12 +770,12 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
                                         'one': my_map["one"],
                                         ConceptParts.BODY: DoNotResolve(value='twenty one')}
 
-    @pytest.mark.parametrize("bar_expr", [
+    @pytest.mark.parametrize("bar_expr, expected", [
-        ConceptExpression("foo"),
+        (ConceptExpression("foo"), {}),
-        OrderedChoice(ConceptExpression("foo"), StrMatch("one")),
+        (OrderedChoice(ConceptExpression("foo"), StrMatch("one")), {'one': ['1002']}),
-        Sequence(StrMatch("one"), ConceptExpression("foo"), StrMatch("two"))
+        (Sequence(StrMatch("one"), ConceptExpression("foo"), StrMatch("two")), {'one': ['1001', '1002']})
     ])
-    def test_i_can_detect_infinite_recursion(self, bar_expr):
+    def test_i_can_detect_infinite_recursion(self, bar_expr, expected):
         my_map = {
             "foo": self.bnf_concept("foo", ConceptExpression("bar")),
             "bar": self.bnf_concept("bar", bar_expr),
@@ -669,14 +785,64 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         parser.context = context
         parser.sheerka = sheerka
 
-        parsing_expression = parser.get_parsing_expression(my_map["foo"])
+        # every obvious cyclic recursion are removed from concept_by_first_keyword dict
+        parser.init_from_concepts(context, my_map.values())
+        assert parser.concepts_by_first_keyword == expected
+
+        # get_parsing_expression() also returns CHICKEN_AND_EGG
+        parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
         assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
         assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
 
-        parsing_expression = parser.get_parsing_expression(my_map["bar"])
+        parsing_expression = parser.get_parsing_expression(context, my_map["bar"])
         assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
         assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG)
 
+    def test_i_can_detect_longer_infinite_recursion(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", ConceptExpression("bar")),
+            "bar": self.bnf_concept("bar", ConceptExpression("baz")),
+            "baz": self.bnf_concept("baz", ConceptExpression("qux")),
+            "qux": self.bnf_concept("qux", ConceptExpression("foo")),
+        }
+
+        sheerka, context, parser = self.init_parser(my_map, singleton=True)
+        parser.context = context
+        parser.sheerka = sheerka
+
+        # every obvious cyclic recursion are removed from concept_by_first_keyword dict
+        parser.init_from_concepts(context, my_map.values())
+        assert parser.concepts_by_first_keyword == {}
+
+        parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
+        assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
+        assert sheerka.isinstance(parser.concepts_grammars.get(my_map["foo"].id), BuiltinConcepts.CHICKEN_AND_EGG)
+        assert parser.concepts_grammars.get(my_map["foo"].id).body == {"1001", "1002", "1003", "1004"}
+
+        assert sheerka.isinstance(parser.concepts_grammars.get(my_map["bar"].id), BuiltinConcepts.CHICKEN_AND_EGG)
+        assert sheerka.isinstance(parser.concepts_grammars.get(my_map["baz"].id), BuiltinConcepts.CHICKEN_AND_EGG)
+        assert sheerka.isinstance(parser.concepts_grammars.get(my_map["qux"].id), BuiltinConcepts.CHICKEN_AND_EGG)
+
+    @pytest.mark.parametrize("expr, expected", [
+        (OrderedChoice(StrMatch("bar"), ConceptExpression("foo")), False),
+        (OrderedChoice(ConceptExpression("foo"), StrMatch("bar")), True),
+        (OrderedChoice(Sequence(StrMatch("bar"), ConceptExpression("foo")), StrMatch("baz")), False),
+        (OrderedChoice(Sequence(ConceptExpression("foo"), StrMatch("bar")), StrMatch("baz")), True)
+    ])
+    def test_i_can_detect_ordered_choice_infinite_recursion(self, expr, expected):
+        my_map = {
+            "foo": self.bnf_concept("foo", expr),
+        }
+
+        sheerka, context, parser = self.init_parser(my_map, singleton=True)
+        parser.init_from_concepts(context, my_map.values())
+        parser.context = context
+        parser.sheerka = sheerka
+
+        res = parser.get_parsing_expression(context, my_map["foo"])
+        assert sheerka.isinstance(res, BuiltinConcepts.CHICKEN_AND_EGG) == expected
+
+
     def test_i_can_get_parsing_expression_when_concept_isa(self):
         my_map = {
             "one": Concept("one"),
@@ -690,15 +856,87 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         sheerka.set_isa(context, sheerka.new("one"), my_map["number"])
         sheerka.set_isa(context, sheerka.new("twenty"), my_map["number"])
 
-        parsing_expression = parser.get_parsing_expression(my_map["twenties"])
+        parser.concepts_grammars.clear()  # make sure parsing expression is created from scratch
+
+        parsing_expression = parser.get_parsing_expression(context, my_map["twenties"])
         assert parsing_expression == Sequence(
             ConceptExpression(my_map["twenty"], rule_name="twenty"),
             ConceptExpression(my_map["number"], rule_name="number"))
 
-        assert parsing_expression.nodes[0].nodes == [StrMatch("twenty")]
+        assert len(parsing_expression.nodes) == len(parsing_expression.elements)
-        assert isinstance(parsing_expression.nodes[1].nodes[0], OrderedChoice)
+        twenty_nodes = parsing_expression.nodes[0].nodes
-        assert ConceptExpression(my_map["one"], rule_name="one") in parsing_expression.nodes[1].nodes[0].elements
+        assert twenty_nodes == [StrMatch("twenty")]
-        assert ConceptExpression(my_map["twenty"], rule_name="twenty") in parsing_expression.nodes[1].nodes[0].elements
+
+        number_nodes = parsing_expression.nodes[1].nodes
+        assert len(number_nodes) == 1
+        assert isinstance(number_nodes[0], OrderedChoice)
+        assert len(number_nodes[0].nodes) == len(number_nodes[0].elements)
+        assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes
+        assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes
+
+        assert my_map["number"].id not in parser.concepts_grammars
+
+    #
+    # def test_i_cannot_get_parsing_expression_when_concept_is_part_of_a_group(self):
+    #     """
+    #     In this test, twenties isa number
+    #     # So 'number' in Sequence(thirty, number) will spawn 'twenties' which, because there is no other indication,
+    #     # will create an infinite loop
+    #     :return:
+    #     """
+    #     my_map = {
+    #         "one": Concept("one"),
+    #         "twenty": Concept("twenty"),
+    #         "number": Concept("number"),
+    #         "twenties": self.bnf_concept("twenties", Sequence(ConceptExpression("twenty"), ConceptExpression("number")))
+    #     }
+    #     sheerka, context, parser = self.init_parser(my_map, singleton=True)
+    #     parser.context = context
+    #     parser.sheerka = sheerka
+    #     sheerka.set_isa(context, sheerka.new("one"), my_map["number"])
+    #     sheerka.set_isa(context, sheerka.new("twenty"), my_map["number"])
+    #     sheerka.set_isa(context, sheerka.new("twenties"), my_map["number"])  # <- twenties is also a number
+    #
+    #     parser.concepts_grammars.clear()  # make sure parsing expression is created from scratch
+    #
+    #     parsing_expression = parser.get_parsing_expression(context, my_map["twenties"])
+    #     assert sheerka.isinstance(parsing_expression, BuiltinConcepts.CHICKEN_AND_EGG)
+    #     assert parsing_expression.body == {my_map["twenties"].id, my_map["number"].id}
+    #
+    #     assert isinstance(parser.concepts_grammars.get(my_map["one"].id), ParsingExpression)
+    #     assert isinstance(parser.concepts_grammars.get(my_map["twenty"].id), ParsingExpression)
+
+    def test_i_can_get_parsing_expression_when_concept_is_part_of_a_group(self):
+        my_map = {
+            "one": Concept("one"),
+            "twenty": Concept("twenty"),
+            "number": Concept("number"),
+            "twenties": self.bnf_concept("twenties", Sequence(ConceptExpression("twenty"), ConceptExpression("number")))
+        }
+        sheerka, context, parser = self.init_parser(my_map, singleton=True)
+        parser.context = context
+        parser.sheerka = sheerka
+        sheerka.set_isa(context, sheerka.new("one"), my_map["number"])
+        sheerka.set_isa(context, sheerka.new("twenty"), my_map["number"])
+        sheerka.set_isa(context, sheerka.new("twenties"), my_map["number"])  # <- twenties is also a number
+
+        parser.concepts_grammars.clear()  # make sure parsing expression is created from scratch
+
+        parsing_expression = parser.get_parsing_expression(context, my_map["twenties"])
+        assert parsing_expression == Sequence(
+            ConceptExpression(my_map["twenty"], rule_name="twenty"),
+            ConceptExpression(my_map["number"], rule_name="number"))
+
+        assert len(parsing_expression.nodes) == len(parsing_expression.elements)
+        twenty_nodes = parsing_expression.nodes[0].nodes
+        assert twenty_nodes == [StrMatch("twenty")]
+
+        number_nodes = parsing_expression.nodes[1].nodes
+        assert len(number_nodes) == 1
+        assert isinstance(number_nodes[0], OrderedChoice)
+        assert len(number_nodes[0].nodes) == len(number_nodes[0].elements)
+        assert ConceptExpression(my_map["one"], rule_name="one") in number_nodes[0].nodes
+        assert ConceptExpression(my_map["twenty"], rule_name="twenty") in number_nodes[0].nodes
 
     def test_i_can_get_parsing_expression_when_sequence_of_concept(self):
         my_map = {
@@ -709,7 +947,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         parser.context = context
         parser.sheerka = sheerka
 
-        parsing_expression = parser.get_parsing_expression(my_map["two_ones"])
+        parsing_expression = parser.get_parsing_expression(context, my_map["two_ones"])
         assert parsing_expression == Sequence(
             ConceptExpression(my_map["one"], rule_name="one"),
             ConceptExpression(my_map["one"], rule_name="one"))
@@ -726,7 +964,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         self.validate_get_concepts_sequences(my_map, text, expected)
 
-    def test_i_can_recognize_unknown_then_they_look_like_known(self):
+    def test_i_can_recognize_unknown_when_they_look_like_known(self):
         my_map = {
             "one two": self.bnf_concept("one two", Sequence("one", "two")),
             "three": self.bnf_concept("three")
@@ -752,15 +990,13 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert len(sequence) == 1
 
     @pytest.mark.parametrize("parser_input, expected_status, expected", [
-        ("one", True, [CNC("bnf one", source="one")]),  # the bnf one is chosen
+        ("baz", True, [CNC("bnf baz", source="baz")]),  # the bnf one is chosen
-        ("one two", True, [CN("one and two", source="one two")]),
+        ("foo bar", True, [CNC("foo then bar", source="foo bar", foo="foo", bar="bar")]),
-        ("three three three", True, [CN("one or more three", source="three three three")]),
+        ("bar", True, [CNC("foo or bar", source="bar", bar="bar", body="bar")]),
-        ("twenty two", True, [CN("twenties", source="twenty two")]),
+        ("one plus two", True, [CNC("plus", source="one plus two", one="one", two="two")]),
-        ("twenty four", True, [CN("twenties", source="twenty four")]),
+        ("twenty one", True, [CNC("t1", source="twenty one", unit="one", one="one")]),
-        ("twenty one", False, [UTN("twenty "), CN("bnf one", source="one")]),
-        ("twenty two + 1", True, [CN("twenties", source="twenty two"), " + 1"]),
     ])
-    def test_i_can_parse(self, parser_input, expected_status, expected):
+    def test_i_can_parse_simple_expressions(self, parser_input, expected_status, expected):
         sheerka, context, parser = self.init_parser(init_from_sheerka=True)
 
         res = parser.parse(context, ParserInput(parser_input))
@@ -772,6 +1008,167 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert concepts_nodes == expected_array
 
+    def test_i_can_when_multiple_times_the_same_variable(self):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+
+        text = "foo foo foo"
+        expected_array = compute_expected_array(cmap, text, [CNC("one or more foo", source=text)])
+        expected_array[0].compiled["foo"] = [cmap["foo"], cmap["foo"], cmap["foo"]]
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.value
+        concepts_nodes = res.value.value
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert concepts_nodes == expected_array
+
+    def test_i_can_test_when_expression_references_other_expressions(self):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+
+        text = "twenty four"
+        expected = CNC("t2",
+                       source=text,
+                       unit=CC("three_four",
+                               source="four",
+                               four=CC("four", body=DoNotResolve("four")),
+                               body=CC("four", body=DoNotResolve("four"))),
+                       four="four")
+        expected_array = compute_expected_array(cmap, text, [expected])
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.value
+        concepts_nodes = res.value.value
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert concepts_nodes == expected_array
+
+    # def test_i_cannot_parse_bnf_concept_mixed_with_isa_concepts(self):
+    #     sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+    #
+    #     # thirties = cls.update_bnf(context, Concept("thirties",
+    #     #                                            definition="thirty number",
+    #     #                                            where="number < 10",
+    #     #                                            body="thirty + number").def_var("thirty").def_var("number"))
+    #     # with thirties isa number
+    #     # So number in 'thirty number' will spawn 'thirties' which, because there is no other indication, will
+    #     # create an infinite loop
+    #
+    #     text = "thirty one"
+    #     expected = CNC("thirties",
+    #                    source=text,
+    #                    number=CC("number",
+    #                              source="one",
+    #                              one=CC("one", body=DoNotResolve("one")),
+    #                              body=CC("one", body=DoNotResolve("one"))),
+    #                    one=CC("one", body=DoNotResolve("one")),
+    #                    thirty="thirty")
+    #     expected_array = compute_expected_array(cmap, text, [expected])
+    #
+    #     res = parser.parse(context, ParserInput(text))
+    #     not_for_me = res.value
+    #     reason = res.value.body
+    #
+    #     assert not res.status
+    #     assert sheerka.isinstance(not_for_me, BuiltinConcepts.NOT_FOR_ME)
+    #     assert sheerka.isinstance(reason, BuiltinConcepts.CHICKEN_AND_EGG)
+    #     assert reason.body == {cmap["thirties"].id, cmap["number"].id}
+
+    def test_i_can_parse_bnf_concept_mixed_with_isa_concepts(self):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+
+        # thirties = cls.update_bnf(context, Concept("thirties",
+        #                                            definition="thirty number",
+        #                                            where="number < 10",
+        #                                            body="thirty + number").def_var("thirty").def_var("number"))
+
+        text = "thirty one"
+        expected = CNC("thirties",
+                       source=text,
+                       number=CC("number",
+                                 source="one",
+                                 one=CC("one", body=DoNotResolve("one")),
+                                 body=CC("one", body=DoNotResolve("one"))),
+                       one=CC("one", body=DoNotResolve("one")),
+                       thirty="thirty")
+        expected_array = compute_expected_array(cmap, text, [expected])
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.value
+        concepts_nodes = res.value.value
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert concepts_nodes == expected_array
+
+    def test_i_can_parse_bnf_concept_mixed_with_isa_concepts_2(self):
+        # this time, three is a number, and also part of  three_four, even if it is not relevant in t3
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+
+        text = "thirty three"
+        expected = CNC("thirties",
+                       source=text,
+                       number=CC("number",
+                                 source="three",
+                                 three=CC("three", body=DoNotResolve("three")),
+                                 body=CC("three", body=DoNotResolve("three"))),
+                       three=CC("three", body=DoNotResolve("three")),
+                       thirty="thirty")
+        expected_array = compute_expected_array(cmap, text, [expected])
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.value
+        concepts_nodes = res.value.value
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert concepts_nodes == expected_array
+
+    def test_i_can_parse_bnf_concept_mixed_with_isa_after_restart(self):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+        sheerka.concepts_grammars.clear()  # simulate restart
+        for c in cmap.values():
+            sheerka.get_by_id(c.id).bnf = None
+
+        text = "thirty three"
+        expected = CNC("thirties",
+                       source=text,
+                       number=CC("number",
+                                 source="three",
+                                 three=CC("three", body=DoNotResolve("three")),
+                                 body=CC("three", body=DoNotResolve("three"))),
+                       three=CC("three", body=DoNotResolve("three")),
+                       thirty="thirty")
+        expected_array = compute_expected_array(cmap, text, [expected])
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.value
+        concepts_nodes = res.value.value
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert concepts_nodes == expected_array
+
+        text = "forty one"
+        expected = CNC("forties",
+                       source=text,
+                       number=CC("number",
+                                 source="one",
+                                 one=CC("one", body=DoNotResolve("one")),
+                                 body=CC("one", body=DoNotResolve("one"))),
+                       one=CC("one", body=DoNotResolve("one")),
+                       forty="forty")
+        expected_array = compute_expected_array(cmap, text, [expected])
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.value
+        concepts_nodes = res.value.value
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert concepts_nodes == expected_array
+
     def test_i_can_parse_when_keyword(self):
         sheerka, context, parser = self.init_parser(init_from_sheerka=True)
 
@@ -800,10 +1197,80 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         parser_result = res.value
         concepts_nodes = res.value.value
 
-        assert res.status == True
+        assert res.status
         assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert concepts_nodes == expected_array
 
+    def test_i_can_parse_descent_grammar(self):
+        my_map = {
+            "factor": Concept("factor", definition="1 | 2 | 3"),
+            "term": Concept("term", definition="factor ('*' factor)*"),
+            "expr": Concept("expr", definition="term ('+' term)*"),
+        }
+
+        sheerka, context, parser = self.init_parser(my_map, singleton=True)
+        parser.init_from_concepts(context, my_map.values())
+
+        text = "1 + 2 * 3"
+
+        res = parser.parse(context, ParserInput(text))
+        parser_result = res.value
+        concepts_nodes = res.value.value
+
+        factor = my_map["factor"]
+        term = my_map["term"]
+        expr = my_map["expr"]
+
+        assert res.status
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert concepts_nodes == [CNC(expr,
+                                      term=[CC(term,
+                                               body=CC(factor, body=DoNotResolve("1")),
+                                               factor=CC(factor, body=DoNotResolve("1"))),
+                                            CC(term,
+                                               body=DoNotResolve("2 * 3"),
+                                               factor=[
+                                                   CC(factor, body=DoNotResolve("2")),
+                                                   CC(factor, body=DoNotResolve("3")),
+                                               ])],
+                                      factor=[
+                                          CC(factor, body=DoNotResolve("1")),
+                                          CC(factor, body=DoNotResolve("2")),
+                                          CC(factor, body=DoNotResolve("3"))],
+                                      body=DoNotResolve("1 + 2 * 3"))]
+
+    def test_i_can_parse_recursive_descent_grammar(self):
+        my_map = {
+            "factor": Concept("factor", definition="1 | 2 | 3"),
+            "term": self.bnf_concept("term", OrderedChoice(
+                Sequence(ConceptExpression("factor"), StrMatch("*"), ConceptExpression("term")),
+                ConceptExpression("factor"))),
+            "expr": self.bnf_concept("expr", OrderedChoice(
+                Sequence(ConceptExpression("term"), StrMatch("+"), ConceptExpression("expr")),
+                ConceptExpression("term"))),
+        }
+        sheerka, context, parser = self.init_parser(my_map, singleton=True)
+        parser.init_from_concepts(context, my_map.values())
+
+        text = "1 + 2 * 3"
+
+        res = parser.parse(context, ParserInput(text))
+        # concepts_nodes = res.value.value is too complicated to be validated
+        assert res.status
+
+    def test_i_can_parse_simple_recursive_grammar(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(StrMatch("foo"),
+                                                    OrderedChoice(StrMatch("bar"), ConceptExpression("foo")))),
+        }
+
+        sheerka, context, parser = self.init_parser(my_map, singleton=True)
+        parser.init_from_concepts(context, my_map.values())
+
+        assert parser.parse(context, ParserInput("foo bar")).status
+        assert parser.parse(context, ParserInput("foo foo foo bar")).status
+        assert not parser.parse(context, ParserInput("foo baz")).status
+
     # @pytest.mark.parametrize("parser_input, expected", [
     #     ("one", [
     #         (True, [CNC("bnf_one", source="one", one="one", body="one")]),

tests/parsers/test_SyaNodeParser.py

@@ -54,10 +54,10 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         cmap["plus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
         cmap["mult"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
         cmap["minus"].set_prop(BuiltinConcepts.ASSOCIATIVITY, "right")
-        TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].is_greater_than(context,
+        TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context,
                                                                                               BuiltinConcepts.PRECEDENCE,
                                                                                               cmap["mult"], cmap["plus"])
-        TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].is_greater_than(context,
+        TestSyaNodeParser.sheerka.services[SheerkaComparisonManager.NAME].set_is_greater_than(context,
                                                                                               BuiltinConcepts.PRECEDENCE,
                                                                                               cmap["mult"], cmap["minus"])
 

tests/sheerkapickle/test_sheerka_handlers.py

@@ -201,7 +201,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, user_input)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == user_input
-        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "22"], "user_name": "my_user_name", "text": "my_text"}'
+        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "23"], "user_name": "my_user_name", "text": "my_text"}'
 
     def test_i_can_encode_decode_user_input_when_tokens(self):
         sheerka = self.get_sheerka()
@@ -213,7 +213,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, user_input)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == sheerka.new(BuiltinConcepts.USER_INPUT, body=text, user_name="my_user_name")
-        assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "22"], "user_name": "my_user_name", "text": "{text}"' + '}'
+        assert to_string == '{' + f'"_sheerka/obj": "core.builtin_concepts.UserInputConcept", "concept/id": ["__USER_INPUT", "23"], "user_name": "my_user_name", "text": "{text}"' + '}'
 
     def test_i_can_encode_decode_return_value(self):
         sheerka = self.get_sheerka()
@@ -223,7 +223,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         to_string = sheerkapickle.encode(sheerka, ret_val)
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == ret_val
-        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "27"], "who": "who", "status": true, "value": 10}'
+        assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept", "concept/id": ["__RETURN_VALUE", "28"], "who": "who", "status": true, "value": 10}'
 
     def test_i_can_encode_decode_return_value_with_parent(self):
         sheerka = self.get_sheerka()
@@ -236,7 +236,7 @@ class TestSheerkaPickleHandler(TestUsingMemoryBasedSheerka):
         decoded = sheerkapickle.decode(sheerka, to_string)
         assert decoded == ret_val
         assert decoded.parents == ret_val.parents
-        id_str = ', "concept/id": ["__RETURN_VALUE", "27"]'
+        id_str = ', "concept/id": ["__RETURN_VALUE", "28"]'
         parents_str = '[{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "parent_who", "status": true, "value": "10"}, {"_sheerka/id": 1}]'
         assert to_string == '{"_sheerka/obj": "core.builtin_concepts.ReturnValueConcept"' + id_str + ', "who": "who", "status": true, "value": 10, "parents": ' + parents_str + '}'