Fixed #72 : Exception when get_results(id=10)

Fixed #74 : Keyword parameters are no longer recognized when a concept that redefines equality is created
Fixed #118 : RecursionError: maximum recursion depth exceeded
Fixed #119 : PreventCircularReferenceEvaluator
Fixed #121 : Plural are not updated when new elements are added
Fixed #123 : BaseCache : Values in cache can be evicted before being committed
Fixed #105 : TOO_MANY_ERROR is not the relevant error when results are filtered

sheerka_backup/202109.sb

@@ -0,0 +1,3 @@
+def concept animal
+def concept dog
+global_truth(a dog is an animal)

sheerka_backup/full.sb

@@ -3,6 +3,7 @@
 #import python
 #import numbers
 #import adjectives
+#import 202109
 
 
 

src/cache/BaseCache.py

@@ -1,6 +1,6 @@
 from threading import RLock
 
-from core.global_symbols import NotFound, Removed
+from core.global_symbols import NotFound
 from core.utils import sheerka_deepcopy
 
 MAX_INITIALIZED_KEY = 100
@@ -97,9 +97,6 @@ class BaseCache:
         :return:
         """
         with self._lock:
-            if self._max_size and self._current_size >= self._max_size:
-                self.evict(self._max_size - self._current_size + 1)
-
             if self._put(key, value, alt_sdp):
                 self._current_size += 1
 
@@ -227,19 +224,29 @@ class BaseCache:
         """
         with self._lock:
             nb_items = self._current_size if self._current_size < nb_items else nb_items
-            nb_to_delete = nb_items
+            to_remove = []
-            while nb_items > 0:
+            iter_cache = iter(self._cache)
-                key = next(iter(self._cache))
+            try:
+                while nb_items > 0:
+                    key = next(iter_cache)
+                    if key in self.to_add or key in self.to_remove:
+                        continue  # cannot remove an item that is not yet committed
+                    else:
+                        to_remove.append(key)
+                        nb_items -= 1
+            except StopIteration:
+                pass
+
+            for key in to_remove:
                 del (self._cache[key])
                 try:
                     self._initialized_keys.remove(key)
                 except KeyError:
                     pass
-                nb_items -= 1
 
-            self._current_size -= nb_to_delete
+            self._current_size -= len(to_remove)
 
-        return nb_to_delete
+        return len(to_remove)
 
     def evict_by_key(self, predicate):
         """
@@ -359,14 +366,17 @@ class BaseCache:
                     simple_copy = False
 
                 if value is not NotFound:
-                    self._cache[key] = value if simple_copy else sheerka_deepcopy(value)
+                    value = value if simple_copy else sheerka_deepcopy(value)
-                    value = self._cache[key]
+                    self._cache[key] = value
 
                     # update _current_size
                     if isinstance(value, (list, set)):
                         self._current_size += len(value)
                     else:
                         self._current_size += 1
+
+                    if self._max_size and self._current_size > self._max_size:
+                        self.evict(self._current_size - self._max_size)
             else:
                 value = self._default
             self._initialized_keys.add(key)
@@ -383,4 +393,4 @@ class BaseCache:
         pass
 
     def _delete(self, key, value, alt_sdp):
-        raise NotImplementedError()
+        raise NotImplementedError("_delete BaseCache")

src/cache/DictionaryCache.py

@@ -80,7 +80,7 @@ class DictionaryCache(BaseCache):
         return False
 
     def _delete(self, key, value, alt_sdp):
-        raise NotImplementedError()
+        raise NotImplementedError("_delete DictionaryCache")
 
     def _count_items(self):
         self._current_size = 0

src/core/builtin_helpers.py

@@ -532,7 +532,7 @@ def get_lexer_nodes(return_values, start, tokens):
                 lexer_nodes.append([RuleNode(rule, start, end, tokens, ret_val.body.source)])
 
         else:
-            raise NotImplementedError()
+            raise NotImplementedError(f"get_lexer_nodes who={who}")
 
     return lexer_nodes
 
@@ -605,7 +605,7 @@ def get_lexer_nodes_using_positions(return_values, positions):
             lexer_nodes.append(node)
 
         else:
-            raise NotImplementedError()
+            raise NotImplementedError(f"get_lexer_nodes_using_positions {who=}")
 
     return lexer_nodes
 
@@ -692,7 +692,7 @@ def update_compiled(context, concept, errors, parsers=None):
 
             elif isinstance(v, SourceCodeNode):
                 if not v.return_value:
-                    raise NotImplementedError("SourceCodeNode")
+                    raise NotImplementedError("_validate_concept SourceCodeNode ret val is False")
                 c.get_compiled()[k] = [v.return_value]
 
             elif isinstance(v, SourceCodeWithConceptNode):

src/core/concept.py

@@ -7,7 +7,7 @@ from typing import List
 import core.utils
 from core.builtin_concepts_ids import BuiltinDynamicAttrs
 from core.global_symbols import NotInit
-from core.tokenizer import Tokenizer, TokenKind
+from core.tokenizer import TokenKind, Tokenizer
 
 PROPERTIES_FOR_DIGEST = ("name", "key",
                          "definition", "definition_type",
@@ -346,9 +346,8 @@ class Concept:
     def body(self):
         return self.get_value(ConceptParts.BODY)
 
-    def get_atomic_def(self):
+    def is_dynamic(self):
-        tokens = [t for t in Tokenizer(self.key, yield_eof=False) if t.type != TokenKind.VAR_DEF]
+        return self.id and "-" in self.id
-        return core.utils.get_text_from_tokens(tokens).strip()
 
     def get_origin(self):
         """

src/core/sheerka/ExecutionContext.py

@@ -456,7 +456,7 @@ class ExecutionContext:
         """
         return list(self.search(predicate, None, False))
 
-    def search(self, predicate=None, get_obj=None, start_with_self=False, stop=None):
+    def search(self, predicate=None, get_obj=None, start_with_self=False, stop=None, only_first=False):
         """
         Iter thru execution context parent and return the list of obj
         :param predicate: what execution context to keep
@@ -469,6 +469,8 @@ class ExecutionContext:
         while current:
             if predicate is None or predicate(current):
                 yield current if get_obj is None else get_obj(current)
+                if only_first:
+                    break
 
             if stop and stop(current):
                 break

src/core/sheerka/Sheerka.py

@@ -614,7 +614,7 @@ class Sheerka(Concept):
         concept.get_hints().is_evaluated = True  # because we have manually set the variables
         return concept
 
-    def new_dynamic(self, concept_or_key, id_suffix, name=None, props=None, attrs=None):
+    def new_dynamic(self, concept_or_key, id_suffix, name=None, props=None, attrs=None, body=None):
         """
         Create a dynamic concept with the given props and attrs
         A dynamic concept is a concept that is not declared by 'def concept' but can be deduced from another one
@@ -625,6 +625,7 @@ class Sheerka(Concept):
         :param name: new name and key for the concept
         :param props:
         :param attrs:
+        :param body:
         :return:
         """
         if not isinstance(concept_or_key, Concept):
@@ -649,6 +650,9 @@ class Sheerka(Concept):
             for k, v in attrs.items():
                 concept.set_value(k, v)
 
+        if body:
+            concept.get_metadata().body = body
+
         return concept
 
     def push_ontology(self, context, name, cache_only=False):

src/core/sheerka/services/SheerkaConceptManager.py

@@ -1,6 +1,6 @@
 import re
 from dataclasses import dataclass
-from typing import Set, List, Union
+from typing import List, Set, Union
 
 import core.utils
 from cache.Cache import Cache
@@ -9,14 +9,14 @@ from cache.DictionaryCache import DictionaryCache
 from cache.ListIfNeededCache import ListIfNeededCache
 from cache.SetCache import SetCache
 from core.builtin_concepts import ErrorConcept
-from core.builtin_concepts_ids import BuiltinConcepts, AllBuiltinConcepts, BuiltinUnique
+from core.builtin_concepts_ids import AllBuiltinConcepts, BuiltinConcepts, BuiltinUnique
-from core.builtin_helpers import ensure_concept, ensure_bnf
+from core.builtin_helpers import ensure_bnf, ensure_concept
-from core.concept import Concept, DEFINITION_TYPE_DEF, DEFINITION_TYPE_BNF, freeze_concept_attrs, ConceptMetadata, \
+from core.concept import Concept, ConceptMetadata, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, VARIABLE_PREFIX, \
-    VARIABLE_PREFIX
+    freeze_concept_attrs
-from core.global_symbols import EVENT_CONCEPT_CREATED, NotInit, NotFound, ErrorObj, EVENT_CONCEPT_DELETED, NoFirstToken, \
+from core.global_symbols import CONCEPT_COMPARISON_CONTEXT, EVENT_CONCEPT_CREATED, EVENT_CONCEPT_DELETED, \
-    EVENT_CONCEPT_MODIFIED, CONCEPT_COMPARISON_CONTEXT
+    EVENT_CONCEPT_MODIFIED, ErrorObj, NoFirstToken, NotFound, NotInit
 from core.sheerka.services.sheerka_service import BaseService
-from core.tokenizer import Tokenizer, TokenKind
+from core.tokenizer import TokenKind, Tokenizer
 from parsers.BnfNodeParser import RegExDef
 from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
 
@@ -133,6 +133,7 @@ class SheerkaConceptManager(BaseService):
         self.sheerka.bind_service_method(self.NAME, self.get_concepts_bnf_definitions, False, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.clear_bnf_definition, True, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.set_precedence, True)
+        self.sheerka.bind_service_method(self.NAME, self.get_plural_concept_value, False)
 
         register_concept_cache = self.sheerka.om.register_concept_cache
 
@@ -1294,3 +1295,11 @@ class SheerkaConceptManager(BaseService):
             for variable in [v for v in visitor.variables if isinstance(v, ParameterVariable)]:
                 if variable.name not in concept.get_metadata().parameters:
                     concept.get_metadata().parameters.append(variable.name)
+
+    @staticmethod
+    def get_plural_concept_value(context, concept):
+        underlying_concept = concept.get_prop(BuiltinConcepts.PLURAL)
+        if context.sheerka.isaset(context, underlying_concept):
+            return context.sheerka.get_set_elements(context, underlying_concept)
+
+        return None

src/core/sheerka/services/SheerkaDebugManager.py

@@ -600,7 +600,7 @@ class SheerkaDebugManager(BaseService):
         elif item_type == self.CONCEPTS_DEBUG_TYPE:
             self.registered_concepts.append((service, method, item))
         else:
-            raise NotImplementedError()
+            raise NotImplementedError(f"register_debug {item_type=}")
 
     def register_debug_vars(self, service, method, item):
         return self.register_debug(self.VARS_DEBUG_TYPE, service, method, item)
@@ -643,7 +643,7 @@ class SheerkaDebugManager(BaseService):
         elif item_type == self.CONCEPTS_DEBUG_TYPE:
             lst = self.registered_concepts
         else:
-            raise NotImplementedError()
+            raise NotImplementedError(f"filter_registered_debug {item_type=}")
 
         for registered in lst:
             if service != "*" and service != registered[0]:

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -230,6 +230,12 @@ class SheerkaEvaluateConcept(BaseService):
         :param possible_variables: concepts that must be considered as variables
         :return:
         """
+        def get_filtered_by_prevent_circular_reference(return_values):
+            for ret_val in return_values:
+                from evaluators.PreventCircularReferenceEvaluator import PreventCircularReferenceEvaluator
+                if ret_val.who == PreventCircularReferenceEvaluator.NAME:
+                    return ret_val.body.body
+            return None
 
         def parse_token_concept(s):
             """
@@ -252,6 +258,7 @@ class SheerkaEvaluateConcept(BaseService):
             """
             parsers_to_use = INIT_AST_QUESTION_PARSERS if p in [ConceptParts.WHERE,
                                                                 ConceptParts.PRE] else INIT_AST_PARSERS
+            recursion_detected = False
             while True:
                 return_value = current_context.sheerka.parse_unrecognized(current_context,
                                                                           s,
@@ -262,8 +269,24 @@ class SheerkaEvaluateConcept(BaseService):
                                                                           possible_variables=possible_variables)
 
                 if not return_value.status:
-                    if current_context.preprocess:
+                    # Dealing with circular reference is not easy.
-                        raise ChickenAndEggException(context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c}))
+                    # Let's take for example 'def concept q from q ? as question(q)'
+                    # 'q' is the name of the concept, but also the name of the parameter
+                    # When chicken_and_egg_err is found with a concept parser (Exact, Sya, Bnf...) we must disable it
+                    # to let a chance to the Python parser (which can handle concepts parameters)
+                    sheerka = context.sheerka
+                    if sheerka.has_error(context, return_value, __type="ChickenAndEggException"):
+                        recursion_detected = True
+                        filtered = get_filtered_by_prevent_circular_reference(return_value.body.body)
+                        recursive_parsers = list(SheerkaEvaluateConcept.get_recursive_definitions(context, c, filtered))
+                        if len(recursive_parsers):
+                            desc = f"Removing parsers {recursive_parsers}"
+                            current_context = current_context.push(context.action, context.action_context, desc=desc)
+                            for recursive_parser in recursive_parsers:
+                                current_context.add_preprocess(recursive_parser.name, enabled=False)
+                            continue
+                    elif recursion_detected:
+                        raise ChickenAndEggException(sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body={c}))
                     else:
                         raise FailedToCompileError([return_value.body])
 
@@ -529,7 +552,7 @@ class SheerkaEvaluateConcept(BaseService):
                     return evaluated
 
             elif isinstance(to_resolve, Rule):
-                raise NotImplementedError()  # how to resolve rules ?
+                raise NotImplementedError(f"evaluate_concept.resolve() {to_resolve=}")  # how to resolve rules ?
 
             # otherwise, execute all return values to find out what is the value
             else:
@@ -772,7 +795,7 @@ class SheerkaEvaluateConcept(BaseService):
             # TODO : Validate the POST condition
             #
 
-            if ConceptParts.BODY in all_metadata_to_eval and not failed_to_evaluate_body:
+            if ConceptParts.BODY in all_metadata_to_eval and not failed_to_evaluate_body and not concept.is_dynamic():
                 concept.get_hints().is_evaluated = True
 
             # # update the cache for concepts with no variables

src/core/sheerka/services/SheerkaExecute.py

@@ -406,7 +406,7 @@ class SheerkaExecute(BaseService):
         if isinstance(text, ParserInput):
             return text.as_text()
 
-        raise NotImplementedError()
+        raise NotImplementedError(f"get_input_as_text({text=})")
 
     def get_parser_input(self, text, tokens=None):
         """
@@ -835,6 +835,15 @@ class SheerkaExecute(BaseService):
         :param possible_variables: concepts that must be considered as variables
         :return:
         """
+
+        # def filter_for_circular_reference(return_values, concept):
+        #     for r in return_values:
+        #         if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT):
+        #             body = r.body.body[0] if isinstance(r.body.body, list) and len(r.body.body) == 1 else r.body.body
+        #             if hasattr(body, "get_concept") and body.get_concept().id == concept.id:
+        #                 continue
+        #         yield r
+
         sheerka = context.sheerka
 
         if prop:
@@ -864,6 +873,14 @@ class SheerkaExecute(BaseService):
                                    sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
             res = sheerka.execute(sub_context, to_parse, PARSE_STEPS)
 
+            # # before user defined filtering, remove the return values that may lead to circular reference
+            # in_recursion = list(context.search(predicate=lambda c: c.action == BuiltinConcepts.EVALUATING_CONCEPT,
+            #                                    get_obj=lambda c: c.action_context,
+            #                                    only_first=True))
+            #
+            # if in_recursion:
+            #     res = list(filter_for_circular_reference(res, in_recursion[0]))
+
             if filter_func:
                 res = filter_func(sub_context, res)
 

src/core/sheerka/services/SheerkaRuleManager.py

@@ -749,7 +749,7 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
                 elif type(c) == NegatedCondition:
                     res.append(Condition(c.identifier, c.attribute, c.value))
                 else:
-                    raise NotImplementedError
+                    raise NotImplementedError("rete - negate_conditions")
             else:
                 res.append(NegatedConjunctiveConditions(*conditions_))
 

src/evaluators/MultipleErrorsEvaluator.py

@@ -30,9 +30,11 @@ class MultipleErrorsEvaluator(AllReturnValuesEvaluator):
                 to_process = True
                 self.eaten.append(ret)
             elif not ret.status and ret.who.startswith(self.PREFIX):
-                nb_evaluators_in_error += 1
-                self.return_values_in_error.append(ret)
                 self.eaten.append(ret)
+                if not context.sheerka.isinstance(ret.body, BuiltinConcepts.FILTERED):
+                    nb_evaluators_in_error += 1
+                    self.return_values_in_error.append(ret)
+
             elif not ret.status and ret.who.startswith(BaseParser.PREFIX):
                 self.eaten.append(ret)
             # else:

src/evaluators/OneErrorEvaluator.py

@@ -14,13 +14,16 @@ class OneErrorEvaluator(AllReturnValuesEvaluator):
     def __init__(self):
         super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 30)
         self.return_value_in_error = None
+        self.return_value_filtered = None
 
     def reset(self):
         super().reset()
         self.return_value_in_error = None
+        self.return_value_filtered = None
 
     def matches(self, context, return_values):
         nb_evaluators_in_error = 0
+        nb_evaluators_filtered = 0
         to_process = False
 
         for ret in return_values:
@@ -30,9 +33,13 @@ class OneErrorEvaluator(AllReturnValuesEvaluator):
                 to_process = True
                 self.eaten.append(ret)
             elif not ret.status and ret.who.startswith(self.PREFIX):
-                nb_evaluators_in_error += 1
-                self.return_value_in_error = ret
                 self.eaten.append(ret)
+                if context.sheerka.isinstance(ret.body, BuiltinConcepts.FILTERED):
+                    nb_evaluators_filtered += 1
+                    self.return_value_filtered = ret
+                else:
+                    nb_evaluators_in_error += 1
+                    self.return_value_in_error = ret
             elif not ret.status and ret.who.startswith(BaseParser.PREFIX):
                 self.eaten.append(ret)
             # else:
@@ -40,11 +47,12 @@ class OneErrorEvaluator(AllReturnValuesEvaluator):
             # They won't be part of result nor part of the parent
             # --> So they will be handled by other evaluators
 
-        return to_process and nb_evaluators_in_error == 1
+        return to_process and (nb_evaluators_in_error == 1 or nb_evaluators_filtered == 1)
 
     def eval(self, context, return_values):
         context.log(f"1 return value in error, {len(self.eaten)} item(s) eaten", who=self)
         context.log(f"{self.return_value_in_error}", who=self)
 
         sheerka = context.sheerka
-        return sheerka.ret(self.name, False, self.return_value_in_error.value, parents=self.eaten.copy())
+        to_return = self.return_value_in_error.value if self.return_value_in_error else self.return_value_filtered.body
+        return sheerka.ret(self.name, False, to_return, parents=self.eaten.copy())

src/evaluators/PreventCircularReferenceEvaluator.py

@@ -0,0 +1,59 @@
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.services.sheerka_service import ChickenAndEggException
+from evaluators.BaseEvaluator import AllReturnValuesEvaluator
+
+
+class PreventCircularReferenceEvaluator(AllReturnValuesEvaluator):
+    """"
+    when
+    -----
+    def concept a or b as a or b
+    def concept or form a or b as or b
+
+    During the creation of the ast, for the body,
+    the parsers will find these two concepts, that will be evaluated by an AFTER_PARSING evaluator
+    And it will create an infinite loop
+    """
+    NAME = "PreventCircularReference"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_PARSING], 99, enabled=True)
+        self.concept = None
+
+    def reset(self):
+        super().reset()
+        self.concept = None
+
+    def matches(self, context: ExecutionContext, return_values):
+        # first find the concept under evaluation
+        concepts = list(context.search(predicate=lambda c: c.action == BuiltinConcepts.EVALUATING_CONCEPT,
+                                       get_obj=lambda c: c.action_context,
+                                       only_first=True))
+        if not concepts:
+            return False
+
+        self.concept = concepts[0]
+        return True
+
+    def eval(self, context: ExecutionContext, return_values):
+        to_filter = []
+        for r in return_values:
+            # find concept parsers
+            if r.status and context.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT):
+                body = r.body.body[0] if isinstance(r.body.body, list) and len(r.body.body) == 1 else r.body.body
+                if hasattr(body, "get_concept"):
+                    concept = body.get_concept()
+                    if concept.id == self.concept.id and concept.name not in concept.variables():
+                        # There is a variable with the same name as the concept
+                        # During evaluation, inner variables take precedence other concepts
+                        # So there won't be any cyclic reference, the variable will be picked
+                        to_filter.append(r)
+
+        if not to_filter:
+            return None
+
+        filtered = context.sheerka.new(BuiltinConcepts.FILTERED,
+                                       filtered=to_filter,
+                                       reason=ChickenAndEggException(self.concept))
+        return context.sheerka.ret(self.NAME, False, filtered, parents=to_filter)

src/parsers/BaseExpressionParser.py

@@ -541,10 +541,10 @@ class BaseExpressionParser(BaseParser):
         return ret
 
     def parse_input(self, context, parser_input: ParserInput, error_sink: ErrorSink):
-        raise NotImplementedError
+        raise NotImplementedError()
 
     def parse_tokens_stop_condition(self, token, parser_input):
-        raise NotImplementedError
+        raise NotImplementedError()
 
     def parse_tokens(self, context, parser_input, error_sink, stop_condition, expr_parser):
         def stop():

src/parsers/BaseNodeParser.py

@@ -40,7 +40,7 @@ class LexerNode(Node):
         pass
 
     def to_short_str(self):
-        raise NotImplementedError
+        raise NotImplementedError()
 
     def get_source_to_parse(self):
         return self.source

src/parsers/BnfNodeParser.py

@@ -610,7 +610,7 @@ class VariableExpression(ParsingExpression):
         if isinstance(node, (SourceCodeNode, SourceCodeWithConceptNode)):
             return node.return_value
 
-        raise NotImplementedError()
+        raise NotImplementedError(f"VariableExpression.get_resolved({node=})")
 
     @staticmethod
     def get_nodes_sequences_from_tokens(parser_helper, start, end, tokens):
@@ -986,7 +986,7 @@ class UnorderedGroup(Repetition):
     """
 
     def _parse(self, parser):
-        raise NotImplementedError()
+        raise NotImplementedError("UnorderedGroup.parse()")
 
     # def __repr__(self):
     #     to_str = ", ".join(repr(n) for n in self.elements)

src/parsers/FormatRuleActionParser.py

@@ -202,7 +202,7 @@ class FormatRuleActionParser(IterParser):
             return [get_text(i) for i in list_or_dict_of_tokens]
         if isinstance(list_or_dict_of_tokens, dict):
             return {k: get_text(v) for k, v in list_or_dict_of_tokens.items()}
-        raise NotImplementedError("")
+        raise NotImplementedError(f"FormatRuleActionParser.to_text({list_or_dict_of_tokens=})")
 
     def to_value(self, tokens):
         """

src/parsers/SequenceNodeParser.py

@@ -480,14 +480,10 @@ class SequenceNodeParser(BaseNodeParser):
         plural_concepts = [self.sheerka.new_dynamic(c,
                                                     BuiltinConcepts.PLURAL,
                                                     name=token.value,
+                                                    body="get_plural_concept_value(self)",
                                                     props={BuiltinConcepts.PLURAL: c})
                            for c in concepts]
 
-        for concept in plural_concepts:
-            underlying_concept = concept.get_prop(BuiltinConcepts.PLURAL)
-            if self.sheerka.isaset(self.context, underlying_concept):
-                concept.get_metadata().body = f"get_set_elements(c:|{underlying_concept.id}:)"
-
         return plural_concepts
 
     @staticmethod

src/parsers/SyaNodeParser.py

@@ -499,10 +499,10 @@ class BaseSyaParser:
         self._state = self.all_states.get_state(state)
 
     def get_unrecognized_tokens_requests_cache(self):
-        raise NotImplementedError()
+        raise NotImplementedError(f"BaseSyaParser.get_unrecognized_tokens_requests_cache()")
 
     def get_tokens_parser(self):
-        raise NotImplementedError()
+        raise NotImplementedError(f"BaseSyaParser.get_tokens_parser()")
 
     def add_debug(self, text, is_error=False, **kwargs):
         args = {"token": self.parser_input.token,

src/sdp/sheerkaSerializer.py

@@ -329,4 +329,4 @@ class CustomTypeSerializer(BaseSerializer):
             return NotFound
         elif value == Removed.value:
             return Removed
-        raise NotImplemented(value)
+        raise NotImplemented(f"CustomTypeSerializer.load({value})")

src/sheerkarete/network.py

@@ -312,7 +312,7 @@ class ReteNetwork:
         if hasattr(rule, "get_rete_disjunctions"):
             return rule.get_rete_disjunctions()
 
-        raise NotImplementedError("")
+        raise NotImplementedError(f"get_rete_conditions({rule=})")
 
     def add_rule(self, rule: Rule):
 

tests/BaseTest.py

@@ -296,5 +296,13 @@ class BaseTest:
     def activate_debug(context, pattern="Sya.*.*"):
         sheerka = context.sheerka
         sheerka.set_debug(context, True)
-        sheerka.set_debug_var(context, pattern)
         sheerka.set_debug_logger_definition(ListDebugLogger)
+
+        if isinstance(pattern, list):
+            for p in pattern:
+                sheerka.set_debug_var(context, p)
+        else:
+            sheerka.set_debug_var(context, pattern)
+
+        # the see the logs, do not forget to add
+        # logs = sheerka.get_debugger_logs()

tests/cache/test_cache.py

@@ -1,4 +1,5 @@
 import pytest
+
 from cache.BaseCache import MAX_INITIALIZED_KEY
 from cache.Cache import Cache
 from cache.CacheManager import CacheManager
@@ -9,7 +10,6 @@ from cache.ListIfNeededCache import ListIfNeededCache
 from cache.SetCache import SetCache
 from core.concept import Concept
 from core.global_symbols import NotFound, Removed
-
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.cache import FakeSdp
 
@@ -65,21 +65,60 @@ class TestCache(TestUsingMemoryBasedSheerka):
         assert len(cache) == 2
         assert cache.copy() == {"key": "another value", "key2": "value2"}
 
-    def test_i_can_evict(self):
+    def test_i_do_not_evict_when_put(self):
         maxsize = 5
         cache = Cache(max_size=5)
 
-        for key in range(maxsize):
+        for key in range(maxsize + 2):
             cache.put(key, key)
 
+        assert len(cache) == maxsize + 2
+        assert cache.copy() == {
+            0: 0,
+            1: 1,
+            2: 2,
+            3: 3,
+            4: 4,
+            5: 5,
+            6: 6,
+        }
+
+    def test_i_can_evict_when_get(self):
+        maxsize = 5
+        cache = Cache(max_size=5, default=lambda k: k)
+
+        for key in range(maxsize + 2):
+            cache.get(key)
+
         assert len(cache) == maxsize
-        assert cache.has(0)
+        assert cache.copy() == {
+            2: 2,
+            3: 3,
+            4: 4,
+            5: 5,
+            6: 6,
+        }
 
-        for key in range(maxsize, maxsize * 2):
+    def test_i_do_not_evict_when_items_are_not_committed(self):
+        maxsize = 5
+        cache = Cache(max_size=5, default=lambda k: k)
+
+        for key in range(maxsize + 2):
             cache.put(key, key)
 
-            assert len(cache) == maxsize
+        assert len(cache) == maxsize + 2
-            assert not cache.has(key - maxsize)
+
+        cache.get(-1)
+        assert len(cache) == maxsize + 2
+        assert cache.copy() == {
+            0: 0,
+            1: 1,
+            2: 2,
+            3: 3,
+            4: 4,
+            5: 5,
+            6: 6,
+        }
 
     def test_i_can_get_a_value_from_alt_sdp(self):
         cache = Cache(sdp=FakeSdp(get_value=lambda cache_name, key: NotFound)).auto_configure("cache_name")
@@ -424,17 +463,6 @@ class TestCache(TestUsingMemoryBasedSheerka):
         assert cache.to_add == {"some_value"}
         assert cache.to_remove == {"some_other_value"}
 
-    def test_max_size_is_respected_when_populate(self):
-        items = [("1", "1"), ("2", "2"), ("3", "3"), ("4", "4"), ("5", "5")]
-        cache = Cache(max_size=3)
-
-        cache.populate(lambda: items, lambda item: item[0])
-
-        assert len(cache) == 3
-        assert cache.get("3") == ("3", "3")
-        assert cache.get("4") == ("4", "4")
-        assert cache.get("5") == ("5", "5")
-
     def test_i_can_get_all(self):
         items = [("1", "1"), ("2", "2"), ("3", "3")]
         cache = Cache()

tests/core/test_ExecutionContext.py

@@ -127,6 +127,9 @@ class TestExecutionContext(TestUsingMemoryBasedSheerka):
             stop=lambda ec: ec.obj == "skip",
             start_with_self=True,
             get_obj=lambda ec: ec.obj)) == ["obj_abbb", "skip"]
+        assert list(abbb.search(only_first=True)) == [abb]
+        assert list(abbb.search(start_with_self=True, only_first=True)) == [abbb]
+        assert list(abbb.search(lambda ec: ec.obj != "skip", only_first=True)) == [ab]
 
     def test_i_can_deactivate_push(self):
         sheerka = self.get_sheerka()

tests/core/test_SheerkaEvaluateConcept.py

@@ -1,9 +1,8 @@
 import pytest
 
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
+from core.builtin_concepts import BuiltinConcepts, ParserResultConcept, ReturnValueConcept
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, \
+from core.concept import Concept, ConceptParts, DEFINITION_TYPE_DEF, DoNotResolve, InfiniteRecursionResolved
-    DEFINITION_TYPE_DEF
+from core.global_symbols import NotFound, NotInit
-from core.global_symbols import NotInit, NotFound
 from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints, SheerkaEvaluateConcept
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
@@ -540,7 +539,8 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.body == NotInit
         assert not evaluated.get_hints().is_evaluated
 
-        evaluated = sheerka.evaluate_concept(context, foo_instance, hints=EvaluationHints(eval_body=True))  # evaluate the body this time
+        evaluated = sheerka.evaluate_concept(context, foo_instance,
+                                             hints=EvaluationHints(eval_body=True))  # evaluate the body this time
         assert isinstance(evaluated.body, bool) and evaluated.body
 
     def test_i_can_apply_intermediate_where_condition_using_python(self):
@@ -1105,3 +1105,24 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         assert sheerka.isinstance(evaluated, foo)
         assert not foo.get_hints().is_evaluated
+
+    def test_dynamic_concepts_are_never_considered_as_evaluated(self):
+        sheerka, context, foo = self.init_concepts("foo")
+        i = 0
+
+        def get_new_id():
+            nonlocal i
+            i += 1
+            return i
+
+        dynamic_foo = sheerka.new_dynamic(foo, "test", body="get_new_id()")
+        context.add_to_short_term_memory("get_new_id", get_new_id)
+
+        evaluated = sheerka.evaluate_concept(context, dynamic_foo, hints=EvaluationHints(eval_body=True))
+        assert evaluated.key == dynamic_foo.key
+        assert evaluated.body == 1
+        assert not evaluated.get_hints().is_evaluated
+
+        evaluated = sheerka.evaluate_concept(context, dynamic_foo, hints=EvaluationHints(eval_body=True))
+        assert evaluated.body == 2
+        assert not evaluated.get_hints().is_evaluated

tests/evaluators/EvaluatorTestsUtils.py

@@ -135,6 +135,11 @@ def cnode_ret_val(concept, source=None, parser=sya):
     return pr_ret_val([cnode], parser=parser, source=source)
 
 
+def concept_ret_val(concept, source=None, parser=exact):
+    source = source or concept.name
+    return pr_ret_val(concept, parser=parser, source=source)
+
+
 def new_concept(key, **kwargs):
     res = Concept(key=key, name=key, is_builtin=False, is_unique=False)
     for k, v in kwargs.items():

tests/evaluators/test_MultipleErrorsEvaluator.py

@@ -1,11 +1,10 @@
 import pytest
 
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept
 from evaluators.BaseEvaluator import BaseEvaluator
 from evaluators.MultipleErrorsEvaluator import MultipleErrorsEvaluator
 from parsers.BaseParser import BaseParser
-
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -13,8 +12,8 @@ def r(value, status=True):
     return ReturnValueConcept(value, status, value)
 
 
-def eval_false(name):
+def eval_false(name, value="value"):
-    return ReturnValueConcept(BaseEvaluator.PREFIX + name, False, "value")
+    return ReturnValueConcept(BaseEvaluator.PREFIX + name, False, value)
 
 
 def eval_true(name):
@@ -32,7 +31,7 @@ def parser_true(name):
 reduce_requested = ReturnValueConcept(
     "some_name",
     True,
-    Concept(name=BuiltinConcepts.REDUCE_REQUESTED,  key=BuiltinConcepts.REDUCE_REQUESTED))
+    Concept(name=BuiltinConcepts.REDUCE_REQUESTED, key=BuiltinConcepts.REDUCE_REQUESTED))
 
 
 class TestMultipleErrorsEvaluator(TestUsingMemoryBasedSheerka):
@@ -96,3 +95,21 @@ class TestMultipleErrorsEvaluator(TestUsingMemoryBasedSheerka):
 
         assert a_successful_concept not in res.parents
         assert a_concept_in_error not in res.parents
+
+    def test_filtered_results_are_not_considered_as_error(self):
+        sheerka, context = self.init_concepts()
+
+        filtered = sheerka.new(BuiltinConcepts.FILTERED)
+        filtered_ret_val = eval_false("Filter", filtered)
+        false_1 = eval_false("one")
+        false_2 = eval_false("two")
+
+        evaluator = MultipleErrorsEvaluator()
+        assert not evaluator.matches(context, [filtered_ret_val, false_1, reduce_requested])
+
+        evaluator.reset()
+        return_values = [filtered_ret_val, false_1, false_2, reduce_requested]
+        assert evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+        assert not res.status
+        assert res.parents == return_values

tests/evaluators/test_OneErrorEvaluator.py

@@ -1,9 +1,9 @@
 import pytest
 
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept
+from evaluators.BaseEvaluator import BaseEvaluator
 from evaluators.OneErrorEvaluator import OneErrorEvaluator
-
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -11,6 +11,10 @@ def r(value, status=True):
     return ReturnValueConcept(value, status, value)
 
 
+def eval_false(name, value="value"):
+    return ReturnValueConcept(BaseEvaluator.PREFIX + name, False, value)
+
+
 reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
 
 
@@ -72,3 +76,34 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka):
 
         assert a_successful_concept not in res.parents
         assert a_concept_in_error not in res.parents
+
+    def test_i_can_manage_filtered_return_values(self):
+        sheerka, context = self.init_concepts()
+
+        filtered = sheerka.new(BuiltinConcepts.FILTERED)
+        filtered_ret_val = eval_false("Filter", filtered)
+        false_1 = eval_false("one")
+
+        evaluator = OneErrorEvaluator()
+        return_values = [filtered_ret_val, false_1, reduce_requested]
+        assert evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+        assert not res.status
+        assert res.body == false_1.body
+        assert res.parents == return_values
+
+        evaluator.reset()
+        return_values = [filtered_ret_val, reduce_requested]
+        assert evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+        assert not res.status
+        assert res.body == filtered
+        assert res.parents == return_values
+
+        evaluator.reset()
+        return_values = [false_1, reduce_requested]
+        assert evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+        assert not res.status
+        assert res.body == false_1.body
+        assert res.parents == return_values

tests/evaluators/test_PreventCircularReferenceEvaluator.py

@@ -0,0 +1,89 @@
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept, DEFINITION_TYPE_DEF
+from core.sheerka.services.sheerka_service import ChickenAndEggException
+from evaluators.PreventCircularReferenceEvaluator import PreventCircularReferenceEvaluator
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.evaluators.EvaluatorTestsUtils import cnode_ret_val, concept_ret_val, python_ret_val
+
+
+class TestPreventCircularReferenceEvaluator(TestUsingMemoryBasedSheerka):
+    def test_i_can_match(self):
+        sheerka, context, foo = self.init_concepts("foo")
+
+        level1 = context.push(BuiltinConcepts.TESTING, None)
+        level2 = level1.push(BuiltinConcepts.EVALUATING_CONCEPT, foo)
+        level3 = level2.push(BuiltinConcepts.TESTING, None)
+        level4 = level3.push(BuiltinConcepts.TESTING, None)
+
+        assert not PreventCircularReferenceEvaluator().matches(context, [])
+        assert not PreventCircularReferenceEvaluator().matches(level1, [])
+        assert not PreventCircularReferenceEvaluator().matches(level2, [])  # only detect sub context
+        assert PreventCircularReferenceEvaluator().matches(level3, [])
+        assert PreventCircularReferenceEvaluator().matches(level4, [])
+
+    def test_i_can_eval(self):
+        sheerka, context, foo, bar = self.init_concepts("foo", "bar")
+
+        level1 = context.push(BuiltinConcepts.EVALUATING_CONCEPT, foo)
+        level2 = level1.push(BuiltinConcepts.TESTING, None)
+
+        python = python_ret_val("1 + 1")
+        cnode_foo = cnode_ret_val(foo)
+        exact_foo = concept_ret_val(foo)
+        exact_bar = concept_ret_val(bar)
+
+        # no circular
+        return_values = [python, exact_bar]
+        evaluator = PreventCircularReferenceEvaluator()
+        assert evaluator.matches(level2, return_values)
+        assert evaluator.eval(level2, return_values) is None
+
+        # circular with exact concept parser
+        return_values = [python, exact_foo]
+        evaluator = PreventCircularReferenceEvaluator()
+        assert evaluator.matches(level2, return_values)
+        ret = evaluator.eval(level2, return_values)
+        assert not ret.status
+        assert sheerka.isinstance(ret.body, BuiltinConcepts.FILTERED)
+        assert ret.body.filtered == [exact_foo]
+        assert isinstance(ret.body.reason, ChickenAndEggException)
+        assert ret.parents == [exact_foo]
+
+        # circular with node concept (can by Sya or Sequence. It does not matter)
+        return_values = [python, cnode_foo]
+        evaluator = PreventCircularReferenceEvaluator()
+        assert evaluator.matches(level2, return_values)
+        ret = evaluator.eval(level2, return_values)
+        assert not ret.status
+        assert sheerka.isinstance(ret.body, BuiltinConcepts.FILTERED)
+        assert ret.body.filtered == [cnode_foo]
+        assert isinstance(ret.body.reason, ChickenAndEggException)
+        assert ret.parents == [cnode_foo]
+
+        # circular when multiple concepts
+        return_values = [python, exact_foo, cnode_foo]
+        evaluator = PreventCircularReferenceEvaluator()
+        assert evaluator.matches(level2, return_values)
+        ret = evaluator.eval(level2, return_values)
+        assert not ret.status
+        assert sheerka.isinstance(ret.body, BuiltinConcepts.FILTERED)
+        assert ret.body.filtered == [exact_foo, cnode_foo]
+        assert isinstance(ret.body.reason, ChickenAndEggException)
+        assert ret.parents == [exact_foo, cnode_foo]
+
+    def test_i_cannot_eval_when_the_name_of_the_concept_is_also_a_parameter_of_the_concept(self):
+        sheerka, context, q = self.init_concepts(
+            Concept("q", definition="q ?", definition_type=DEFINITION_TYPE_DEF).def_var("q")
+        )
+
+        level1 = context.push(BuiltinConcepts.EVALUATING_CONCEPT, q)
+        level2 = level1.push(BuiltinConcepts.TESTING, None)
+
+        python = python_ret_val("1 + 1")
+        cnode_q = cnode_ret_val(q)
+        exact_q = concept_ret_val(q)
+
+        return_values = [python, cnode_q, exact_q]
+        evaluator = PreventCircularReferenceEvaluator()
+        assert evaluator.matches(level2, return_values)
+        assert evaluator.eval(level2, return_values) is None

tests/non_reg/test_sheerka_non_reg.py

@@ -8,7 +8,7 @@ from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
 from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
 from evaluators.PythonEvaluator import PythonEvalError
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import CMV, CC, compare_with_test_object, CB
+from tests.parsers.parsers_utils import CB, CC, CMV, compare_with_test_object
 
 
 class TestSheerkaNonRegMemory(TestUsingMemoryBasedSheerka):
@@ -684,10 +684,12 @@ as:
         assert res[0].body == 22
 
         res = sheerka.evaluate_user_input("twenty three")
-        assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED)
+        assert len(res) == 1
+        assert not res[0].status
 
         res = sheerka.evaluate_user_input("eval twenty three")
-        assert sheerka.has_error(context, res, __type=BuiltinConcepts.CONDITION_FAILED)
+        assert len(res) == 1
+        assert not res[0].status
 
     def test_i_can_manage_some_type_of_infinite_recursion(self):
         sheerka = self.get_sheerka()
@@ -1422,4 +1424,3 @@ as:
 
         assert len(res) == 1
         assert res[0].status
-

tests/non_reg/test_sheerka_non_reg2.py

@@ -1,3 +1,5 @@
+from core.builtin_concepts import ConceptEvalError
+from core.builtin_helpers import only_successful
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -77,3 +79,66 @@ class TestSheerkaNonRegMemory2(TestUsingMemoryBasedSheerka):
         rex = sheerka.new("rex")
         dog = sheerka.new("dog")
         assert sheerka.isa(rex, dog)
+
+    def test_fixing_maximum_recursion_depth_exceeded(self):
+        init = [
+            "def concept x and y pre is_question() as x and y",
+            "def concept x or y pre is_question() as x or y",
+            "def concept or from a or b as a or b",
+            "def concept and",
+        ]
+        sheerka = self.init_scenario(init)
+        context = self.get_context(sheerka)
+
+        res = sheerka.evaluate_user_input("or or and")
+        res = only_successful(context, res)
+        assert len(res.body.body) == 3
+        # note that there will be only one result when the sya node parser will fix its duplicate results issue
+
+    def test_121_plural_are_not_updated_when_new_elements_are_added(self):
+        init = [
+            "def concept animal",
+            "def concept dog",
+            "def concept a x is an y as set_isa(x, y)",
+            "eval animals",
+            "global_truth(a dog is an animal)",
+        ]
+        sheerka = self.init_scenario(init)
+
+        res = sheerka.evaluate_user_input("eval animals")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == [sheerka.new("dog")]
+
+    def test_105_TOO_MANY_ERROR_is_not_the_relevant_error_when_results_are_filtered(self):
+        init = [
+            "def concept foo",
+            "def concept bar",
+            "def concept x is a y pre is_question() def_var x def_var y",
+            "def concept x is a y as raise NotImplementedError() def_var x def_var y",
+        ]
+        sheerka = self.init_scenario(init)
+        res = sheerka.evaluate_user_input("eval foo is a bar")
+
+        assert len(res) == 1
+        assert not res[0].status
+        assert isinstance(res[0].body, ConceptEvalError)
+
+    def test_74_keyword_parameters_are_no_longer_recognized_when_a_concept_that_redefines_equality_is_created(self):
+        init = [
+            "def concept a=b as a=b",
+        ]
+        sheerka = self.init_scenario(init)
+        bag = {}
+
+        def test_function(**kwargs):
+            nonlocal bag
+            bag.update(kwargs)
+
+        sheerka.add_to_short_term_memory(None, "test_function", test_function)
+        res = sheerka.evaluate_user_input("test_function(id=11)")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert bag["id"] == 11

tests/parsers/parsers_utils.py

@@ -61,7 +61,7 @@ class ExprTestObj:
         return list(Tokenizer(source, yield_eof=False)), to_skip
 
     def get_expr_node(self, full_text_as_tokens=None):
-        raise NotImplementedError
+        raise NotImplementedError()
 
     @staticmethod
     def safe_get_expr_node(obj, full_text_as_tokens):

tests/parsers/test_SequenceNodeParser.py

@@ -482,4 +482,4 @@ class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
         assert len(lexer_nodes) == 1
         concept_found = lexer_nodes[0].concept
 
-        assert concept_found.get_metadata().body == "get_set_elements(c:|1003:)"
+        assert concept_found.get_metadata().body == "get_plural_concept_value(self)"