Fixed #125: SheerkaErrorManager

Fixed #135: Change services service priorities
Fixed #136: ErrorManager: Implement recognize_error
Fixed #137: BNFNodeParser : Error when parsing regex with sub parsers
Fixed #138: get_last_errors(): real errors sources are lost
Fixed #139: OneError return value removes the origin of the error
Fixed #140: Concept variables are not correctly handled when parsing sub expression
Fixed #143: Implement has_unknown_concepts()

sheerka_backup/admin.sb

@@ -2,7 +2,11 @@
 push_ontology("admin")
 def concept explain as get_results(id=0, depth=2) auto_eval True
 def concept explain last as get_last_results(id=0, depth=2) auto_eval True
+def concept explain last results as get_last_results(id=0, depth=2) auto_eval True
 def concept explain x as get_results(id=x, depth=3) auto_eval True
+def concept explain last errors as get_last_errors(level=0) auto_eval True
+def concept explain errors as get_errors(level=0) auto_eval True
+def concept explain error x as get_errors(id=x, depth=99) where isinstance(x, int) auto_eval True
 
 def concept precedence a > precedence b as set_is_greater_than(__PRECEDENCE, a, b, 'Sya') auto_eval True
 
@@ -16,13 +20,15 @@ def concept debug off as set_debug(False) auto_eval True
 def concept activate debug on x as set_debug_var(x) auto_eval True
 def concept activate debug on x id=y as set_debug_var(x, y) auto_eval True
 def concept debug x as set_debug_var(x) auto_eval True
+def concept activate concept debug on x as set_debug_concept(x) auto_eval True
 
 def concept debug var x as set_debug_var(x) auto_eval True
 def concept debug variable x as set_debug_var(variable=x) auto_eval True
 def concept debug method x as set_debug_var(method=x) auto_eval True
 def concept debug service x as set_debug_var(service=x) auto_eval True
 
-def concept deactivate debug on x as debug_var(x, enabled=False) where x auto_eval True
+def concept deactivate debug on x as set_debug_var(x, enabled=False) where x auto_eval True
+def concept deactivate concept debug on x as set_debug_concept(x, enabled=False) auto_eval True
 
 def concept activate return values processing as set_var("sheerka.enable_process_return_values", True) auto_eval True
 def concept deactivate return values processing as set_var("sheerka.enable_process_return_values", False) auto_eval True

sheerka_backup/python.sb

@@ -1,6 +1,7 @@
 push_ontology("english")
 def concept x is a string pre is_question() as isinstance(x, str)
-def concept x is a integer pre is_question() as isinstance(x, int)
+def concept x is an integer pre is_question() as isinstance(x, int)
+def concept x is an int pre is_question() as isinstance(x, int)
 def concept x starts with y pre is_question() where x is a string as x.startswith(y)
 def concept sha256 from bnf r'[a-f0-9]{64}'
 def concept sha512 from bnf r'[a-f0-9]{128}'

src/cache/FastCache.py

@@ -16,6 +16,15 @@ class FastCache:
     def __contains__(self, item):
         return self.has(item)
 
+    def __iter__(self):
+        yield from self.cache
+
+    def __next__(self):
+        return next(iter(self.cache))
+
+    def __len__(self):
+        return len(self.cache)
+
     def put(self, key, value):
         if len(self.cache) == self.max_size:
             del self.cache[self.lru.pop(0)]

src/core/builtin_concepts.py

@@ -165,6 +165,9 @@ class ParserResultConcept(Concept):
         from parsers.BaseParser import BaseParser
         return parser.name if isinstance(parser, BaseParser) else str(parser)
 
+    def get_error(self):
+        return None
+
 
 class RuleEvaluationResultConcept(Concept):
     """
@@ -273,6 +276,9 @@ class FilteredConcept(Concept):
         self.set_value("reason", reason)
         self._hints.is_evaluated = True
 
+    def get_error(self):
+        return self.get_value('reason')
+
 
 class ConceptAlreadyInSet(Concept, ErrorObj):
     ALL_ATTRIBUTES = ["concept", "concept_set"]
@@ -312,9 +318,9 @@ class PropertyAlreadyDefined(Concept, ErrorObj):
 
 
 class ConditionFailed(Concept, ErrorObj):
-    ALL_ATTRIBUTES = ["condition", "concept", "prop", "reason"]
+    ALL_ATTRIBUTES = ["condition", "concept", "prop", "reason", "args"]
 
-    def __init__(self, condition=None, concept=None, prop=None, reason=None):
+    def __init__(self, condition=None, concept=None, prop=None, reason=None, args=None):
         Concept.__init__(self,
                          BuiltinConcepts.CONDITION_FAILED,
                          True,
@@ -325,10 +331,15 @@ class ConditionFailed(Concept, ErrorObj):
         self.set_value("concept", concept)
         self.set_value("prop", prop)
         self.set_value("reason", reason)
+        self.set_value("args", args)
         self._hints.is_evaluated = True
 
     def __repr__(self):
-        return f"ConditionFailed(condition='{self.body}', concept='{self.concept}', prop='{self.prop}')"
+        text = f"ConditionFailed(condition='{self.body}', concept='{self.concept}', prop='{self.prop}'"
+        if self.args:
+            for k, v in self.args.items():
+                text += f", {k}={v}"
+        return text + ")"
 
 
 class NotForMeConcept(Concept):  # Not considered as an error ?
@@ -348,6 +359,9 @@ class NotForMeConcept(Concept):  # Not considered as an error ?
     def __repr__(self):
         return f"NotForMeConcept(source={self.body}, reason={self.get_value('reason')})"
 
+    def get_error(self):
+        return self.get_value('reason')
+
 
 class ExplanationConcept(Concept):
     ALL_ATTRIBUTES = ["digest", "command", "title", "instructions", "execution_result"]

src/core/builtin_helpers.py

@@ -930,11 +930,11 @@ def get_inner_body(context, concept):
         return concept.body
 
 
-def get_possible_variables_from_concept(context, concept):
+def get_possible_parameters_from_concept(context, concept):
     """
     Given concept definition,
     gives the variables of the concept that can be considered as a parameter in another function
-    >>> gpvfc = get_possible_variables_from_concept
+    >>> gpvfc = get_possible_parameters_from_concept
     >>> assert gpvfc(Concept("a plus b").def_var("a").def_var("b")) == {"a", "b"}
     >>> assert gpvfc(Concept("twenties", definition="twenty (one|two)=n").def_var("n")) == set()
     :param context:

src/core/global_symbols.py

@@ -70,6 +70,12 @@ class ErrorObj:
     pass
 
 
+class SheerkaException(Exception, ErrorObj):
+    def __init__(self, value):
+        super().__init__(value)
+        self.value = value
+
+
 CURRENT_OBJ = "__obj"
 
 
@@ -100,3 +106,23 @@ INIT_AST_QUESTION_PARSERS = ["Expression"]
 DEFAULT_EVALUATORS = ["Python", "Concept", "LexerNode", "Expression", "ValidateConcept", "ResolveAmbiguity"]
 
 OBJECTS_COUNTER_KEY = "__#OBJECTS_COUNTER_KEY#__"
+
+
+class ErrorItem:
+    """"
+    Description of an error
+    """
+
+    def __init__(self, identifier: int, level: int, root, error: object, children=None):
+        self.id = identifier
+        self.level = level
+        self.root = root
+        self.error = error
+        self.children = children
+
+    def __repr__(self):
+        return f"ErrorItem(id={self.id}, level={self.level}, error={self.error}, source={self.source})"
+
+    @property
+    def source(self):
+        return self.root.who if hasattr(self.root, "who") else str(self.root)

src/core/sheerka/Sheerka.py

@@ -102,7 +102,8 @@ class Sheerka(Concept):
 
         self.services = {}  # sheerka plugins
 
-        self.builtin_cache = {}  # cache for builtin concepts
+        self.builtin_cache = {}  # cache for builtin concepts <concept name, Python class>
+        self.builtin_cache_by_class_name = {}  # <Python Class's name, concept name>
         self.parsers = {}  # cache for builtin parsers
         self.evaluators = []  # cache for builtin evaluators
 
@@ -203,7 +204,7 @@ class Sheerka(Concept):
             self.bind_service_method(self.name, self.objvalue, False)
 
             self.om = SheerkaOntologyManager(self, root_folder, self.cache_only)
-            self.builtin_cache = self.get_builtins_classes_as_dict()
+            self.builtin_cache, self.builtin_cache_by_class_name = self.get_builtins_classes_as_dict()
 
             self.initialize_caching()
             self.get_builtin_parsers()
@@ -849,7 +850,7 @@ class Sheerka(Concept):
         if isinstance(b, tuple):
             for item in b:
                 item_key = item.key if isinstance(item, Concept) else str(item)
-                if a.key == item_key.key:
+                if a.key == item_key:
                     return True
             return False
 
@@ -903,12 +904,14 @@ class Sheerka(Concept):
 
     @staticmethod
     def get_builtins_classes_as_dict():
-        res = {}
+        by_concept_name = {}
+        by_class_name = {}
         for c in core.utils.get_classes("core.builtin_concepts"):
             if issubclass(c, Concept) and c != Concept:
-                res[c()._metadata.key] = c
-
-        return res
+                concept = c()
+                by_concept_name[concept._metadata.key] = c
+                by_class_name[c.__name__] = concept._metadata.name
+        return by_concept_name, by_class_name
 
     @staticmethod
     def init_logging(debug, loggers):
@@ -993,13 +996,6 @@ class Sheerka(Concept):
 
         return concept
 
-    @staticmethod
-    def deepdiff(a, b):
-        from deepdiff import DeepDiff
-        ddiff = DeepDiff(a, b, ignore_order=True)
-        print(ddiff)
-        return ddiff
-
 
 def to_profile():
     sheerka = Sheerka()

src/core/sheerka/services/SheerkaConceptManager.py

@@ -1184,14 +1184,13 @@ class SheerkaConceptManager(BaseService):
         """
         Go thru all the declared regular expressions and try to see if there is a match
         :param expr:
-        :param pos:
+        :param pos: starting pos (try to match at this starting position)
         :return:
         """
         result = []
         for compiled_regex, concept_ids in self.compiled_concepts_by_regex:
             if compiled_regex.match(expr, pos):
                 result.extend([self.sheerka.get_by_id(concept_id) for concept_id in concept_ids])
-
         return result
 
     def get_concepts_bnf_definitions(self):

src/core/sheerka/services/SheerkaDebugManager.py

@@ -417,8 +417,16 @@ class ListDebugLogger(BaseDebugLogger):
         if not self.should_i_log_concept(concept.id):
             return
 
-        concept_id = f"{concept.id}{text}" if text else f"{concept.id}"
-        self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_CONCEPT, concept_id, False, kwargs))
+        raw = kwargs.pop('raw', None)
+        str_vars = raw if raw else pp.pformat(kwargs) if kwargs else ""
+        text = " - " + text if text is not None else ""
+        colon = ": " if str_vars else ""
+        str_text = f"..concept#{concept.id}{text}{colon}"
+
+        self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_CONCEPT,
+                                                    str_text + str_vars,
+                                                    False,
+                                                    kwargs))
 
 
 class TeeDebugLogger(BaseDebugLogger):

src/core/sheerka/services/SheerkaErrorManager.py

@@ -1,78 +1,121 @@
 import core.builtin_helpers
 import core.utils
+from cache.FastCache import FastCache
 from core.builtin_concepts_ids import BuiltinConcepts, BuiltinErrors
 from core.concept import Concept
-from core.global_symbols import ErrorObj, NotInit
+from core.global_symbols import EVENT_USER_INPUT_EVALUATED, ErrorItem, ErrorObj, NotInit, SheerkaException
 from core.sheerka.services.sheerka_service import BaseService
+from core.tokenizer import TokenKind, Tokenizer
 
+MAX_EXECUTION_HISTORY = 100
+CURRENT_ERRORS_EVENT_ID = "current_errors_event_id"
+CURRENT_ERRORS_EVENT_MESSAGE = "current_errors_event_message"
+LAST_DETECTED_ERRORS_EVENT_ID = "last_detected_errors_event_id"
+LAST_DETECTED_ERRORS_EVENT_MESSAGE = "last_detected_errors_event_message"
 
-class ErrorItem:
-    def __init__(self, level: int, parent, error: object):
-        self.level = level
-        self.parent = parent
-        self.error = error
+LAST_UNKNOWN_CONCEPTS = "__last_unknown_concepts"
 
 
 class SheerkaErrorManager(BaseService):
     NAME = "Error"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka, order=1)
+        super().__init__(sheerka, order=6)
+        self.last_detected_errors_event_id = None  # updated every time an user input fails
+        self.last_detected_errors_event_message = None  # updated every time an user input fails
+
+        self.current_errors_event_id = None  # updated only when get_last_errors() is called
+        self.current_errors_event_message = None  # updated only when get_last_errors() is called
+
+        self.last_return_values = None
+
+        self.state_vars = [CURRENT_ERRORS_EVENT_ID,
+                           CURRENT_ERRORS_EVENT_MESSAGE,
+                           LAST_DETECTED_ERRORS_EVENT_ID,
+                           LAST_DETECTED_ERRORS_EVENT_MESSAGE]
+
+        self.executions_contexts_cache = FastCache(MAX_EXECUTION_HISTORY)
+        self.get_unknown_names_chain = [self._get_unknown_from_unknown_concept,
+                                        self._get_unknown_from_name_error,
+                                        self._get_unknown_from_undefined_symbol]
 
     def initialize(self):
-        self.sheerka.bind_service_method(self.NAME, self.get_errors, False)
+        self.sheerka.bind_service_method(self.NAME, self.get_obj_errors, False, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.has_error, False)
         self.sheerka.bind_service_method(self.NAME, self.get_error_cause, False)
+        self.sheerka.bind_service_method(self.NAME, self.get_errors, False)
+        self.sheerka.bind_service_method(self.NAME, self.get_last_errors, False)
+        self.sheerka.bind_service_method(self.NAME, self.recognize_error, False)
+        self.sheerka.bind_service_method(self.NAME, self.has_unknown_concepts, False)
 
-    def get_errors(self, context, obj, **kwargs):
+        self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.on_user_input_evaluated)
+
+    def initialize_deferred(self, context, is_first_time):
+        self.restore_values(*self.state_vars)
+
+    def reset_state(self):
+        self.last_detected_errors_event_id = None
+        self.last_detected_errors_event_message = None
+        self.current_errors_event_id = None
+        self.current_errors_event_message = None
+        self.last_return_values = None
+
+    def is_error(self, obj):
+        if isinstance(obj, (ErrorObj, Exception)):
+            return True
+
+        if isinstance(obj, Concept) and obj.get_metadata().is_builtin and obj.key in BuiltinErrors:
+            return True
+
+        if self.sheerka.isinstance(obj, BuiltinConcepts.NOT_FOR_ME):
+            return True  # In this situation, we want NOT_FOR_ME to appear in the list of errors
+
+        return False
+
+    def get_obj_errors(self, context, obj, predicate=None, **kwargs):
         """
-        Browse obj, looking for error
+        Quickly retrieve all errors from obj (which can be a list)
+        All errors are flatten. No hierarchy
         :param context:
         :param obj:
+        :param predicate: if defined, filtering function
         :param kwargs: if defined, specialize the error (example __type="PythonEvalError")
         :return:
         """
 
-        def is_error(_obj):
-            if isinstance(_obj, (ErrorObj, Exception)):
-                return True
-
-            if isinstance(_obj, Concept) and _obj.get_metadata().is_builtin and _obj.key in BuiltinErrors:
-                return True
-
-            return False
-
-        def inner_get_errors(_obj, level, parent):
+        def inner_get_errors(_obj):
             if self.sheerka.isinstance(_obj, BuiltinConcepts.RETURN_VALUE) and _obj.status:
                 return []
 
             if isinstance(_obj, (list, set, tuple)):
-                return core.utils.flatten([inner_get_errors(o, level, parent) for o in _obj])
+                return core.utils.flatten([inner_get_errors(o) for o in _obj])
+
+            if self.is_error(_obj):
+                if hasattr(_obj, "get_error"):
+                    return [_obj] + inner_get_errors(_obj.get_error())
+
+                elif isinstance(_obj, Concept) and _obj.body not in (NotInit, None):
+                    return [_obj] + inner_get_errors(_obj.body)
 
-            if is_error(_obj):
-                #error_item = ErrorItem(level, parent, _obj)
-                error_item = _obj
-                if isinstance(_obj, Concept) and _obj.body not in (NotInit, None):
-                    return [error_item] + inner_get_errors(_obj.body, level + 1, error_item)
-                if isinstance(_obj, ErrorObj) and hasattr(_obj, "get_error"):
-                    return [error_item] + inner_get_errors(_obj.get_error(), level + 1, error_item)
                 else:
-                    return [error_item]
+                    return [_obj]
 
             if self.sheerka.isinstance(_obj, BuiltinConcepts.FILTERED):
-                return inner_get_errors(_obj.reason, level, parent)
+                return inner_get_errors(_obj.reason)
 
             if isinstance(_obj, Concept) and _obj.body != NotInit:
-                return inner_get_errors(_obj.body, level, parent)
+                return inner_get_errors(_obj.body)
 
             return []
 
-        errors = inner_get_errors(obj, 0, None)
-        #return self.sheerka.filter_objects(context, errors, mapping=lambda o: o.error, **kwargs)
-        return self.sheerka.filter_objects(context, errors, **kwargs)
+        errors = inner_get_errors(obj)
 
-    def has_error(self, context, obj, **kwargs):
-        errors = self.get_errors(context, obj, **kwargs)
+        predicate = self.enhance_error_predicate(predicate)
+
+        return self.sheerka.filter_objects(context, errors, predicate=predicate, **kwargs)
+
+    def has_error(self, context, obj, predicate=None, **kwargs):
+        errors = self.get_obj_errors(context, obj, predicate, **kwargs)
         return len(errors) > 0
 
     def get_error_cause(self, obj):
@@ -87,3 +130,253 @@ class SheerkaErrorManager(BaseService):
             return res[0]
         else:
             return res
+
+    def get_errors_items(self, returns_values):
+        """
+        Returns a list or ErrorItems
+        They are numbered and have hierarchy
+        :param returns_values:
+        :return:
+        """
+        identifier = 0
+
+        def inner_get_errors_items(obj, level, root):
+            nonlocal identifier
+
+            if self.is_error(obj) or hasattr(obj, "get_error"):
+
+                error_item = ErrorItem(identifier, level, root, obj)
+                identifier += 1
+
+                if hasattr(obj, "get_error"):
+                    error_item.children = list(inner_get_errors_items(obj.get_error(), level + 1, root))
+
+                elif isinstance(obj, Concept) and obj.body not in (NotInit, None):
+                    error_item.children = list(inner_get_errors_items(obj.body, level + 1, root))
+
+                else:
+                    error_item.children = []
+
+                yield error_item
+
+            elif isinstance(obj, Concept) and obj.body != NotInit:
+                if self.sheerka.isinstance(obj, BuiltinConcepts.RETURN_VALUE) and not obj.status:
+                    root_to_use = obj
+                else:
+                    root_to_use = root
+                yield from inner_get_errors_items(obj.body, level, root_to_use)
+
+            elif hasattr(obj, "__iter__") and not isinstance(obj, str):
+                for item in obj:
+                    yield from inner_get_errors_items(item, level, root)
+
+        for ret_val in [r for r in returns_values if not r.status]:
+            yield from inner_get_errors_items(ret_val.body, 0, ret_val)
+
+    def get_all_error_items(self, return_values):
+        def _as_list(error_items):
+            for error_item in error_items:
+                yield error_item
+                yield from _as_list(error_item.children)
+
+        return _as_list(self.get_errors_items(return_values))
+
+    def get_return_values_in_error(self):
+        if self.current_errors_event_id is None:
+            error = self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"})
+            raise SheerkaException(error)
+
+        # search in history
+        if self.current_errors_event_id in self.executions_contexts_cache:
+            execution_result = self.executions_contexts_cache.get(self.current_errors_event_id)
+        else:
+            try:
+                execution_result = self.sheerka.om.current_sdp().load_result(self.current_errors_event_id)
+                self.executions_contexts_cache.put(self.current_errors_event_id, execution_result)
+            except FileNotFoundError as ex:
+                return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"digest": self.current_errors_event_id})
+
+        return [ret for ret in execution_result.values["return_values"] if not ret.status]
+
+    def get_errors(self, context, predicate=None, **kwargs):
+        """
+        Returns
+        :param context:
+        :param predicate:
+        :param kwargs:
+        :return:
+        """
+        try:
+            in_error = self.get_return_values_in_error()
+            error_items = self.get_all_error_items(in_error)
+            depth = kwargs.pop("depth", None)
+            depth = kwargs.pop("recursion_depth", depth)
+
+            filtered = self.filter_error_items(context, list(error_items), predicate, **kwargs)
+
+            explanation = self.sheerka.new(BuiltinConcepts.EXPLANATION,
+                                           digest=self.current_errors_event_id,
+                                           command=self.current_errors_event_message,
+                                           body=filtered)
+
+            # add format instructions if applicable
+            if depth is not None:
+                explanation.set_format_instr(recursion_depth=depth, recurse_on="children")
+
+            return explanation
+        except SheerkaException as ex:
+            return ex.value
+
+    def get_last_errors(self, context, predicate=None, **kwargs):
+        """
+
+        :param context:
+        :param predicate:
+        :param kwargs:
+        :return:
+        """
+        if self.last_detected_errors_event_id is None:
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"})
+
+        self.current_errors_event_id = self.last_detected_errors_event_id
+        self.current_errors_event_message = self.last_detected_errors_event_message
+        self.store_values(context, CURRENT_ERRORS_EVENT_ID, CURRENT_ERRORS_EVENT_MESSAGE)
+        return self.get_errors(context, predicate, **kwargs)
+
+    def recognize_error(self, context, predicate=None, **kwargs):
+        """
+        Browse the last errors to see if the given error can be recognized
+        :param context:
+        :param predicate:
+        :param kwargs:
+        :return:
+        """
+        if self.last_return_values is None:
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"})
+
+        error_items = self.get_all_error_items(self.last_return_values)
+        predicate = self.enhance_error_predicate(predicate)
+        filtered = self.filter_error_items(context, list(error_items), predicate, **kwargs)
+
+        if len(filtered) == 0:
+            return False
+
+        return True
+
+    def has_unknown_concepts(self, context):
+        """
+        Browse the last errors looking for unknown concepts
+        :param context:
+        :return:
+        """
+        if self.last_return_values is None:
+            return self.sheerka.new(BuiltinConcepts.NOT_FOUND, body={"query": "get_last_error"})
+
+        errors = self.get_obj_errors(context, self.last_return_values)
+
+        # use all the known ways to detect unrecognized concepts
+        unknown_concepts = set()
+        for get_unknown_names in self.get_unknown_names_chain:
+            items = get_unknown_names(context, errors)
+            if items:
+                unknown_concepts.update(items)
+
+        if len(unknown_concepts) > 0:
+            # use all the ways to reduce the list ?
+            errors_names = self._remove_recognized(unknown_concepts)
+
+            if len(errors_names) == 1:
+                self.sheerka.set_in_memory(context, LAST_UNKNOWN_CONCEPTS, next(iter(errors_names)))
+            else:
+                self.sheerka.set_in_memory(context, LAST_UNKNOWN_CONCEPTS, errors_names)
+            return True
+
+    def _get_unknown_from_unknown_concept(self, context, errors):
+        from_unknown = self.sheerka.filter_objects(context, errors, __type=BuiltinConcepts.UNKNOWN_CONCEPT)
+        return [e.body for e in from_unknown] if from_unknown else None
+
+    def _get_unknown_from_name_error(self, context, errors):
+        from_name_error = self.sheerka.filter_objects(context, errors, __type="NameError")
+        if not from_name_error:
+            return None
+
+        return [list(Tokenizer(e.args[0], yield_eof=False))[2].strip_quote for e in from_name_error]
+
+    def _get_unknown_from_undefined_symbol(self, context, errors):
+        from_undefined_error = self.sheerka.filter_objects(context, errors, __type="UndefinedSymbolError")
+        return [e.symbol for e in from_undefined_error] if from_undefined_error else None
+
+    def _remove_recognized(self, errors_names):
+        """
+        parse error names a split it if some parts are known concept
+        ex 'dress' is a known concept
+        'red dress' will not be recognized, but only 'red' should be in error
+        :param errors_names:
+        :return:
+        """
+        # remove parts that are known
+        res = set()
+        unrecognized = []
+        for error_name in [e.strip() for e in errors_names]:
+            unrecognized.clear()
+
+            for token in Tokenizer(error_name, yield_eof=False):
+                if (token.type == TokenKind.WHITESPACE and
+                        len(unrecognized) > 0 and
+                        unrecognized[-1].type != TokenKind.WHITESPACE):
+                    unrecognized.append(token)
+                    continue
+
+                elif self.sheerka.fast_resolve(token.value, return_new=False):
+                    unrecognized_name = core.utils.get_text_from_tokens(unrecognized).strip()
+                    if unrecognized_name:
+                        res.add(unrecognized_name)
+                    unrecognized.clear()
+
+                else:
+                    unrecognized.append(token)
+
+            if unrecognized:
+                unrecognized_name = core.utils.get_text_from_tokens(unrecognized).strip()
+                if unrecognized_name:
+                    res.add(unrecognized_name)
+
+        return res
+
+    def on_user_input_evaluated(self, execution_context):
+        """
+        Track user input result in error
+        :param execution_context:
+        :return:
+        """
+        self.last_return_values = [ret for ret in execution_context.values["return_values"] if not ret.status]
+        if self.last_return_values:
+            self.last_detected_errors_event_id = execution_context.event.get_digest()
+            self.last_detected_errors_event_message = execution_context.event.message
+            self.store_values(execution_context, LAST_DETECTED_ERRORS_EVENT_ID, LAST_DETECTED_ERRORS_EVENT_MESSAGE)
+            self.executions_contexts_cache.put(execution_context.event.get_digest(), execution_context)
+
+    def filter_error_items(self, context, error_items, predicate=None, **kwargs):
+        error_items_args = {}
+        for key in ["id", "level", "source"]:
+            if key in kwargs:
+                error_items_args[key] = kwargs.pop(key)
+
+        filtered = self.sheerka.filter_objects(context, list(error_items), **error_items_args)
+        return self.sheerka.filter_objects(context, list(filtered), lambda x: x.error, predicate, **kwargs)
+
+    def enhance_error_predicate(self, predicate):
+        if isinstance(predicate, type):
+            if issubclass(predicate, Concept):
+                c = predicate()
+                predicate = f"get_type(self) == {core.utils.get_safe_str_value(c.name)}"
+            else:
+                predicate = f"get_type(self) == {core.utils.get_safe_str_value(predicate.__name__)}"
+        elif isinstance(predicate, str) and " " not in predicate.strip():
+            if predicate in self.sheerka.builtin_cache_by_class_name:
+                concept_name = self.sheerka.builtin_cache_by_class_name[predicate]
+                predicate = f"get_type(self) == {core.utils.get_safe_str_value(concept_name)}"
+            else:
+                predicate = f"get_type(self) == {core.utils.get_safe_str_value(predicate)}"
+
+        return predicate

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import ensure_bnf, ensure_concept, expect_one, get_parsed_concept, is_only_successful, \
+from core.builtin_helpers import ensure_concept, expect_one, get_parsed_concept, is_only_successful, \
     only_successful, variables_in_context
 from core.concept import AllConceptParts, Concept, ConceptParts, DoNotResolve, InfiniteRecursionResolved, \
     concept_part_value
@@ -15,6 +15,7 @@ from core.utils import unstr_concept
 from parsers.BaseExpressionParser import TrueifyVisitor
 from parsers.BaseNodeParser import ConceptNode
 from sheerkapython.ExprToConditions import ExprToConditionsVisitor
+from sheerkapython.python_wrapper import get_possible_variables_from_concept
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,
@@ -174,7 +175,8 @@ class SheerkaEvaluateConcept(BaseService):
             parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(trueified_where)
             parsed = ExpressionParser(auto_compile=False).parse(context, parser_input)
 
-            expr_to_cond_python_visitor = ExprToConditionsVisitor(context)
+            possible_variables = get_possible_variables_from_concept(context, concept)
+            expr_to_cond_python_visitor = ExprToConditionsVisitor(context, known_variables=possible_variables)
             conditions = expr_to_cond_python_visitor.get_conditions(parsed.body.body)
             return WhereClauseDef(concept, concept.get_metadata().where, trueified_where, var_name, conditions)
         except FailedToCompileError:
@@ -392,7 +394,8 @@ class SheerkaEvaluateConcept(BaseService):
                                 body=where_clause_def.clause,
                                 concept=where_clause_def.concept,
                                 prop=where_clause_def.prop,
-                                reason=reason)
+                                reason=reason,
+                                args=where_clause_def.concept.variables())
 
     def manage_infinite_recursion(self, context):
         """
@@ -434,15 +437,7 @@ class SheerkaEvaluateConcept(BaseService):
         # def concept foo a as 'foo' a where a == 'baz'
         # In the second concept (foo) as is a still a concept, but also a variable in the where clause
 
-        ensure_bnf(context, concept)
-        if concept.get_bnf():
-            from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
-            visitor = BnfNodeConceptExpressionVisitor()
-            visitor.visit(concept.get_bnf())
-            possible_variables = [c.name if isinstance(c, Concept) else c for c in visitor.references]
-        else:
-            possible_variables = None
-
+        possible_variables = get_possible_variables_from_concept(context, concept)
         for part_key in AllConceptParts:
             if part_key in concept.get_compiled():
                 continue

src/core/sheerka/services/SheerkaExecute.py

@@ -44,11 +44,17 @@ class ParserInput:
             if len(tokens) == 0:
                 self.tokens = [Token(TokenKind.EOF, "", 0, 1, 1)]
             elif (last_token := tokens[-1]).type != TokenKind.EOF:
+                if isinstance(last_token.value, str):
+                    offset = len(last_token.value)
+                else:
+                    # KSI 2021-10-23
+                    # Not really true. So it will be to really handled later
+                    offset = 1
                 self.tokens = tokens + [Token(TokenKind.EOF,
                                               "",
-                                              last_token.index + 1,
+                                              last_token.index + offset,
                                               last_token.line,
-                                              last_token.column + 1)]
+                                              last_token.column + offset)]
             else:
                 self.tokens = tokens
 
@@ -259,7 +265,6 @@ class SheerkaExecute(BaseService):
         self.sheerka.bind_service_method(self.NAME, self.execute, True, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.execute_rules, True, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.parse_unrecognized, False, visible=False)
-        self.sheerka.bind_service_method(self.NAME, self.parse_function, False, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.parse_python, False, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.parse_expression, False, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.clear_parser_cache, True)
@@ -893,39 +898,6 @@ class SheerkaExecute(BaseService):
             sub_context.add_values(return_values=res)
             return res
 
-    def parse_function(self, context, source, tokens=None, start=0):
-        """
-        Helper function that parses what is supposed to be a function
-        :param context:
-        :param source:
-        :param tokens:
-        :param start: start index for the source code node
-        :return:
-        """
-        from parsers.BaseNodeParser import SourceCodeWithConceptNode
-
-        sheerka = context.sheerka
-        from parsers.FunctionParserOld import FunctionParserOld
-        parser = FunctionParserOld()
-        desc = f"Parsing function '{source}'"
-        with context.push(BuiltinConcepts.PARSE_CODE, source, desc=desc) as sub_context:
-            sheerka_execution = sheerka.services[SheerkaExecute.NAME]
-            res = parser.parse(sub_context, sheerka_execution.get_parser_input(source, tokens))
-
-            if not isinstance(res, list):
-                res = [res]
-
-            for r in [r for r in res if sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT)]:
-                r.body.body.start += start
-                r.body.body.end += start
-
-                if isinstance(r.body.body, SourceCodeWithConceptNode):
-                    for n in [r.body.body.first, r.body.body.last] + r.body.body.nodes:
-                        n.start += start
-                        n.end += start
-
-        return res
-
     def parse_python(self, context, source, desc=None):
         """
         Helper function that parses what is known to be Python source code

src/core/sheerka/services/SheerkaHistoryManager.py

@@ -4,7 +4,7 @@ from core.global_symbols import NotInit
 from core.sheerka.services.sheerka_service import BaseService
 from sdp.sheerkaDataProvider import Event
 
-hist = namedtuple("HistoryTest", "text status")  # tests purposes only
+hist = namedtuple("HistoryTest", "message status")  # tests purposes only
 
 
 class History:
@@ -37,7 +37,7 @@ class History:
             return True
 
         if isinstance(other, hist):
-            return self.event.message == other.text and self.status == other.status
+            return self.event.message == other.message and self.status == other.status
 
         if not isinstance(other, History):
             return False

src/core/sheerka/services/SheerkaMemory.py

@@ -44,13 +44,14 @@ class SheerkaMemory(BaseService):
         self.sheerka.bind_service_method(self.NAME, self.add_to_short_term_memory, True, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.remove_context, True, as_name="clear_short_term_memory",
                                          visible=False)
-        self.sheerka.bind_service_method(self.NAME, self.add_to_memory, True)
         self.sheerka.bind_service_method(self.NAME, self.add_many_to_short_term_memory, True, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.get_from_memory, False)
         self.sheerka.bind_service_method(self.NAME, self.get_last_from_memory, False)
         self.sheerka.bind_service_method(self.NAME, self.register_object, True, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.unregister_object, True, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.commit_registered_objects, True, visible=False)
+        self.sheerka.bind_service_method(self.NAME, self.add_to_memory, True)
+        self.sheerka.bind_service_method(self.NAME, self.set_in_memory, True)
         self.sheerka.bind_service_method(self.NAME, self.memory, False)
         self.sheerka.bind_service_method(self.NAME, self.mem, False)
 
@@ -123,7 +124,8 @@ class SheerkaMemory(BaseService):
 
     def add_to_memory(self, context, key, concept):
         """
-        Adds an object to memory
+        Adds an object to memory.
+        Create a list if the entry already exists
         :param context:
         :param key:
         :param concept:
@@ -157,6 +159,28 @@ class SheerkaMemory(BaseService):
                                                                            context.event.date.timestamp(),
                                                                            concept))
 
+    def set_in_memory(self, context, key, value):
+        """
+        Add an element in memory
+        Replace the old one if the entry already exists
+        :param context:
+        :param key:
+        :param value:
+        :return:
+        """
+        last = self.sheerka.om.get(SheerkaMemory.OBJECTS_ENTRY, key)
+        if last is NotFound:
+            self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(),
+                                                                               context.event.date.timestamp(),
+                                                                               value))
+            return
+
+        # replace with the new one
+        self.sheerka.om.delete(SheerkaMemory.OBJECTS_ENTRY, key, last)
+        self.sheerka.om.put(SheerkaMemory.OBJECTS_ENTRY, key, MemoryObject(context.event.get_digest(),
+                                                                           context.event.date.timestamp(),
+                                                                           value))
+
     def get_from_memory(self, context, key):
         """"
         """

src/core/sheerka/services/SheerkaOut.py

@@ -33,7 +33,7 @@ class SheerkaOut(BaseService):
         if valid_rules:
             if len(valid_rules) > 1:
                 # TODO manage when too many rules
-                print("TODO: TOO MANY RULES !!!!!")
+                print("TODO: TOO MANY RULES !!!!!", [r.id for r in valid_rules])
                 pass
 
             rule = valid_rules[0]

src/core/sheerka/services/SheerkaQueryManager.py

@@ -20,7 +20,7 @@ class SheerkaQueryManager(BaseService):
     MAPPING_PREFIX = "__xxx__map__xx__"
 
     def __init__(self, sheerka):
-        super().__init__(sheerka, order=16)
+        super().__init__(sheerka, order=5)
         self.queries = FastCache()
         self.conditions = FastCache()
         self.lexer = Lexer()
@@ -31,10 +31,9 @@ class SheerkaQueryManager(BaseService):
         self.sheerka.bind_service_method(self.NAME, self.select_objects, False)
         self.sheerka.bind_service_method(self.NAME, self.collect_attributes, False)
 
-        self.sheerka.bind_service_method(self.NAME, self.filter_objects, False)
+        self.sheerka.bind_service_method(self.NAME, self.where_on_objects, False, as_name="pipe_where")
         self.sheerka.bind_service_method(self.NAME, self.select_objects, False, as_name="pipe_select")
         self.sheerka.bind_service_method(self.NAME, self.collect_attributes, False, as_name="pipe_props")
-        self.sheerka.bind_service_method(self.NAME, self.where_on_objects, False, as_name="pipe_where")
 
         self.sheerka.register_debug_vars(SheerkaQueryManager.NAME, "filter_objects", "query")
 
@@ -62,6 +61,15 @@ class SheerkaQueryManager(BaseService):
             local_namespace[current_variable_name] = v
 
             if k == "__type":
+                if isinstance(v, type):
+                    if issubclass(v, Concept):
+                        c = v()
+                        local_namespace[current_variable_name] = c.name
+                    else:
+                        local_namespace[current_variable_name] = v.__name__
+                elif v in self.sheerka.builtin_cache_by_class_name:
+                    local_namespace[current_variable_name] = self.sheerka.builtin_cache_by_class_name[v]
+
                 conditions.append(f"get_type({self_ident}) == {current_variable_name}")
 
             elif k in ("__self", "_"):
@@ -97,7 +105,7 @@ class SheerkaQueryManager(BaseService):
         :param mapping: mapping to execute on each object before applying the predicate (lambda obj:obj)
         :param predicate:
         :param kwargs:
-        :return:
+        :return: empty list when nothing is found
         """
 
         debugger = context.get_debugger(SheerkaQueryManager.NAME, "filter_objects")

src/core/sheerka/services/SheerkaResultManager.py

@@ -1,11 +1,11 @@
 import ast
 import os
 
-from cache.Cache import Cache
+from cache.FastCache import FastCache
 from core.builtin_concepts import BuiltinConcepts
-from core.global_symbols import EVENT_USER_INPUT_EVALUATED, EVENT_CONCEPT_CREATED, NotFound, EVENT_CONCEPT_MODIFIED, \
-    EVENT_CONCEPT_DELETED, EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_CREATED, EVENT_RULE_DELETED, \
-    EVENT_RULE_PRECEDENCE_MODIFIED, SHEERKA_BACKUP_FOLDER, SHEERKA_BACKUP_FILE
+from core.global_symbols import EVENT_CONCEPT_CREATED, EVENT_CONCEPT_DELETED, EVENT_CONCEPT_MODIFIED, \
+    EVENT_CONCEPT_PRECEDENCE_MODIFIED, EVENT_RULE_CREATED, EVENT_RULE_DELETED, EVENT_RULE_PRECEDENCE_MODIFIED, \
+    EVENT_USER_INPUT_EVALUATED, NotFound, SHEERKA_BACKUP_FILE, SHEERKA_BACKUP_FOLDER
 from core.sheerka.services.sheerka_service import BaseService
 from core.utils import CONSOLE_COLORS_MAP as CCM
 from core.utils import as_bag
@@ -22,7 +22,7 @@ class SheerkaResultManager(BaseService):
     def __init__(self, sheerka, page_size=30):
         super().__init__(sheerka)
         self.page_size = page_size
-        self.executions_contexts_cache = Cache(MAX_EXECUTION_HISTORY)
+        self.executions_contexts_cache = FastCache(MAX_EXECUTION_HISTORY)
         self.last_execution = None
         self.last_created_concept = None
         self.last_created_concept_id = None
@@ -42,7 +42,7 @@ class SheerkaResultManager(BaseService):
         self.sheerka.bind_service_method(self.NAME, self.get_last_error, False, as_name="last_err")
 
         self.sheerka.subscribe(EVENT_USER_INPUT_EVALUATED, self.user_input_evaluated)
-        self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.new_concept_created)
+        self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.on_new_concept_created)
 
         self.sheerka.subscribe(EVENT_CONCEPT_CREATED, self.on_global_state_is_modified)
         self.sheerka.subscribe(EVENT_CONCEPT_MODIFIED, self.on_global_state_is_modified)
@@ -144,7 +144,7 @@ class SheerkaResultManager(BaseService):
 
             # add format instructions if applicable
             if (depth := kwargs.get("depth", None)) is not None or \
-                (depth := kwargs.get("recursion_depth", None)) is not None:
+                    (depth := kwargs.get("recursion_depth", None)) is not None:
                 explanation.set_format_instr(recursion_depth=depth, recurse_on="_children")
 
             return explanation
@@ -331,7 +331,7 @@ class SheerkaResultManager(BaseService):
 
         return self.last_errors
 
-    def new_concept_created(self, context, concept):
+    def on_new_concept_created(self, context, concept):
         """
         Subscriber (callback) when a new concept is created
         :param context:
@@ -380,8 +380,8 @@ class SheerkaResultManager(BaseService):
 
     def backup_command(self, execution_context):
         if (self.sheerka.during_restore or
-            not execution_context.is_state_modified() or
-            not self.sheerka.enable_commands_backup):
+                not execution_context.is_state_modified() or
+                not self.sheerka.enable_commands_backup):
             return
 
         folder = os.getenv(SHEERKA_BACKUP_FOLDER)

src/core/sheerka/services/SheerkaRuleManager.py

@@ -13,8 +13,8 @@ from core.rule import ACTION_TYPE_PRINT, Rule
 from core.sheerka.Sheerka import RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME
 from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, UnknownVariableError
 from core.tokenizer import Token, TokenKind
-from parsers.BaseExpressionParser import AndNode, ComparisonNode, ExpressionVisitor, \
-    FunctionNodeOld, NameExprNode, NotNode, VariableNode
+from parsers.BaseExpressionParser import AndNode, ComparisonNode, ExpressionVisitor, NameExprNode, \
+    NotNode, VariableNode
 from parsers.BaseNodeParser import ConceptNode
 from parsers.FormatRuleActionParser import FormatRuleActionParser
 from parsers.LogicalOperatorParser import LogicalOperatorParser
@@ -271,9 +271,12 @@ class SheerkaRuleManager(BaseService):
         """
         Remove a rule
         """
+        original_rule = rule
         rule = self.resolve_rule(context, rule)
         if rule is None:
-            return
+            sheerka = context.sheerka
+            err = sheerka.new(BuiltinConcepts.NOT_FOUND, body={"rule": original_rule})
+            return sheerka.ret(self.NAME, False, err)
 
         # rule will be deleted. publish the event first, as the rule may not be available after
         self.sheerka.publish(context, EVENT_RULE_DELETED, rule)
@@ -292,6 +295,26 @@ class SheerkaRuleManager(BaseService):
 
     def init_builtin_rules(self, context):
         # self.sheerka.init_log.debug("Initializing default rules")
+        rules_by_names = {}
+
+        def is_less_than(r1, r2):
+            self.sheerka.set_is_less_than(context,
+                                          BuiltinConcepts.PRECEDENCE,
+                                          rules_by_names[r1], rules_by_names[r2],
+                                          RULE_COMPARISON_CONTEXT)
+
+        def is_greater_than(r1, r2):
+            self.sheerka.set_is_greater_than(context,
+                                             BuiltinConcepts.PRECEDENCE,
+                                             rules_by_names[r1], rules_by_names[r2],
+                                             RULE_COMPARISON_CONTEXT)
+
+        def is_greatest(r1):
+            self.sheerka.set_is_greatest(context,
+                                         BuiltinConcepts.PRECEDENCE,
+                                         rules_by_names[r1],
+                                         RULE_COMPARISON_CONTEXT)
+
         rules = [
             # index=[0] in code, id=1 Rule #2 in debug
             Rule("print", "Print return values", "__rets", "list(__rets)"),
@@ -336,25 +359,24 @@ class SheerkaRuleManager(BaseService):
                  "isinstance(__ret_container, BuiltinConcepts.MULTIPLE_SUCCESS)",
                  "list(__ret_container.body)"),
 
-            # # index=[9] in code, id=10 in debug
-            # Rule("print", "Display simple result when only one success",
-            #      "len(__rets)==1 and __rets[0].status == True",
-            #      "{__rets[0].body}"),
+            # index=[9] in code, id=10 in debug
+            Rule("print", "Print ErrorItem",
+                 "isinstance(__obj, ErrorItem)",
+                 "[{id:3}] {__tab}green(source) red(error)"),
         ]
 
         for r in rules:
             self.create_new_rule(context, r)
+            rules_by_names[r.name] = r
 
-        self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[2], RULE_COMPARISON_CONTEXT)
-        self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[3], RULE_COMPARISON_CONTEXT)
-        self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[5], RULE_COMPARISON_CONTEXT)
-        self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[6], RULE_COMPARISON_CONTEXT)
-        self.sheerka.set_is_less_than(context, BuiltinConcepts.PRECEDENCE, rules[1], rules[8], RULE_COMPARISON_CONTEXT)
-        self.sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, rules[7], rules[6],
-                                         RULE_COMPARISON_CONTEXT)
-        self.sheerka.set_is_greater_than(context, BuiltinConcepts.PRECEDENCE, rules[7], rules[5],
-                                         RULE_COMPARISON_CONTEXT)
-        self.sheerka.set_is_greatest(context, BuiltinConcepts.PRECEDENCE, rules[0], RULE_COMPARISON_CONTEXT)
+        is_less_than("Print ReturnValue", "Failed ReturnValue in red")
+        is_less_than("Print ReturnValue", "List explanations")
+        is_less_than("Print ReturnValue", "Display formatted list")
+        is_less_than("Print ReturnValue", "Display formatted dict")
+        is_less_than("Print ReturnValue", "Display multiple success")
+        is_greater_than("Display multiple outputs", "Display formatted dict")
+        is_greater_than("Display multiple outputs", "Display formatted list")
+        is_greatest("Print return values")
 
     def get_rule_by_id(self, rule_id):
         """
@@ -585,7 +607,7 @@ class GetConditionExprVisitor(ExpressionVisitor):
 
         if return_body:
             if not res.status:
-                python_eval_error = self.context.sheerka.get_errors(self.context, res, __type="PythonEvalError")
+                python_eval_error = self.context.sheerka.get_obj_errors(self.context, res, __type="PythonEvalError")
                 if python_eval_error and isinstance(python_eval_error[0].error, NameError):
                     raise UnknownVariableError(python_eval_error[0].source)
                 raise FailedToCompileError(res.body)
@@ -706,15 +728,6 @@ class ReteConditionExprVisitor(GetConditionExprVisitor):
         else:
             raise FailedToCompileError([expr_node])
 
-    def visit_FunctionNodeOld(self, expr_node: FunctionNodeOld):
-        if expr_node.first.value == "recognize(":
-            if not isinstance(expr_node.parameters[0].value, VariableNode):
-                return FailedToCompileError([f"Cannot recognize '{expr_node.parameters[0].value}'"])
-
-            return self.recognize_concept(expr_node.parameters[0].value.unpack(),
-                                          expr_node.parameters[1].value,
-                                          {})
-
     def visit_NotNode(self, expr_node: NotNode):
 
         def get_sub_conditions(conditions_):

src/core/utils.py

@@ -876,27 +876,6 @@ def escape_str(x):
     return x
 
 
-def new_array(size):
-    res = []
-    for _ in range(size):
-        res.append(0)
-
-    return res
-
-
-class NextIdManager:
-    """
-    solely return the next integer
-    """
-
-    def __init__(self):
-        self.id = -1
-
-    def get_next_id(self):
-        self.id += 1
-        return self.id
-
-
 def compute_hash(obj):
     try:
         if isinstance(obj, (list, tuple)):

src/evaluators/OneErrorEvaluator.py

@@ -54,5 +54,5 @@ class OneErrorEvaluator(AllReturnValuesEvaluator):
         context.log(f"{self.return_value_in_error}", who=self)
 
         sheerka = context.sheerka
-        to_return = self.return_value_in_error.value if self.return_value_in_error else self.return_value_filtered.body
+        to_return = self.return_value_in_error if self.return_value_in_error else self.return_value_filtered
         return sheerka.ret(self.name, False, to_return, parents=self.eaten.copy())

src/parsers/BaseExpressionParser.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from itertools import product
-from typing import List, Union
+from typing import List
 
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.sheerka.services.SheerkaExecute import ParserInput
@@ -572,72 +572,6 @@ class FunctionNode(ExprNode):
                             self.parameters.clone())
 
 
-@dataclass()
-class FunctionParameter:
-    """
-    class the represent result of the parameter parsing
-    """
-    value: NameExprNode  # value parsed
-    separator: NameExprNode = None  # holds the value and the position of the separator
-
-    def add_sep(self, start, end, tokens):
-        self.separator = NameExprNode(start, end, tokens)
-
-    def value_to_unrecognized(self):
-        return UnrecognizedTokensNode(self.value.start, self.value.end, self.value.tokens).fix_source()
-
-    def separator_to_unrecognized(self):
-        if self.separator is None:
-            return None
-        return UnrecognizedTokensNode(self.separator.start, self.separator.end, self.separator.tokens).fix_source()
-
-    def clone(self):
-        return FunctionParameter(self.value.clone(), self.separator.clone())
-
-
-class FunctionNodeOld(ExprNode):
-
-    def __init__(self, start, end, tokens,
-                 first: NameExprNode, last: NameExprNode, parameters: Union[None, List[FunctionParameter]]):
-        super().__init__(start, end, tokens)
-        self.first = first
-        self.last = last
-        self.parameters = parameters
-
-    def __eq__(self, other):
-        if id(self) == id(other):
-            return True
-
-        if not isinstance(other, FunctionNodeOld):
-            return False
-
-        return (self.first == other.first and
-                self.last == other.last and
-                self.parameters == other.parameters)
-
-    def __hash__(self):
-        return hash((self.first, self.last, self.parameters))
-
-    def __repr__(self):
-        return f"FunctionNodeOld(start={self.start}, end={self.end}, {self.first!r} {self.last} {self.parameters!r})"
-
-    def __str__(self):
-        return f"{self.first} {self.parameters} {self.last}"
-
-    def clone(self):
-        if self.parameters is not None:
-            parameters = [p.clone for p in self.parameters]
-        else:
-            parameters = None
-
-        return FunctionNodeOld(self.start,
-                               self.end,
-                               self.tokens.copy(),
-                               self.first.clone(),
-                               self.last.clone(),
-                               parameters)
-
-
 @dataclass()
 class Comprehension:
     target: ExprNode
@@ -1013,12 +947,6 @@ class IsAQuestionVisitor(ExpressionVisitor):
             return True
         return None
 
-    def visit_FunctionNodeOld(self, expr_node: FunctionNodeOld):
-        if tokens_are_matching(expr_node.tokens, is_question_tokens) or \
-                tokens_are_matching(expr_node.tokens, in_context_tokens):
-            return True
-        return None
-
     def visit_FunctionNode(self, expr_node: FunctionNode):
         if tokens_are_matching(expr_node.tokens, is_question_tokens) or \
                 tokens_are_matching(expr_node.tokens, in_context_tokens):

src/parsers/BaseNodeParser.py

@@ -1,8 +1,10 @@
 from dataclasses import dataclass
+from typing import Union
 
 import core.utils
 from cache.FastCache import FastCache
 from core import builtin_helpers
+from core.concept import Concept
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.tokenizer import Token, TokenKind
 from core.var_ref import VariableRef
@@ -483,11 +485,13 @@ class VariableNode(LexerNode):
 @dataclass()
 class GrammarErrorNode(ParsingError):
     message: str
+    concept: Union[Concept, None] = None
 
 
 @dataclass()
 class NoMatchingTokenError(ParsingError):
     pos: int
+    concept: Union[Concept, None] = None
 
 
 class UnrecognizedTokensCache:

src/parsers/BnfNodeParser.py

@@ -10,7 +10,7 @@ import re
 from collections import defaultdict
 from dataclasses import dataclass, field
 from operator import attrgetter
-from typing import List
+from typing import List, Union
 
 import core.builtin_helpers
 import core.utils
@@ -23,11 +23,21 @@ from core.tokenizer import Token, TokenKind, Tokenizer
 from core.utils import CONSOLE_COLORS_MAP as CCM
 from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, GrammarErrorNode, NoMatchingTokenError, RuleNode, \
     SourceCodeNode, SourceCodeWithConceptNode, UnrecognizedTokensCache, UnrecognizedTokensNode
+from parsers.BaseParser import ParsingError
 
 PARSERS = ["Sequence", "Sya", "Python"]
 VARIABLE_EXPR_PARSER = ["Sequence", "Sya", "Python", "Bnf"]
 
 
+@dataclass
+class UndefinedSymbolError(ParsingError):
+    """
+    When parsing a concept, returns an error when a symbol is not recognized as a concept, a rule, or python code...
+    """
+    symbol: str
+    concept: Union[Concept, None] = None
+
+
 @dataclass(eq=True)
 class RegExDef:
     to_match: str = None
@@ -469,6 +479,9 @@ class VariableExpression(ParsingExpression):
     def __hash__(self):
         return hash(("VariableExpression", self.rule_name))
 
+    def debug_prefix(self, parser_helper):
+        return super().debug_prefix("VariableExpression", parser_helper)
+
     def init_parsing(self):
         """
         Get the instance of the following VariableExpression if they exists,
@@ -485,6 +498,9 @@ class VariableExpression(ParsingExpression):
     def get_nodes_sequences_when_variables_are_first(self, parser_helper):
         if len(parser_helper.sequence) < len(self.expected_variables):
             # variable(s) is/are expected. But nothing found
+            if parser_helper.debugger.is_enabled():
+                debug_prefix = self.debug_prefix(parser_helper)
+                parser_helper.debug_concept(debug_prefix, color="red", raw="Failed! Too few elements")
             return None
 
         # only take the requested number of variables
@@ -499,6 +515,11 @@ class VariableExpression(ParsingExpression):
         end = parser_helper.get_last_token_pos()
         nodes_sequences = self.get_nodes_sequences_from_tokens(parser_helper, start, end, tokens)
         if not nodes_sequences:
+            unrecognized = core.utils.get_text_from_tokens(tokens)
+            parser_helper.add_error(UndefinedSymbolError(unrecognized, parser_helper.get_current_concept()))
+            if parser_helper.debugger.is_enabled():
+                debug_prefix = self.debug_prefix(parser_helper)
+                parser_helper.debug_concept(debug_prefix, color="red", raw=f"Failed to recognize '{unrecognized}'")
             return nodes_sequences
 
         # only take the requested number of variables
@@ -527,7 +548,7 @@ class VariableExpression(ParsingExpression):
     def _parse(self, parser_helper):
 
         if parser_helper.debugger.is_enabled():
-            debug_prefix = self.debug_prefix("VariableExpression", parser_helper)
+            debug_prefix = self.debug_prefix(parser_helper)
             debug_vars = {"pos": parser_helper.pos,
                           "expected variables": self.expected_variables,
                           "next to match": self.next_node_to_parse}
@@ -541,8 +562,14 @@ class VariableExpression(ParsingExpression):
         else:
             nodes_sequences = self.get_nodes_sequences_when_variables_are_in_between(parser_helper)
 
-        if nodes_sequences is None or self.has_unrecognized(nodes_sequences):
-            # nothing is recognized or only part is recognized
+        if nodes_sequences is None:
+            # nothing is recognized
+            return None
+
+        if self.has_unrecognized(nodes_sequences):
+            if parser_helper.debugger.is_enabled():
+                debug_prefix = self.debug_prefix(parser_helper)
+                parser_helper.debug_concept(debug_prefix, color="red", raw="Some parts are not recognized")
             return None
 
         all_results = []
@@ -558,7 +585,7 @@ class VariableExpression(ParsingExpression):
 
                 resolved = self.get_resolved(node)
                 if resolved is None:
-                    parser_helper.errors.append(f"Failed to recognize {node.source}")
+                    parser_helper.add_error(UndefinedSymbolError(node.source))
                     break
 
                 ptree_nodes.append(TerminalNode(variable_expr, node.start, node.end, node.source, resolved))
@@ -575,6 +602,11 @@ class VariableExpression(ParsingExpression):
                 all_results.append(ptree_nodes)
 
         if len(all_results) == 0:
+            if parser_helper.debugger.is_enabled():
+                debug_prefix = self.debug_prefix(parser_helper)
+                parser_helper.debug_concept(debug_prefix, color="red",
+                                            raw="No result matching the expected number of variables",
+                                            nodes_sequences=nodes_sequences)
             return None
 
         # every seems to be fine. We can pop the nodes from parser_helper used as variable
@@ -618,8 +650,9 @@ class VariableExpression(ParsingExpression):
             return None
 
         utn = UnrecognizedTokensNode(start, end, tokens)
-        nodes_sequences = parser_helper.parser.cache2.get_lexer_nodes_from_unrecognized(parser_helper.parser.context,
-                                                                                        utn)
+        nodes_sequences = parser_helper.parser.variable_expr_cache.get_lexer_nodes_from_unrecognized(
+            parser_helper.parser.context,
+            utn)
         return nodes_sequences
 
     @staticmethod
@@ -1043,21 +1076,24 @@ class StrMatch(Match):
 
         if parser_helper.debugger.is_enabled():
             debug_prefix = self.debug_prefix("StrMatch", parser_helper)
-            debug_text = f"pos={parser_helper.pos}, to_match={self.to_match}, token={token.str_value}"
-            parser_helper.debug_concept(debug_prefix, raw=f"{CCM['green']}{debug_text}{CCM['reset']}")
+            debug_vars = {"pos": parser_helper.pos,
+                          "to_match": self.to_match,
+                          "token": token.str_value}
+            debug_text = self.debug_to_raw(debug_vars)
+            parser_helper.debug_concept(debug_prefix, color="cyan", raw=debug_text)
 
         m = token.str_value.lower() == self.to_match.lower() if self.ignore_case \
             else token.strip_quote == self.to_match
 
         if m:
             if parser_helper.debugger.is_enabled():
-                parser_helper.debug_concept(debug_prefix, raw=f"{CCM['green']}{debug_text}{CCM['reset']}")
+                parser_helper.debug_concept(debug_prefix, color="green", raw=f"matched")
             node = TerminalNode(self, parser_helper.pos, parser_helper.pos, token.str_value, token.str_value)
             parser_helper.next_token(self.skip_white_space)
             return node
 
         if parser_helper.debugger.is_enabled():
-            parser_helper.debug_concept(debug_prefix, raw=f"{CCM['red']}{debug_text}{CCM['reset']}")
+            parser_helper.debug_concept(debug_prefix, color="red", raw=f"not matched")
         return None
 
 
@@ -1110,17 +1146,18 @@ class RegExMatch(Match):
         #     debug_text = f"pos={parser_helper.pos}, to_match={self.to_match}, text={text_debug}"
         #     parser_helper.debug_concept(debug_prefix, raw=f"{CCM['green']}{debug_text}{CCM['reset']}")
 
-        m = self.regex.match(text, parser_helper.token.index)
+        m = self.regex.match(text, parser_helper.token.index - parser_helper.token_offset)
         if m:
             matched = m.group()
 
             # TODO: Add debug info here
 
             if matched:
-                # the match is only valid if it fits with the actual tokens
+                # the match is only valid it ends at the end of a token
                 next_pos = parser_helper.get_next_matching_pos(m.end())
                 if next_pos is NotFound:
-                    parser_helper.errors.append(NoMatchingTokenError(m.end()))
+                    current_concept = parser_helper.get_current_concept()
+                    parser_helper.add_error(NoMatchingTokenError(m.end(), concept=current_concept))
                     return None
 
                 node = TerminalNode(self, parser_helper.pos, next_pos - 1, matched, matched)
@@ -1242,11 +1279,11 @@ class HasAChoiceExpressionVisitor(ParsingExpressionVisitor):
 
 
 class BnfConceptParserHelper:
-    def __init__(self, parser, debugger):
+    def __init__(self, parser, token_offset, debugger):
         self.parser = parser
         self.debugger = debugger
         self.debug = []  # keep track of the tokens
-        self.errors = []  # sink of errors
+        self.errors = []  # sink of errors, sorted by bnf concepts
         self.sequence = []  # output. List of lexer nodes correctly parsed
         self.concepts = []  # stack of concepts being processed (fed by ConceptExpression)
         self.concepts_ids = []  # ids if the concept to increase speed
@@ -1260,6 +1297,7 @@ class BnfConceptParserHelper:
         self.forked = []
 
         self.token = None
+        self.token_offset = token_offset
         self.pos = -1
 
     def __repr__(self):
@@ -1282,6 +1320,12 @@ class BnfConceptParserHelper:
         if len(self.concepts) <= 2:
             self.debugger.debug_concept(self.concepts[0], text, **kwargs)
 
+    def get_current_concept(self):
+        return self.concepts[-1] if len(self.concepts) > 0 else None
+
+    def add_error(self, error):
+        self.errors.append(error)
+
     def get_current_rule_name(self):
         for rule_name in reversed(self.rules_names):
             if rule_name:
@@ -1328,15 +1372,26 @@ class BnfConceptParserHelper:
     def get_next_matching_pos(self, token_index):
         """
         Given the token, tries to find a token (within the remaining tokens) that matches the index
+        Why do we do that ?
+        The regex matching is done against the text of the parser input, not its tokens
+        With this function, we check that the matching found correspond to an exact number of tokens.
+        Token is "abcdef" and we manage to match "abc"
+        -> 'abc' falls in the middle of the token, so it's not a valid match
         :param token_index:
         :return:
         """
         current = self.pos
+        token_index += self.token_offset  # add the offset when using sub parser input
         while current <= self.parser.parser_input.end:
             if self.parser.parser_input.tokens[current].index == token_index:
                 return current
             current += 1
 
+        # last chance when dealing with sub parser input (that do not end with token(EOF))
+        if current < self.parser.parser_input.length:
+            if self.parser.parser_input.tokens[current].index == token_index:
+                return current
+
         # No matching token
         return NotFound
 
@@ -1384,7 +1439,7 @@ class BnfConceptParserHelper:
                 error_msg = f"Failed to parse concept '{concept}'"
                 if parsing_expression is not None:
                     error_msg += f". Reason: '{parsing_expression}'"
-                self.errors.append(GrammarErrorNode(error_msg))
+                self.add_error(GrammarErrorNode(error_msg, concept=concept))
                 return
 
             self.pos = self.parser.parser_input.pos
@@ -1468,7 +1523,7 @@ class BnfConceptParserHelper:
         self.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, [])
 
     def clone(self):
-        clone = BnfConceptParserHelper(self.parser, self.debugger)
+        clone = BnfConceptParserHelper(self.parser, self.token_offset, self.debugger)
         clone.debug = self.debug[:]
         clone.errors = self.errors[:]
         clone.sequence = self.sequence[:]
@@ -1636,8 +1691,11 @@ class BnfNodeParser(BaseNodeParser):
         else:
             self.concepts_grammars = Cache()
 
+        # this cache is for unrecognized tokens that cannot be BNF expression
         self.cache = UnrecognizedTokensCache(PARSERS)
-        self.cache2 = UnrecognizedTokensCache(VARIABLE_EXPR_PARSER)
+
+        # This cache is for variable expression. It can be anything
+        self.variable_expr_cache = UnrecognizedTokensCache(VARIABLE_EXPR_PARSER)
         self.ignore_case = True
 
     @staticmethod
@@ -1730,10 +1788,12 @@ class BnfNodeParser(BaseNodeParser):
 
             return list1 + list2
 
+        sheerka = context.sheerka
         forked = []
         debugger = context.get_debugger(self.NAME, "parse")
         debugger.debug_entering(source=self.parser_input.as_text())
-        concept_parser_helpers = [BnfConceptParserHelper(self, debugger)]
+        token_offset = self.parser_input.tokens[self.parser_input.start].index
+        concept_parser_helpers = [BnfConceptParserHelper(self, token_offset, debugger)]
 
         while self.parser_input.next_token(False):
 
@@ -1753,8 +1813,8 @@ class BnfNodeParser(BaseNodeParser):
                         debugger.debug_log(debug_prefix + ", all parsers are locked. Nothing to do.")
                     continue
 
-                by_token = context.sheerka.get_concepts_by_first_token(token, self._is_eligible, strip_quotes=False)
-                by_regex = context.sheerka.get_concepts_by_first_regex(self.parser_input.sub_text, token.index)
+                by_token = sheerka.get_concepts_by_first_token(token, self._is_eligible, strip_quotes=False)
+                by_regex = sheerka.get_concepts_by_first_regex(self.parser_input.sub_text, token.index - token_offset)
 
                 concepts = _merge(by_token, by_regex)
 
@@ -2120,7 +2180,7 @@ class BnfNodeParser(BaseNodeParser):
         debugger = context.get_debugger(self.NAME, "parse")
         if debugger.is_enabled:
             debugger.debug_var("stats", self.cache.to_dict())
-            #debugger.debug_var("stats", self.cache2.to_dict())
+            # debugger.debug_var("stats", self.variable_expr_cache.to_dict())
 
         if valid_parser_helpers is None:
             return self.sheerka.ret(

src/parsers/ExpressionParser.py

@@ -3,11 +3,9 @@ from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.ArithmericOperatorParser import ArithmeticOperatorParser
 from parsers.BaseExpressionParser import BaseExpressionParser, ChoiceParser
 from parsers.FunctionParser import FunctionParser
-from parsers.FunctionParserOld import FunctionParserOld
 from parsers.ListParser import ListParser
 from parsers.LogicalOperatorParser import LogicalOperatorParser
 from parsers.RelationalOperatorParser import RelationalOperatorParser
-from parsers.VariableOrNamesParser import VariableOrNamesParser
 from sheerkapython.ExprToConditions import ExprToConditionsVisitor
 from sheerkapython.ExprToPython import PythonExprVisitor
 
@@ -19,24 +17,19 @@ class ExpressionParser(BaseExpressionParser):
 
     NAME = "Expression"
 
-    def __init__(self, auto_compile=True, old_style=False, known_variables=None, **kwargs):
+    def __init__(self, auto_compile=True, known_variables=None, **kwargs):
         super().__init__(ExpressionParser.NAME,
                          0,
                          True,
                          yield_eof=False,
                          hints={BuiltinConcepts.EVAL_QUESTION_REQUESTED: 60})
-        if old_style:
-            self.variable_parser = VariableOrNamesParser()
-            self.function_parser = FunctionParserOld(expr_parser=self, tokens_parser=self.variable_parser)
-            self.relational_parser = RelationalOperatorParser(old_style=old_style, expr_parser=self.function_parser)
-            self.logical_parser = LogicalOperatorParser(expr_parser=self.relational_parser)
-        else:
-            function_parser = FunctionParser(expr_parser=self)
-            list_parser = ListParser(expr_parser=self)
-            choice_parser = ChoiceParser(list_parser, function_parser)
-            arithmetic_parser = ArithmeticOperatorParser(expr_parser=self, function_parser=choice_parser)
-            relational_parser = RelationalOperatorParser(expr_parser=arithmetic_parser)
-            self.logical_parser = LogicalOperatorParser(expr_parser=relational_parser)
+
+        function_parser = FunctionParser(expr_parser=self)
+        list_parser = ListParser(expr_parser=self)
+        choice_parser = ChoiceParser(list_parser, function_parser)
+        arithmetic_parser = ArithmeticOperatorParser(expr_parser=self, function_parser=choice_parser)
+        relational_parser = RelationalOperatorParser(expr_parser=arithmetic_parser)
+        self.logical_parser = LogicalOperatorParser(expr_parser=relational_parser)
 
         self.auto_compile = auto_compile
         self.known_variables = known_variables

src/parsers/FunctionParserOld.py

@@ -1,343 +0,0 @@
-from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import get_lexer_nodes_from_unrecognized, update_compiled
-from core.concept import Concept
-from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import TokenKind
-from core.utils import get_n_clones
-from parsers.BaseExpressionParser import NameExprNode, FunctionNodeOld, FunctionParameter, BaseExpressionParser
-from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode
-from parsers.BaseParser import UnexpectedTokenParsingError, UnexpectedEofParsingError, ErrorSink
-from parsers.BnfNodeParser import BnfNodeParser
-from parsers.PythonWithConceptsParser import PythonWithConceptsParser
-from parsers.RuleParser import RuleParser
-from parsers.SequenceNodeParser import SequenceNodeParser
-from parsers.SyaNodeParser import SyaNodeParser
-
-PARSERS = [RuleParser.NAME,
-           SequenceNodeParser.NAME,
-           BnfNodeParser.NAME,
-           SyaNodeParser.NAME]
-
-
-class FunctionParserOld(BaseExpressionParser):
-    """
-    The parser will be used to parse func(x, y, z)
-    where x, y and z can be source code, concepts or other functions
-    It will return a  SourceCodeNode or SourceCodeNodeWithConcept
-    """
-
-    NAME = "FunctionOld"
-
-    def __init__(self, sep=",", longest_concepts_only=True, strict=True, **kwargs):
-        """
-
-        :param sep:
-        :param longest_concepts_only: When multiples concepts are found, only keep the longest one
-        so 'twenty one' will resolve to [[c:twenty one:]], not [[c:twenty one:], [c:twenty:, c:one:]]
-        :param strict: Only allow expression that start by a function definition so 'xxx f(x)' will be refused
-        :param kwargs:
-        """
-        super().__init__(self.NAME, 55, enabled=False)
-        self.sep = sep
-        self.longest_concepts_only = longest_concepts_only
-        self.strict = strict
-        self.expr_parser = kwargs.get("expr_parser", None)
-        self.tokens_parser = kwargs.get("tokens_parser", None)
-
-    def function_parser_get_return_value_body(self, context, source, source_code_node):
-        if source_code_node.error_when_parsing:
-            return context.sheerka.new(BuiltinConcepts.ERROR,
-                                       body=source_code_node.error_when_parsing)
-
-        return context.sheerka.new(BuiltinConcepts.PARSER_RESULT,
-                                   parser=self,
-                                   source=source,
-                                   body=source_code_node,
-                                   try_parsed=source_code_node)
-
-    def parse(self, context, parser_input: ParserInput):
-        ret = super().parse(context, parser_input)
-
-        if ret is None:
-            return None
-
-        if not ret.status:
-            return ret
-
-        # FunctionParserOld returns LexerNodes, rather than an ExprNode
-        # I know that is is not very logical, but at the beginning, the FunctionParserOld was
-        #   uses exclusively by the SyaNodeParser.
-        # It has been refactored to fit in ExpressionParser. So it has two main usages
-        node = ret.body.body
-        source_code_nodes = self.to_source_code_node(context, node)
-        res = []
-        for source_code_node in source_code_nodes:
-            body = self.function_parser_get_return_value_body(context, parser_input.as_text(), source_code_node)
-            res.append(context.sheerka.ret(self.name, source_code_node.python_node is not None, body))
-
-        return res[0] if len(res) == 1 else res
-
-    def parse_input(self, context, parser_input, error_sink):
-        # when FunctionParserOld is used by LexerNode or SheerkaExecute, it must fail if no function is found
-        # when it is used by ExpressionParser, it must default to VariableOrNamesParser
-        # KSI 20210910 - Not quite quite it's the best approach. It seems a little bit complicated
-        pos = parser_input.pos
-        res = self.parse_function(context, parser_input, error_sink)
-        if (not res or error_sink.has_error) and self.tokens_parser:
-            parser_input.seek(pos)
-            error_sink.clear()
-            return self.tokens_parser.parse_input(context, parser_input, error_sink)
-
-        return res
-
-    def parse_function(self, context, parser_input, error_sink):
-        unrecognized_tokens_start = unrecognized_tokens_end = parser_input.pos
-        if not self.strict:
-            # eat everything that is not part of the function
-            token = parser_input.token
-            while token.type != TokenKind.EOF:
-                if token.type == TokenKind.IDENTIFIER and parser_input.the_token_after().type == TokenKind.LPAR:
-                    break
-                else:
-                    unrecognized_tokens_end += 1
-                    parser_input.next_token(skip_whitespace=False)
-                    token = parser_input.token
-
-        start = parser_input.pos
-        token = parser_input.token
-        if token.type != TokenKind.IDENTIFIER:
-            error_sink.add_error(UnexpectedTokenParsingError(f"{token.repr_value} is not a identifier",
-                                                             token,
-                                                             [TokenKind.IDENTIFIER]))
-            return None
-
-        if not parser_input.next_token():
-            error_sink.add_error(UnexpectedEofParsingError(f"while parsing left parenthesis"))
-            return None
-
-        token = parser_input.token
-        if token.type != TokenKind.LPAR:
-            error_sink.add_error(UnexpectedTokenParsingError(f"{token.repr_value} is not a left parenthesis",
-                                                             token,
-                                                             [TokenKind.LPAR]))
-            return None
-
-        start_node = NameExprNode(start, start + 1, parser_input.tokens[start:start + 2])
-        if not parser_input.next_token():
-            error_sink.add_error(UnexpectedEofParsingError(f"after left parenthesis"))
-            return FunctionNodeOld(start, start + 1, [], start_node, None, None)
-
-        params = self.parse_parameters(context, parser_input, error_sink)
-        if error_sink.has_error:
-            return FunctionNodeOld(start, parser_input.pos, [], start_node, None, params)
-
-        token = parser_input.token
-        if not token or token.type != TokenKind.RPAR:
-            error_sink.add_error(UnexpectedTokenParsingError(f"Right parenthesis not found",
-                                                             token,
-                                                             [TokenKind.RPAR]))
-            return FunctionNodeOld(start, parser_input.pos, [], start_node, None, params)
-
-        function_node = FunctionNodeOld(start,
-                                        parser_input.pos,
-                                        parser_input.tokens[start:parser_input.pos + 1],
-                                        start_node,
-                                        NameExprNode(parser_input.pos, parser_input.pos, [token]),
-                                        params)
-
-        parser_input.next_token()  # do not forget to eat the trailing parenthesis
-
-        # if unrecognized_tokens_end != unrecognized_tokens_start:
-        #     if self.expr_parser:
-        #         sub_parser = parser_input.sub_part(unrecognized_tokens_start,
-        #                                            unrecognized_tokens_end,
-        #                                            yield_oef=False).reset()
-        #         sub_parser.next_token()
-        #         expr_node = self.expr_parser(context, parser_input, error_sink)
-        #     else:
-        #         expr_node = NameExprNode(unrecognized_tokens_start,
-        #                                  unrecognized_tokens_end,
-        #                                  parser_input.tokens[unrecognized_tokens_start: unrecognized_tokens_end + 1])
-        #
-        #     return SequenceNode(unrecognized_tokens_start, function_node.end,
-        #                         parser_input.tokens[unrecognized_tokens_start: function_node.end + 1],
-        #                         expr_node, function_node)
-
-        return function_node
-
-    def parse_parameters(self, context, parser_input, error_sink):
-        nodes = []
-        while True:
-            param_value = self.parse_parameter_value(context, parser_input, error_sink)
-            if not param_value:
-                break
-
-            function_parameter = FunctionParameter(param_value)
-            nodes.append(function_parameter)
-
-            token = parser_input.token
-            if token.type == TokenKind.EOF:
-                error_sink.add_error(UnexpectedEofParsingError(f"while parsing parameters"))
-                return None
-
-            if token.type == TokenKind.RPAR:
-                break
-
-            if token.value == self.sep:
-                sep_pos = parser_input.pos
-                has_next = parser_input.next_token()  # it's before add_sep() to capture trailing whitespace
-                function_parameter.add_sep(sep_pos,
-                                           parser_input.pos - 1,
-                                           parser_input.tokens[sep_pos: parser_input.pos])
-                if not has_next:
-                    break
-
-        return nodes
-
-    def parse_parameter_value(self, context, parser_input, error_sink):
-        # check if the parameter is a function
-        start_pos = parser_input.pos
-        new_error_sink = ErrorSink()
-        func = self.parse_function(context, parser_input, new_error_sink)
-        if func and not new_error_sink.has_error:
-            return func
-
-        # otherwise, eat until LPAR or separator
-        parser_input.seek(start_pos)
-        return self.parse_tokens(context,
-                                 parser_input,
-                                 error_sink,
-                                 self.parse_tokens_stop_condition,
-                                 self.expr_parser,
-                                 self.expr_parser)
-
-    def parse_tokens_stop_condition(self, token, parser_input):
-        return token.value == self.sep
-
-    def to_source_code_node(self, context, function_node: FunctionNodeOld):
-        python_parser = PythonWithConceptsParser()
-
-        def update_source_code_node(scn, nodes, sep):
-            if hasattr(nodes, "__iter__"):
-                for n in nodes:
-                    scn.add_node(n)
-            else:
-                scn.add_node(nodes)
-
-            if sep:
-                scn.add_node(sep.to_unrecognized())
-
-        def get_errors_from_python_parsing(parsing_res):
-            if parsing_res.status:
-                return None
-
-            if context.sheerka.isinstance(parsing_res.body, BuiltinConcepts.NOT_FOR_ME):
-                return parsing_res.body.reason
-            else:
-                return parsing_res.body.body
-
-        if len(function_node.parameters) == 0:
-            # validate the source
-            nodes_to_parse = [function_node.first.to_unrecognized(), function_node.last.to_unrecognized()]
-            python_parsing_res = python_parser.parse_nodes(context, nodes_to_parse)
-            python_node = python_parsing_res.body.body if python_parsing_res.status else None
-
-            return [SourceCodeNode(start=function_node.first.start,
-                                   end=function_node.last.end,
-                                   tokens=function_node.first.tokens + function_node.last.tokens,
-                                   python_node=python_node,
-                                   return_value=python_parsing_res,
-                                   error_when_parsing=get_errors_from_python_parsing(python_parsing_res))]
-
-        res = [SourceCodeWithConceptNode(function_node.first.to_unrecognized(), function_node.last.to_unrecognized())]
-
-        # try to recognize every parameter, one by one
-        for param in function_node.parameters:
-            if isinstance(param.value, NameExprNode):
-                # try to recognize concepts
-                unrecognized = param.value.to_unrecognized()
-                nodes_sequences = get_lexer_nodes_from_unrecognized(context,
-                                                                    unrecognized,
-                                                                    PARSERS)
-            else:
-                # the parameter is also a function
-                nodes_sequences = self.to_source_code_node(context, param.value)
-
-            if self.longest_concepts_only:
-                nodes_sequences = self.get_longest_concepts(nodes_sequences)
-
-            if nodes_sequences is None:
-                # no concept found
-                for source_code_node in res:
-                    update_source_code_node(source_code_node, unrecognized, param.separator)
-
-            elif len(nodes_sequences) == 1:
-                # only one result
-                # It is the same code than when there are multiple results
-                # But here, we save the creation of the tmp_res object (not sure it worth it)
-                for source_code_node in res:
-                    update_source_code_node(source_code_node, nodes_sequences[0], param.separator)
-            else:
-                # multiple result, make the cartesian product
-                tmp_res = []
-                for source_code_node in res:
-                    instances = get_n_clones(source_code_node, len(nodes_sequences))
-                    tmp_res.extend(instances)
-                    for instance, node_sequence in zip(instances, nodes_sequences):
-                        update_source_code_node(instance, node_sequence, param.separator)
-                res = tmp_res
-
-        # check if it is a valid source code
-        for source_code_node in res:
-            source_code_node.fix_all_pos()
-            source_code_node.pseudo_fix_source()
-
-            python_parsing_res = python_parser.parse_nodes(context, source_code_node.get_all_nodes())
-            if python_parsing_res.status:
-                source_code_node.python_node = python_parsing_res.body.body
-                source_code_node.return_value = python_parsing_res
-
-                # make sure that concepts found can be evaluated
-                errors = []
-                for c in [c for c in source_code_node.python_node.objects.values() if isinstance(c, Concept)]:
-                    update_compiled(context, c, errors)
-
-                if errors:
-                    source_code_node.error_when_parsing = errors
-
-            else:
-                source_code_node.error_when_parsing = get_errors_from_python_parsing(python_parsing_res)
-
-        return res
-
-    @staticmethod
-    def get_longest_concepts(nodes_sequences):
-        """
-        The longest sequences are the ones that have the less number of concepts
-        For example
-        'twenty one' resolves to
-            [c:twenty one:]
-            [c:twenty:, c:one:]
-        [c:twenty one:] has only one concept, so it's the longest one (two tokens against one token twice)
-        :param nodes_sequences:
-        :return:
-        """
-        if nodes_sequences is None:
-            return None
-
-        res = []
-        min_len = -1
-        for current_sequence in nodes_sequences:
-            # awful hack to remove when NodeSequence and ConceptSequence will be implemented
-            current_len = len(current_sequence) if hasattr(current_sequence, "__len__") else 1
-            if len(res) == 0:
-                res.append(current_sequence)
-                min_len = current_len
-            elif current_len == min_len:
-                res.append(current_sequence)
-            elif current_len < min_len:
-                res.clear()
-                res.append(current_sequence)
-                min_len = current_len
-
-        return res

src/parsers/RelationalOperatorParser.py

@@ -1,6 +1,6 @@
 from core.tokenizer import TokenKind
-from parsers.BaseExpressionParser import ComparisonNode, ComparisonType, \
-    ParenthesisNode, BaseExpressionParser, open_parenthesis_mapping
+from parsers.BaseExpressionParser import BaseExpressionParser, ComparisonNode, ComparisonType, ParenthesisNode, \
+    open_parenthesis_mapping
 from parsers.BaseParser import UnexpectedTokenParsingError
 from parsers.ListParser import ListParser
 
@@ -13,10 +13,9 @@ class RelationalOperatorParser(BaseExpressionParser):
 
     NAME = "RelationalOperator"
 
-    def __init__(self, old_style=False, **kwargs):
+    def __init__(self, **kwargs):
         super().__init__(self.NAME, 60, False, yield_eof=True)
         self.expr_parser = kwargs.get("expr_parser", None)
-        self.old_style = old_style
         self.list_parser = ListParser()
 
     def parse_input(self, context, parser_input, error_sink):
@@ -36,29 +35,18 @@ class RelationalOperatorParser(BaseExpressionParser):
         if (comp := self.eat_comparison(parser_input)) is None:
             return left
 
-        if self.old_style:
+        if comp in (ComparisonType.IN, ComparisonType.NOT_IN):
+            token = parser_input.token
+            if token.type not in open_parenthesis_mapping:
+                error_sink.add_error(UnexpectedTokenParsingError(f"Expected parenthesis", token, [TokenKind.LPAR]))
+            right = self.list_parser.parse_input(context, parser_input, error_sink)
+        else:
             right = self.parse_tokens(context,
                                       parser_input,
                                       error_sink,
                                       self.parse_tokens_stop_condition,
                                       self.expr_parser,
                                       self)
-            if comp == ComparisonType.IN and not isinstance(right, ParenthesisNode):
-                t = right.tokens[0]
-                error_sink.add_error(UnexpectedTokenParsingError(f"Expected parenthesis", t, [TokenKind.LPAR]))
-        else:
-            if comp in (ComparisonType.IN, ComparisonType.NOT_IN):
-                token = parser_input.token
-                if token.type not in open_parenthesis_mapping:
-                    error_sink.add_error(UnexpectedTokenParsingError(f"Expected parenthesis", token, [TokenKind.LPAR]))
-                right = self.list_parser.parse_input(context, parser_input, error_sink)
-            else:
-                right = self.parse_tokens(context,
-                                          parser_input,
-                                          error_sink,
-                                          self.parse_tokens_stop_condition,
-                                          self.expr_parser,
-                                          self)
 
         end = right.end if right else parser_input.pos
 

src/parsers/SyaNodeParser.py

@@ -1007,8 +1007,11 @@ class SyaNodeParser(BaseNodeParser):
             result.append(tokens_parser)
         elif tokens_parser.has_sya_concept and tokens_parser.has_error():
             in_error.append(tokens_parser)
+            for err in tokens_parser.errors:
+                tokens_parser.debugger.debug_log(f"Validation Error: {err}", True)
         else:
             tokens_parser.errors.append(NoSyaConceptFound())
+            tokens_parser.debugger.debug_log(f"Validation Error: No Sya Found", True)
             not_for_me.append(tokens_parser)
 
         # recurse on the forks

src/sdp/sheerkaDataProvider.py

@@ -52,7 +52,7 @@ class Event(object):
         if self._digest:
             return self._digest
 
-        if self.message == "" and self.user_id == "":
+        if self.user_id == "" and (self.message == "" or self.message.startswith("xxxTESTxxx::")):
             self._digest = "xxx"  # to speed unit tests
             return self._digest
 

src/sheerkapython/BaseExprTransform.py

@@ -57,14 +57,15 @@ check_existence = ExprTransformHints(check_variable_existence=True,
 
 
 class BaseExprTransform(ExpressionVisitorWithHint):
-    def __init__(self, context, obj_counter):
+    def __init__(self, context, obj_counter, known_variables=None):
         self.context = context
         self.obj_counter = obj_counter
         self.objects_by_id = {}
         self.objects_by_name = {}
         self.errors = {}
+        self.known_variables = known_variables if known_variables is not None else set()
         from parsers.ExpressionParser import ExpressionParser
-        self.expression_parser = ExpressionParser(auto_compile=False, old_style=False)
+        self.expression_parser = ExpressionParser(auto_compile=False)
 
     def visit_NotNode(self, expr_node: NotNode, hint: ExprTransformHints):
         """
@@ -482,7 +483,12 @@ class BaseExprTransform(ExpressionVisitorWithHint):
         identifier = self.get_object_identifier(concept)
         objects = {}
 
-        concept_variables = get_variables_from_concept_asts(self.context, concept, variables, parameters_only=True)
+        known_variables = self.known_variables.copy()
+        known_variables.update(variables)
+        concept_variables = get_variables_from_concept_asts(self.context,
+                                                            concept,
+                                                            known_variables,
+                                                            parameters_only=True)
         parameters_to_compute = {}
         for var_name, default_value in [(k, v) for k, v in concept.get_metadata().variables if k in concept_variables]:
             if isinstance(concept.get_compiled()[var_name], Concept):

src/sheerkapython/ExprToConditions.py

@@ -8,9 +8,8 @@ from sheerkapython.python_wrapper import is_variable
 
 class ExprToConditionsVisitor(BaseExprTransform):
     def __init__(self, context, obj_counter=0, known_variables=None):
-        super().__init__(context, obj_counter)
+        super().__init__(context, obj_counter, known_variables)
         self.intermediate_variables = {}
-        self.known_variables = known_variables if known_variables is not None else set()
 
     def get_conditions(self, expr_node):
         # first transform expr_node into list of conjunctions

src/sheerkapython/ExprToPython.py

@@ -3,7 +3,7 @@ from itertools import product
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.sheerka.services.sheerka_service import FailedToCompileError
 from core.tokenizer import TokenKind
-from parsers.BaseExpressionParser import AndNode, FunctionNodeOld, ListComprehensionNode, NameExprNode, VariableNode, \
+from parsers.BaseExpressionParser import AndNode, ListComprehensionNode, NameExprNode, VariableNode, \
     end_parenthesis_mapping, open_parenthesis_mapping
 from sheerkapython.BaseExprTransform import BaseExprTransform, ExprTransformHints, do_not_eval_source_hint, \
     is_a_question_hint, not_a_question_hint, wrap_concept_call_hint
@@ -128,19 +128,3 @@ class PythonExprVisitor(BaseExprTransform):
         :return:
         """
         return self.visit_or_or_and_node("or", expr_node, hint)
-
-    def visit_FunctionNodeOld(self, expr_node: FunctionNodeOld, hint: ExprTransformHints):
-        visitor_objects = []
-        source = expr_node.get_source()
-
-        parameters_objects = []
-        for parameter in expr_node.parameters:
-            parameters_objects.append(self.visit(parameter.value, hint))
-
-        for parameters in product(*parameters_objects):
-            visitor_objects.append(self.create_function_old(source,
-                                                            expr_node.first.get_source(),
-                                                            expr_node.last.get_source(),
-                                                            parameters))
-
-        return visitor_objects

src/sheerkapython/python_wrapper.py

@@ -6,7 +6,7 @@ from core.ast_helpers import UnreferencedVariablesVisitor
 from core.builtin_concepts import ReturnValueConcept
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.concept import AllConceptParts, Concept
-from core.global_symbols import ErrorObj, NotFound, NotInit, SyaAssociativity
+from core.global_symbols import ErrorItem, ErrorObj, NotFound, NotInit, SyaAssociativity
 from core.rule import Rule
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.services.SheerkaAdmin import SheerkaAdmin
@@ -110,8 +110,14 @@ sheerka_globals = {
     "get_type": get_type,
     "hasattr": sheerka_hasattr,
     "getattr": sheerka_getattr,
+    "ErrorItem": ErrorItem
 }
 
+# Adds all concepts that have their own class definition
+for c in core.utils.get_classes("core.builtin_concepts"):
+    if issubclass(c, Concept) and c != Concept:
+        sheerka_globals[c.__name__] = c
+
 
 def inject_context(context):
     """
@@ -285,6 +291,7 @@ def get_variables_from_concept_asts(context, concept, known_variables, parameter
     """
     From a given concept that is already compiled,
     browse the compiled to see if there is any symbol that is unknown, eg variable
+    It is used to detect all mandatory variables before concept evaluation
     :param context:
     :param concept:
     :param known_variables:
@@ -348,6 +355,25 @@ def get_variables_from_concept_asts(context, concept, known_variables, parameter
     return variables
 
 
+def get_possible_variables_from_concept(context, concept):
+    """
+    Given a concept, get its symbols that may be considered as variables for other concepts
+    :param context:
+    :param concept:
+    :return:
+    """
+    possible_variables = set(concept.get_metadata().parameters)
+
+    core.builtin_helpers.ensure_bnf(context, concept)
+    if concept.get_bnf():
+        from parsers.BnfNodeParser import BnfNodeConceptExpressionVisitor
+        visitor = BnfNodeConceptExpressionVisitor()
+        visitor.visit(concept.get_bnf())
+        possible_variables.update([c.name if isinstance(c, Concept) else c for c in visitor.references])
+
+    return possible_variables
+
+
 def is_variable(context, name):
     """
     tells whether or not the name can be a variable

tests/BaseTest.py

@@ -48,7 +48,7 @@ class InitTestHelper:
                         c.get_metadata().definition_type = DEFINITION_TYPE_BNF
                     else:
                         raise Exception(f"Error in bnf definition '{c.get_metadata().definition}'",
-                                        self.sheerka.get_errors(self.context, res))
+                                        self.sheerka.get_obj_errors(self.context, res))
 
             self._update_concept_parameters(c)
             if create_new:
@@ -92,8 +92,7 @@ class InitTestHelper:
             if create_new:
                 res = self.sheerka.create_new_rule(self.context, rule)
                 if not res.status:
-                    raise Exception(f"Error in rule definition '{res.body}'",
-                                    self.sheerka.get_errors(res))
+                    raise Exception(f"Error in rule definition '{res.body}'", self.sheerka.get_obj_errors(res))
                 self.items.append(res.body.body)
             else:
                 self.items.append(rule)
@@ -294,16 +293,26 @@ class BaseTest:
         return [ret_val for ret_val in return_values if ret_val.status]
 
     @staticmethod
-    def activate_debug(context, pattern="Sya.*.*"):
+    def activate_debug(context, pattern="Sya.*.*", debug_var=True, debug_concept=False, debug_rule=False):
         sheerka = context.sheerka
         sheerka.set_debug(context, True)
         sheerka.set_debug_logger_definition(ListDebugLogger)
 
         if isinstance(pattern, list):
             for p in pattern:
-                sheerka.set_debug_var(context, p)
+                if debug_var:
+                    sheerka.set_debug_var(context, p)
+                if debug_concept:
+                    sheerka.set_debug_concept(context, p)
+                if debug_rule:
+                    sheerka.set_debug_rule(context, p)
         else:
-            sheerka.set_debug_var(context, pattern)
+            if debug_var:
+                sheerka.set_debug_var(context, pattern)
+            if debug_concept:
+                sheerka.set_debug_concept(context, pattern)
+            if debug_rule:
+                sheerka.set_debug_rule(context, pattern)
 
         # the see the logs, do not forget to add
         # logs = sheerka.get_debugger_logs()

tests/TestUsingFileBasedSheerka.py

@@ -1,8 +1,10 @@
+import shutil
+from os import path
+
 from conftest import SHEERKA_TEST_FOLDER
 from core.global_symbols import EVENT_ONTOLOGY_CREATED
 from core.sheerka.Sheerka import Sheerka
 from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager
-
 from tests.BaseTest import BaseTest
 
 
@@ -45,3 +47,18 @@ class TestUsingFileBasedSheerka(BaseTest):
 
         self.sheerka.push_ontology(self.context, ontology_name, cache_only=cache_only)
         return TestUsingFileBasedSheerka.sheerka
+
+    @staticmethod
+    def commit(context):
+        sheerka = context.sheerka
+        event = context.event
+        sheerka.om.save_event(event)
+
+        if sheerka.om.is_dirty():
+            sheerka.om.commit(context)
+
+    @staticmethod
+    def reset_hard_test_env():
+        if path.exists(SHEERKA_TEST_FOLDER):
+            shutil.rmtree(SHEERKA_TEST_FOLDER)
+            TestUsingFileBasedSheerka.sheerka = None

tests/core/test_SheerkaErrorManager.py

@@ -0,0 +1,463 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, UnknownConcept
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.SheerkaConceptManager import ValueNotFound
+from core.sheerka.services.SheerkaErrorManager import LAST_UNKNOWN_CONCEPTS, SheerkaErrorManager
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.sheerka.services.SheerkaRuleManager import NoConditionFound
+from evaluators.PythonEvaluator import PythonEvaluator
+from parsers.BaseParser import ParsingError, UnexpectedEofParsingError
+from parsers.BnfNodeParser import BnfNodeParser
+from parsers.ExactConceptParser import ExactConceptParser
+from parsers.PythonParser import PythonErrorNode, PythonParser
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def inner_errors(lst):
+    return [e.error for e in lst]
+
+
+class TestSheerkaErrorManger(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("obj, expected", [
+        ("a string", []),
+        (True, []),
+        (False, []),
+        (Concept("foo"), []),
+        (Concept("foo", body=False).auto_init(), []),
+        (UnknownConcept(), [UnknownConcept()]),
+        (Concept("foo", body=UnknownConcept()).auto_init(), [UnknownConcept()]),
+        (PythonErrorNode("msg", None), [PythonErrorNode("msg", None)])
+    ])
+    def test_i_can_get_error_for_simple_objects(self, obj, expected):
+        sheerka, context = self.init_test().unpack()
+
+        assert sheerka.get_obj_errors(context, obj) == expected
+
+    def test_i_can_get_error_when_builtin_concept_in_error(self):
+        sheerka, context = self.init_test().unpack()
+
+        obj = sheerka.new(BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED)
+        assert sheerka.get_obj_errors(context, obj) == [obj]
+
+    def test_i_can_get_error_when_return_value(self):
+        sheerka, context = self.init_test().unpack()
+
+        error = sheerka.err("an error")
+        ret_val = ReturnValueConcept("Test", False, sheerka.err("an error"))
+        assert sheerka.get_obj_errors(context, ret_val) == [error]
+
+    def test_i_can_get_inner_error(self):
+        sheerka, context = self.init_test().unpack()
+
+        error = sheerka.err("an error")
+        ret_val = ReturnValueConcept("Test", False, sheerka.err("an error"))
+        assert sheerka.get_obj_errors(context, ret_val) == [error]
+
+    def test_i_can_get_error_when_embedded_errors(self):
+        sheerka, context = self.init_test().unpack()
+
+        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
+        not_an_error = sheerka.new(BuiltinConcepts.AUTO_EVAL)
+        error = sheerka.err([concept_eval_error, unknown_concept, not_an_error])
+
+        ret_val = ReturnValueConcept("Test", False, error)
+        errors_found = sheerka.get_obj_errors(context, ret_val)
+
+        assert errors_found == [error, concept_eval_error, unknown_concept]
+
+    def test_i_can_get_embedded_errors_from_get_error_method(self):
+        sheerka, context = self.init_test().unpack()
+
+        embedded_error = NotImplementedError()
+        error = sheerka.new(BuiltinConcepts.NOT_FOR_ME, body="some text", reason=embedded_error)
+        ret_val = ReturnValueConcept("Test", False, error)
+        errors_found = sheerka.get_obj_errors(context, ret_val)
+
+        assert errors_found == [error, embedded_error]
+
+    def test_i_can_get_error_from_list(self):
+        sheerka, context = self.init_test().unpack()
+
+        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
+        not_an_error = sheerka.new(BuiltinConcepts.AUTO_EVAL)
+        error = sheerka.err([concept_eval_error, unknown_concept, not_an_error])
+        ret_val_1 = ReturnValueConcept("Test", False, error)
+
+        python_error = PythonErrorNode("msg", Exception())
+        value_not_found = ValueNotFound("item", "value")
+        multiple_error = sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=[python_error, value_not_found])
+        ret_val_2 = ReturnValueConcept("Test", False, multiple_error)
+
+        errors_found = sheerka.get_obj_errors(context, [ret_val_1, ret_val_2])
+
+        assert errors_found == [error, concept_eval_error, unknown_concept,
+                                multiple_error, python_error, value_not_found]
+
+    def test_i_can_filter_error_by_concept_key(self):
+        sheerka, context = self.init_test().unpack()
+
+        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
+        python_error = PythonErrorNode("msg", Exception())
+        error = sheerka.err([concept_eval_error, unknown_concept, python_error])
+        ret_val = ReturnValueConcept("Test", False, error)
+
+        errors_found = sheerka.get_obj_errors(context, ret_val, __type=BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        assert errors_found == [concept_eval_error]
+
+    def test_i_can_filter_error_by_class_name(self):
+        sheerka, context = self.init_test().unpack()
+
+        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
+        python_error = PythonErrorNode("msg", Exception())
+        error = sheerka.err([concept_eval_error, unknown_concept, python_error])
+        ret_val = ReturnValueConcept("Test", False, error)
+
+        errors_found = sheerka.get_obj_errors(context, ret_val, __type="PythonErrorNode")
+        assert errors_found == [python_error]
+
+    def test_i_can_filter_error_by_concept_attribute(self):
+        sheerka, context = self.init_test().unpack()
+
+        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref")
+        python_error = PythonErrorNode("msg", Exception())
+        error = sheerka.err([concept_eval_error, unknown_concept, python_error])
+        ret_val = ReturnValueConcept("Test", False, error)
+
+        errors_found = sheerka.get_obj_errors(context, ret_val, concept_ref="a_concept_ref")
+        assert errors_found == [unknown_concept]
+
+    def test_i_can_filter_error_by_class_attribute(self):
+        sheerka, context = self.init_test().unpack()
+
+        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref")
+        python_error = PythonErrorNode("error source", Exception())
+        error = sheerka.err([concept_eval_error, unknown_concept, python_error])
+        ret_val = ReturnValueConcept("Test", False, error)
+
+        errors_found = sheerka.get_obj_errors(context, ret_val, source="error source")
+        assert errors_found == [python_error]
+
+    def test_i_can_filter_error_by_exception_name_and_type(self):
+        sheerka, context = self.init_test().unpack()
+
+        from evaluators.PythonEvaluator import PythonEvalError
+        name_error = NameError("foo")
+        python_eval_error = PythonEvalError(name_error, "foo", None, None)
+        error = sheerka.err([python_eval_error])
+        ret_val = ReturnValueConcept("Test", False, error)
+
+        errors_found = sheerka.get_obj_errors(context, ret_val, __type="NameError")
+        assert errors_found == [name_error]
+
+        errors_found = sheerka.get_obj_errors(context, ret_val, __type=NameError)
+        assert errors_found == [name_error]
+
+    def test_i_can_filter_error_on_multiple_criteria(self):
+        sheerka, context = self.init_test().unpack()
+
+        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref")
+        value_not_found = ValueNotFound("an_item", "a value")
+        error = sheerka.err([concept_eval_error, unknown_concept, value_not_found])
+        ret_val = ReturnValueConcept("Test", False, error)
+
+        errors_found = sheerka.get_obj_errors(context, ret_val, __type="ValueNotFound", item="an_item", value="a value")
+        assert errors_found == [value_not_found]
+
+    def test_i_cannot_get_error_when_return_value_s_status_is_true(self):
+        sheerka, context = self.init_test().unpack()
+
+        ret_val = ReturnValueConcept("Test", True, sheerka.err("an error"))
+        assert sheerka.get_obj_errors(context, ret_val) == []
+
+    def test_i_can_get_error_items(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+
+        simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo"))
+        ret1 = sheerka.ret("Test1", False, simple_error)
+
+        sub_error_is_an_exception = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=NotImplementedError())
+        ret2 = sheerka.ret("Test2", False, sub_error_is_an_exception)
+
+        inner_error = sheerka.err([ParsingError(), UnexpectedEofParsingError("EOF", 0)])
+        sub_error_is_an_error = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=inner_error)
+        ret3 = sheerka.ret("Test3", False, sub_error_is_an_error)
+
+        parsing_error = sheerka.new(BuiltinConcepts.PARSER_RESULT,
+                                    parser=PythonParser(),
+                                    source="not a valid source",
+                                    body="Not totally valid parsing")
+        ret4 = sheerka.ret("Test4", False, parsing_error)
+
+        multiple_level_errors1 = sheerka.err("Err1")
+        multiple_level_errors2 = sheerka.err(multiple_level_errors1)
+        multiple_level_errors3 = sheerka.err(multiple_level_errors2)
+        ret5 = sheerka.ret("Test5", False, multiple_level_errors3)
+
+        filtered = sheerka.new(BuiltinConcepts.FILTERED, reason="not really given")
+        ret6 = sheerka.ret("Test6", False, filtered)
+
+        ret7 = sheerka.ret("Test5", True, None)
+
+        error_items = list(service.get_errors_items([ret1, ret2, ret3, ret4, ret5, ret6, ret7]))
+        assert len(error_items) == 6
+
+        # ret1 : simple_error
+        assert len(error_items[0].children) == 0
+
+        # ret2 : sub_error_is_an_exception
+        assert len(error_items[1].children) == 1
+        assert len(error_items[1].children[0].children) == 0
+
+        # ret3 : inner_error
+        assert len(error_items[2].children) == 1
+        assert len(error_items[2].children[0].children) == 2
+        assert len(error_items[2].children[0].children[0].children) == 0
+        assert len(error_items[2].children[0].children[1].children) == 0
+
+        # ret4 : parsing_error
+        assert len(error_items[3].children) == 0
+
+        # ret5 : multiple_level_errors
+        assert len(error_items[4].children) == 1
+        assert len(error_items[4].children[0].children) == 1
+        assert len(error_items[4].children[0].children[0].children) == 0
+
+        # ret6 Filtered without a real reason
+        assert len(error_items[5].children) == 0
+
+    def test_i_can_get_all_error_items(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+
+        simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo"))
+        ret1 = sheerka.ret("Test1", False, simple_error)
+
+        inner_error = sheerka.err([ParsingError(), UnexpectedEofParsingError("EOF", 0)])
+        sub_error_is_an_error = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=inner_error)
+        ret2 = sheerka.ret("Test3", False, sub_error_is_an_error)
+
+        multiple_level_errors1 = sheerka.err("Err1")
+        multiple_level_errors2 = sheerka.err(multiple_level_errors1)
+        multiple_level_errors3 = sheerka.err(multiple_level_errors2)
+        ret3 = sheerka.ret("Test5", False, multiple_level_errors3)
+
+        all_error_items = list(service.get_all_error_items([ret1, ret2, ret3]))
+
+        assert len(all_error_items) == 8
+
+    def test_i_can_keep_the_correct_error_source(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+
+        simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo"))
+        ret1 = sheerka.ret("Source1", False, simple_error)
+
+        sub_error_is_an_exception = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=NotImplementedError())
+        ret2 = sheerka.ret("Source2", False, sub_error_is_an_exception)
+
+        multiple_errors = sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=[ret1, ret2])
+        ret3 = sheerka.ret("Source3", False, multiple_errors)
+
+        error_items = list(service.get_errors_items([ret3]))
+        assert len(error_items) == 1
+        assert error_items[0].source == "Source3"
+        assert error_items[0].children[0].source == "Source1"
+        assert error_items[0].children[1].source == "Source2"
+
+    def test_i_can_get_filtered_error_explanation(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+
+        simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo"))
+        ret1 = sheerka.ret("Test1", False, simple_error)
+
+        sub_error_is_an_exception = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=NotImplementedError())
+        ret2 = sheerka.ret("Test2", False, sub_error_is_an_exception)
+        context.add_values(return_values=[ret1, ret2])
+        service.on_user_input_evaluated(context)
+
+        res = sheerka.get_last_errors(context)
+        assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
+        assert len(res.body) == 3
+
+        res = sheerka.get_last_errors(context, __type=BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(res, BuiltinConcepts.EXPLANATION)
+        assert len(res.body) == 1
+
+    def test_i_cannot_recognize_error_when_no_error(self):
+        sheerka, context = self.init_test().unpack()
+        res = sheerka.recognize_error(context, "Some error")
+        assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
+
+    def test_i_can_recognized_error_when_error_is_a_builtin_concept(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+
+        simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo"))
+        ret1 = sheerka.ret("Test1", False, simple_error)
+        context.add_values(return_values=[ret1])
+        service.on_user_input_evaluated(context)
+
+        assert sheerka.recognize_error(context, __type=BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.recognize_error(context, __type=UnknownConcept)
+        assert sheerka.recognize_error(context, __type="UnknownConcept")
+        assert sheerka.recognize_error(context, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.recognize_error(context, UnknownConcept)
+        assert sheerka.recognize_error(context, "UnknownConcept")
+        assert sheerka.recognize_error(context, f"self.name == '{BuiltinConcepts.UNKNOWN_CONCEPT}'")
+        assert sheerka.recognize_error(context, f"get_type(self) == '{BuiltinConcepts.UNKNOWN_CONCEPT}'")
+
+    def test_i_can_recognized_error_when_error_is_a_error_ErrorObj(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+
+        simple_error = sheerka.new(BuiltinConcepts.ERROR, body=NoConditionFound())
+        ret1 = sheerka.ret("Test1", False, simple_error)
+        context.add_values(return_values=[ret1])
+        service.on_user_input_evaluated(context)
+
+        assert sheerka.recognize_error(context, __type=NoConditionFound)
+        assert sheerka.recognize_error(context, __type="NoConditionFound")
+
+    def test_i_cannot_recognize_unknown_concept_when_no_error(self):
+        sheerka, context = self.init_test().unpack()
+
+        res = sheerka.has_unknown_concepts(context)
+        assert sheerka.isinstance(res, BuiltinConcepts.NOT_FOUND)
+
+    def test_i_can_manage_direct_unknown_concept(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+        exact_concept_parser = ExactConceptParser()
+
+        # a simple unknown concept
+        ret = exact_concept_parser.parse(context, ParserInput("foo bar"))
+        context.add_values(return_values=[ret])
+        service.on_user_input_evaluated(context)
+
+        assert sheerka.has_unknown_concepts(context)
+        assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == "foo bar"
+
+    def test_i_can_manage_multiple_direct_unknown_concept(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+        exact_concept_parser = ExactConceptParser()
+
+        # a simple unknown concept
+        ret1 = exact_concept_parser.parse(context, ParserInput("foo"))
+        ret2 = exact_concept_parser.parse(context, ParserInput("bar"))
+        context.add_values(return_values=[ret1, ret2])
+        service.on_user_input_evaluated(context)
+
+        assert sheerka.has_unknown_concepts(context)
+        assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == {"foo", "bar"}
+
+    @pytest.mark.parametrize("source, expected", [
+        ("foo bar", "foo"),
+        ("bar foo", "foo"),
+        ("foo bar baz", {"foo", "baz"}),
+        ("my foo bar my baz", {"my foo", "my baz"}),
+    ])
+    def test_i_can_manage_unknown_concept_when_parts_are_recognized(self, source, expected):
+        sheerka, context, foo = self.init_concepts("bar")
+        service = sheerka.services[SheerkaErrorManager.NAME]
+        exact_concept_parser = ExactConceptParser()
+
+        ret1 = exact_concept_parser.parse(context, ParserInput(source))
+        context.add_values(return_values=[ret1])
+        service.on_user_input_evaluated(context)
+
+        assert sheerka.has_unknown_concepts(context)
+        assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == expected
+
+    def test_i_can_manage_multiple_unknown_concept_when_parts_are_recognized(self):
+        sheerka, context, foo = self.init_concepts("bar")
+        service = sheerka.services[SheerkaErrorManager.NAME]
+        exact_concept_parser = ExactConceptParser()
+
+        ret1 = exact_concept_parser.parse(context, ParserInput("foo bar"))
+        ret2 = exact_concept_parser.parse(context, ParserInput("my foo bar my baz"))
+        context.add_values(return_values=[ret1, ret2])
+        service.on_user_input_evaluated(context)
+
+        assert sheerka.has_unknown_concepts(context)
+        assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == {"foo", "my foo", "my baz"}
+
+    def test_i_can_manage_unknown_concept_from_name_error(self):
+        sheerka, context = self.init_test().unpack()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+        python_parser = PythonParser()
+        python_evaluator = PythonEvaluator()
+
+        #
+        ret = python_parser.parse(context, ParserInput("foo"))
+        ret = python_evaluator.eval(context, ret)
+        context.add_values(return_values=[ret])
+        service.on_user_input_evaluated(context)
+
+        assert sheerka.has_unknown_concepts(context)
+        assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == "foo"
+
+    def test_i_can_manage_unknown_concept_from_bnf_error(self):
+        sheerka, context, quantify_x = self.init_concepts(
+            Concept("quantify x", definition="('one' | 'two')=unit x").def_var("x"),
+            create_new=True
+        )
+        service = sheerka.services[SheerkaErrorManager.NAME]
+        bnf_node_parser = BnfNodeParser()
+
+        # a simple unknown concept
+        ret = bnf_node_parser.parse(context, ParserInput("one foo"))
+
+        context.add_values(return_values=[ret])
+        service.on_user_input_evaluated(context)
+
+        assert sheerka.has_unknown_concepts(context)
+        assert sheerka.get_from_memory(context, LAST_UNKNOWN_CONCEPTS).obj == "foo"
+
+
+class TestFileBasedSheerkaErrorManger(TestUsingFileBasedSheerka):
+    def test_i_can_remember_last_error(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(message="TestingErrorManagement::get_last_errors()")
+        service = sheerka.services[SheerkaErrorManager.NAME]
+
+        simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo"))
+        ret1 = sheerka.ret("Test1", False, simple_error)
+
+        sub_error_is_an_exception = sheerka.new(BuiltinConcepts.NOT_FOR_ME, reason=NotImplementedError())
+        ret2 = sheerka.ret("Test2", False, sub_error_is_an_exception)
+
+        context.add_values(return_values=[ret1, ret2])
+        service.on_user_input_evaluated(context)
+        new_context = self.get_context(sheerka)
+        explanations = sheerka.get_last_errors(new_context)
+        self.commit(context)
+
+        assert sheerka.isinstance(explanations, BuiltinConcepts.EXPLANATION)
+        assert len(explanations.body) == 3
+        assert service.last_detected_errors_event_id == context.event.get_digest()
+        assert service.last_detected_errors_event_message == context.event.message
+        assert service.current_errors_event_id == context.event.get_digest()
+        assert service.current_errors_event_message == context.event.message
+
+        sheerka = self.new_sheerka_instance()
+        service = sheerka.services[SheerkaErrorManager.NAME]
+        assert service.last_detected_errors_event_id == context.event.get_digest()
+        assert service.last_detected_errors_event_message == context.event.message
+        assert service.current_errors_event_id == context.event.get_digest()
+        assert service.current_errors_event_message == context.event.message
+
+        self.reset_hard_test_env()

tests/core/test_SheerkaHistoryManager.py

@@ -21,7 +21,9 @@ class TestSheerkaHistoryManager(TestUsingFileBasedSheerka):
         sheerka.evaluate_user_input("five")
 
         h = list(service.history(-1))  # all
-        assert h == [
+        transformed_h = [hist(item.event.message, item.status) for item in h]
+
+        assert transformed_h == [
             hist("five", True),
             hist("def concept five as 5", True),
             hist("four", True),
@@ -37,13 +39,15 @@ class TestSheerkaHistoryManager(TestUsingFileBasedSheerka):
         ]
 
         h = list(service.history(2))
-        assert h == [
+        transformed_h = [hist(item.event.message, item.status) for item in h]
+        assert transformed_h == [
             hist("five", True),
             hist("def concept five as 5", True)
         ]
 
         h = list(service.history(2, 2))
-        assert h == [
+        transformed_h = [hist(item.event.message, item.status) for item in h]
+        assert transformed_h == [
             hist("four", True),
             hist("def concept four as 4", True),
         ]

tests/core/test_SheerkaMemory.py

@@ -3,11 +3,10 @@ from core.builtin_helpers import ensure_evaluated
 from core.concept import Concept
 from core.global_symbols import NotFound
 from core.sheerka.ExecutionContext import ExecutionContext
-from core.sheerka.services.SheerkaMemory import SheerkaMemory, MemoryObject
-
+from core.sheerka.services.SheerkaMemory import MemoryObject, SheerkaMemory
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import CV, compare_with_test_object, CB
+from tests.parsers.parsers_utils import CB, CV, compare_with_test_object
 
 
 class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
@@ -92,6 +91,33 @@ class TestSheerkaMemory(TestUsingMemoryBasedSheerka):
                                                                                   foo)}
         assert id(sheerka.get_from_memory(context, "a").obj) == id(foo)
 
+        # another object is appended
+        bar = Concept("bar")
+        sheerka.add_to_memory(context, "a", bar)
+        assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {
+            "a": [
+                MemoryObject(context.event.get_digest(), context.event.date.timestamp(), foo),
+                MemoryObject(context.event.get_digest(), context.event.date.timestamp(), bar)
+            ]}
+
+    def test_i_can_set_and_retrieve_from_memory(self):
+        sheerka, context = self.init_test().unpack()
+
+        assert sheerka.get_from_memory(context, "a") is NotFound
+
+        foo = Concept("foo")
+        sheerka.set_in_memory(context, "a", foo)
+        assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(),
+                                                                                  context.event.date.timestamp(),
+                                                                                  foo)}
+        assert id(sheerka.get_from_memory(context, "a").obj) == id(foo)
+
+        # another object replace the first one
+        sheerka.set_in_memory(context, "a", "foo")
+        assert sheerka.om.copy(SheerkaMemory.OBJECTS_ENTRY) == {"a": MemoryObject(context.event.get_digest(),
+                                                                                  context.event.date.timestamp(),
+                                                                                  "foo")}
+
     def test_i_can_use_memory_with_a_string(self):
         sheerka, context = self.init_test().unpack()
 

tests/core/test_SheerkaQueryManager.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 
 import pytest
 
+from core.builtin_concepts import NotFoundConcept
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.concept import Concept
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -143,6 +144,28 @@ class TestSheerkaQueryManager(TestUsingMemoryBasedSheerka):
         res = sheerka.filter_objects(context, lst, mapping=lambda o: o.prop1, predicate="self is a concept")
         assert res == [lst[1], lst[3]]
 
+    def test_i_can_filter_objects_by_type(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", "a12"), Concept("foo"), NotImplementedError(), NotFoundConcept()]
+
+        assert sheerka.filter_objects(context, lst, __type="A") == [lst[0]]
+        assert sheerka.filter_objects(context, lst, __type="foo") == [lst[1]]
+
+        assert sheerka.filter_objects(context, lst, __type="NotImplementedError") == [lst[2]]
+        assert sheerka.filter_objects(context, lst, __type=NotImplementedError) == [lst[2]]
+
+        assert sheerka.filter_objects(context, lst, __type=BuiltinConcepts.NOT_FOUND) == [lst[3]]
+        assert sheerka.filter_objects(context, lst, __type=NotFoundConcept) == [lst[3]]
+        assert sheerka.filter_objects(context, lst, __type="NotFoundConcept") == [lst[3]]
+
+    def test_empty_list_is_returned_when_nothing_is_found(self):
+        sheerka, context = self.init_test().unpack()
+        lst = [A("a11", "a12")]
+
+        assert sheerka.filter_objects(context, lst, __type="foo") == []
+        assert sheerka.filter_objects(context, [], __type="foo") == []
+        assert sheerka.filter_objects(context, lst, predicate="self.name == 'bar'") == []
+
     def test_i_must_select_object_property_using_string(self):
         sheerka, context = self.init_test().unpack()
 

tests/core/test_SheerkaRuleManager.py

@@ -345,23 +345,23 @@ class TestSheerkaRuleManager(TestUsingMemoryBasedSheerka):
         ("not __ret.status", "__ret", [NEGCOND("#__x_00__|__name__|'__ret.status'")]),
         ("__ret and not __ret.status", "__ret",
          ["#__x_00__|__name__|'__ret'", NEGCOND("#__x_01__|__name__|'__ret.status'")]),
-        ("not recognize(__ret.body, hello sheerka)", "__ret", ["#__x_00__|__name__|'__ret'",
-                                                               NCCOND(["#__x_00__|body|#__x_01__",
-                                                                       "#__x_01__|__is_concept__|True",
-                                                                       "#__x_01__|key|'hello __var__0'",
-                                                                       "#__x_01__|a|'__sheerka__'"])]),
         ("__ret and not __error", "__ret", ["#__x_00__|__name__|'__ret'", NEGCOND("#__x_01__|__name__|'__error'")]),
-        ("not recognize(self, hello sheerka)", "__ret", ["#__x_00__|__name__|'self'",
-                                                         NCCOND(["#__x_00__|__is_concept__|True",
-                                                                 "#__x_00__|key|'hello __var__0'",
-                                                                 "#__x_00__|a|'__sheerka__'"])]),
+        # ("not recognize(__ret.body, hello sheerka)", "__ret", ["#__x_00__|__name__|'__ret'",
+        #                                                        NCCOND(["#__x_00__|body|#__x_01__",
+        #                                                                "#__x_01__|__is_concept__|True",
+        #                                                                "#__x_01__|key|'hello __var__0'",
+        #                                                                "#__x_01__|a|'__sheerka__'"])]),
+        # ("not recognize(self, hello sheerka)", "__ret", ["#__x_00__|__name__|'self'",
+        #                                                  NCCOND(["#__x_00__|__is_concept__|True",
+        #                                                          "#__x_00__|key|'hello __var__0'",
+        #                                                          "#__x_00__|a|'__sheerka__'"])]),
     ])
     def test_i_can_get_rete_using_not(self, expression, bag_key, expected):
         sheerka, context, greetings, foo = self.init_test().with_concepts(
             Concept("greetings", definition="hello a", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
             Concept("foo"),
         ).unpack()
-        parser = ExpressionParser(old_style=True)
+        parser = ExpressionParser()
         expected_conditions = get_rete_conditions(*expected)
 
         error_sink = ErrorSink()

tests/core/test_SheerkaRuleManagerRulesCompilation.py

@@ -25,7 +25,7 @@ class BaseTestSheerkaRuleManagerRulesCompilation(TestUsingMemoryBasedSheerka):
 
     @staticmethod
     def get_conditions(context, expression):
-        parser = ExpressionParser(old_style=True)
+        parser = ExpressionParser()
         error_sink = ErrorSink()
         parser_input = ParserInput(expression)
         parser.reset_parser_input(parser_input, error_sink)
@@ -927,7 +927,7 @@ class TestSheerkaRuleManagerRulesCompilationRecognizeConcept(BaseTestSheerkaRule
             Concept("my best friend"),
             create_new=True
         ).unpack()
-        parser = ExpressionParser(old_style=True)
+        parser = ExpressionParser()
         expected = get_rete_conditions(*expected_as_list_of_str)
 
         error_sink = ErrorSink()

tests/core/test_sheerka.py

@@ -3,14 +3,13 @@ import os
 import pytest
 
 from conftest import SHEERKA_TEST_FOLDER
-from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UnknownConcept, UserInputConcept
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
 from core.builtin_concepts_ids import AllBuiltinConcepts
 from core.concept import Concept, ConceptParts, PROPERTIES_TO_SERIALIZE, get_concept_attrs
 from core.global_symbols import NotInit
 from core.sheerka.Sheerka import BASE_NODE_PARSER_CLASS, Sheerka
-from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, ValueNotFound
+from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.tokenizer import Token, TokenKind
-from parsers.PythonParser import PythonErrorNode
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -44,21 +43,25 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
 
     def test_i_can_list_builtin_concepts(self):
         sheerka = self.get_sheerka()
-        builtins = list(sheerka.get_builtins_classes_as_dict())
+        builtins, _ = sheerka.get_builtins_classes_as_dict()
 
-        assert str(BuiltinConcepts.ERROR) in builtins
-        assert str(BuiltinConcepts.RETURN_VALUE) in builtins
+        assert BuiltinConcepts.ERROR in builtins
+        assert BuiltinConcepts.RETURN_VALUE in builtins
 
     def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string(self):
         """
         Checks that a concept can be found its name
         even when there are variables in the name (ex 'hello + a' or 'a + b' )
+        Also returns the mapping between concept name and specific class name
         :return:
         """
         sheerka = self.get_sheerka()
-        for key in sheerka.get_builtins_classes_as_dict():
+        builtins_by_key, builtins_by_class_name = sheerka.get_builtins_classes_as_dict()
+        for key in builtins_by_key:
             assert sheerka.get_by_key(key) is not None
-            assert sheerka.get_by_key(str(key)) is not None
+
+        for k, v in builtins_by_class_name.items():
+            assert type(sheerka.get_by_key(v)).__name__ == k
 
     def test_i_cannot_get_when_key_is_none(self):
         sheerka = self.get_sheerka()
@@ -98,7 +101,8 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert ret.value == "value"
 
         # check the others
-        for key, concept_class in sheerka.get_builtins_classes_as_dict().items():
+        builtins_by_key, _ = sheerka.get_builtins_classes_as_dict()
+        for key, concept_class in builtins_by_key.items():
             assert isinstance(sheerka.get_by_key(key), concept_class)
 
     def test_i_can_instantiate_a_builtin_concept_when_no_specific_class(self):
@@ -426,151 +430,6 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED)
 
-    @pytest.mark.parametrize("obj, expected", [
-        ("a string", []),
-        (True, []),
-        (False, []),
-        (Concept("foo"), []),
-        (Concept("foo", body=False).auto_init(), []),
-        (UnknownConcept(), [UnknownConcept()]),
-        (Concept("foo", body=UnknownConcept()).auto_init(), [UnknownConcept()]),
-        (PythonErrorNode("msg", None), [PythonErrorNode("msg", None)])
-    ])
-    def test_i_can_get_error_for_simple_objects(self, obj, expected):
-        sheerka, context = self.init_test().unpack()
-
-        assert sheerka.get_errors(context, obj) == expected
-
-    def test_i_can_get_error_when_builtin_concept_in_error(self):
-        sheerka, context = self.init_test().unpack()
-
-        obj = sheerka.new(BuiltinConcepts.ONTOLOGY_ALREADY_DEFINED)
-        assert sheerka.get_errors(context, obj) == [obj]
-
-    def test_i_can_get_error_when_return_value(self):
-        sheerka, context = self.init_test().unpack()
-
-        error = sheerka.err("an error")
-        ret_val = ReturnValueConcept("Test", False, sheerka.err("an error"))
-        assert sheerka.get_errors(context, ret_val) == [error]
-
-    def test_i_can_get_inner_error(self):
-        sheerka, context = self.init_test().unpack()
-
-        error = sheerka.err("an error")
-        ret_val = ReturnValueConcept("Test", False, sheerka.err("an error"))
-        assert sheerka.get_errors(context, ret_val) == [error]
-
-    def test_i_can_get_error_when_embedded_errors(self):
-        sheerka, context = self.init_test().unpack()
-
-        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
-        not_an_error = sheerka.new(BuiltinConcepts.AUTO_EVAL)
-        error = sheerka.err([concept_eval_error, unknown_concept, not_an_error])
-
-        ret_val = ReturnValueConcept("Test", False, error)
-        errors_found = sheerka.get_errors(context, ret_val)
-
-        assert errors_found == [error, concept_eval_error, unknown_concept]
-
-    def test_i_can_get_error_from_list(self):
-        sheerka, context = self.init_test().unpack()
-
-        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
-        not_an_error = sheerka.new(BuiltinConcepts.AUTO_EVAL)
-        error = sheerka.err([concept_eval_error, unknown_concept, not_an_error])
-        ret_val_1 = ReturnValueConcept("Test", False, error)
-
-        python_error = PythonErrorNode("msg", Exception())
-        value_not_found = ValueNotFound("item", "value")
-        multiple_error = sheerka.new(BuiltinConcepts.MULTIPLE_ERRORS, body=[python_error, value_not_found])
-        ret_val_2 = ReturnValueConcept("Test", False, multiple_error)
-
-        errors_found = sheerka.get_errors(context, [ret_val_1, ret_val_2])
-
-        assert errors_found == [error, concept_eval_error, unknown_concept,
-                                multiple_error, python_error, value_not_found]
-
-    def test_i_can_filter_error_by_concept_key(self):
-        sheerka, context = self.init_test().unpack()
-
-        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
-        python_error = PythonErrorNode("msg", Exception())
-        error = sheerka.err([concept_eval_error, unknown_concept, python_error])
-        ret_val = ReturnValueConcept("Test", False, error)
-
-        errors_found = sheerka.get_errors(context, ret_val, __type=BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        assert errors_found == [concept_eval_error]
-
-    def test_i_can_filter_error_by_class_name(self):
-        sheerka, context = self.init_test().unpack()
-
-        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
-        python_error = PythonErrorNode("msg", Exception())
-        error = sheerka.err([concept_eval_error, unknown_concept, python_error])
-        ret_val = ReturnValueConcept("Test", False, error)
-
-        errors_found = sheerka.get_errors(context, ret_val, __type="PythonErrorNode")
-        assert errors_found == [python_error]
-
-    def test_i_can_filter_error_by_concept_attribute(self):
-        sheerka, context = self.init_test().unpack()
-
-        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref")
-        python_error = PythonErrorNode("msg", Exception())
-        error = sheerka.err([concept_eval_error, unknown_concept, python_error])
-        ret_val = ReturnValueConcept("Test", False, error)
-
-        errors_found = sheerka.get_errors(context, ret_val, concept_ref="a_concept_ref")
-        assert errors_found == [unknown_concept]
-
-    def test_i_can_filter_error_by_class_attribute(self):
-        sheerka, context = self.init_test().unpack()
-
-        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref")
-        python_error = PythonErrorNode("error source", Exception())
-        error = sheerka.err([concept_eval_error, unknown_concept, python_error])
-        ret_val = ReturnValueConcept("Test", False, error)
-
-        errors_found = sheerka.get_errors(context, ret_val, source="error source")
-        assert errors_found == [python_error]
-
-    def test_i_can_filter_error_by_exception_name(self):
-        sheerka, context = self.init_test().unpack()
-
-        from evaluators.PythonEvaluator import PythonEvalError
-        name_error = NameError("foo")
-        python_eval_error = PythonEvalError(name_error, "foo", None, None)
-        error = sheerka.err([python_eval_error])
-        ret_val = ReturnValueConcept("Test", False, error)
-
-        errors_found = sheerka.get_errors(context, ret_val, __type="NameError")
-        assert errors_found == [name_error]
-
-    def test_i_can_filter_error_on_multiple_criteria(self):
-        sheerka, context = self.init_test().unpack()
-
-        concept_eval_error = sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        unknown_concept = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, concept_ref="a_concept_ref")
-        value_not_found = ValueNotFound("an_item", "a value")
-        error = sheerka.err([concept_eval_error, unknown_concept, value_not_found])
-        ret_val = ReturnValueConcept("Test", False, error)
-
-        errors_found = sheerka.get_errors(context, ret_val, __type="ValueNotFound", item="an_item", value="a value")
-        assert errors_found == [value_not_found]
-
-    def test_i_cannot_get_error_when_return_value_s_status_is_true(self):
-        sheerka, context = self.init_test().unpack()
-
-        ret_val = ReturnValueConcept("Test", True, sheerka.err("an error"))
-        assert sheerka.get_errors(context, ret_val) == []
-
     def test_i_can_add_and_remove_global_context_hints(self):
         sheerka, context = self.init_test().unpack()
 
@@ -592,7 +451,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         ).unpack()
 
         res = sheerka.evaluate_user_input("eval foo", "testing_user")
-        assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONDITION_FAILED)  # sanity check
+        assert sheerka.isinstance(res[0].value.value, BuiltinConcepts.CONDITION_FAILED)  # sanity check
 
         sheerka.add_to_context(BuiltinConcepts.EVAL_GLOBAL_TRUTH_REQUESTED)
         res = sheerka.evaluate_user_input("eval foo", "testing_user")
@@ -600,7 +459,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
 
         sheerka.remove_fom_context(BuiltinConcepts.EVAL_GLOBAL_TRUTH_REQUESTED)
         res = sheerka.evaluate_user_input("eval foo", "testing_user")
-        assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONDITION_FAILED)
+        assert sheerka.isinstance(res[0].value.value, BuiltinConcepts.CONDITION_FAILED)
 
 
 class TestSheerkaUsingFileBasedSheerka(TestUsingFileBasedSheerka):

tests/core/test_sheerka_ontology.py

@@ -1316,11 +1316,11 @@ class TestSheerkaOntology(TestUsingMemoryBasedSheerka):
         sheerka.create_new_rule(context, Rule(ACTION_TYPE_EXEC, "rule3", "id3.attr3 == 'value'", "True"))
 
         sheerka.pop_ontology(context)
-        assert rules_by_ontology_from_cache() == {'#unit_test#': {'10'}}
-        assert ontologies_from_cache() == {'10': '#unit_test#'}
+        assert rules_by_ontology_from_cache() == {'#unit_test#': {'11'}}
+        assert ontologies_from_cache() == {'11': '#unit_test#'}
 
         # check that the 'rule is deleted' events are raised
-        assert events_raised == {'11', '12'}
+        assert events_raised == {'12', '13'}
 
     # def test_i_can_list_by_key_when_dictionaries(self):
     #     sheerka = self.get_sheerka(cache_only=False)
@@ -1664,6 +1664,6 @@ class TestSheerkaOntologyWithFileBasedSheerka(TestUsingFileBasedSheerka):
         rules_from_db = sheerka.om.self_cache_manager.sdp.get(SheerkaOntologyManager.RULES_BY_ONTOLOGY_ENTRY)
         del rules_from_db["__default__"]
         assert rules_from_db == {
-            '#unit_test#': {'10'},
-            'another ontology': {'13'},
-            'new ontology': {'12'}}
+            '#unit_test#': {'11'},
+            'another ontology': {'14'},
+            'new ontology': {'13'}}

tests/evaluators/test_OneErrorEvaluator.py

@@ -49,7 +49,8 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka):
         res = evaluator.eval(context, return_values)
 
         assert not res.status
-        assert res.body == "evaluators.one error"
+        assert context.sheerka.isinstance(res.body, BuiltinConcepts.RETURN_VALUE)
+        assert res.body.body == "evaluators.one error"
         assert len(res.parents) == 4
 
     def test_unwanted_return_values_are_not_eaten(self):
@@ -71,7 +72,8 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka):
         res = evaluator.eval(context, return_values)
 
         assert not res.status
-        assert res.body == "evaluators.one error"
+        assert context.sheerka.isinstance(res.body, BuiltinConcepts.RETURN_VALUE)
+        assert res.body.body == "evaluators.one error"
         assert len(res.parents) == 4
 
         assert a_successful_concept not in res.parents
@@ -89,7 +91,7 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluator.matches(context, return_values)
         res = evaluator.eval(context, return_values)
         assert not res.status
-        assert res.body == false_1.body
+        assert res.body.body == false_1.body
         assert res.parents == return_values
 
         evaluator.reset()
@@ -97,7 +99,7 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluator.matches(context, return_values)
         res = evaluator.eval(context, return_values)
         assert not res.status
-        assert res.body == filtered
+        assert res.body.body == filtered
         assert res.parents == return_values
 
         evaluator.reset()
@@ -105,5 +107,5 @@ class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluator.matches(context, return_values)
         res = evaluator.eval(context, return_values)
         assert not res.status
-        assert res.body == false_1.body
+        assert res.body.body == false_1.body
         assert res.parents == return_values

tests/evaluators/test_PythonEvaluator.py

@@ -11,7 +11,7 @@ from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
 from evaluators.PythonEvaluator import PythonEvaluator, PythonEvalError
 from parsers.BaseNodeParser import SourceCodeNode, SourceCodeWithConceptNode
-from parsers.FunctionParserOld import FunctionParserOld
+from parsers.FunctionParser import FunctionParser
 from parsers.PythonParser import PythonNode, PythonParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import CB, compare_with_test_object
@@ -318,7 +318,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         context.add_to_short_term_memory("get_obj_name", get_obj_name)
 
-        parsed = FunctionParserOld().parse(context, ParserInput("get_obj_name(r:|1:)"))
+        parsed = FunctionParser().parse(context, ParserInput("get_obj_name(r:|1:)"))
         python_evaluator = PythonEvaluator()
         evaluated = python_evaluator.eval(context, parsed)
 
@@ -344,7 +344,7 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         context.add_to_short_term_memory("return_return_value", return_return_value)
 
-        parsed = FunctionParserOld().parse(context, ParserInput(method))
+        parsed = FunctionParser().parse(context, ParserInput(method))
         python_evaluator = PythonEvaluator()
         evaluated = python_evaluator.eval(context, parsed)
         ret_val = return_return_value(expected_status)
@@ -371,9 +371,9 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("parser, value", [
         (PythonParser(), "3"),
-        (FunctionParserOld(), "3"),
+        (FunctionParser(), "3"),
         (PythonParser(), 3),
-        (FunctionParserOld(), 3),
+        (FunctionParser(), 3),
     ])
     def test_i_can_eval_unresolved_rules(self, parser, value):
         context = self.get_context()

tests/evaluators/test_ValidateConceptEvaluator.py

@@ -9,6 +9,7 @@ from evaluators.ValidateConceptEvaluator import ValidateConceptEvaluator
 from parsers.BaseNodeParser import ConceptNode
 from parsers.BaseParser import BaseParser
 from parsers.BnfNodeParser import BnfNodeParser
+from parsers.ExactConceptParser import ExactConceptParser
 from parsers.SyaNodeParser import SyaNodeParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.evaluators.EvaluatorTestsUtils import p_ret_val, pr_ret_val, ret_val
@@ -185,3 +186,15 @@ class TestValidateConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.FILTERED)
         assert res.body.body == ret_val.body.body
+
+    def test_i_can_eval_exact_parser_concepts(self):
+        sheerka, context, explain_x = self.init_concepts(
+            Concept("explain x", where="isinstance(x, int)").def_var("x")
+        )
+        evaluator = ValidateConceptEvaluator()
+
+        ret_val = ExactConceptParser().parse(context, ParserInput("explain 1"))[0]
+
+        assert evaluator.matches(context, ret_val)
+        res = evaluator.eval(context, ret_val)
+        assert sheerka.isinstance(res.body.body, explain_x)

tests/non_reg/test_sheerka_non_reg.py

@@ -166,7 +166,7 @@ as:
 
         assert len(res) == 1
         assert not res[0].status
-        assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+        assert sheerka.isinstance(res[0].value.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
 
     @pytest.mark.parametrize("text", [
         "",
@@ -740,7 +740,7 @@ as:
         res = sheerka.evaluate_user_input("eval foobar")  # where clause is forced
         assert len(res) == 1  # error
         assert not res[0].status
-        assert sheerka.isinstance(res[0].body, BuiltinConcepts.CONDITION_FAILED)
+        assert sheerka.isinstance(res[0].body.value, BuiltinConcepts.CONDITION_FAILED)
 
     @pytest.mark.skip("Not ready for that")
     def test_i_can_manage_missing_variables_from_bnf_parsing(self):
@@ -921,15 +921,15 @@ as:
         res = sheerka.evaluate_user_input("eval foo plus one")
 
         assert not res[0].status
-        assert context.sheerka.isinstance(res[0].body, BuiltinConcepts.CONCEPT_EVAL_ERROR)
-        assert context.sheerka.isinstance(res[0].body.body, BuiltinConcepts.ERROR)
+        assert context.sheerka.isinstance(res[0].body.body, BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        assert context.sheerka.isinstance(res[0].body.body.body, BuiltinConcepts.ERROR)
 
-        error0 = res[0].body.body.body[0]
+        error0 = res[0].body.body.body.body[0]
         assert isinstance(error0, PythonEvalError)
         assert isinstance(error0.error, TypeError)
         assert error0.error.args[0] == "unsupported operand type(s) for +: 'Concept' and 'int'"
 
-        error1 = res[0].body.body.body[1]
+        error1 = res[0].body.body.body.body[1]
         assert isinstance(error1, PythonEvalError)
         assert isinstance(error1.error, TypeError)
         assert error1.error.args[0] == 'can only concatenate str (not "int") to str'

tests/non_reg/test_sheerka_non_reg2.py

@@ -123,7 +123,7 @@ class TestSheerkaNonRegMemory2(TestUsingMemoryBasedSheerka):
 
         assert len(res) == 1
         assert not res[0].status
-        assert isinstance(res[0].body, ConceptEvalError)
+        assert isinstance(res[0].body.body, ConceptEvalError)
 
     def test_74_keyword_parameters_are_no_longer_recognized_when_a_concept_that_redefines_equality_is_created(self):
         init = [

tests/non_reg/test_sheerka_non_reg_out.py

@@ -1,3 +1,7 @@
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.SheerkaErrorManager import SheerkaErrorManager
+from core.utils import no_color_str
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -200,4 +204,39 @@ two: (1002)two
         assert captured.out == """DebugItem(type=vars, setting=Sya.parsers.*, context_id=45, debug_id=None, context_children=False, debug_children=False (enabled=True))
 DebugItem(type=concepts, setting=c:|1015.*.*, context_id=13, debug_id=None, context_children=True, debug_children=False (enabled=True))
 DebugItem(type=rules, setting=Out.*.*, context_id=None, debug_id=None, context_children=False, debug_children=False (enabled=True))
+"""
+
+    def test_i_can_display_errors(self, capsys):
+        sheerka = self.get_sheerka()
+        context = self.get_context(message="xxxTESTxxx::get_last_errors()")
+        service = sheerka.services[SheerkaErrorManager.NAME]
+
+        simple_error = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=Concept("foo"))
+        ret1 = sheerka.ret("Test1", False, simple_error)
+
+        multiple_level_errors1 = sheerka.err("Err1")
+        multiple_level_errors2 = sheerka.err(multiple_level_errors1)
+        multiple_level_errors3 = sheerka.err(multiple_level_errors2)
+        ret2 = sheerka.ret("Test5", False, multiple_level_errors3)
+
+        context.add_values(return_values=[ret1, ret2])
+        service.on_user_input_evaluated(context)
+
+        sheerka.enable_process_return_values = True
+        sheerka.evaluate_user_input("get_last_errors(level=0)")
+
+        captured = capsys.readouterr()
+        colorless_captured = no_color_str(captured.out)
+        assert colorless_captured == """xxx : xxxTESTxxx::get_last_errors()
+[  0] Test1 (47)__UNKNOWN_CONCEPT: (None)foo
+[  1] Test5 (46)__ERROR: (46)__ERROR: (46)__ERROR: Err1
+"""
+
+        sheerka.evaluate_user_input("get_last_errors(id=1, depth=3)")
+        captured = capsys.readouterr()
+        colorless_captured = no_color_str(captured.out)
+        assert colorless_captured == """xxx : xxxTESTxxx::get_last_errors()
+[  1] Test5 (46)__ERROR: (46)__ERROR: (46)__ERROR: Err1
+[  2]   Test5 (46)__ERROR: (46)__ERROR: Err1
+[  3]     Test5 (46)__ERROR: Err1
 """

tests/parsers/parsers_utils.py

@@ -10,12 +10,10 @@ from core.tokenizer import Token, TokenKind, Tokenizer
 from core.utils import get_text_from_tokens, str_concept, tokens_index
 from parsers.BaseExpressionParser import AndNode, BinaryNode, ComparisonNode, ComparisonType, Comprehension, \
     FunctionNode, \
-    FunctionParameter, \
     ListComprehensionNode, ListNode, NameExprNode, \
     NotNode, OrNode, SequenceNode, VariableNode, t_comma
 from parsers.BaseNodeParser import ConceptNode, RuleNode, SourceCodeNode, SourceCodeWithConceptNode, \
     UnrecognizedTokensNode
-from parsers.FunctionParserOld import FunctionNodeOld
 from parsers.PythonParser import PythonNode
 from sheerkapython.python_wrapper import sheerka_globals
 from sheerkarete.common import V
@@ -387,99 +385,6 @@ class LC(ExprTestObj):  # for List Comprehension node
         return ListComprehensionNode(start, end, full_text_as_tokens[start: end + 1], element, comprehensions)
 
 
-class FNOld(ExprTestObj):
-    """
-    Test class only
-    It matches with FunctionNodeOld but with less constraints
-
-    Thereby,
-    FNOld("first", "last", ["param1," ...]) can be compared to
-    FunctionNodeOld(NameExprNode("first"), NameExprNode("second"),  [FunctionParameter(NamesNodes("param1"), NamesNodes(", ")])
-
-    Note that FunctionParameter can easily be defined with a single string
-        * "param" -> FunctionParameter(NameExprNode("param"), None)
-        * "param, " -> FunctionParameter(NameExprNode("param"), NameExprNode(", "))
-    For more complicated situations, you can use a tuple (value, sep) to define the value part and the separator part
-    """
-
-    def __init__(self, first, last, parameters):
-        self.first = first
-        self.last = last
-        self.parameters = []
-        for param in parameters:
-            if isinstance(param, tuple):
-                self.parameters.append(param)
-            elif isinstance(param, str) and (pos := param.find(",")) != -1:
-                self.parameters.append((param[:pos], param[pos:]))
-            else:
-                self.parameters.append((param, None))
-
-    def __repr__(self):
-        res = self.first
-        for param in self.parameters:
-            if param[1]:
-                res += f"{param[0]}{param[1]} "
-            else:
-                res += f"{param[0]}"
-        return res + self.last
-
-    def __eq__(self, other):
-        if id(self) == id(other):
-            return True
-
-        if isinstance(other, FNOld):
-            return self.first == other.first and self.last == other.last and self.parameters == other.parameters
-
-        return False
-
-    def __hash__(self):
-        return hash((self.first, self.last, self.parameters))
-
-    def transform_real_obj(self, other, get_test_obj_delegate):
-        if isinstance(other, FNOld):
-            return other
-
-        if isinstance(other, FunctionNodeOld):
-            params = []
-            for self_parameter, other_parameter in zip(self.parameters, other.parameters):
-                if isinstance(self_parameter[0], str):
-                    value = other_parameter.value.value
-                else:
-                    value = get_test_obj_delegate(other_parameter.value, self_parameter[0])
-                sep = other_parameter.separator.value if other_parameter.separator else None
-                params.append((value, sep))
-
-            return FNOld(other.first.value, other.last.value, params)
-
-        raise Exception(f"Expecting FunctionNodeOld but received {other=}")
-
-    def get_expr_node(self, full_text_as_tokens, default_expr_obj):
-        start, end = self.get_pos_from_source(self.first, full_text_as_tokens)
-        first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
-        start, end = self.get_pos_from_source(self.last, full_text_as_tokens)
-        last = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
-        parameters = []
-        for param_value, sep in self.parameters:
-            if isinstance(param_value, str):
-                start, end = self.get_pos_from_source(param_value, full_text_as_tokens)
-                param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
-            else:
-                param_as_expr_node = param_value.get_expr_node(full_text_as_tokens, default_expr_obj)
-
-            if sep:
-                sep_tokens = Tokenizer(sep, yield_eof=False)
-                start = param_as_expr_node.end + 1
-                end = start + len(list(sep_tokens)) - 1
-                sep_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
-            else:
-                sep_as_expr_node = None
-
-            parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
-
-        start, end = first.start, last.end
-        return FunctionNodeOld(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
-
-
 class HelperWithPos:
     def __init__(self, start=None, end=None):
         self.start = start

tests/parsers/test_BnfNodeParser.py

@@ -9,6 +9,7 @@ from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.SheerkaIsAManager import SheerkaIsAManager
+from core.tokenizer import Tokenizer
 from parsers.BaseNodeParser import NoMatchingTokenError
 from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor, BnfNodeParser, ConceptExpression, Match, NonTerminalNode, \
@@ -1075,6 +1076,44 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         expected = [CNC("foo", "one onyx", x=CC("one"))]
         self.validate_get_concepts_sequences(my_map, text, expected)
 
+    def test_i_can_match_regex_when_sub_parser_input(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", RegExMatch("[a-f0-9]{4}")),
+        }
+        text = "begin 0af0 end"
+        parser_input = ParserInput(text).reset()
+        sub_parser = parser_input.sub_part(2, 3)
+
+        sheerka, context, parser = self.init_parser(my_map)
+
+        res = parser.parse(context, sub_parser)
+
+        assert res.status
+        concept_nodes = res.body.body
+
+        expected = [CN("foo", "0af0")]
+        actual, expected = tests.parsers.parsers_utils.prepare_nodes_comparison(my_map, text, concept_nodes, expected)
+        assert actual == expected
+
+    def test_i_can_match_regex_when_from_tokens(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", RegExMatch("[a-f0-9]{4}")),
+        }
+        text = "begin 0af0 end"
+        tokens = list(Tokenizer(text))
+        parser_input = ParserInput(None, tokens[2:4]).reset()
+
+        sheerka, context, parser = self.init_parser(my_map)
+
+        res = parser.parse(context, parser_input)
+
+        assert res.status
+        concept_nodes = res.body.body
+
+        expected = [CN("foo", "0af0")]
+        actual, expected = tests.parsers.parsers_utils.prepare_nodes_comparison(my_map, "0af0", concept_nodes, expected)
+        assert actual == expected
+
     def test_i_can_reuse_the_same_variable(self):
         # in this test, the variable appears several times, but only once in concept.compiled
         my_map = {
@@ -1831,7 +1870,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
-        assert res.body.reason == [NoMatchingTokenError(4)]
+        assert res.body.reason == [NoMatchingTokenError(4, concept=foo)]
 
     @pytest.mark.parametrize("text", [
         "one",
@@ -1852,7 +1891,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         (Sequence(StrMatch("foo"), VariableExpression("x")), "foo one"),
         (Sequence(StrMatch("foo"), VariableExpression("x"), StrMatch("bar")), "foo one bar"),
     ])
-    def test_i_cannot_parse_variable_when_unrecognized_nodes(self, bnf, text):
+    def test_i_cannot_parse_variable_when_unrecognized_nodes(self, bnf, text): 
         sheerka, context, foo = self.init_test().with_concepts(
             self.bnf_concept("foo", Sequence(VariableExpression("x"), StrMatch("shoe")))
         ).unpack()

tests/parsers/test_DefConceptParser.py

@@ -16,7 +16,7 @@ from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Se
 from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode, CannotHandleParsingError
 from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode
 from parsers.ExpressionParser import ExpressionParser
-from parsers.FunctionParserOld import FunctionParserOld
+from parsers.FunctionParser import FunctionParser
 from parsers.PythonParser import PythonParser, PythonNode
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import compute_expected_array, SCWC, compare_with_test_object, CIO
@@ -77,7 +77,7 @@ def get_concept_part(part, use_expression=False):
             status=True,
             value=ParserResultConcept(
                 source=part.source,
-                parser=FunctionParserOld(),
+                parser=FunctionParser(),
                 value=nodes[0],
                 try_parsed=nodes[0]))
 

tests/parsers/test_FunctionParserOld.py

@@ -1,249 +0,0 @@
-import pytest
-
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
-from core.sheerka.services.SheerkaExecute import ParserInput
-from parsers.BaseNodeParser import SourceCodeWithConceptNode
-from parsers.BaseParser import ErrorSink
-from parsers.FunctionParserOld import FunctionParserOld
-from parsers.PythonParser import PythonErrorNode
-from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import compute_expected_array, SCN, SCWC, CN, UTN, CNC, RN, FNOld, get_test_obj, \
-    get_expr_node_from_test_node
-
-cmap = {
-    "one": Concept("one"),
-    "two": Concept("two"),
-    "twenties": Concept("twenties", definition="'twenty' (one|two)=unit").def_var("unit"),
-    "plus": Concept("a plus b").def_var("a").def_var("b"),
-}
-
-
-class TestFunctionParserOld(TestUsingMemoryBasedSheerka):
-    shared_ontology = None
-
-    @classmethod
-    def setup_class(cls):
-        init_test_helper = cls().init_test(cache_only=False, ontology="#TestFunctionParserOld#")
-        sheerka, context, *updated = init_test_helper.with_concepts(*cmap.values(), create_new=True).unpack()
-        for i, concept_name in enumerate(cmap):
-            cmap[concept_name] = updated[i]
-
-        cls.shared_ontology = sheerka.get_ontology(context)
-        sheerka.pop_ontology(context)
-
-    def init_parser(self, my_concepts_map=None, **kwargs):
-        if my_concepts_map is None:
-            sheerka, context = self.init_test().unpack()
-            sheerka.add_ontology(context, self.shared_ontology)
-        else:
-            sheerka, context, *updated = self.init_test().with_concepts(*my_concepts_map.values(), **kwargs).unpack()
-            for i, pair in enumerate(my_concepts_map):
-                my_concepts_map[pair] = updated[i]
-
-        parser = FunctionParserOld()
-        return sheerka, context, parser
-
-    def init_parser_with_source(self, source):
-        sheerka, context, parser = self.init_parser()
-        error_sink = ErrorSink()
-        parser_input = ParserInput(source)
-        parser.reset_parser_input(parser_input, error_sink)
-        return sheerka, context, parser, parser_input, error_sink
-
-    def test_i_can_detect_empty_expression(self):
-        sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, ParserInput(""))
-
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.IS_EMPTY)
-
-    def test_input_must_be_a_parser_input(self):
-        sheerka, context, parser = self.init_parser()
-        parser.parse(context, "not a parser input") is None
-
-    def test_i_cannot_parse_when_not_a_function(self):
-        sheerka, context, parser = self.init_parser()
-        res = parser.parse(context, ParserInput("not a function"))
-
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
-
-    @pytest.mark.parametrize("expression, expected", [
-        ("func()", FNOld("func(", ")", [])),
-        ("concept(one)", FNOld("concept(", ")", ["one"])),
-        ("func(one)", FNOld("func(", ")", ["one"])),
-        ("func(a long two, 'three', ;:$*)", FNOld("func(", ")", ["a long two, ", "'three', ", ";:$*"])),
-        ("func(func1(one), two, func2(func3(), func4(three)))", FNOld("func(", (")", 4), [
-            (FNOld("func1(", ")", ["one"]), ", "),
-            "two, ",
-            (FNOld("func2(", (")", 3), [
-                (FNOld("func3(", (")", 1), []), ", "),
-                (FNOld("func4(", (")", 2), ["three"]), None),
-            ]), None)
-        ])),
-        ("func(r:|1:)", FNOld("func(", ")", ["r:|1:"]))
-    ])
-    def test_i_can_parse_function(self, expression, expected):
-        sheerka, context, parser, parser_input, error_sink = self.init_parser_with_source(expression)
-        expected = get_expr_node_from_test_node(expression, expected)
-
-        parsed = parser.parse_input(context, parser_input, error_sink)
-
-        assert not error_sink.has_error
-        assert parsed == expected
-
-    @pytest.mark.parametrize("text, expected", [
-        ("func()", SCN("func()")),
-        (" func()", SCN("func()")),
-        ("func(one)", SCWC("func(", ")", CN("one"))),
-        ("func(one, unknown, two)", SCWC("func(", ")", CN("one"), ", ", UTN("unknown"), (", ", 1), CN("two"))),
-        ("func(one, twenty two)", SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))),
-        ("func(one plus two, three)", SCWC("func(", ")", CNC("plus", a="one", b="two"), ", ", UTN("three"))),
-        ("func(func1(one), two)", SCWC("func(", (")", 1), SCWC("func1(", ")", "one"), ", ", "two"))
-    ])
-    def test_i_can_parse(self, text, expected):
-        sheerka, context, parser = self.init_parser()
-        resolved_expected = compute_expected_array(cmap, text, [expected])[0]
-        res = parser.parse(context, ParserInput(text))
-        parser_result = res.body
-        expression = res.body.body
-
-        assert res.status
-        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
-        transformed_expression = get_test_obj(expression, resolved_expected)
-        assert transformed_expression == resolved_expected
-        assert expression.python_node is not None
-        assert expression.return_value is not None
-
-    def test_i_can_parse_when_multiple_results_when_requested(self):
-        # the previous output was
-        # [
-        #   SCWC("func(", ")", "one", ", ", "twenty ", "two"),
-        #   SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))
-        #  ]
-        # But the first one is now filtered out, as it's not a valid python function call
-        sheerka, context, parser = self.init_parser()
-        parser.longest_concepts_only = False
-        text = "func(one, twenty two)"
-        expected = [SCWC("func(", ")", "one", ", ", CN("twenties", "twenty two"))]
-        resolved_expected = compute_expected_array(cmap, text, expected)
-
-        results = parser.parse(context, ParserInput(text))
-
-        assert len(results) == 2
-
-        res = results[0]
-        assert not res.status
-        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
-        assert len(res.body.body) == 1
-        assert (res.body.body[0], PythonErrorNode)
-
-        res = results[1]
-        parser_result = res.body
-        expressions = res.body.body
-        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
-        transformed_expressions = get_test_obj(expressions, resolved_expected[0])
-        assert transformed_expressions == resolved_expected[0]
-
-    def test_i_can_parse_when_the_parameter_is_not_a_concept(self):
-        """
-        It's not a concept, but it can be a valid short term memory object
-        :return:
-        """
-        sheerka, context, parser = self.init_parser()
-        text = "func(unknown_concept)"
-
-        res = parser.parse(context, ParserInput(text))
-        expected = [SCWC("func(", ")", "unknown_concept")]
-        resolved_expected = compute_expected_array(cmap, text, expected)
-
-        assert res.status
-        parsed = res.body.body
-        transformed_parsed = get_test_obj([parsed], resolved_expected)
-        assert transformed_parsed == resolved_expected
-
-    def test_i_can_parse_when_the_concept_is_not_found(self):
-        """
-        We do not check yet if it's a valid concept
-        If you find a cheap way to do so, simply remove this test
-        :return:
-        """
-        sheerka, context, parser = self.init_parser()
-        text = "func(c:|xxx:)"
-
-        res = parser.parse(context, ParserInput(text))
-
-        assert res.status
-
-    def test_i_can_parse_when_rules(self):
-        sheerka, context, parser = self.init_parser()
-        text = "func(r:|1:)"
-        expected = SCWC("func(", ")", RN("1"))
-        resolved_expected = compute_expected_array(cmap, text, [expected])[0]
-
-        res = parser.parse(context, ParserInput(text))
-        parser_result = res.body
-        expression = res.body.body
-        transformed_expression = get_test_obj(expression, resolved_expected)
-
-        assert res.status
-        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
-        assert transformed_expression == resolved_expected
-        assert expression.python_node is not None
-        assert expression.return_value is not None
-
-    def test_i_can_parse_when_the_parameter_is_a_dynamic_concept(self):
-        sheerka, context, parser = self.init_parser()
-
-        text = "func(ones)"
-        res = parser.parse(context, ParserInput(text))
-
-        assert res.status
-        assert isinstance(res.body.body, SourceCodeWithConceptNode)
-        assert res.body.body.python_node.source == 'func(__C__ones__1001___PLURAL__C__)'
-        assert "__C__ones__1001___PLURAL__C__" in res.body.body.python_node.objects
-
-    @pytest.mark.parametrize("text, expected_error_type", [
-        ("one", BuiltinConcepts.NOT_FOR_ME),  # no function found
-        ("$*!", BuiltinConcepts.NOT_FOR_ME),  # no function found
-        ("func(", BuiltinConcepts.ERROR),  # function found, but incomplete
-        ("func(one", BuiltinConcepts.ERROR),  # function found, but incomplete
-        ("func(one, two, ", BuiltinConcepts.ERROR),  # function found, but incomplete
-        ("func(one) and func(two)", BuiltinConcepts.ERROR),  # to many function
-        ("one func(one)", BuiltinConcepts.NOT_FOR_ME),  # function not found ! (as it is not the first)
-        ("func(a=b, c)", BuiltinConcepts.ERROR),  # function found, but cannot be parsed
-        ("func(one two)", BuiltinConcepts.ERROR),  # function found, but cannot be parsed
-    ])
-    def test_i_cannot_parse(self, text, expected_error_type):
-        sheerka, context, parser = self.init_parser()
-
-        res = parser.parse(context, ParserInput(text))
-
-        assert not res.status
-        assert sheerka.isinstance(res.body, expected_error_type)
-
-    @pytest.mark.parametrize("sequence, expected", [
-        (None, None),
-        ([["a"]], [["a"]]),
-        ([["a"], ["b", "c"]], [["a"]]),
-        ([["b", "c"], ["a"]], [["a"]]),
-        ([["b", "c"], ["a"], ["d", "e"], ["f"]], [["a"], ["f"]]),
-    ])
-    def test_i_can_get_the_longest_concept_sequence(self, sequence, expected):
-        assert FunctionParserOld.get_longest_concepts(sequence) == expected
-
-    def test_concepts_found_are_fully_initialized(self):
-        sheerka, context, parser = self.init_parser()
-
-        res = parser.parse(context, ParserInput("func(one plus three)"))
-        concept = res.body.body.nodes[0].concept
-
-        assert res.status
-        assert isinstance(concept.get_compiled()["a"], Concept)
-
-        # three is not recognized,
-        # so it will be transformed into list of ReturnValueConcept that indicate how to recognized it
-        assert isinstance(concept.get_compiled()["b"], list)
-        for item in concept.get_compiled()["b"]:
-            assert sheerka.isinstance(item, BuiltinConcepts.RETURN_VALUE)

tests/parsers/test_SyaNodeParser.py

@@ -9,8 +9,7 @@ from core.sheerka.Sheerka import RECOGNIZED_BY_KEY
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Token, TokenKind, Tokenizer, comparable_tokens
 from core.utils import get_text_from_tokens
-from parsers.BaseExpressionParser import BinaryNode, FunctionNode, FunctionNodeOld, FunctionParameter, ListNode, \
-    NameExprNode, VariableNode
+from parsers.BaseExpressionParser import BinaryNode, FunctionNode, ListNode, NameExprNode, VariableNode
 from parsers.BaseNodeParser import ConceptNode, SourceCodeNode, UnrecognizedTokensNode
 from parsers.PythonParser import PythonNode
 from parsers.SyaNodeParser import FunctionDetected, NoSyaConceptFound, NotEnoughParameters, SyaConceptParser, \
@@ -1564,33 +1563,6 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         _stack, _expected = prepare_nodes_comparison(concepts_map, text, lexer_nodes, expected)
         assert _stack == _expected
 
-    def test_i_can_parse_when_function_old_style_expr_tokens(self):
-        sheerka, context, parser = self.init_parser()
-
-        text = "one plus func(twenty two)"
-        tokens = list(Tokenizer(text, yield_eof=False))
-        fun_token = tokens[4]
-        expr = FunctionNodeOld(4, 9, tokens[4:10],
-                               NameExprNode(4, 5, tokens[4:6]),
-                               NameExprNode(9, 9, tokens[9:10]),
-                               [FunctionParameter(NameExprNode(6, 8, tokens[6:9]), None)])
-        tokens[4:] = [Token(TokenKind.EXPR, expr, fun_token.index, fun_token.line, fun_token.column)]
-        res = parser.parse(context, ParserInput(None, tokens=tokens))
-        wrapper = res.body
-        lexer_nodes = res.body.body
-
-        assert res.status
-        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-
-        expected_ret_val = RETVAL("func(twenty two)", objects={"__o_00__": CIO("twenties", source="twenty two")})
-        expected = [CNC("plus", a=CC("one"), b=[expected_ret_val], source=text)]
-        _stack, _expected = prepare_nodes_comparison(cmap, text, lexer_nodes, expected)
-        assert _stack == _expected
-
-        # check the metadata
-        expected_concept = lexer_nodes[0].concept
-        assert expected_concept.get_metadata().variables == [("a", "one"), ("b", "func(twenty two)")]
-
     def test_i_can_parse_when_function_style_expr_tokens(self):
         sheerka, context, parser = self.init_parser()
 

tests/parsers/test_parsers_utils.py

@@ -1,13 +1,13 @@
 from core.concept import Concept, ConceptParts
 from core.global_symbols import NotInit
-from core.rule import Rule, ACTION_TYPE_EXEC
+from core.rule import ACTION_TYPE_EXEC, Rule
 from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.BaseNodeParser import RuleNode
 from parsers.BnfNodeParser import BnfNodeParser
-from parsers.FunctionParserOld import FunctionParserOld
+from parsers.FunctionParser import FunctionParser
 from parsers.SyaNodeParser import SyaNodeParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import get_test_obj, CNC, CC, CN, SCN, SCWC, UTN, RN, CB
+from tests.parsers.parsers_utils import CB, CC, CN, CNC, RN, SCN, get_test_obj
 
 
 class TestParsersUtils(TestUsingMemoryBasedSheerka):
@@ -133,7 +133,7 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka):
     def test_i_can_get_test_obj_when_SCN(self):
         sheerka, context = self.init_test().unpack()
 
-        parser = FunctionParserOld()
+        parser = FunctionParser()
         scn = parser.parse(context, ParserInput("test()")).body.body
 
         scn_res = get_test_obj(scn, SCN("", start=0, end=1))
@@ -145,29 +145,6 @@ class TestParsersUtils(TestUsingMemoryBasedSheerka):
         assert isinstance(scn_res, SCN)
         assert scn_res == SCN("test()", start=None, end=None)
 
-    def test_i_can_get_test_obj_when_SCWC(self):
-        sheerka, context = self.init_test().unpack()
-
-        parser = FunctionParserOld()
-        scwc = parser.parse(context, ParserInput("test(param1, test2())")).body.body
-
-        scwc_res = get_test_obj(scwc, SCWC(UTN(""), UTN(""), UTN(""), UTN(""), SCN("", None, 0, 0)))
-        assert isinstance(scwc_res, SCWC)
-        expected = SCWC(UTN("test(", 0, 1),
-                        UTN(")", 8, 8),
-                        UTN("param1", 2, 2),
-                        UTN(", ", 3, 4),
-                        SCN("test2()", None, 5, 7))
-        expected.start = 0
-        expected.end = 8
-        assert scwc_res == expected
-
-        assert isinstance(scwc_res.first, UTN)
-        assert isinstance(scwc_res.last, UTN)
-        assert isinstance(scwc_res.content[0], UTN)
-        assert isinstance(scwc_res.content[1], UTN)
-        assert isinstance(scwc_res.content[2], SCN)
-
     def test_i_can_get_test_obj_when_RN(self):
         rule = Rule(ACTION_TYPE_EXEC, "test_rule", "True", "True")
         rn = RuleNode(rule, 1, 1, source="r:|xxx:")

tests/sheerkapython/test_ExprToConditionsVisitor.py

@@ -87,7 +87,7 @@ class TestExprToConditionsVisitor(TestUsingMemoryBasedSheerka):
 
     @staticmethod
     def get_conditions_from_expression(context, expression, parser=None):
-        parser = parser or ExpressionParser(old_style=False)
+        parser = parser or ExpressionParser()
         error_sink = ErrorSink()
         parser_input = ParserInput(expression)
         parser.reset_parser_input(parser_input, error_sink)
@@ -293,6 +293,21 @@ class TestExprToConditionsVisitor(TestUsingMemoryBasedSheerka):
         resolved_expected_objects = {k: cmap[v].id for k, v in {"__o_00__": "two"}.items()}
         assert resolved_objects == resolved_expected_objects
 
+    def test_i_can_manage_possible_variable_indication(self):
+        my_map = {
+            "isa": Concept("x is an int").def_var("x")  # no pre condition
+        }
+        sheerka, context = self.initialize_test(my_map)
+        parser = ExpressionParser(known_variables={"x"})
+        conditions = self.get_conditions_from_expression(context, "x is an int", parser)
+
+        python_source = conditions[0].return_value.body.body.source
+        assert python_source == "call_concept(__o_00__, x=x)"
+
+        resolved_objects = {k: v.id for k, v in conditions[0].objects.items()}
+        resolved_expected_objects = {k: my_map[v].id for k, v in {"__o_00__": "isa"}.items()}
+        assert resolved_objects == resolved_expected_objects
+
     def test_i_can_parse_when_variables_are_missing(self):
         sheerka, context = self.initialize_test()
         expression = "x equals 1"

tests/sheerkapython/test_PythonExprVisitor.py

@@ -17,7 +17,7 @@ class TestPythonExprVisitor(TestUsingMemoryBasedSheerka):
 
     @staticmethod
     def get_expr_node(context, expression, parser=None):
-        parser = parser or ExpressionParser(old_style=False)
+        parser = parser or ExpressionParser()
         error_sink = ErrorSink()
         parser_input = ParserInput(expression)
         parser.reset_parser_input(parser_input, error_sink)